From 6c3c4a7d0b9e1d302d15b85e05f91c5f6a620404 Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Fri, 13 Jun 2025 17:24:13 +0200 Subject: [PATCH 001/424] Bind self-types in checkmember for decorated classmethods (#19025) Fixes #19023. Fixes #18993. --- mypy/checkmember.py | 8 +++++- test-data/unit/check-selftype.test | 41 ++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/mypy/checkmember.py b/mypy/checkmember.py index 502251b3960cc..0c2c92fc6904e 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -1230,7 +1230,7 @@ def analyze_class_attribute_access( is_trivial_self = node.node.is_trivial_self if isinstance(t, FunctionLike) and is_classmethod and not is_trivial_self: t = check_self_arg(t, mx.self_type, False, mx.context, name, mx.msg) - result = add_class_tvars( + t = add_class_tvars( t, isuper, is_classmethod, @@ -1238,6 +1238,12 @@ def analyze_class_attribute_access( original_vars=original_vars, is_trivial_self=is_trivial_self, ) + if is_decorated and not is_staticmethod: + t = expand_self_type_if_needed( + t, mx, cast(Decorator, node.node).var, itype, is_class=is_classmethod + ) + + result = t # __set__ is not called on class objects. if not mx.is_lvalue: result = analyze_descriptor_access(result, mx) diff --git a/test-data/unit/check-selftype.test b/test-data/unit/check-selftype.test index cb7e5a9fac71a..12d6133ec83f7 100644 --- a/test-data/unit/check-selftype.test +++ b/test-data/unit/check-selftype.test @@ -2219,3 +2219,44 @@ class B: class C(A, B): # OK: both methods take Self pass [builtins fixtures/tuple.pyi] + +[case testSelfInFuncDecoratedClassmethod] +from collections.abc import Callable +from typing import Self, TypeVar + +T = TypeVar("T") + +def debug(make: Callable[[type[T]], T]) -> Callable[[type[T]], T]: + return make + +class Foo: + @classmethod + @debug + def make(cls) -> Self: + return cls() + +class Bar(Foo): ... + +reveal_type(Foo.make()) # N: Revealed type is "__main__.Foo" +reveal_type(Foo().make()) # N: Revealed type is "__main__.Foo" +reveal_type(Bar.make()) # N: Revealed type is "__main__.Bar" +reveal_type(Bar().make()) # N: Revealed type is "__main__.Bar" +[builtins fixtures/tuple.pyi] + +[case testSelfInClassDecoratedClassmethod] +from typing import Callable, Generic, TypeVar, Self + +T = TypeVar("T") + +class W(Generic[T]): + def __init__(self, fn: Callable[..., T]) -> None: ... + def __call__(self) -> T: ... + +class Check: + @W + def foo(self) -> Self: + ... + +reveal_type(Check.foo()) # N: Revealed type is "def () -> __main__.Check" +reveal_type(Check().foo()) # N: Revealed type is "__main__.Check" +[builtins fixtures/tuple.pyi] From 2996c914708d4a4d6896a9a9d6da8a8fdee243ed Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Fri, 13 Jun 2025 17:25:30 +0200 Subject: [PATCH 002/424] Preserve literals when joining Literal and Instance with matching last_known_value (#19279) Discovered in #19225. --- mypy/join.py | 2 ++ test-data/unit/check-literal.test | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/mypy/join.py b/mypy/join.py index a012a633dfa3c..099df02680f06 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -625,6 +625,8 @@ def visit_literal_type(self, t: LiteralType) -> ProperType: if self.s.fallback.type.is_enum and t.fallback.type.is_enum: return mypy.typeops.make_simplified_union([self.s, t]) return join_types(self.s.fallback, t.fallback) + elif isinstance(self.s, Instance) and self.s.last_known_value == t: + return t else: return join_types(self.s, t.fallback) diff --git a/test-data/unit/check-literal.test b/test-data/unit/check-literal.test index d91b257b0096c..f995332643af7 100644 --- a/test-data/unit/check-literal.test +++ b/test-data/unit/check-literal.test @@ -2976,3 +2976,22 @@ x: Type[Literal[1]] # E: Type[...] can't contain "Literal[...]" y: Type[Union[Literal[1], Literal[2]]] # E: Type[...] can't contain "Union[Literal[...], Literal[...]]" z: Type[Literal[1, 2]] # E: Type[...] can't contain "Union[Literal[...], Literal[...]]" [builtins fixtures/tuple.pyi] + +[case testJoinLiteralAndInstance] +from typing import Generic, TypeVar, Literal + +T = TypeVar("T") + +class A(Generic[T]): ... + +def f(a: A[T], t: T) -> T: ... +def g(a: T, t: A[T]) -> T: ... + +def check(obj: A[Literal[1]]) -> None: + reveal_type(f(obj, 1)) # N: Revealed type is "Literal[1]" + reveal_type(f(obj, '')) # E: Cannot infer type argument 1 of "f" \ + # N: Revealed type is "Any" + reveal_type(g(1, obj)) # N: Revealed type is "Literal[1]" + reveal_type(g('', obj)) # E: Cannot infer type argument 1 of "g" \ + # N: Revealed type is "Any" +[builtins fixtures/tuple.pyi] From 2ba79cba94c8a416c16877c1532932662ea20d40 Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Fri, 13 Jun 2025 17:36:36 +0200 Subject: [PATCH 003/424] Use union of current context and left side for right side narrowing of binary ops (#19249) Fixes #12001. Fixes #6898. Fixes #15368. Improves #17790 and #11508. When encountering `a {and,or} b`, we used to use `a` as primary context for `b` inference. This results in weird errors when `a` and `b` are completely unrelated, and in many cases return type/assignment type context can do much better. Inferring to union should be harmless in most cases, so use union of `a` and current context instead. --- mypy/checkexpr.py | 14 +++++++++++++- test-data/unit/check-inference-context.test | 21 +++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index edc3ac70fa541..4ca55e1679e40 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -4288,7 +4288,9 @@ def check_boolean_op(self, e: OpExpr, context: Context) -> Type: ): self.msg.unreachable_right_operand(e.op, e.right) - right_type = self.analyze_cond_branch(right_map, e.right, expanded_left_type) + right_type = self.analyze_cond_branch( + right_map, e.right, self._combined_context(expanded_left_type) + ) if left_map is None and right_map is None: return UninhabitedType() @@ -5919,6 +5921,16 @@ def analyze_cond_branch( self.chk.push_type_map(map) return self.accept(node, type_context=context, allow_none_return=allow_none_return) + def _combined_context(self, ty: Type | None) -> Type | None: + ctx_items = [] + if ty is not None: + ctx_items.append(ty) + if self.type_context and self.type_context[-1] is not None: + ctx_items.append(self.type_context[-1]) + if ctx_items: + return make_simplified_union(ctx_items) + return None + # # Helpers # diff --git a/test-data/unit/check-inference-context.test b/test-data/unit/check-inference-context.test index 0aa67b2bf7f34..67ae22a369b19 100644 --- a/test-data/unit/check-inference-context.test +++ b/test-data/unit/check-inference-context.test @@ -1510,3 +1510,24 @@ def mymin( def check(paths: Iterable[str], key: Callable[[str], int]) -> Union[str, None]: return mymin(paths, key=key, default=None) [builtins fixtures/tuple.pyi] + +[case testBinaryOpInferenceContext] +from typing import Literal, TypeVar + +T = TypeVar("T") + +def identity(x: T) -> T: + return x + +def check1(use: bool, val: str) -> "str | Literal[True]": + return use or identity(val) + +def check2(use: bool, val: str) -> "str | bool": + return use or identity(val) + +def check3(use: bool, val: str) -> "str | Literal[False]": + return use and identity(val) + +def check4(use: bool, val: str) -> "str | bool": + return use and identity(val) +[builtins fixtures/tuple.pyi] From 4373e057219f4a8f511eeb8678b2863d532885e1 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 13 Jun 2025 17:27:37 +0100 Subject: [PATCH 004/424] Fix tests in master (#19293) --- mypy/checkmember.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mypy/checkmember.py b/mypy/checkmember.py index 0c2c92fc6904e..1633eaf529836 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -1221,6 +1221,9 @@ def analyze_class_attribute_access( is_classmethod = (is_decorated and cast(Decorator, node.node).func.is_class) or ( isinstance(node.node, SYMBOL_FUNCBASE_TYPES) and node.node.is_class ) + is_staticmethod = (is_decorated and cast(Decorator, node.node).func.is_static) or ( + isinstance(node.node, SYMBOL_FUNCBASE_TYPES) and node.node.is_static + ) t = get_proper_type(t) is_trivial_self = False if isinstance(node.node, Decorator): From 5081c59b9c0c7ebe7070c62a4aeaf3d0de203a24 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 14 Jun 2025 00:21:35 +0100 Subject: [PATCH 005/424] Handle corner case: protocol vs classvar vs descriptor (#19277) Ref https://github.com/python/mypy/issues/19274 This is a bit ugly. But I propose to have this "hot-fix" until we have a proper overhaul of instance vs class variables. To be clear: attribute access already works correctly (on both `P` and `Type[P]`), but subtyping returns false because of ```python elif (IS_CLASSVAR in subflags) != (IS_CLASSVAR in superflags): return False ``` --- docs/source/protocols.rst | 47 +++++++++++++++++++++++++++++ mypy/subtypes.py | 12 +++++++- test-data/unit/check-protocols.test | 44 +++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 1 deletion(-) diff --git a/docs/source/protocols.rst b/docs/source/protocols.rst index ed8d94f62ef15..258cd4b0de564 100644 --- a/docs/source/protocols.rst +++ b/docs/source/protocols.rst @@ -352,6 +352,53 @@ the parameters are positional-only. Example (using the legacy syntax for generic copy_a = copy_b # OK copy_b = copy_a # Also OK +Binding of types in protocol attributes +*************************************** + +All protocol attributes annotations are treated as externally visible types +of those attributes. This means that for example callables are not bound, +and descriptors are not invoked: + +.. code-block:: python + + from typing import Callable, Protocol, overload + + class Integer: + @overload + def __get__(self, instance: None, owner: object) -> Integer: ... + @overload + def __get__(self, instance: object, owner: object) -> int: ... + # + + class Example(Protocol): + foo: Callable[[object], int] + bar: Integer + + ex: Example + reveal_type(ex.foo) # Revealed type is Callable[[object], int] + reveal_type(ex.bar) # Revealed type is Integer + +In other words, protocol attribute types are handled as they would appear in a +``self`` attribute annotation in a regular class. If you want some protocol +attributes to be handled as though they were defined at class level, you should +declare them explicitly using ``ClassVar[...]``. Continuing previous example: + +.. code-block:: python + + from typing import ClassVar + + class OtherExample(Protocol): + # This style is *not recommended*, but may be needed to reuse + # some complex callable types. Otherwise use regular methods. + foo: ClassVar[Callable[[object], int]] + # This may be needed to mimic descriptor access on Type[...] types, + # otherwise use a plain "bar: int" style. + bar: ClassVar[Integer] + + ex2: OtherExample + reveal_type(ex2.foo) # Revealed type is Callable[[], int] + reveal_type(ex2.bar) # Revealed type is int + .. _predefined_protocols_reference: Predefined protocol reference diff --git a/mypy/subtypes.py b/mypy/subtypes.py index acb41609fdc55..a5e6938615e75 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -1457,7 +1457,8 @@ def get_member_flags(name: str, itype: Instance, class_obj: bool = False) -> set flags = {IS_VAR} if not v.is_final: flags.add(IS_SETTABLE) - if v.is_classvar: + # TODO: define cleaner rules for class vs instance variables. + if v.is_classvar and not is_descriptor(v.type): flags.add(IS_CLASSVAR) if class_obj and v.is_inferred: flags.add(IS_CLASSVAR) @@ -1465,6 +1466,15 @@ def get_member_flags(name: str, itype: Instance, class_obj: bool = False) -> set return set() +def is_descriptor(typ: Type | None) -> bool: + typ = get_proper_type(typ) + if isinstance(typ, Instance): + return typ.type.get("__get__") is not None + if isinstance(typ, UnionType): + return all(is_descriptor(item) for item in typ.relevant_items()) + return False + + def find_node_type( node: Var | FuncBase, itype: Instance, diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index f330aa4ecc028..c6c2c5f8da980 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -4602,3 +4602,47 @@ def deco(fn: Callable[[], T]) -> Callable[[], list[T]]: ... @deco def defer() -> int: ... [builtins fixtures/list.pyi] + +[case testProtocolClassValDescriptor] +from typing import Any, Protocol, overload, ClassVar, Type + +class Desc: + @overload + def __get__(self, instance: None, owner: object) -> Desc: ... + @overload + def __get__(self, instance: object, owner: object) -> int: ... + def __get__(self, instance, owner): + pass + +class P(Protocol): + x: ClassVar[Desc] + +class C: + x = Desc() + +t: P = C() +reveal_type(t.x) # N: Revealed type is "builtins.int" +tt: Type[P] = C +reveal_type(tt.x) # N: Revealed type is "__main__.Desc" + +bad: P = C # E: Incompatible types in assignment (expression has type "type[C]", variable has type "P") \ + # N: Following member(s) of "C" have conflicts: \ + # N: x: expected "int", got "Desc" + +[case testProtocolClassValCallable] +from typing import Any, Protocol, overload, ClassVar, Type, Callable + +class P(Protocol): + foo: Callable[[object], int] + bar: ClassVar[Callable[[object], int]] + +class C: + foo: Callable[[object], int] + bar: ClassVar[Callable[[object], int]] + +t: P = C() +reveal_type(t.foo) # N: Revealed type is "def (builtins.object) -> builtins.int" +reveal_type(t.bar) # N: Revealed type is "def () -> builtins.int" +tt: Type[P] = C +reveal_type(tt.foo) # N: Revealed type is "def (builtins.object) -> builtins.int" +reveal_type(tt.bar) # N: Revealed type is "def (builtins.object) -> builtins.int" From 28f06004745049d81d8803e3f893f694d8a25ed6 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 16 Jun 2025 23:59:15 +0100 Subject: [PATCH 006/424] Support properties with generic setters (#19298) This is yet another followup for `checkmember` work. This handles a niche use case, but it is still used in the wild, and this restores parity between logic for descriptors with `__set__()` and properties with custom setters. --- mypy/checkmember.py | 8 +++++++- test-data/unit/check-generics.test | 29 +++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/mypy/checkmember.py b/mypy/checkmember.py index 1633eaf529836..8f62fee699c02 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -968,7 +968,13 @@ def expand_and_bind_callable( # TODO: a decorated property can result in Overloaded here. assert isinstance(expanded, CallableType) if var.is_settable_property and mx.is_lvalue and var.setter_type is not None: - # TODO: use check_call() to infer better type, same as for __set__(). + if expanded.variables: + type_ctx = mx.rvalue or TempNode(AnyType(TypeOfAny.special_form), context=mx.context) + _, inferred_expanded = mx.chk.expr_checker.check_call( + expanded, [type_ctx], [ARG_POS], mx.context + ) + expanded = get_proper_type(inferred_expanded) + assert isinstance(expanded, CallableType) if not expanded.arg_types: # This can happen when accessing invalid property from its own body, # error will be reported elsewhere. diff --git a/test-data/unit/check-generics.test b/test-data/unit/check-generics.test index 809c3c4eca480..8839dfb954f4a 100644 --- a/test-data/unit/check-generics.test +++ b/test-data/unit/check-generics.test @@ -3628,3 +3628,32 @@ def draw_none( takes_int_str_none(c2) takes_int_str_none(c3) [builtins fixtures/tuple.pyi] + +[case testPropertyWithGenericSetter] +from typing import TypeVar + +class B: ... +class C(B): ... +T = TypeVar("T", bound=B) + +class Test: + @property + def foo(self) -> list[C]: ... + @foo.setter + def foo(self, val: list[T]) -> None: ... + +t1: Test +t2: Test + +lb: list[B] +lc: list[C] +li: list[int] + +t1.foo = lb +t1.foo = lc +t1.foo = li # E: Value of type variable "T" of "foo" of "Test" cannot be "int" + +t2.foo = [B()] +t2.foo = [C()] +t2.foo = [1] # E: Value of type variable "T" of "foo" of "Test" cannot be "int" +[builtins fixtures/property.pyi] From ed88d823f1fd2c572d8600d2c461bff1a86237c1 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 17 Jun 2025 10:56:08 +0100 Subject: [PATCH 007/424] Skip existing when retrying upload-pypi.py (#19305) The upload script has been pretty flaky for me. Now rerunning skips the wheels that have already been uploaded, so the upload should always eventually finish after enough retries. Work on #19174. --- misc/upload-pypi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/upload-pypi.py b/misc/upload-pypi.py index c9db475c14b47..8ea86bbea584b 100644 --- a/misc/upload-pypi.py +++ b/misc/upload-pypi.py @@ -108,7 +108,7 @@ def tmp_twine() -> Iterator[Path]: def upload_dist(dist: Path, dry_run: bool = True) -> None: with tmp_twine() as twine: files = [item for item in dist.iterdir() if item_ok_for_pypi(item.name)] - cmd: list[Any] = [twine, "upload"] + cmd: list[Any] = [twine, "upload", "--skip-existing"] cmd += files if dry_run: print("[dry run] " + " ".join(map(str, cmd))) From 4cda52d4be08d52d0aec741dc7c0812b50136ec6 Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Wed, 18 Jun 2025 19:40:00 +0200 Subject: [PATCH 008/424] Include tuple fallback in constraints built from tuple types (#19100) Fixes #19093. --- mypy/constraints.py | 5 +++++ test-data/unit/check-typevar-tuple.test | 16 ++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/mypy/constraints.py b/mypy/constraints.py index 2936185562031..9eeea3cb2c262 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -1335,6 +1335,11 @@ def visit_tuple_type(self, template: TupleType) -> list[Constraint]: res.extend( infer_constraints(template_items[i], actual_items[i], self.direction) ) + res.extend( + infer_constraints( + template.partial_fallback, actual.partial_fallback, self.direction + ) + ) return res elif isinstance(actual, AnyType): return self.infer_against_any(template.items, actual) diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index 0f69d0a56f475..862fd9ff5fb06 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -2628,3 +2628,19 @@ def fn(f: Callable[[*tuple[T]], int]) -> Callable[[*tuple[T]], int]: ... def test(*args: Unpack[tuple[T]]) -> int: ... reveal_type(fn(test)) # N: Revealed type is "def [T] (T`1) -> builtins.int" [builtins fixtures/tuple.pyi] + +[case testConstraintsIncludeTupleFallback] +from typing import Generic, TypeVar +from typing_extensions import TypeVarTuple, Unpack + +T = TypeVar("T") +Ts = TypeVarTuple("Ts") +_FT = TypeVar("_FT", bound=type) + +def identity(smth: _FT) -> _FT: + return smth + +@identity +class S(tuple[Unpack[Ts]], Generic[T, Unpack[Ts]]): + def f(self, x: T, /) -> T: ... +[builtins fixtures/tuple.pyi] From ce5f12726f1555be2de3794c740235cc2492f50f Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Wed, 18 Jun 2025 13:43:03 -0400 Subject: [PATCH 009/424] support Callable / callable Protocols in suggest decorator unwarpping (#19072) Resolves #18940 while I feel like just accepting any identity function would be good enough I expanded the check to explicitly allow Callables and Protocol callables --------- Co-authored-by: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> --- mypy/suggestions.py | 14 ++++++++++- test-data/unit/fine-grained-suggest.test | 32 ++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/mypy/suggestions.py b/mypy/suggestions.py index a662dd7b98e9e..673076729ffad 100644 --- a/mypy/suggestions.py +++ b/mypy/suggestions.py @@ -229,6 +229,18 @@ def is_implicit_any(typ: Type) -> bool: return isinstance(typ, AnyType) and not is_explicit_any(typ) +def _arg_accepts_function(typ: ProperType) -> bool: + return ( + # TypeVar / Callable + isinstance(typ, (TypeVarType, CallableType)) + or + # Protocol with __call__ + isinstance(typ, Instance) + and typ.type.is_protocol + and typ.type.get_method("__call__") is not None + ) + + class SuggestionEngine: """Engine for finding call sites and suggesting signatures.""" @@ -658,7 +670,7 @@ def extract_from_decorator(self, node: Decorator) -> FuncDef | None: for ct in typ.items: if not ( len(ct.arg_types) == 1 - and isinstance(ct.arg_types[0], TypeVarType) + and _arg_accepts_function(get_proper_type(ct.arg_types[0])) and ct.arg_types[0] == ct.ret_type ): return None diff --git a/test-data/unit/fine-grained-suggest.test b/test-data/unit/fine-grained-suggest.test index 7034b5e48943d..f2db85c05f182 100644 --- a/test-data/unit/fine-grained-suggest.test +++ b/test-data/unit/fine-grained-suggest.test @@ -651,6 +651,38 @@ foo3('hello hello') (str) -> str == +[case testSuggestInferFuncDecorator6] +# suggest: foo.f +[file foo.py] +from __future__ import annotations + +from typing import Callable, Protocol, TypeVar +from typing_extensions import ParamSpec + +P = ParamSpec('P') +R = TypeVar('R') +R_co = TypeVar('R_co', covariant=True) + +class Proto(Protocol[P, R_co]): + def __call__(self, *args: P.args, **kwargs: P.kwargs) -> R_co: ... + +def dec1(f: Callable[P, R]) -> Callable[P, R]: ... +def dec2(f: Callable[..., R]) -> Callable[..., R]: ... +def dec3(f: Proto[P, R_co]) -> Proto[P, R_co]: ... + +@dec1 +@dec2 +@dec3 +def f(x): + return x + +f('hi') + +[builtins fixtures/isinstancelist.pyi] +[out] +(str) -> str +== + [case testSuggestFlexAny1] # suggest: --flex-any=0.4 m.foo # suggest: --flex-any=0.7 m.foo From 5c196d1e3801ba57bac3461882ea318fa1ada7e9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 18 Jun 2025 10:50:12 -0700 Subject: [PATCH 010/424] Sync typeshed (#19301) Source commit: https://github.com/python/typeshed/commit/ecd5141cc036366cc9e3ca371096d6a14b0ccd13 --- mypy/typeshed/stdlib/VERSIONS | 1 + mypy/typeshed/stdlib/_hashlib.pyi | 70 +++++++++-- mypy/typeshed/stdlib/_socket.pyi | 3 +- mypy/typeshed/stdlib/_zstd.pyi | 96 ++++++++++++++ mypy/typeshed/stdlib/asyncio/tasks.pyi | 26 ++-- mypy/typeshed/stdlib/bz2.pyi | 11 +- .../stdlib/compression/_common/_streams.pyi | 3 +- .../stdlib/compression/zstd/__init__.pyi | 87 +++++++++++++ .../stdlib/compression/zstd/_zstdfile.pyi | 117 ++++++++++++++++++ .../stdlib/email/_header_value_parser.pyi | 7 +- mypy/typeshed/stdlib/fractions.pyi | 35 ++++-- mypy/typeshed/stdlib/genericpath.pyi | 7 +- mypy/typeshed/stdlib/gzip.pyi | 13 +- mypy/typeshed/stdlib/lzma.pyi | 9 +- mypy/typeshed/stdlib/ntpath.pyi | 17 ++- mypy/typeshed/stdlib/posixpath.pyi | 20 ++- mypy/typeshed/stdlib/socket.pyi | 4 + mypy/typeshed/stdlib/tarfile.pyi | 17 ++- mypy/typeshed/stdlib/tkinter/__init__.pyi | 4 +- mypy/typeshed/stdlib/typing_extensions.pyi | 2 +- mypy/typeshed/stdlib/zoneinfo/__init__.pyi | 10 +- 21 files changed, 482 insertions(+), 77 deletions(-) create mode 100644 mypy/typeshed/stdlib/_zstd.pyi create mode 100644 mypy/typeshed/stdlib/compression/zstd/__init__.pyi create mode 100644 mypy/typeshed/stdlib/compression/zstd/_zstdfile.pyi diff --git a/mypy/typeshed/stdlib/VERSIONS b/mypy/typeshed/stdlib/VERSIONS index 1ecd8af645595..c86bbb3146670 100644 --- a/mypy/typeshed/stdlib/VERSIONS +++ b/mypy/typeshed/stdlib/VERSIONS @@ -76,6 +76,7 @@ _warnings: 3.0- _weakref: 3.0- _weakrefset: 3.0- _winapi: 3.3- +_zstd: 3.14- abc: 3.0- aifc: 3.0-3.12 annotationlib: 3.14- diff --git a/mypy/typeshed/stdlib/_hashlib.pyi b/mypy/typeshed/stdlib/_hashlib.pyi index 746b1657e2dbf..8b7ef52cdffdf 100644 --- a/mypy/typeshed/stdlib/_hashlib.pyi +++ b/mypy/typeshed/stdlib/_hashlib.pyi @@ -60,19 +60,63 @@ def compare_digest(a: ReadableBuffer, b: ReadableBuffer, /) -> bool: ... def compare_digest(a: AnyStr, b: AnyStr, /) -> bool: ... def get_fips_mode() -> int: ... def hmac_new(key: bytes | bytearray, msg: ReadableBuffer = b"", digestmod: _DigestMod = None) -> HMAC: ... -def new(name: str, string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> HASH: ... -def openssl_md5(string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> HASH: ... -def openssl_sha1(string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> HASH: ... -def openssl_sha224(string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> HASH: ... -def openssl_sha256(string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> HASH: ... -def openssl_sha384(string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> HASH: ... -def openssl_sha512(string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> HASH: ... -def openssl_sha3_224(string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> HASH: ... -def openssl_sha3_256(string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> HASH: ... -def openssl_sha3_384(string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> HASH: ... -def openssl_sha3_512(string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> HASH: ... -def openssl_shake_128(string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> HASHXOF: ... -def openssl_shake_256(string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> HASHXOF: ... + +if sys.version_info >= (3, 13): + def new( + name: str, data: ReadableBuffer = b"", *, usedforsecurity: bool = True, string: ReadableBuffer | None = None + ) -> HASH: ... + def openssl_md5( + data: ReadableBuffer = b"", *, usedforsecurity: bool = True, string: ReadableBuffer | None = None + ) -> HASH: ... + def openssl_sha1( + data: ReadableBuffer = b"", *, usedforsecurity: bool = True, string: ReadableBuffer | None = None + ) -> HASH: ... + def openssl_sha224( + data: ReadableBuffer = b"", *, usedforsecurity: bool = True, string: ReadableBuffer | None = None + ) -> HASH: ... + def openssl_sha256( + data: ReadableBuffer = b"", *, usedforsecurity: bool = True, string: ReadableBuffer | None = None + ) -> HASH: ... + def openssl_sha384( + data: ReadableBuffer = b"", *, usedforsecurity: bool = True, string: ReadableBuffer | None = None + ) -> HASH: ... + def openssl_sha512( + data: ReadableBuffer = b"", *, usedforsecurity: bool = True, string: ReadableBuffer | None = None + ) -> HASH: ... + def openssl_sha3_224( + data: ReadableBuffer = b"", *, usedforsecurity: bool = True, string: ReadableBuffer | None = None + ) -> HASH: ... + def openssl_sha3_256( + data: ReadableBuffer = b"", *, usedforsecurity: bool = True, string: ReadableBuffer | None = None + ) -> HASH: ... + def openssl_sha3_384( + data: ReadableBuffer = b"", *, usedforsecurity: bool = True, string: ReadableBuffer | None = None + ) -> HASH: ... + def openssl_sha3_512( + data: ReadableBuffer = b"", *, usedforsecurity: bool = True, string: ReadableBuffer | None = None + ) -> HASH: ... + def openssl_shake_128( + data: ReadableBuffer = b"", *, usedforsecurity: bool = True, string: ReadableBuffer | None = None + ) -> HASHXOF: ... + def openssl_shake_256( + data: ReadableBuffer = b"", *, usedforsecurity: bool = True, string: ReadableBuffer | None = None + ) -> HASHXOF: ... + +else: + def new(name: str, string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> HASH: ... + def openssl_md5(string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> HASH: ... + def openssl_sha1(string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> HASH: ... + def openssl_sha224(string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> HASH: ... + def openssl_sha256(string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> HASH: ... + def openssl_sha384(string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> HASH: ... + def openssl_sha512(string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> HASH: ... + def openssl_sha3_224(string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> HASH: ... + def openssl_sha3_256(string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> HASH: ... + def openssl_sha3_384(string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> HASH: ... + def openssl_sha3_512(string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> HASH: ... + def openssl_shake_128(string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> HASHXOF: ... + def openssl_shake_256(string: ReadableBuffer = b"", *, usedforsecurity: bool = True) -> HASHXOF: ... + def hmac_digest(key: bytes | bytearray, msg: ReadableBuffer, digest: str) -> bytes: ... def pbkdf2_hmac( hash_name: str, password: ReadableBuffer, salt: ReadableBuffer, iterations: int, dklen: int | None = None diff --git a/mypy/typeshed/stdlib/_socket.pyi b/mypy/typeshed/stdlib/_socket.pyi index 06a8a2ba5fa06..41fdce87ec14d 100644 --- a/mypy/typeshed/stdlib/_socket.pyi +++ b/mypy/typeshed/stdlib/_socket.pyi @@ -229,6 +229,8 @@ if sys.platform != "win32": IP_RECVOPTS: int IP_RECVRETOPTS: int IP_RETOPTS: int +if sys.version_info >= (3, 13) and sys.platform == "linux": + CAN_RAW_ERR_FILTER: int if sys.version_info >= (3, 14): IP_RECVTTL: int @@ -246,7 +248,6 @@ if sys.version_info >= (3, 14): TCP_QUICKACK: int if sys.platform == "linux": - CAN_RAW_ERR_FILTER: int IP_FREEBIND: int IP_RECVORIGDSTADDR: int VMADDR_CID_LOCAL: int diff --git a/mypy/typeshed/stdlib/_zstd.pyi b/mypy/typeshed/stdlib/_zstd.pyi new file mode 100644 index 0000000000000..0648d898448b6 --- /dev/null +++ b/mypy/typeshed/stdlib/_zstd.pyi @@ -0,0 +1,96 @@ +from _typeshed import ReadableBuffer +from collections.abc import Mapping +from compression.zstd import CompressionParameter, DecompressionParameter +from typing import Final, Literal, final +from typing_extensions import Self, TypeAlias + +ZSTD_CLEVEL_DEFAULT: Final = 3 +ZSTD_DStreamOutSize: Final = 131072 +ZSTD_btlazy2: Final = 6 +ZSTD_btopt: Final = 7 +ZSTD_btultra: Final = 8 +ZSTD_btultra2: Final = 9 +ZSTD_c_chainLog: Final = 103 +ZSTD_c_checksumFlag: Final = 201 +ZSTD_c_compressionLevel: Final = 100 +ZSTD_c_contentSizeFlag: Final = 200 +ZSTD_c_dictIDFlag: Final = 202 +ZSTD_c_enableLongDistanceMatching: Final = 160 +ZSTD_c_hashLog: Final = 102 +ZSTD_c_jobSize: Final = 401 +ZSTD_c_ldmBucketSizeLog: Final = 163 +ZSTD_c_ldmHashLog: Final = 161 +ZSTD_c_ldmHashRateLog: Final = 164 +ZSTD_c_ldmMinMatch: Final = 162 +ZSTD_c_minMatch: Final = 105 +ZSTD_c_nbWorkers: Final = 400 +ZSTD_c_overlapLog: Final = 402 +ZSTD_c_searchLog: Final = 104 +ZSTD_c_strategy: Final = 107 +ZSTD_c_targetLength: Final = 106 +ZSTD_c_windowLog: Final = 101 +ZSTD_d_windowLogMax: Final = 100 +ZSTD_dfast: Final = 2 +ZSTD_fast: Final = 1 +ZSTD_greedy: Final = 3 +ZSTD_lazy: Final = 4 +ZSTD_lazy2: Final = 5 + +_ZstdCompressorContinue: TypeAlias = Literal[0] +_ZstdCompressorFlushBlock: TypeAlias = Literal[1] +_ZstdCompressorFlushFrame: TypeAlias = Literal[2] + +@final +class ZstdCompressor: + CONTINUE: Final = 0 + FLUSH_BLOCK: Final = 1 + FLUSH_FRAME: Final = 2 + def __init__( + self, level: int | None = None, options: Mapping[int, int] | None = None, zstd_dict: ZstdDict | None = None + ) -> None: ... + def compress( + self, /, data: ReadableBuffer, mode: _ZstdCompressorContinue | _ZstdCompressorFlushBlock | _ZstdCompressorFlushFrame = 0 + ) -> bytes: ... + def flush(self, /, mode: _ZstdCompressorFlushBlock | _ZstdCompressorFlushFrame = 2) -> bytes: ... + @property + def last_mode(self) -> _ZstdCompressorContinue | _ZstdCompressorFlushBlock | _ZstdCompressorFlushFrame: ... + +@final +class ZstdDecompressor: + def __init__(self, zstd_dict: ZstdDict | None = None, options: Mapping[int, int] | None = None) -> None: ... + def decompress(self, /, data: ReadableBuffer, max_length: int = -1) -> bytes: ... + @property + def eof(self) -> bool: ... + @property + def needs_input(self) -> bool: ... + @property + def unused_data(self) -> bytes: ... + +@final +class ZstdDict: + def __init__(self, dict_content: bytes, /, *, is_raw: bool = False) -> None: ... + def __len__(self, /) -> int: ... + @property + def as_digested_dict(self) -> tuple[Self, int]: ... + @property + def as_prefix(self) -> tuple[Self, int]: ... + @property + def as_undigested_dict(self) -> tuple[Self, int]: ... + @property + def dict_content(self) -> bytes: ... + @property + def dict_id(self) -> int: ... + +class ZstdError(Exception): ... + +def finalize_dict( + custom_dict_bytes: bytes, samples_bytes: bytes, samples_sizes: tuple[int, ...], dict_size: int, compression_level: int, / +) -> bytes: ... +def get_frame_info(frame_buffer: ReadableBuffer) -> tuple[int, int]: ... +def get_frame_size(frame_buffer: ReadableBuffer) -> int: ... +def get_param_bounds(parameter: int, is_compress: bool) -> tuple[int, int]: ... +def set_parameter_types(c_parameter_type: type[CompressionParameter], d_parameter_type: type[DecompressionParameter]) -> None: ... +def train_dict(samples_bytes: bytes, samples_sizes: tuple[int, ...], dict_size: int, /) -> bytes: ... + +zstd_version: Final[str] +zstd_version_number: Final[int] diff --git a/mypy/typeshed/stdlib/asyncio/tasks.pyi b/mypy/typeshed/stdlib/asyncio/tasks.pyi index e42151213e69c..a088e95af653d 100644 --- a/mypy/typeshed/stdlib/asyncio/tasks.pyi +++ b/mypy/typeshed/stdlib/asyncio/tasks.pyi @@ -423,6 +423,25 @@ if sys.version_info >= (3, 12): else: def current_task(loop: AbstractEventLoop | None = None) -> Task[Any] | None: ... +if sys.version_info >= (3, 14): + def eager_task_factory( + loop: AbstractEventLoop | None, + coro: _TaskCompatibleCoro[_T_co], + *, + name: str | None = None, + context: Context | None = None, + eager_start: bool = True, + ) -> Task[_T_co]: ... + +elif sys.version_info >= (3, 12): + def eager_task_factory( + loop: AbstractEventLoop | None, + coro: _TaskCompatibleCoro[_T_co], + *, + name: str | None = None, + context: Context | None = None, + ) -> Task[_T_co]: ... + if sys.version_info >= (3, 12): _TaskT_co = TypeVar("_TaskT_co", bound=Task[Any], covariant=True) @@ -451,10 +470,3 @@ if sys.version_info >= (3, 12): def create_eager_task_factory( custom_task_constructor: _CustomTaskConstructor[_TaskT_co], ) -> _EagerTaskFactoryType[_TaskT_co]: ... - def eager_task_factory( - loop: AbstractEventLoop | None, - coro: _TaskCompatibleCoro[_T_co], - *, - name: str | None = None, - context: Context | None = None, - ) -> Task[_T_co]: ... diff --git a/mypy/typeshed/stdlib/bz2.pyi b/mypy/typeshed/stdlib/bz2.pyi index 0f9d00fbc633e..dce6187a2da10 100644 --- a/mypy/typeshed/stdlib/bz2.pyi +++ b/mypy/typeshed/stdlib/bz2.pyi @@ -2,7 +2,8 @@ import sys from _bz2 import BZ2Compressor as BZ2Compressor, BZ2Decompressor as BZ2Decompressor from _typeshed import ReadableBuffer, StrOrBytesPath, WriteableBuffer from collections.abc import Iterable -from typing import IO, Literal, Protocol, SupportsIndex, TextIO, overload +from io import TextIOWrapper +from typing import IO, Literal, Protocol, SupportsIndex, overload from typing_extensions import Self, TypeAlias if sys.version_info >= (3, 14): @@ -48,7 +49,7 @@ def open( encoding: str | None = None, errors: str | None = None, newline: str | None = None, -) -> TextIO: ... +) -> TextIOWrapper: ... @overload def open( filename: _WritableFileobj, @@ -66,7 +67,7 @@ def open( encoding: str | None = None, errors: str | None = None, newline: str | None = None, -) -> TextIO: ... +) -> TextIOWrapper: ... @overload def open( filename: StrOrBytesPath, @@ -84,7 +85,7 @@ def open( encoding: str | None = None, errors: str | None = None, newline: str | None = None, -) -> TextIO: ... +) -> TextIOWrapper: ... @overload def open( filename: StrOrBytesPath | _ReadableFileobj | _WritableFileobj, @@ -93,7 +94,7 @@ def open( encoding: str | None = None, errors: str | None = None, newline: str | None = None, -) -> BZ2File | TextIO: ... +) -> BZ2File | TextIOWrapper: ... class BZ2File(BaseStream, IO[bytes]): def __enter__(self) -> Self: ... diff --git a/mypy/typeshed/stdlib/compression/_common/_streams.pyi b/mypy/typeshed/stdlib/compression/_common/_streams.pyi index 6303a9b1d460c..b8463973ec671 100644 --- a/mypy/typeshed/stdlib/compression/_common/_streams.pyi +++ b/mypy/typeshed/stdlib/compression/_common/_streams.pyi @@ -1,10 +1,11 @@ from _typeshed import Incomplete, WriteableBuffer from collections.abc import Callable from io import DEFAULT_BUFFER_SIZE, BufferedIOBase, RawIOBase -from typing import Any, Protocol +from typing import Any, Protocol, type_check_only BUFFER_SIZE = DEFAULT_BUFFER_SIZE +@type_check_only class _Reader(Protocol): def read(self, n: int, /) -> bytes: ... def seekable(self) -> bool: ... diff --git a/mypy/typeshed/stdlib/compression/zstd/__init__.pyi b/mypy/typeshed/stdlib/compression/zstd/__init__.pyi new file mode 100644 index 0000000000000..24a9633c488e6 --- /dev/null +++ b/mypy/typeshed/stdlib/compression/zstd/__init__.pyi @@ -0,0 +1,87 @@ +import enum +from _typeshed import ReadableBuffer +from collections.abc import Iterable, Mapping +from compression.zstd._zstdfile import ZstdFile, open +from typing import Final, final + +import _zstd +from _zstd import ZstdCompressor, ZstdDecompressor, ZstdDict, ZstdError, get_frame_size, zstd_version + +__all__ = ( + # compression.zstd + "COMPRESSION_LEVEL_DEFAULT", + "compress", + "CompressionParameter", + "decompress", + "DecompressionParameter", + "finalize_dict", + "get_frame_info", + "Strategy", + "train_dict", + # compression.zstd._zstdfile + "open", + "ZstdFile", + # _zstd + "get_frame_size", + "zstd_version", + "zstd_version_info", + "ZstdCompressor", + "ZstdDecompressor", + "ZstdDict", + "ZstdError", +) + +zstd_version_info: Final[tuple[int, int, int]] +COMPRESSION_LEVEL_DEFAULT: Final = _zstd.ZSTD_CLEVEL_DEFAULT + +class FrameInfo: + decompressed_size: int + dictionary_id: int + def __init__(self, decompressed_size: int, dictionary_id: int) -> None: ... + +def get_frame_info(frame_buffer: ReadableBuffer) -> FrameInfo: ... +def train_dict(samples: Iterable[ReadableBuffer], dict_size: int) -> ZstdDict: ... +def finalize_dict(zstd_dict: ZstdDict, /, samples: Iterable[ReadableBuffer], dict_size: int, level: int) -> ZstdDict: ... +def compress( + data: ReadableBuffer, level: int | None = None, options: Mapping[int, int] | None = None, zstd_dict: ZstdDict | None = None +) -> bytes: ... +def decompress(data: ReadableBuffer, zstd_dict: ZstdDict | None = None, options: Mapping[int, int] | None = None) -> bytes: ... +@final +class CompressionParameter(enum.IntEnum): + compression_level = _zstd.ZSTD_c_compressionLevel + window_log = _zstd.ZSTD_c_windowLog + hash_log = _zstd.ZSTD_c_hashLog + chain_log = _zstd.ZSTD_c_chainLog + search_log = _zstd.ZSTD_c_searchLog + min_match = _zstd.ZSTD_c_minMatch + target_length = _zstd.ZSTD_c_targetLength + strategy = _zstd.ZSTD_c_strategy + enable_long_distance_matching = _zstd.ZSTD_c_enableLongDistanceMatching + ldm_hash_log = _zstd.ZSTD_c_ldmHashLog + ldm_min_match = _zstd.ZSTD_c_ldmMinMatch + ldm_bucket_size_log = _zstd.ZSTD_c_ldmBucketSizeLog + ldm_hash_rate_log = _zstd.ZSTD_c_ldmHashRateLog + content_size_flag = _zstd.ZSTD_c_contentSizeFlag + checksum_flag = _zstd.ZSTD_c_checksumFlag + dict_id_flag = _zstd.ZSTD_c_dictIDFlag + nb_workers = _zstd.ZSTD_c_nbWorkers + job_size = _zstd.ZSTD_c_jobSize + overlap_log = _zstd.ZSTD_c_overlapLog + def bounds(self) -> tuple[int, int]: ... + +@final +class DecompressionParameter(enum.IntEnum): + window_log_max = _zstd.ZSTD_d_windowLogMax + def bounds(self) -> tuple[int, int]: ... + +@final +class Strategy(enum.IntEnum): + fast = _zstd.ZSTD_fast + dfast = _zstd.ZSTD_dfast + greedy = _zstd.ZSTD_greedy + lazy = _zstd.ZSTD_lazy + lazy2 = _zstd.ZSTD_lazy2 + btlazy2 = _zstd.ZSTD_btlazy2 + btopt = _zstd.ZSTD_btopt + btultra = _zstd.ZSTD_btultra + btultra2 = _zstd.ZSTD_btultra2 diff --git a/mypy/typeshed/stdlib/compression/zstd/_zstdfile.pyi b/mypy/typeshed/stdlib/compression/zstd/_zstdfile.pyi new file mode 100644 index 0000000000000..045b2d35acfe0 --- /dev/null +++ b/mypy/typeshed/stdlib/compression/zstd/_zstdfile.pyi @@ -0,0 +1,117 @@ +from _typeshed import ReadableBuffer, StrOrBytesPath, SupportsWrite, WriteableBuffer +from collections.abc import Mapping +from compression._common import _streams +from compression.zstd import ZstdDict +from io import TextIOWrapper, _WrappedBuffer +from typing import Literal, overload, type_check_only +from typing_extensions import TypeAlias + +from _zstd import ZstdCompressor, _ZstdCompressorFlushBlock, _ZstdCompressorFlushFrame + +__all__ = ("ZstdFile", "open") + +_ReadBinaryMode: TypeAlias = Literal["r", "rb"] +_WriteBinaryMode: TypeAlias = Literal["w", "wb", "x", "xb", "a", "ab"] +_ReadTextMode: TypeAlias = Literal["rt"] +_WriteTextMode: TypeAlias = Literal["wt", "xt", "at"] + +@type_check_only +class _FileBinaryRead(_streams._Reader): + def close(self) -> None: ... + +@type_check_only +class _FileBinaryWrite(SupportsWrite[bytes]): + def close(self) -> None: ... + +class ZstdFile(_streams.BaseStream): + FLUSH_BLOCK = ZstdCompressor.FLUSH_BLOCK + FLUSH_FRAME = ZstdCompressor.FLUSH_FRAME + + @overload + def __init__( + self, + file: StrOrBytesPath | _FileBinaryRead, + /, + mode: _ReadBinaryMode = "r", + *, + level: None = None, + options: Mapping[int, int] | None = None, + zstd_dict: ZstdDict | None = None, + ) -> None: ... + @overload + def __init__( + self, + file: StrOrBytesPath | _FileBinaryWrite, + /, + mode: _WriteBinaryMode, + *, + level: int | None = None, + options: Mapping[int, int] | None = None, + zstd_dict: ZstdDict | None = None, + ) -> None: ... + def write(self, data: ReadableBuffer, /) -> int: ... + def flush(self, mode: _ZstdCompressorFlushBlock | _ZstdCompressorFlushFrame = 1) -> bytes: ... # type: ignore[override] + def read(self, size: int | None = -1) -> bytes: ... + def read1(self, size: int | None = -1) -> bytes: ... + def readinto(self, b: WriteableBuffer) -> int: ... + def readinto1(self, b: WriteableBuffer) -> int: ... + def readline(self, size: int | None = -1) -> bytes: ... + def seek(self, offset: int, whence: int = 0) -> int: ... + def peek(self, size: int = -1) -> bytes: ... + @property + def name(self) -> str | bytes: ... + @property + def mode(self) -> Literal["rb", "wb"]: ... + +@overload +def open( + file: StrOrBytesPath | _FileBinaryRead, + /, + mode: _ReadBinaryMode = "rb", + *, + level: None = None, + options: Mapping[int, int] | None = None, + zstd_dict: ZstdDict | None = None, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, +) -> ZstdFile: ... +@overload +def open( + file: StrOrBytesPath | _FileBinaryWrite, + /, + mode: _WriteBinaryMode, + *, + level: int | None = None, + options: Mapping[int, int] | None = None, + zstd_dict: ZstdDict | None = None, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, +) -> ZstdFile: ... +@overload +def open( + file: StrOrBytesPath | _WrappedBuffer, + /, + mode: _ReadTextMode, + *, + level: None = None, + options: Mapping[int, int] | None = None, + zstd_dict: ZstdDict | None = None, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, +) -> TextIOWrapper: ... +@overload +def open( + file: StrOrBytesPath | _WrappedBuffer, + /, + mode: _WriteTextMode, + *, + level: int | None = None, + options: Mapping[int, int] | None = None, + zstd_dict: ZstdDict | None = None, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, +) -> TextIOWrapper: ... diff --git a/mypy/typeshed/stdlib/email/_header_value_parser.pyi b/mypy/typeshed/stdlib/email/_header_value_parser.pyi index a8abfead92172..95ada186c4ec8 100644 --- a/mypy/typeshed/stdlib/email/_header_value_parser.pyi +++ b/mypy/typeshed/stdlib/email/_header_value_parser.pyi @@ -1,4 +1,3 @@ -import sys from collections.abc import Iterable, Iterator from email.errors import HeaderParseError, MessageDefect from email.policy import Policy @@ -22,10 +21,8 @@ NLSET: Final[set[str]] # Added in Python 3.9.20, 3.10.15, 3.11.10, 3.12.5 SPECIALSNL: Final[set[str]] -if sys.version_info >= (3, 10): - # Added in Python 3.10.17, 3.11.12, 3.12.9, 3.13.2 (may still be backported to 3.9) - def make_quoted_pairs(value: Any) -> str: ... - +# Added in Python 3.9.23, 3.10.17, 3.11.12, 3.12.9, 3.13.2 +def make_quoted_pairs(value: Any) -> str: ... def quote_string(value: Any) -> str: ... rfc2047_matcher: Pattern[str] diff --git a/mypy/typeshed/stdlib/fractions.pyi b/mypy/typeshed/stdlib/fractions.pyi index 83592eb583366..16259fcfadc7c 100644 --- a/mypy/typeshed/stdlib/fractions.pyi +++ b/mypy/typeshed/stdlib/fractions.pyi @@ -107,16 +107,31 @@ class Fraction(Rational): def __rdivmod__(a, b: int | Fraction) -> tuple[int, Fraction]: ... @overload def __rdivmod__(a, b: float) -> tuple[float, Fraction]: ... - @overload - def __pow__(a, b: int) -> Fraction: ... - @overload - def __pow__(a, b: float | Fraction) -> float: ... - @overload - def __pow__(a, b: complex) -> complex: ... - @overload - def __rpow__(b, a: float | Fraction) -> float: ... - @overload - def __rpow__(b, a: complex) -> complex: ... + if sys.version_info >= (3, 14): + @overload + def __pow__(a, b: int, modulo: None = None) -> Fraction: ... + @overload + def __pow__(a, b: float | Fraction, modulo: None = None) -> float: ... + @overload + def __pow__(a, b: complex, modulo: None = None) -> complex: ... + else: + @overload + def __pow__(a, b: int) -> Fraction: ... + @overload + def __pow__(a, b: float | Fraction) -> float: ... + @overload + def __pow__(a, b: complex) -> complex: ... + if sys.version_info >= (3, 14): + @overload + def __rpow__(b, a: float | Fraction, modulo: None = None) -> float: ... + @overload + def __rpow__(b, a: complex, modulo: None = None) -> complex: ... + else: + @overload + def __rpow__(b, a: float | Fraction) -> float: ... + @overload + def __rpow__(b, a: complex) -> complex: ... + def __pos__(a) -> Fraction: ... def __neg__(a) -> Fraction: ... def __abs__(a) -> Fraction: ... diff --git a/mypy/typeshed/stdlib/genericpath.pyi b/mypy/typeshed/stdlib/genericpath.pyi index 9d87c48fd5200..3caed77a661ac 100644 --- a/mypy/typeshed/stdlib/genericpath.pyi +++ b/mypy/typeshed/stdlib/genericpath.pyi @@ -2,7 +2,7 @@ import os import sys from _typeshed import BytesPath, FileDescriptorOrPath, StrOrBytesPath, StrPath, SupportsRichComparisonT from collections.abc import Sequence -from typing import Literal, overload +from typing import Literal, NewType, overload from typing_extensions import LiteralString __all__ = [ @@ -17,6 +17,7 @@ __all__ = [ "samefile", "sameopenfile", "samestat", + "ALLOW_MISSING", ] if sys.version_info >= (3, 12): __all__ += ["islink"] @@ -57,3 +58,7 @@ if sys.version_info >= (3, 13): def isjunction(path: StrOrBytesPath) -> bool: ... def isdevdrive(path: StrOrBytesPath) -> bool: ... def lexists(path: StrOrBytesPath) -> bool: ... + +# Added in Python 3.9.23, 3.10.18, 3.11.13, 3.12.11, 3.13.4 +_AllowMissingType = NewType("_AllowMissingType", object) +ALLOW_MISSING: _AllowMissingType diff --git a/mypy/typeshed/stdlib/gzip.pyi b/mypy/typeshed/stdlib/gzip.pyi index 883456b1ddc3d..34ae92b4d8ed6 100644 --- a/mypy/typeshed/stdlib/gzip.pyi +++ b/mypy/typeshed/stdlib/gzip.pyi @@ -1,6 +1,6 @@ import sys import zlib -from _typeshed import ReadableBuffer, SizedBuffer, StrOrBytesPath +from _typeshed import ReadableBuffer, SizedBuffer, StrOrBytesPath, WriteableBuffer from io import FileIO, TextIOWrapper from typing import Final, Literal, Protocol, overload from typing_extensions import TypeAlias @@ -157,8 +157,17 @@ class GzipFile(BaseStream): def seek(self, offset: int, whence: int = 0) -> int: ... def readline(self, size: int | None = -1) -> bytes: ... + if sys.version_info >= (3, 14): + def readinto(self, b: WriteableBuffer) -> int: ... + def readinto1(self, b: WriteableBuffer) -> int: ... + class _GzipReader(DecompressReader): def __init__(self, fp: _ReadableFileobj) -> None: ... -def compress(data: SizedBuffer, compresslevel: int = 9, *, mtime: float | None = None) -> bytes: ... +if sys.version_info >= (3, 14): + def compress(data: SizedBuffer, compresslevel: int = 9, *, mtime: float = 0) -> bytes: ... + +else: + def compress(data: SizedBuffer, compresslevel: int = 9, *, mtime: float | None = None) -> bytes: ... + def decompress(data: ReadableBuffer) -> bytes: ... diff --git a/mypy/typeshed/stdlib/lzma.pyi b/mypy/typeshed/stdlib/lzma.pyi index b066d222466ba..b7ef607b75cbf 100644 --- a/mypy/typeshed/stdlib/lzma.pyi +++ b/mypy/typeshed/stdlib/lzma.pyi @@ -35,7 +35,8 @@ from _lzma import ( is_check_supported as is_check_supported, ) from _typeshed import ReadableBuffer, StrOrBytesPath -from typing import IO, Literal, TextIO, overload +from io import TextIOWrapper +from typing import IO, Literal, overload from typing_extensions import Self, TypeAlias if sys.version_info >= (3, 14): @@ -144,7 +145,7 @@ def open( encoding: str | None = None, errors: str | None = None, newline: str | None = None, -) -> TextIO: ... +) -> TextIOWrapper: ... @overload def open( filename: StrOrBytesPath, @@ -157,7 +158,7 @@ def open( encoding: str | None = None, errors: str | None = None, newline: str | None = None, -) -> TextIO: ... +) -> TextIOWrapper: ... @overload def open( filename: _PathOrFile, @@ -170,7 +171,7 @@ def open( encoding: str | None = None, errors: str | None = None, newline: str | None = None, -) -> LZMAFile | TextIO: ... +) -> LZMAFile | TextIOWrapper: ... def compress( data: ReadableBuffer, format: int = 1, check: int = -1, preset: int | None = None, filters: _FilterChain | None = None ) -> bytes: ... diff --git a/mypy/typeshed/stdlib/ntpath.pyi b/mypy/typeshed/stdlib/ntpath.pyi index ebe305ef708c2..074df075b9727 100644 --- a/mypy/typeshed/stdlib/ntpath.pyi +++ b/mypy/typeshed/stdlib/ntpath.pyi @@ -1,6 +1,8 @@ import sys from _typeshed import BytesPath, StrOrBytesPath, StrPath from genericpath import ( + ALLOW_MISSING as ALLOW_MISSING, + _AllowMissingType, commonprefix as commonprefix, exists as exists, getatime as getatime, @@ -89,6 +91,7 @@ __all__ = [ "sameopenfile", "samestat", "commonpath", + "ALLOW_MISSING", ] if sys.version_info >= (3, 12): __all__ += ["isjunction", "splitroot"] @@ -108,16 +111,10 @@ def join(path: StrPath, /, *paths: StrPath) -> str: ... def join(path: BytesPath, /, *paths: BytesPath) -> bytes: ... if sys.platform == "win32": - if sys.version_info >= (3, 10): - @overload - def realpath(path: PathLike[AnyStr], *, strict: bool = False) -> AnyStr: ... - @overload - def realpath(path: AnyStr, *, strict: bool = False) -> AnyStr: ... - else: - @overload - def realpath(path: PathLike[AnyStr]) -> AnyStr: ... - @overload - def realpath(path: AnyStr) -> AnyStr: ... + @overload + def realpath(path: PathLike[AnyStr], *, strict: bool | _AllowMissingType = False) -> AnyStr: ... + @overload + def realpath(path: AnyStr, *, strict: bool | _AllowMissingType = False) -> AnyStr: ... else: realpath = abspath diff --git a/mypy/typeshed/stdlib/posixpath.pyi b/mypy/typeshed/stdlib/posixpath.pyi index 3313667f1781b..84e1b1e028bde 100644 --- a/mypy/typeshed/stdlib/posixpath.pyi +++ b/mypy/typeshed/stdlib/posixpath.pyi @@ -2,6 +2,8 @@ import sys from _typeshed import AnyOrLiteralStr, BytesPath, FileDescriptorOrPath, StrOrBytesPath, StrPath from collections.abc import Iterable from genericpath import ( + ALLOW_MISSING as ALLOW_MISSING, + _AllowMissingType, commonprefix as commonprefix, exists as exists, getatime as getatime, @@ -61,6 +63,7 @@ __all__ = [ "relpath", "commonpath", ] +__all__ += ["ALLOW_MISSING"] if sys.version_info >= (3, 12): __all__ += ["isjunction", "splitroot"] if sys.version_info >= (3, 13): @@ -122,19 +125,10 @@ def join(a: LiteralString, /, *paths: LiteralString) -> LiteralString: ... def join(a: StrPath, /, *paths: StrPath) -> str: ... @overload def join(a: BytesPath, /, *paths: BytesPath) -> bytes: ... - -if sys.version_info >= (3, 10): - @overload - def realpath(filename: PathLike[AnyStr], *, strict: bool = False) -> AnyStr: ... - @overload - def realpath(filename: AnyStr, *, strict: bool = False) -> AnyStr: ... - -else: - @overload - def realpath(filename: PathLike[AnyStr]) -> AnyStr: ... - @overload - def realpath(filename: AnyStr) -> AnyStr: ... - +@overload +def realpath(filename: PathLike[AnyStr], *, strict: bool | _AllowMissingType = False) -> AnyStr: ... +@overload +def realpath(filename: AnyStr, *, strict: bool | _AllowMissingType = False) -> AnyStr: ... @overload def relpath(path: LiteralString, start: LiteralString | None = None) -> LiteralString: ... @overload diff --git a/mypy/typeshed/stdlib/socket.pyi b/mypy/typeshed/stdlib/socket.pyi index 1ee006235ee6c..b4fa4381a72ca 100644 --- a/mypy/typeshed/stdlib/socket.pyi +++ b/mypy/typeshed/stdlib/socket.pyi @@ -773,6 +773,10 @@ if sys.platform == "linux": if sys.version_info < (3, 11): from _socket import CAN_RAW_ERR_FILTER as CAN_RAW_ERR_FILTER + __all__ += ["CAN_RAW_ERR_FILTER"] + if sys.version_info >= (3, 13): + from _socket import CAN_RAW_ERR_FILTER as CAN_RAW_ERR_FILTER + __all__ += ["CAN_RAW_ERR_FILTER"] if sys.platform == "linux": diff --git a/mypy/typeshed/stdlib/tarfile.pyi b/mypy/typeshed/stdlib/tarfile.pyi index 31094f87872dc..a18ef0b823f9b 100644 --- a/mypy/typeshed/stdlib/tarfile.pyi +++ b/mypy/typeshed/stdlib/tarfile.pyi @@ -38,6 +38,8 @@ if sys.version_info >= (3, 12): "AbsolutePathError", "LinkOutsideDestinationError", ] +if sys.version_info >= (3, 13): + __all__ += ["LinkFallbackError"] _FilterFunction: TypeAlias = Callable[[TarInfo, str], TarInfo | None] _TarfileFilter: TypeAlias = Literal["fully_trusted", "tar", "data"] | _FilterFunction @@ -550,7 +552,14 @@ class TarFile: filter: _TarfileFilter | None = ..., ) -> None: ... def _extract_member( - self, tarinfo: TarInfo, targetpath: str, set_attrs: bool = True, numeric_owner: bool = False + self, + tarinfo: TarInfo, + targetpath: str, + set_attrs: bool = True, + numeric_owner: bool = False, + *, + filter_function: _FilterFunction | None = None, + extraction_root: str | None = None, ) -> None: ... # undocumented def extractfile(self, member: str | TarInfo) -> IO[bytes] | None: ... def makedir(self, tarinfo: TarInfo, targetpath: StrOrBytesPath) -> None: ... # undocumented @@ -559,6 +568,9 @@ class TarFile: def makefifo(self, tarinfo: TarInfo, targetpath: StrOrBytesPath) -> None: ... # undocumented def makedev(self, tarinfo: TarInfo, targetpath: StrOrBytesPath) -> None: ... # undocumented def makelink(self, tarinfo: TarInfo, targetpath: StrOrBytesPath) -> None: ... # undocumented + def makelink_with_filter( + self, tarinfo: TarInfo, targetpath: StrOrBytesPath, filter_function: _FilterFunction, extraction_root: str + ) -> None: ... # undocumented def chown(self, tarinfo: TarInfo, targetpath: StrOrBytesPath, numeric_owner: bool) -> None: ... # undocumented def chmod(self, tarinfo: TarInfo, targetpath: StrOrBytesPath) -> None: ... # undocumented def utime(self, tarinfo: TarInfo, targetpath: StrOrBytesPath) -> None: ... # undocumented @@ -607,6 +619,9 @@ class AbsoluteLinkError(FilterError): class LinkOutsideDestinationError(FilterError): def __init__(self, tarinfo: TarInfo, path: str) -> None: ... +class LinkFallbackError(FilterError): + def __init__(self, tarinfo: TarInfo, path: str) -> None: ... + def fully_trusted_filter(member: TarInfo, dest_path: str) -> TarInfo: ... def tar_filter(member: TarInfo, dest_path: str) -> TarInfo: ... def data_filter(member: TarInfo, dest_path: str) -> TarInfo: ... diff --git a/mypy/typeshed/stdlib/tkinter/__init__.pyi b/mypy/typeshed/stdlib/tkinter/__init__.pyi index 2a4657f86ce13..db0e34d737a62 100644 --- a/mypy/typeshed/stdlib/tkinter/__init__.pyi +++ b/mypy/typeshed/stdlib/tkinter/__init__.pyi @@ -4,7 +4,7 @@ from _typeshed import Incomplete, MaybeNone, StrOrBytesPath from collections.abc import Callable, Iterable, Mapping, Sequence from tkinter.constants import * from tkinter.font import _FontDescription -from types import TracebackType +from types import GenericAlias, TracebackType from typing import Any, ClassVar, Generic, Literal, NamedTuple, Protocol, TypedDict, TypeVar, overload, type_check_only from typing_extensions import TypeAlias, TypeVarTuple, Unpack, deprecated @@ -308,6 +308,8 @@ class Event(Generic[_W_co]): type: EventType widget: _W_co delta: int + if sys.version_info >= (3, 14): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... def NoDefaultRoot() -> None: ... diff --git a/mypy/typeshed/stdlib/typing_extensions.pyi b/mypy/typeshed/stdlib/typing_extensions.pyi index 07cd57ebc18f3..3f7c257120814 100644 --- a/mypy/typeshed/stdlib/typing_extensions.pyi +++ b/mypy/typeshed/stdlib/typing_extensions.pyi @@ -697,6 +697,6 @@ class Sentinel: if sys.version_info >= (3, 14): def __or__(self, other: Any) -> UnionType: ... # other can be any type form legal for unions def __ror__(self, other: Any) -> UnionType: ... # other can be any type form legal for unions - else: + elif sys.version_info >= (3, 10): def __or__(self, other: Any) -> _SpecialForm: ... # other can be any type form legal for unions def __ror__(self, other: Any) -> _SpecialForm: ... # other can be any type form legal for unions diff --git a/mypy/typeshed/stdlib/zoneinfo/__init__.pyi b/mypy/typeshed/stdlib/zoneinfo/__init__.pyi index 35381758a1b7e..e9f54fbf2a26c 100644 --- a/mypy/typeshed/stdlib/zoneinfo/__init__.pyi +++ b/mypy/typeshed/stdlib/zoneinfo/__init__.pyi @@ -1,3 +1,4 @@ +import sys from collections.abc import Iterable from datetime import datetime, timedelta, tzinfo from typing_extensions import Self @@ -17,8 +18,13 @@ class ZoneInfo(tzinfo): def __new__(cls, key: str) -> Self: ... @classmethod def no_cache(cls, key: str) -> Self: ... - @classmethod - def from_file(cls, fobj: _IOBytes, /, key: str | None = None) -> Self: ... + if sys.version_info >= (3, 12): + @classmethod + def from_file(cls, file_obj: _IOBytes, /, key: str | None = None) -> Self: ... + else: + @classmethod + def from_file(cls, fobj: _IOBytes, /, key: str | None = None) -> Self: ... + @classmethod def clear_cache(cls, *, only_keys: Iterable[str] | None = None) -> None: ... def tzname(self, dt: datetime | None, /) -> str | None: ... From b62957b992ec1da37c42ad37e8ed23fc51644b3e Mon Sep 17 00:00:00 2001 From: Brian Schubert Date: Wed, 18 Jun 2025 14:17:03 -0400 Subject: [PATCH 011/424] Fix missing error when redeclaring type variable in nested generic class (#18883) Closes #10479 Fixes a case where mypy doesn't warn about an inner generic class using a type variable by the same name as an outer generic class if the inner generic class doesn't declare the type variable using `Generic`, `Protocol`, or PEP-695 syntax. --- mypy/message_registry.py | 4 ++++ mypy/semanal.py | 8 ++++++++ mypy/typeanal.py | 7 ++----- test-data/unit/semanal-errors.test | 6 ++++++ 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/mypy/message_registry.py b/mypy/message_registry.py index 609f968a8c654..3c7745876f87b 100644 --- a/mypy/message_registry.py +++ b/mypy/message_registry.py @@ -353,6 +353,10 @@ def with_additional_msg(self, info: str) -> ErrorMessage: "TypeVar constraint type cannot be parametrized by type variables", codes.MISC ) +TYPE_VAR_REDECLARED_IN_NESTED_CLASS: Final = ErrorMessage( + 'Type variable "{}" is bound by an outer class', codes.VALID_TYPE +) + TYPE_ALIAS_WITH_YIELD_EXPRESSION: Final = ErrorMessage( "Yield expression cannot be used within a type alias", codes.SYNTAX ) diff --git a/mypy/semanal.py b/mypy/semanal.py index d70abe911fead..704aa91d1d12e 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -2374,6 +2374,14 @@ def tvar_defs_from_tvars( tvar_expr.default = tvar_expr.default.accept( TypeVarDefaultTranslator(self, tvar_expr.name, context) ) + # PEP-695 type variables that are redeclared in an inner scope are warned + # about elsewhere. + if not tvar_expr.is_new_style and not self.tvar_scope.allow_binding( + tvar_expr.fullname + ): + self.fail( + message_registry.TYPE_VAR_REDECLARED_IN_NESTED_CLASS.format(name), context + ) tvar_def = self.tvar_scope.bind_new(name, tvar_expr) if last_tvar_name_with_default is not None and not tvar_def.has_default(): self.msg.tvar_without_default_type( diff --git a/mypy/typeanal.py b/mypy/typeanal.py index eeb5d3c52ac68..b0d11759303c4 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -1844,11 +1844,8 @@ def bind_function_type_variables( defs = [] for name, tvar in typevars: if not self.tvar_scope.allow_binding(tvar.fullname): - self.fail( - f'Type variable "{name}" is bound by an outer class', - defn, - code=codes.VALID_TYPE, - ) + err_msg = message_registry.TYPE_VAR_REDECLARED_IN_NESTED_CLASS.format(name) + self.fail(err_msg.value, defn, code=err_msg.code) binding = self.tvar_scope.bind_new(name, tvar) defs.append(binding) diff --git a/test-data/unit/semanal-errors.test b/test-data/unit/semanal-errors.test index fa5cec7959319..1e760799828a8 100644 --- a/test-data/unit/semanal-errors.test +++ b/test-data/unit/semanal-errors.test @@ -1015,6 +1015,12 @@ class A(Generic[T]): # E: Free type variable expected in Generic[...] [out] +[case testRedeclaredTypeVarWithinNestedGenericClass] +from typing import Generic, Iterable, TypeVar +T = TypeVar('T') +class A(Generic[T]): + class B(Iterable[T]): pass # E: Type variable "T" is bound by an outer class + [case testIncludingGenericTwiceInBaseClassList] from typing import Generic, TypeVar T = TypeVar('T') From a48ffed250cc19efee6394eb906e8c7d70288dc5 Mon Sep 17 00:00:00 2001 From: Brian Schubert Date: Wed, 18 Jun 2025 14:25:16 -0400 Subject: [PATCH 012/424] Show name of type variable in "Cannot infer type argument" message (#19290) Fixes #19289 The type argument index currently shown can be wrong if other type arguments were substituted during an earlier pass. Given: ```python def foo[T1, T2]( a: T1, b: T2, c: Callable[[T2], T2], ) -> tuple[T1, T2]: ... def bar(y: float) -> float: ... reveal_type(foo(1, None, bar)) # Expect T1=int, T2= ``` Before: ``` main.py:9: error: Cannot infer type argument 1 of "foo" [misc] main.py:9: note: Revealed type is "tuple[builtins.int, Any]" ``` After: ``` main.py:9: error: Cannot infer type argument to type parameter "T2" of "foo" [misc] main.py:9: note: Revealed type is "tuple[builtins.int, Any]" ``` --- mypy/checkexpr.py | 4 ++-- mypy/messages.py | 9 ++++--- test-data/unit/check-dataclasses.test | 2 +- test-data/unit/check-expressions.test | 4 ++-- test-data/unit/check-generics.test | 6 ++--- test-data/unit/check-inference-context.test | 2 +- test-data/unit/check-inference.test | 24 +++++++++++++++---- test-data/unit/check-literal.test | 4 ++-- test-data/unit/check-overloading.test | 6 ++--- .../unit/check-parameter-specification.test | 2 +- test-data/unit/check-plugin-attrs.test | 4 ++-- test-data/unit/check-protocols.test | 4 ++-- test-data/unit/check-typevar-tuple.test | 8 +++---- 13 files changed, 49 insertions(+), 30 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 4ca55e1679e40..26cb2a35794b7 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -2338,10 +2338,10 @@ def apply_inferred_arguments( # Report error if some of the variables could not be solved. In that # case assume that all variables have type Any to avoid extra # bogus error messages. - for i, inferred_type in enumerate(inferred_args): + for inferred_type, tv in zip(inferred_args, callee_type.variables): if not inferred_type or has_erased_component(inferred_type): # Could not infer a non-trivial type for a type variable. - self.msg.could_not_infer_type_arguments(callee_type, i + 1, context) + self.msg.could_not_infer_type_arguments(callee_type, tv, context) inferred_args = [AnyType(TypeOfAny.from_error)] * len(inferred_args) # Apply the inferred types to the function type. In this case the # return type must be CallableType, since we give the right number of type diff --git a/mypy/messages.py b/mypy/messages.py index 01414f1c7f2bc..13a4facc82b03 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -1370,11 +1370,14 @@ def incompatible_type_application( self.fail(f"Type application has too few types ({s})", context) def could_not_infer_type_arguments( - self, callee_type: CallableType, n: int, context: Context + self, callee_type: CallableType, tv: TypeVarLikeType, context: Context ) -> None: callee_name = callable_name(callee_type) - if callee_name is not None and n > 0: - self.fail(f"Cannot infer type argument {n} of {callee_name}", context) + if callee_name is not None: + self.fail( + f"Cannot infer value of type parameter {format_type(tv, self.options)} of {callee_name}", + context, + ) if callee_name == "": # Invariance in key type causes more of these errors than we would want. self.note( diff --git a/test-data/unit/check-dataclasses.test b/test-data/unit/check-dataclasses.test index 30d8497c9cd28..2ead202bd6af1 100644 --- a/test-data/unit/check-dataclasses.test +++ b/test-data/unit/check-dataclasses.test @@ -705,7 +705,7 @@ class A(Generic[T]): return self.z # E: Incompatible return value type (got "list[T]", expected "T") reveal_type(A) # N: Revealed type is "def [T] (x: T`1, y: T`1, z: builtins.list[T`1]) -> __main__.A[T`1]" -A(1, 2, ["a", "b"]) # E: Cannot infer type argument 1 of "A" +A(1, 2, ["a", "b"]) # E: Cannot infer value of type parameter "T" of "A" a = A(1, 2, [1, 2]) reveal_type(a) # N: Revealed type is "__main__.A[builtins.int]" reveal_type(a.x) # N: Revealed type is "builtins.int" diff --git a/test-data/unit/check-expressions.test b/test-data/unit/check-expressions.test index f3c00627892ec..33271a3cc04c1 100644 --- a/test-data/unit/check-expressions.test +++ b/test-data/unit/check-expressions.test @@ -1878,7 +1878,7 @@ a = {'a': 1} b = {'z': 26, **a} c = {**b} d = {**a, **b, 'c': 3} -e = {1: 'a', **a} # E: Cannot infer type argument 1 of \ +e = {1: 'a', **a} # E: Cannot infer value of type parameter "KT" of \ # N: Try assigning the literal to a variable annotated as dict[, ] f = {**b} # type: Dict[int, int] # E: Unpacked dict entry 0 has incompatible type "dict[str, int]"; expected "SupportsKeysAndGetItem[int, int]" g = {**Thing()} @@ -1893,7 +1893,7 @@ i = {**Thing()} # type: Dict[int, int] # E: Unpacked dict entry 0 has incompat # N: def keys(self) -> Iterable[int] \ # N: Got: \ # N: def keys(self) -> Iterable[str] -j = {1: 'a', **Thing()} # E: Cannot infer type argument 1 of \ +j = {1: 'a', **Thing()} # E: Cannot infer value of type parameter "KT" of \ # N: Try assigning the literal to a variable annotated as dict[, ] [builtins fixtures/dict.pyi] [typing fixtures/typing-medium.pyi] diff --git a/test-data/unit/check-generics.test b/test-data/unit/check-generics.test index 8839dfb954f4a..0be9d918c69f2 100644 --- a/test-data/unit/check-generics.test +++ b/test-data/unit/check-generics.test @@ -584,7 +584,7 @@ def func2(x: SameNode[T]) -> SameNode[T]: return x reveal_type(func2) # N: Revealed type is "def [T] (x: __main__.Node[T`-1, T`-1]) -> __main__.Node[T`-1, T`-1]" -func2(Node(1, 'x')) # E: Cannot infer type argument 1 of "func2" +func2(Node(1, 'x')) # E: Cannot infer value of type parameter "T" of "func2" y = func2(Node('x', 'x')) reveal_type(y) # N: Revealed type is "__main__.Node[builtins.str, builtins.str]" @@ -888,7 +888,7 @@ def fun2(v: Vec[T], scale: T) -> Vec[T]: reveal_type(fun1([(1, 1)])) # N: Revealed type is "builtins.int" fun1(1) # E: Argument 1 to "fun1" has incompatible type "int"; expected "list[tuple[bool, bool]]" -fun1([(1, 'x')]) # E: Cannot infer type argument 1 of "fun1" +fun1([(1, 'x')]) # E: Cannot infer value of type parameter "T" of "fun1" reveal_type(fun2([(1, 1)], 1)) # N: Revealed type is "builtins.list[tuple[builtins.int, builtins.int]]" fun2([('x', 'x')], 'x') # E: Value of type variable "T" of "fun2" cannot be "str" @@ -909,7 +909,7 @@ def f(x: Node[T, T]) -> TupledNode[T]: return Node(x.x, (x.x, x.x)) f(1) # E: Argument 1 to "f" has incompatible type "int"; expected "Node[Never, Never]" -f(Node(1, 'x')) # E: Cannot infer type argument 1 of "f" +f(Node(1, 'x')) # E: Cannot infer value of type parameter "T" of "f" reveal_type(Node('x', 'x')) # N: Revealed type is "a.Node[builtins.str, builtins.str]" [file a.py] diff --git a/test-data/unit/check-inference-context.test b/test-data/unit/check-inference-context.test index 67ae22a369b19..ff726530cf9f0 100644 --- a/test-data/unit/check-inference-context.test +++ b/test-data/unit/check-inference-context.test @@ -1009,7 +1009,7 @@ class D(C): ... def f(x: List[T], y: List[T]) -> List[T]: ... -f([C()], [D()]) # E: Cannot infer type argument 1 of "f" +f([C()], [D()]) # E: Cannot infer value of type parameter "T" of "f" [builtins fixtures/list.pyi] [case testInferTypeVariableFromTwoGenericTypes3] diff --git a/test-data/unit/check-inference.test b/test-data/unit/check-inference.test index 856d430a544c9..90cb7d3799cf6 100644 --- a/test-data/unit/check-inference.test +++ b/test-data/unit/check-inference.test @@ -693,8 +693,8 @@ class A(Generic[T]): pass class B: pass -f(ao, ab) # E: Cannot infer type argument 1 of "f" -f(ab, ao) # E: Cannot infer type argument 1 of "f" +f(ao, ab) # E: Cannot infer value of type parameter "T" of "f" +f(ab, ao) # E: Cannot infer value of type parameter "T" of "f" f(ao, ao) f(ab, ab) @@ -3774,8 +3774,8 @@ reveal_type(f(x, [])) # N: Revealed type is "builtins.str" reveal_type(f(["yes"], [])) # N: Revealed type is "builtins.str" empty: List[NoReturn] -f(x, empty) # E: Cannot infer type argument 1 of "f" -f(["no"], empty) # E: Cannot infer type argument 1 of "f" +f(x, empty) # E: Cannot infer value of type parameter "T" of "f" +f(["no"], empty) # E: Cannot infer value of type parameter "T" of "f" [builtins fixtures/list.pyi] [case testInferenceWorksWithEmptyCollectionsUnion] @@ -4149,3 +4149,19 @@ class Foo: else: self.qux = {} # E: Need type annotation for "qux" (hint: "qux: dict[, ] = ...") [builtins fixtures/dict.pyi] + +[case testConstraintSolvingFailureShowsCorrectArgument] +from typing import Callable, TypeVar + +T1 = TypeVar('T1') +T2 = TypeVar('T2') +def foo( + a: T1, + b: T2, + c: Callable[[T2], T2], +) -> tuple[T1, T2]: ... + +def bar(y: float) -> float: ... + +foo(1, None, bar) # E: Cannot infer value of type parameter "T2" of "foo" +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-literal.test b/test-data/unit/check-literal.test index f995332643af7..3c9290b8dbbba 100644 --- a/test-data/unit/check-literal.test +++ b/test-data/unit/check-literal.test @@ -2989,9 +2989,9 @@ def g(a: T, t: A[T]) -> T: ... def check(obj: A[Literal[1]]) -> None: reveal_type(f(obj, 1)) # N: Revealed type is "Literal[1]" - reveal_type(f(obj, '')) # E: Cannot infer type argument 1 of "f" \ + reveal_type(f(obj, '')) # E: Cannot infer value of type parameter "T" of "f" \ # N: Revealed type is "Any" reveal_type(g(1, obj)) # N: Revealed type is "Literal[1]" - reveal_type(g('', obj)) # E: Cannot infer type argument 1 of "g" \ + reveal_type(g('', obj)) # E: Cannot infer value of type parameter "T" of "g" \ # N: Revealed type is "Any" [builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-overloading.test b/test-data/unit/check-overloading.test index 0ccc8a2a353c6..e427d5b21d40e 100644 --- a/test-data/unit/check-overloading.test +++ b/test-data/unit/check-overloading.test @@ -3370,10 +3370,10 @@ def wrapper() -> None: obj2: Union[W1[A], W2[B]] reveal_type(foo(obj2)) # N: Revealed type is "Union[__main__.A, __main__.B]" - bar(obj2) # E: Cannot infer type argument 1 of "bar" + bar(obj2) # E: Cannot infer value of type parameter "T" of "bar" b1_overload: A = foo(obj2) # E: Incompatible types in assignment (expression has type "Union[A, B]", variable has type "A") - b1_union: A = bar(obj2) # E: Cannot infer type argument 1 of "bar" + b1_union: A = bar(obj2) # E: Cannot infer value of type parameter "T" of "bar" [case testOverloadingInferUnionReturnWithObjectTypevarReturn] from typing import overload, Union, TypeVar, Generic @@ -3496,7 +3496,7 @@ def t_is_same_bound(arg1: T1, arg2: S) -> Tuple[T1, S]: # The arguments in the tuple are swapped x3: Union[List[S], List[Tuple[S, T1]]] y3: S - Dummy[T1]().foo(x3, y3) # E: Cannot infer type argument 1 of "foo" of "Dummy" \ + Dummy[T1]().foo(x3, y3) # E: Cannot infer value of type parameter "S" of "foo" of "Dummy" \ # E: Argument 1 to "foo" of "Dummy" has incompatible type "Union[list[S], list[tuple[S, T1]]]"; expected "list[tuple[T1, Any]]" x4: Union[List[int], List[Tuple[C, int]]] diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index 085f6fe59809b..e53c45b5b512b 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -2135,7 +2135,7 @@ def d(f: Callable[P, None], fn: Callable[Concatenate[Callable[P, None], P], None reveal_type(d(a, f1)) # N: Revealed type is "def (i: builtins.int)" reveal_type(d(a, f2)) # N: Revealed type is "def (i: builtins.int)" -reveal_type(d(b, f1)) # E: Cannot infer type argument 1 of "d" \ +reveal_type(d(b, f1)) # E: Cannot infer value of type parameter "P" of "d" \ # N: Revealed type is "def (*Any, **Any)" reveal_type(d(b, f2)) # N: Revealed type is "def (builtins.int)" [builtins fixtures/paramspec.pyi] diff --git a/test-data/unit/check-plugin-attrs.test b/test-data/unit/check-plugin-attrs.test index 6415b5104296b..00bec13ab16d5 100644 --- a/test-data/unit/check-plugin-attrs.test +++ b/test-data/unit/check-plugin-attrs.test @@ -470,8 +470,8 @@ reveal_type(a) # N: Revealed type is "__main__.A[builtins.int]" reveal_type(a.x) # N: Revealed type is "builtins.list[builtins.int]" reveal_type(a.y) # N: Revealed type is "builtins.int" -A(['str'], 7) # E: Cannot infer type argument 1 of "A" -A([1], '2') # E: Cannot infer type argument 1 of "A" +A(['str'], 7) # E: Cannot infer value of type parameter "T" of "A" +A([1], '2') # E: Cannot infer value of type parameter "T" of "A" [builtins fixtures/list.pyi] diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index c6c2c5f8da980..79207c9aad56d 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -4217,10 +4217,10 @@ def g2(a: Input[bytes], b: Output[bytes]) -> None: f(a, b) def g3(a: Input[str], b: Output[bytes]) -> None: - f(a, b) # E: Cannot infer type argument 1 of "f" + f(a, b) # E: Cannot infer value of type parameter "AnyStr" of "f" def g4(a: Input[bytes], b: Output[str]) -> None: - f(a, b) # E: Cannot infer type argument 1 of "f" + f(a, b) # E: Cannot infer value of type parameter "AnyStr" of "f" [builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index 862fd9ff5fb06..c0c826d09c9e2 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -2372,9 +2372,9 @@ def pointwise_multiply(x: Array[Unpack[Ts]], y: Array[Unpack[Ts]]) -> Array[Unpa def a1(x: Array[int], y: Array[str], z: Array[int, str]) -> None: reveal_type(pointwise_multiply(x, x)) # N: Revealed type is "__main__.Array[builtins.int]" - reveal_type(pointwise_multiply(x, y)) # E: Cannot infer type argument 1 of "pointwise_multiply" \ + reveal_type(pointwise_multiply(x, y)) # E: Cannot infer value of type parameter "Ts" of "pointwise_multiply" \ # N: Revealed type is "__main__.Array[Unpack[builtins.tuple[Any, ...]]]" - reveal_type(pointwise_multiply(x, z)) # E: Cannot infer type argument 1 of "pointwise_multiply" \ + reveal_type(pointwise_multiply(x, z)) # E: Cannot infer value of type parameter "Ts" of "pointwise_multiply" \ # N: Revealed type is "__main__.Array[Unpack[builtins.tuple[Any, ...]]]" def func(x: Array[Unpack[Ts]], *args: Unpack[Ts]) -> Tuple[Unpack[Ts]]: @@ -2382,9 +2382,9 @@ def func(x: Array[Unpack[Ts]], *args: Unpack[Ts]) -> Tuple[Unpack[Ts]]: def a2(x: Array[int, str]) -> None: reveal_type(func(x, 2, "Hello")) # N: Revealed type is "tuple[builtins.int, builtins.str]" - reveal_type(func(x, 2)) # E: Cannot infer type argument 1 of "func" \ + reveal_type(func(x, 2)) # E: Cannot infer value of type parameter "Ts" of "func" \ # N: Revealed type is "builtins.tuple[Any, ...]" - reveal_type(func(x, 2, "Hello", True)) # E: Cannot infer type argument 1 of "func" \ + reveal_type(func(x, 2, "Hello", True)) # E: Cannot infer value of type parameter "Ts" of "func" \ # N: Revealed type is "builtins.tuple[Any, ...]" [builtins fixtures/tuple.pyi] From ae778ccd7dea64f4d3756eb23a127aa9ea275e49 Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Thu, 19 Jun 2025 00:32:19 +0200 Subject: [PATCH 013/424] Fix `dmypy suggest` interaction with `__new__` (#18966) Fixes #18964. `__new__` is special - it has `is_static` set in `semanal.py`, but still has first `cls` argument. This PR tells `dmypy suggest` about that. --- mypy/checker.py | 4 ++-- mypy/nodes.py | 11 +++++++++++ mypy/semanal.py | 4 ++-- mypy/suggestions.py | 2 +- mypy/typeops.py | 2 +- test-data/unit/fine-grained-suggest.test | 20 ++++++++++++++++++++ 6 files changed, 37 insertions(+), 6 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 0639340d30bb0..10683327284bc 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -1362,7 +1362,7 @@ def check_func_def( if typ.type_is: arg_index = 0 # For methods and classmethods, we want the second parameter - if ref_type is not None and (not defn.is_static or defn.name == "__new__"): + if ref_type is not None and defn.has_self_or_cls_argument: arg_index = 1 if arg_index < len(typ.arg_types) and not is_subtype( typ.type_is, typ.arg_types[arg_index] @@ -1382,7 +1382,7 @@ def check_func_def( isinstance(defn, FuncDef) and ref_type is not None and i == 0 - and (not defn.is_static or defn.name == "__new__") + and defn.has_self_or_cls_argument and typ.arg_kinds[0] not in [nodes.ARG_STAR, nodes.ARG_STAR2] ): if defn.is_class or defn.name == "__new__": diff --git a/mypy/nodes.py b/mypy/nodes.py index 2cec4852f31c1..1b6884f04bf57 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -508,6 +508,8 @@ def __init__(self) -> None: self.info = FUNC_NO_INFO self.is_property = False self.is_class = False + # Is this a `@staticmethod` (explicit or implicit)? + # Note: use has_self_or_cls_argument to check if there is `self` or `cls` argument self.is_static = False self.is_final = False self.is_explicit_override = False @@ -524,6 +526,15 @@ def name(self) -> str: def fullname(self) -> str: return self._fullname + @property + def has_self_or_cls_argument(self) -> bool: + """If used as a method, does it have an argument for method binding (`self`, `cls`)? + + This is true for `__new__` even though `__new__` does not undergo method binding, + because we still usually assume that `cls` corresponds to the enclosing class. + """ + return not self.is_static or self.name == "__new__" + OverloadPart: _TypeAlias = Union["FuncDef", "Decorator"] diff --git a/mypy/semanal.py b/mypy/semanal.py index 704aa91d1d12e..8f9d1c4f35d67 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1069,7 +1069,7 @@ def prepare_method_signature(self, func: FuncDef, info: TypeInfo, has_self_type: functype = func.type if func.name == "__new__": func.is_static = True - if not func.is_static or func.name == "__new__": + if func.has_self_or_cls_argument: if func.name in ["__init_subclass__", "__class_getitem__"]: func.is_class = True if not func.arguments: @@ -1602,7 +1602,7 @@ def analyze_function_body(self, defn: FuncItem) -> None: # The first argument of a non-static, non-class method is like 'self' # (though the name could be different), having the enclosing class's # instance type. - if is_method and (not defn.is_static or defn.name == "__new__") and defn.arguments: + if is_method and defn.has_self_or_cls_argument and defn.arguments: if not defn.is_class: defn.arguments[0].variable.is_self = True else: diff --git a/mypy/suggestions.py b/mypy/suggestions.py index 673076729ffad..cfd7413860ec2 100644 --- a/mypy/suggestions.py +++ b/mypy/suggestions.py @@ -486,7 +486,7 @@ def get_suggestion(self, mod: str, node: FuncDef) -> PyAnnotateSignature: if self.no_errors and orig_errors: raise SuggestionFailure("Function does not typecheck.") - is_method = bool(node.info) and not node.is_static + is_method = bool(node.info) and node.has_self_or_cls_argument with state.strict_optional_set(graph[mod].options.strict_optional): guesses = self.get_guesses( diff --git a/mypy/typeops.py b/mypy/typeops.py index e8087a1713ff9..aaa3f91a0798b 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -890,7 +890,7 @@ def callable_type( fdef: FuncItem, fallback: Instance, ret_type: Type | None = None ) -> CallableType: # TODO: somewhat unfortunate duplication with prepare_method_signature in semanal - if fdef.info and (not fdef.is_static or fdef.name == "__new__") and fdef.arg_names: + if fdef.info and fdef.has_self_or_cls_argument and fdef.arg_names: self_type: Type = fill_typevars(fdef.info) if fdef.is_class or fdef.name == "__new__": self_type = TypeType.make_normalized(self_type) diff --git a/test-data/unit/fine-grained-suggest.test b/test-data/unit/fine-grained-suggest.test index f2db85c05f182..3a696ce19c634 100644 --- a/test-data/unit/fine-grained-suggest.test +++ b/test-data/unit/fine-grained-suggest.test @@ -795,6 +795,26 @@ def bar(iany) -> None: (int) -> None == +[case testSuggestNewInit] +# suggest: foo.F.__init__ +# suggest: foo.F.__new__ +[file foo.py] +class F: + def __new__(cls, t): + return super().__new__(cls) + + def __init__(self, t): + self.t = t + +[file bar.py] +from foo import F +def bar(iany) -> None: + F(0) +[out] +(int) -> None +(int) -> Any +== + [case testSuggestColonBasic] # suggest: tmp/foo.py:1 # suggest: tmp/bar/baz.py:2 From 9f455bd9f8f32e166ba748055aafba5f30a21714 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 19 Jun 2025 00:23:49 +0100 Subject: [PATCH 014/424] Generalize class/static method and property alias support (#19297) Fixes https://github.com/python/mypy/issues/6700 This is another followup for the `checkmember` work. Currently, we only support non-instance method aliasing in few very specific cases. I am making this support general. --- mypy/checker.py | 36 +++++++++++++++++----- mypy/checkmember.py | 18 +++++++---- test-data/unit/check-callable.test | 48 ++++++++++++++++++++++++++++++ test-data/unit/check-classes.test | 17 +++++++++++ 4 files changed, 106 insertions(+), 13 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 10683327284bc..9d02bcac84713 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -4400,9 +4400,9 @@ def set_inferred_type(self, var: Var, lvalue: Lvalue, type: Type) -> None: refers to the variable (lvalue). If var is None, do nothing. """ if var and not self.current_node_deferred: - # TODO: should we also set 'is_ready = True' here? var.type = type var.is_inferred = True + var.is_ready = True if var not in self.var_decl_frames: # Used for the hack to improve optional type inference in conditionals self.var_decl_frames[var] = {frame.id for frame in self.binder.frames} @@ -4412,9 +4412,23 @@ def set_inferred_type(self, var: Var, lvalue: Lvalue, type: Type) -> None: self.inferred_attribute_types[lvalue.def_var] = type self.store_type(lvalue, type) p_type = get_proper_type(type) - if isinstance(p_type, CallableType) and is_node_static(p_type.definition): - # TODO: handle aliases to class methods (similarly). - var.is_staticmethod = True + definition = None + if isinstance(p_type, CallableType): + definition = p_type.definition + elif isinstance(p_type, Overloaded): + # Randomly select first item, if items are different, there will + # be an error during semantic analysis. + definition = p_type.items[0].definition + if definition: + if is_node_static(definition): + var.is_staticmethod = True + elif is_classmethod_node(definition): + var.is_classmethod = True + elif is_property(definition): + var.is_property = True + if isinstance(p_type, Overloaded): + # TODO: in theory we can have a property with a deleter only. + var.is_settable_property = True def set_inference_error_fallback_type(self, var: Var, lvalue: Lvalue, type: Type) -> None: """Store best known type for variable if type inference failed. @@ -8531,15 +8545,21 @@ def visit_type_alias_type(self, t: TypeAliasType) -> Type: return t.copy_modified(args=[a.accept(self) for a in t.args]) +def is_classmethod_node(node: Node | None) -> bool | None: + """Find out if a node describes a classmethod.""" + if isinstance(node, FuncDef): + return node.is_class + if isinstance(node, Var): + return node.is_classmethod + return None + + def is_node_static(node: Node | None) -> bool | None: """Find out if a node describes a static function method.""" - if isinstance(node, FuncDef): return node.is_static - if isinstance(node, Var): return node.is_staticmethod - return None @@ -8786,6 +8806,8 @@ def is_static(func: FuncBase | Decorator) -> bool: def is_property(defn: SymbolNode) -> bool: + if isinstance(defn, FuncDef): + return defn.is_property if isinstance(defn, Decorator): return defn.func.is_property if isinstance(defn, OverloadedFuncDef): diff --git a/mypy/checkmember.py b/mypy/checkmember.py index 8f62fee699c02..2a8d09808cfb3 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -371,8 +371,6 @@ def analyze_instance_member_access( signature, mx.self_type, method.is_class, mx.context, name, mx.msg ) signature = bind_self(signature, mx.self_type, is_classmethod=method.is_class) - # TODO: should we skip these steps for static methods as well? - # Since generic static methods should not be allowed. typ = map_instance_to_supertype(typ, method.info) member_type = expand_type_by_instance(signature, typ) freeze_all_type_vars(member_type) @@ -1224,8 +1222,11 @@ def analyze_class_attribute_access( # C[int].x -> int t = erase_typevars(expand_type_by_instance(t, isuper), {tv.id for tv in def_vars}) - is_classmethod = (is_decorated and cast(Decorator, node.node).func.is_class) or ( - isinstance(node.node, SYMBOL_FUNCBASE_TYPES) and node.node.is_class + is_classmethod = ( + (is_decorated and cast(Decorator, node.node).func.is_class) + or (isinstance(node.node, SYMBOL_FUNCBASE_TYPES) and node.node.is_class) + or isinstance(node.node, Var) + and node.node.is_classmethod ) is_staticmethod = (is_decorated and cast(Decorator, node.node).func.is_static) or ( isinstance(node.node, SYMBOL_FUNCBASE_TYPES) and node.node.is_static @@ -1237,7 +1238,12 @@ def analyze_class_attribute_access( is_trivial_self = node.node.func.is_trivial_self and not node.node.decorators elif isinstance(node.node, (FuncDef, OverloadedFuncDef)): is_trivial_self = node.node.is_trivial_self - if isinstance(t, FunctionLike) and is_classmethod and not is_trivial_self: + if ( + isinstance(t, FunctionLike) + and is_classmethod + and not is_trivial_self + and not t.bound() + ): t = check_self_arg(t, mx.self_type, False, mx.context, name, mx.msg) t = add_class_tvars( t, @@ -1406,7 +1412,7 @@ class B(A[str]): pass tvars = original_vars if original_vars is not None else [] if not mx.preserve_type_var_ids: t = freshen_all_functions_type_vars(t) - if is_classmethod: + if is_classmethod and not t.is_bound: if is_trivial_self: t = bind_self_fast(t, mx.self_type) else: diff --git a/test-data/unit/check-callable.test b/test-data/unit/check-callable.test index 39e6c4fa3ff1c..23db0bf50a4ec 100644 --- a/test-data/unit/check-callable.test +++ b/test-data/unit/check-callable.test @@ -587,6 +587,54 @@ class C(B): class B: ... [builtins fixtures/classmethod.pyi] +[case testClassMethodAliasInClass] +from typing import overload + +class C: + @classmethod + def foo(cls) -> int: ... + + bar = foo + + @overload + @classmethod + def foo2(cls, x: int) -> int: ... + @overload + @classmethod + def foo2(cls, x: str) -> str: ... + @classmethod + def foo2(cls, x): + ... + + bar2 = foo2 + +reveal_type(C.bar) # N: Revealed type is "def () -> builtins.int" +reveal_type(C().bar) # N: Revealed type is "def () -> builtins.int" +reveal_type(C.bar2) # N: Revealed type is "Overload(def (x: builtins.int) -> builtins.int, def (x: builtins.str) -> builtins.str)" +reveal_type(C().bar2) # N: Revealed type is "Overload(def (x: builtins.int) -> builtins.int, def (x: builtins.str) -> builtins.str)" +[builtins fixtures/classmethod.pyi] + +[case testPropertyAliasInClassBody] +class A: + @property + def f(self) -> int: ... + + g = f + + @property + def f2(self) -> int: ... + @f2.setter + def f2(self, val: int) -> None: ... + + g2 = f2 + +reveal_type(A().g) # N: Revealed type is "builtins.int" +reveal_type(A().g2) # N: Revealed type is "builtins.int" +A().g = 1 # E: Property "g" defined in "A" is read-only +A().g2 = 1 +A().g2 = "no" # E: Incompatible types in assignment (expression has type "str", variable has type "int") +[builtins fixtures/property.pyi] + [case testCallableUnionCallback] from typing import Union, Callable, TypeVar diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index f4bbaf41dc471..3d99ccb302c6a 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -4588,6 +4588,23 @@ reveal_type(a.a) # N: Revealed type is "def (a: builtins.int)" reveal_type(a.c) # N: Revealed type is "def (a: builtins.int)" [builtins fixtures/staticmethod.pyi] +[case testClassStaticMethodIndirectOverloaded] +from typing import overload +class A: + @overload + @staticmethod + def a(x: int) -> int: ... + @overload + @staticmethod + def a(x: str) -> str: ... + @staticmethod + def a(x): + ... + c = a +reveal_type(A.c) # N: Revealed type is "Overload(def (x: builtins.int) -> builtins.int, def (x: builtins.str) -> builtins.str)" +reveal_type(A().c) # N: Revealed type is "Overload(def (x: builtins.int) -> builtins.int, def (x: builtins.str) -> builtins.str)" +[builtins fixtures/staticmethod.pyi] + [case testClassStaticMethodSubclassing] class A: @staticmethod From d52ce3b3702bc7c194d6c14bca7affad2596e84b Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 19 Jun 2025 00:26:51 +0100 Subject: [PATCH 015/424] Fix constructor type for subclasses of Any (#19295) Fixes https://github.com/python/mypy/issues/9815 Fixes https://github.com/python/mypy/issues/10848 Fixes https://github.com/python/mypy/issues/17781 Also discovered this while working on `checkmember` stuff. Previously, return type of the type object was the class where `__init__()` was defined (if there was an `Any` somewhere in MRO). And since we use return type for attribute access on type objects, it went completely sideways. Fix is simple (we accidentally used `info` instead of `def_info` in one place). --- mypy/typeops.py | 8 ++++---- test-data/unit/check-classes.test | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/mypy/typeops.py b/mypy/typeops.py index aaa3f91a0798b..e84be19465cc7 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -125,7 +125,8 @@ def tuple_fallback(typ: TupleType) -> Instance: ) -def get_self_type(func: CallableType, default_self: Instance | TupleType) -> Type | None: +def get_self_type(func: CallableType, def_info: TypeInfo) -> Type | None: + default_self = fill_typevars(def_info) if isinstance(get_proper_type(func.ret_type), UninhabitedType): return func.ret_type elif func.arg_types and func.arg_types[0] != default_self and func.arg_kinds[0] == ARG_POS: @@ -227,9 +228,8 @@ def type_object_type_from_function( # self-types only in the defining class, similar to __new__ (but not exactly the same, # see comment in class_callable below). This is mostly useful for annotating library # classes such as subprocess.Popen. - default_self = fill_typevars(info) if not is_new and not info.is_newtype: - orig_self_types = [get_self_type(it, default_self) for it in signature.items] + orig_self_types = [get_self_type(it, def_info) for it in signature.items] else: orig_self_types = [None] * len(signature.items) @@ -245,7 +245,7 @@ def type_object_type_from_function( # We need to map B's __init__ to the type (List[T]) -> None. signature = bind_self( signature, - original_type=default_self, + original_type=fill_typevars(info), is_classmethod=is_new, # Explicit instance self annotations have special handling in class_callable(), # we don't need to bind any type variables in them if they are generic. diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index 3d99ccb302c6a..1b3de53567d1f 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -8796,3 +8796,21 @@ class C: C().foo = "no" # E: Incompatible types in assignment (expression has type "str", variable has type "int") C().bar = "fine" [builtins fixtures/property.pyi] + +[case testCorrectConstructorTypeWithAnyFallback] +from typing import Generic, TypeVar + +class B(Unknown): # type: ignore + def __init__(self) -> None: ... +class C(B): ... + +reveal_type(C) # N: Revealed type is "def () -> __main__.C" + +T = TypeVar("T") +class BG(Generic[T], Unknown): # type: ignore + def __init__(self) -> None: ... +class CGI(BG[int]): ... +class CGT(BG[T]): ... + +reveal_type(CGI) # N: Revealed type is "def () -> __main__.CGI" +reveal_type(CGT) # N: Revealed type is "def [T] () -> __main__.CGT[T`1]" From ad570933924b3810ba61d2e4a13eac596f74672b Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 19 Jun 2025 17:27:14 +0100 Subject: [PATCH 016/424] Cleanup generic class variable access (#19292) Fixes https://github.com/python/mypy/issues/5144 Fixes https://github.com/python/mypy/issues/15223 This is related to the work on https://github.com/python/mypy/issues/7724 Now that all attribute access goes through `checkmember.py` there is not much benefit in giving an error at the definition site, especially that it prohibits some valid (and common) use cases, see comments in https://github.com/python/mypy/issues/5144. While looking at this I discovered a bunch of defects in the _implementation_, that I also fix (I am keeping unsafe self-type related logic as is): * We used to erase type vars of the definition class instead of the use class. This caused type variables leaks. * The erasure was inconsistent, so that in some cases we silently erased type variables to `Any` even in allowed use cases * `TypeVarTuple` and `ParamSpec` were not handled as equal to regular type variables (I guess because of old problems with erasing them) --- mypy/checkmember.py | 30 ++++++++++++++----------- mypy/message_registry.py | 1 - mypy/semanal.py | 8 ------- test-data/unit/check-classvar.test | 23 ++++++++++++++++--- test-data/unit/check-selftype.test | 22 ++++++++++++++++++ test-data/unit/check-typevar-tuple.test | 22 ++++++++++++++++++ test-data/unit/semanal-classvar.test | 11 --------- 7 files changed, 81 insertions(+), 36 deletions(-) diff --git a/mypy/checkmember.py b/mypy/checkmember.py index 2a8d09808cfb3..5b5580a648a8b 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -45,7 +45,6 @@ freeze_all_type_vars, function_type, get_all_type_vars, - get_type_vars, make_simplified_union, supported_self_type, tuple_fallback, @@ -1196,31 +1195,36 @@ def analyze_class_attribute_access( if isinstance(node.node, Var): assert isuper is not None + object_type = get_proper_type(mx.self_type) # Check if original variable type has type variables. For example: # class C(Generic[T]): # x: T # C.x # Error, ambiguous access # C[int].x # Also an error, since C[int] is same as C at runtime # Exception is Self type wrapped in ClassVar, that is safe. + prohibit_self = not node.node.is_classvar def_vars = set(node.node.info.defn.type_vars) - if not node.node.is_classvar and node.node.info.self_type: + if prohibit_self and node.node.info.self_type: def_vars.add(node.node.info.self_type) - # TODO: should we include ParamSpec etc. here (i.e. use get_all_type_vars)? - typ_vars = set(get_type_vars(t)) - if def_vars & typ_vars: - # Exception: access on Type[...], including first argument of class methods is OK. - if not isinstance(get_proper_type(mx.original_type), TypeType) or node.implicit: - if node.node.is_classvar: - message = message_registry.GENERIC_CLASS_VAR_ACCESS - else: - message = message_registry.GENERIC_INSTANCE_VAR_CLASS_ACCESS - mx.fail(message) + # Exception: access on Type[...], including first argument of class methods is OK. + prohibit_generic = not isinstance(object_type, TypeType) or node.implicit + if prohibit_generic and def_vars & set(get_all_type_vars(t)): + if node.node.is_classvar: + message = message_registry.GENERIC_CLASS_VAR_ACCESS + else: + message = message_registry.GENERIC_INSTANCE_VAR_CLASS_ACCESS + mx.fail(message) t = expand_self_type_if_needed(t, mx, node.node, itype, is_class=True) + t = expand_type_by_instance(t, isuper) # Erase non-mapped variables, but keep mapped ones, even if there is an error. # In the above example this means that we infer following types: # C.x -> Any # C[int].x -> int - t = erase_typevars(expand_type_by_instance(t, isuper), {tv.id for tv in def_vars}) + if prohibit_generic: + erase_vars = set(itype.type.defn.type_vars) + if prohibit_self and itype.type.self_type: + erase_vars.add(itype.type.self_type) + t = erase_typevars(t, {tv.id for tv in erase_vars}) is_classmethod = ( (is_decorated and cast(Decorator, node.node).func.is_class) diff --git a/mypy/message_registry.py b/mypy/message_registry.py index 3c7745876f87b..381aedfca0598 100644 --- a/mypy/message_registry.py +++ b/mypy/message_registry.py @@ -253,7 +253,6 @@ def with_additional_msg(self, info: str) -> ErrorMessage: 'Cannot override class variable (previously declared on base class "{}") with instance ' "variable" ) -CLASS_VAR_WITH_TYPEVARS: Final = "ClassVar cannot contain type variables" CLASS_VAR_WITH_GENERIC_SELF: Final = "ClassVar cannot contain Self type in generic classes" CLASS_VAR_OUTSIDE_OF_CLASS: Final = "ClassVar can only be used for assignments in class body" diff --git a/mypy/semanal.py b/mypy/semanal.py index 8f9d1c4f35d67..87aef2595caf6 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -5079,14 +5079,6 @@ def check_classvar(self, s: AssignmentStmt) -> None: node.is_classvar = True analyzed = self.anal_type(s.type) assert self.type is not None - if analyzed is not None and set(get_type_vars(analyzed)) & set( - self.type.defn.type_vars - ): - # This means that we have a type var defined inside of a ClassVar. - # This is not allowed by PEP526. - # See https://github.com/python/mypy/issues/11538 - - self.fail(message_registry.CLASS_VAR_WITH_TYPEVARS, s) if ( analyzed is not None and self.type.self_type in get_type_vars(analyzed) diff --git a/test-data/unit/check-classvar.test b/test-data/unit/check-classvar.test index 63bbd7471bc8b..8384e56247934 100644 --- a/test-data/unit/check-classvar.test +++ b/test-data/unit/check-classvar.test @@ -285,7 +285,7 @@ main:3: error: Cannot assign to class variable "x" via instance from typing import ClassVar, Generic, TypeVar T = TypeVar('T') class A(Generic[T]): - x: ClassVar[T] # E: ClassVar cannot contain type variables + x: ClassVar[T] # Error reported at access site @classmethod def foo(cls) -> T: return cls.x # OK @@ -308,7 +308,7 @@ from typing import ClassVar, Generic, Tuple, TypeVar, Union, Type T = TypeVar('T') U = TypeVar('U') class A(Generic[T, U]): - x: ClassVar[Union[T, Tuple[U, Type[U]]]] # E: ClassVar cannot contain type variables + x: ClassVar[Union[T, Tuple[U, Type[U]]]] # Error reported at access site @classmethod def foo(cls) -> Union[T, Tuple[U, Type[U]]]: return cls.x # OK @@ -319,7 +319,9 @@ A[int, str].x # E: Access to generic class variables is ambiguous class Bad(A[int, str]): pass -Bad.x # E: Access to generic class variables is ambiguous +reveal_type(Bad.x) # E: Access to generic class variables is ambiguous \ + # N: Revealed type is "Union[builtins.int, tuple[builtins.str, type[builtins.str]]]" +reveal_type(Bad().x) # N: Revealed type is "Union[builtins.int, tuple[builtins.str, type[builtins.str]]]" class Good(A[int, str]): x = 42 @@ -343,3 +345,18 @@ class C: g: ClassVar[Union[Callable[[C], int], int]] = f reveal_type(C().g) # N: Revealed type is "Union[def () -> builtins.int, builtins.int]" + +[case testGenericSubclassAccessNoLeak] +from typing import ClassVar, Generic, TypeVar + +T = TypeVar("T") +class B(Generic[T]): + x: T + y: ClassVar[T] + +class C(B[T]): ... + +reveal_type(C.x) # E: Access to generic instance variables via class is ambiguous \ + # N: Revealed type is "Any" +reveal_type(C.y) # E: Access to generic class variables is ambiguous \ + # N: Revealed type is "Any" diff --git a/test-data/unit/check-selftype.test b/test-data/unit/check-selftype.test index 12d6133ec83f7..88ca53c8ed66c 100644 --- a/test-data/unit/check-selftype.test +++ b/test-data/unit/check-selftype.test @@ -2220,6 +2220,28 @@ class C(A, B): # OK: both methods take Self pass [builtins fixtures/tuple.pyi] +[case testSelfTypeClassMethodNotSilentlyErased] +from typing import Self, Optional + +class X: + _inst: Optional[Self] = None + @classmethod + def default(cls) -> Self: + reveal_type(cls._inst) # N: Revealed type is "Union[Self`0, None]" + if cls._inst is None: + cls._inst = cls() + return cls._inst + +reveal_type(X._inst) # E: Access to generic instance variables via class is ambiguous \ + # N: Revealed type is "Union[__main__.X, None]" +reveal_type(X()._inst) # N: Revealed type is "Union[__main__.X, None]" + +class Y(X): ... +reveal_type(Y._inst) # E: Access to generic instance variables via class is ambiguous \ + # N: Revealed type is "Union[__main__.Y, None]" +reveal_type(Y()._inst) # N: Revealed type is "Union[__main__.Y, None]" +[builtins fixtures/tuple.pyi] + [case testSelfInFuncDecoratedClassmethod] from collections.abc import Callable from typing import Self, TypeVar diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index c0c826d09c9e2..f44758f7b51b6 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -2629,6 +2629,28 @@ def test(*args: Unpack[tuple[T]]) -> int: ... reveal_type(fn(test)) # N: Revealed type is "def [T] (T`1) -> builtins.int" [builtins fixtures/tuple.pyi] +[case testNoGenericTypeVarTupleClassVarAccess] +from typing import Generic, Tuple, TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +class C(Generic[Unpack[Ts]]): + x: Tuple[Unpack[Ts]] + +reveal_type(C.x) # E: Access to generic instance variables via class is ambiguous \ + # N: Revealed type is "builtins.tuple[Any, ...]" + +class Bad(C[int, int]): + pass +reveal_type(Bad.x) # E: Access to generic instance variables via class is ambiguous \ + # N: Revealed type is "tuple[builtins.int, builtins.int]" +reveal_type(Bad().x) # N: Revealed type is "tuple[builtins.int, builtins.int]" + +class Good(C[int, int]): + x = (1, 1) +reveal_type(Good.x) # N: Revealed type is "tuple[builtins.int, builtins.int]" +reveal_type(Good().x) # N: Revealed type is "tuple[builtins.int, builtins.int]" +[builtins fixtures/tuple.pyi] + [case testConstraintsIncludeTupleFallback] from typing import Generic, TypeVar from typing_extensions import TypeVarTuple, Unpack diff --git a/test-data/unit/semanal-classvar.test b/test-data/unit/semanal-classvar.test index a7bcec0324dc2..8add559bdd27e 100644 --- a/test-data/unit/semanal-classvar.test +++ b/test-data/unit/semanal-classvar.test @@ -207,14 +207,3 @@ class B: pass [out] main:4: error: ClassVar can only be used for assignments in class body - -[case testClassVarWithTypeVariable] -from typing import ClassVar, TypeVar, Generic, List - -T = TypeVar('T') - -class Some(Generic[T]): - error: ClassVar[T] # E: ClassVar cannot contain type variables - nested: ClassVar[List[List[T]]] # E: ClassVar cannot contain type variables - ok: ClassVar[int] -[out] From 96fcd59585d1460440967f5dd6bdc82d43703187 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 19 Jun 2025 22:57:43 +0100 Subject: [PATCH 017/424] Re-widen custom properties after narrowing (#19296) Fixes https://github.com/python/mypy/issues/10399 This is another smaller cleanup for https://github.com/python/mypy/issues/7724. The current logic as documented is IMO correct, for attributes (either properties or custom descriptors) with setter type different from getter type, we narrow the attribute type in an assignment if: * The attribute is "normalizing", i.e. getter type is a subtype of setter type (e.g. `Sequence[Employee]` is normalized to `tuple[Employee, ...]`) * The given r.h.s. type in the assignment is a subtype of getter type (and thus transitively the setter as well), e.g. `tuple[Manager, ...]` vs `tuple[Employee, ...]` in the example above. The problem was that this logic was implemented too literally, as a result assignments that didn't satisfy these two rules were simply ignored (thus making previous narrowing incorrectly "sticky"). In fact, we also need to re-widen previously narrowed types whenever second condition is not satisfied. (I also decided to rename one variable name to make it more obvious.) --------- Co-authored-by: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> --- mypy/checker.py | 18 ++++++++---------- test-data/unit/check-narrowing.test | 19 +++++++++++++++++++ 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 9d02bcac84713..bfacf7f882e0e 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -4594,7 +4594,7 @@ def check_member_assignment( self, lvalue: MemberExpr, instance_type: Type, - attribute_type: Type, + set_lvalue_type: Type, rvalue: Expression, context: Context, ) -> tuple[Type, Type, bool]: @@ -4611,23 +4611,21 @@ def check_member_assignment( if (isinstance(instance_type, FunctionLike) and instance_type.is_type_obj()) or isinstance( instance_type, TypeType ): - rvalue_type, _ = self.check_simple_assignment(attribute_type, rvalue, context) - return rvalue_type, attribute_type, True + rvalue_type, _ = self.check_simple_assignment(set_lvalue_type, rvalue, context) + return rvalue_type, set_lvalue_type, True with self.msg.filter_errors(filter_deprecated=True): get_lvalue_type = self.expr_checker.analyze_ordinary_member_access( lvalue, is_lvalue=False ) - # Special case: if the rvalue_type is a subtype of both '__get__' and '__set__' types, - # and '__get__' type is narrower than '__set__', then we invoke the binder to narrow type + # Special case: if the rvalue_type is a subtype of '__get__' type, and + # '__get__' type is narrower than '__set__', then we invoke the binder to narrow type # by this assignment. Technically, this is not safe, but in practice this is # what a user expects. - rvalue_type, _ = self.check_simple_assignment(attribute_type, rvalue, context) - infer = is_subtype(rvalue_type, get_lvalue_type) and is_subtype( - get_lvalue_type, attribute_type - ) - return rvalue_type if infer else attribute_type, attribute_type, infer + rvalue_type, _ = self.check_simple_assignment(set_lvalue_type, rvalue, context) + rvalue_type = rvalue_type if is_subtype(rvalue_type, get_lvalue_type) else get_lvalue_type + return rvalue_type, set_lvalue_type, is_subtype(get_lvalue_type, set_lvalue_type) def check_indexed_assignment( self, lvalue: IndexExpr, rvalue: Expression, context: Context diff --git a/test-data/unit/check-narrowing.test b/test-data/unit/check-narrowing.test index 3778c5276576e..0443bc8459070 100644 --- a/test-data/unit/check-narrowing.test +++ b/test-data/unit/check-narrowing.test @@ -2593,3 +2593,22 @@ def baz(item: Base) -> None: reveal_type(item) # N: Revealed type is "__main__." item.bar() [builtins fixtures/isinstance.pyi] + +[case testCustomSetterNarrowingReWidened] +class B: ... +class C(B): ... +class C1(B): ... +class D(C): ... + +class Test: + @property + def foo(self) -> C: ... + @foo.setter + def foo(self, val: B) -> None: ... + +t: Test +t.foo = D() +reveal_type(t.foo) # N: Revealed type is "__main__.D" +t.foo = C1() +reveal_type(t.foo) # N: Revealed type is "__main__.C" +[builtins fixtures/property.pyi] From ffb692884f9e84ac2d2ec141996ba07755219090 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Fri, 20 Jun 2025 11:44:22 -0400 Subject: [PATCH 018/424] [mypyc] feat(docs): detail issue with inspect.iscoroutinefunction (#19309) Silly little PR, but this note could have saved me some debugging time. I also opened [an issue](https://github.com/mypyc/mypyc/issues/1110) and will make an attempt to solve the problem sometime in Q3. --- mypyc/doc/differences_from_python.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/mypyc/doc/differences_from_python.rst b/mypyc/doc/differences_from_python.rst index 65ad709677afb..5a230bd984c22 100644 --- a/mypyc/doc/differences_from_python.rst +++ b/mypyc/doc/differences_from_python.rst @@ -317,6 +317,7 @@ non-exhaustive list of what won't work: - Frames of compiled functions can't be inspected using ``inspect`` - Compiled methods aren't considered methods by ``inspect.ismethod`` - ``inspect.signature`` chokes on compiled functions +- ``inspect.iscoroutinefunction`` and ``asyncio.iscoroutinefunction`` will always return False for compiled functions, even those defined with `async def` Profiling hooks and tracing *************************** From 4322d4f443ef7568b1c0f4fec8df9e12e90fd8b9 Mon Sep 17 00:00:00 2001 From: Christoph Tyralla Date: Fri, 20 Jun 2025 17:48:30 +0200 Subject: [PATCH 019/424] Improve the handling of "iteration dependent" errors and notes in finally clauses. (#19270) Fixes #19269 This PR refactors the logic implemented in #19118 (which only targeted repeatedly checked loops) and applies it to repeatedly checked finally clauses. I moved nearly all relevant code to the class `LoopErrorWatcher`, which now has the more general name `IterationErrorWatcher`, to avoid code duplication. However, one duplication is left, which concerns error reporting. It would be nice and easy to move this functionality to `IterationErrorWatcher`, too, but this would result in import cycles, and I am unsure if working with `TYPE_CHECKING` and postponed importing is acceptable in such cases (both for Mypy and Mypyc). After the refactoring, it should not be much effort to apply the logic to other cases where code sections are analysed iteratively. However, the only thing that comes to my mind is the repeated checking of functions with arguments that contain constrained type variables. I will check it. If anyone finds a similar case and the solution is as simple as expected, we could add the fix to this PR, of course. --- mypy/checker.py | 67 ++++++-------- mypy/errors.py | 96 ++++++++++++++++---- test-data/unit/check-narrowing.test | 19 ++++ test-data/unit/check-redefine2.test | 3 +- test-data/unit/check-union-error-syntax.test | 21 +++++ 5 files changed, 148 insertions(+), 58 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index bfacf7f882e0e..e05523a1aa05c 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -25,7 +25,14 @@ from mypy.constraints import SUPERTYPE_OF from mypy.erasetype import erase_type, erase_typevars, remove_instance_last_known_values from mypy.errorcodes import TYPE_VAR, UNUSED_AWAITABLE, UNUSED_COROUTINE, ErrorCode -from mypy.errors import ErrorInfo, Errors, ErrorWatcher, LoopErrorWatcher, report_internal_error +from mypy.errors import ( + ErrorInfo, + Errors, + ErrorWatcher, + IterationDependentErrors, + IterationErrorWatcher, + report_internal_error, +) from mypy.expandtype import expand_type from mypy.literals import Key, extract_var_from_literal_hash, literal, literal_hash from mypy.maptype import map_instance_to_supertype @@ -598,26 +605,15 @@ def accept_loop( # on without bound otherwise) widened_old = len(self.widened_vars) - # one set of `unreachable`, `redundant-expr`, and `redundant-casts` errors - # per iteration step: - uselessness_errors = [] - # one set of unreachable line numbers per iteration step: - unreachable_lines = [] - # one set of revealed types per line where `reveal_type` is used (each - # created set can grow during the iteration): - revealed_types = defaultdict(set) + iter_errors = IterationDependentErrors() iter = 1 while True: with self.binder.frame_context(can_skip=True, break_frame=2, continue_frame=1): if on_enter_body is not None: on_enter_body() - with LoopErrorWatcher(self.msg.errors) as watcher: + with IterationErrorWatcher(self.msg.errors, iter_errors) as watcher: self.accept(body) - uselessness_errors.append(watcher.uselessness_errors) - unreachable_lines.append(watcher.unreachable_lines) - for key, values in watcher.revealed_types.items(): - revealed_types[key].update(values) partials_new = sum(len(pts.map) for pts in self.partial_types) widened_new = len(self.widened_vars) @@ -639,29 +635,10 @@ def accept_loop( if iter == 20: raise RuntimeError("Too many iterations when checking a loop") - # Report only those `unreachable`, `redundant-expr`, and `redundant-casts` - # errors that could not be ruled out in any iteration step: - persistent_uselessness_errors = set() - for candidate in set(itertools.chain(*uselessness_errors)): - if all( - (candidate in errors) or (candidate[2] in lines) - for errors, lines in zip(uselessness_errors, unreachable_lines) - ): - persistent_uselessness_errors.add(candidate) - for error_info in persistent_uselessness_errors: - context = Context(line=error_info[2], column=error_info[3]) - context.end_line = error_info[4] - context.end_column = error_info[5] - self.msg.fail(error_info[1], context, code=error_info[0]) - - # Report all types revealed in at least one iteration step: - for note_info, types in revealed_types.items(): - sorted_ = sorted(types, key=lambda typ: typ.lower()) - revealed = sorted_[0] if len(types) == 1 else f"Union[{', '.join(sorted_)}]" - context = Context(line=note_info[1], column=note_info[2]) - context.end_line = note_info[3] - context.end_column = note_info[4] - self.note(f'Revealed type is "{revealed}"', context) + for error_info in watcher.yield_error_infos(): + self.msg.fail(*error_info[:2], code=error_info[2]) + for note_info in watcher.yield_note_infos(self.options): + self.note(*note_info) # If exit_condition is set, assume it must be False on exit from the loop: if exit_condition: @@ -4960,6 +4937,9 @@ def type_check_raise(self, e: Expression, s: RaiseStmt, optional: bool = False) def visit_try_stmt(self, s: TryStmt) -> None: """Type check a try statement.""" + + iter_errors = None + # Our enclosing frame will get the result if the try/except falls through. # This one gets all possible states after the try block exited abnormally # (by exception, return, break, etc.) @@ -4974,7 +4954,9 @@ def visit_try_stmt(self, s: TryStmt) -> None: self.visit_try_without_finally(s, try_frame=bool(s.finally_body)) if s.finally_body: # First we check finally_body is type safe on all abnormal exit paths - self.accept(s.finally_body) + iter_errors = IterationDependentErrors() + with IterationErrorWatcher(self.msg.errors, iter_errors) as watcher: + self.accept(s.finally_body) if s.finally_body: # Then we try again for the more restricted set of options @@ -4988,8 +4970,15 @@ def visit_try_stmt(self, s: TryStmt) -> None: # type checks in both contexts, but only the resulting types # from the latter context affect the type state in the code # that follows the try statement.) + assert iter_errors is not None if not self.binder.is_unreachable(): - self.accept(s.finally_body) + with IterationErrorWatcher(self.msg.errors, iter_errors) as watcher: + self.accept(s.finally_body) + + for error_info in watcher.yield_error_infos(): + self.msg.fail(*error_info[:2], code=error_info[2]) + for note_info in watcher.yield_note_infos(self.options): + self.msg.note(*note_info) def visit_try_without_finally(self, s: TryStmt, try_frame: bool) -> None: """Type check a try statement, ignoring the finally block. diff --git a/mypy/errors.py b/mypy/errors.py index 7a173f16d1961..41a4de6392366 100644 --- a/mypy/errors.py +++ b/mypy/errors.py @@ -4,13 +4,15 @@ import sys import traceback from collections import defaultdict -from collections.abc import Iterable +from collections.abc import Iterable, Iterator +from itertools import chain from typing import Callable, Final, NoReturn, Optional, TextIO, TypeVar from typing_extensions import Literal, Self, TypeAlias as _TypeAlias from mypy import errorcodes as codes from mypy.error_formatter import ErrorFormatter from mypy.errorcodes import IMPORT, IMPORT_NOT_FOUND, IMPORT_UNTYPED, ErrorCode, mypy_error_codes +from mypy.nodes import Context from mypy.options import Options from mypy.scope import Scope from mypy.util import DEFAULT_SOURCE_OFFSET, is_typeshed_file @@ -219,23 +221,43 @@ def filtered_errors(self) -> list[ErrorInfo]: return self._filtered -class LoopErrorWatcher(ErrorWatcher): - """Error watcher that filters and separately collects `unreachable` errors, - `redundant-expr` and `redundant-casts` errors, and revealed types when analysing - loops iteratively to help avoid making too-hasty reports.""" +class IterationDependentErrors: + """An `IterationDependentErrors` instance serves to collect the `unreachable`, + `redundant-expr`, and `redundant-casts` errors, as well as the revealed types, + handled by the individual `IterationErrorWatcher` instances sequentially applied to + the same code section.""" - # Meaning of the tuple items: ErrorCode, message, line, column, end_line, end_column: - uselessness_errors: set[tuple[ErrorCode, str, int, int, int, int]] + # One set of `unreachable`, `redundant-expr`, and `redundant-casts` errors per + # iteration step. Meaning of the tuple items: ErrorCode, message, line, column, + # end_line, end_column. + uselessness_errors: list[set[tuple[ErrorCode, str, int, int, int, int]]] - # Meaning of the tuple items: function_or_member, line, column, end_line, end_column: + # One set of unreachable line numbers per iteration step. Not only the lines where + # the error report occurs but really all unreachable lines. + unreachable_lines: list[set[int]] + + # One set of revealed types for each `reveal_type` statement. Each created set can + # grow during the iteration. Meaning of the tuple items: function_or_member, line, + # column, end_line, end_column: revealed_types: dict[tuple[str | None, int, int, int, int], set[str]] - # Not only the lines where the error report occurs but really all unreachable lines: - unreachable_lines: set[int] + def __init__(self) -> None: + self.uselessness_errors = [] + self.unreachable_lines = [] + self.revealed_types = defaultdict(set) + + +class IterationErrorWatcher(ErrorWatcher): + """Error watcher that filters and separately collects `unreachable` errors, + `redundant-expr` and `redundant-casts` errors, and revealed types when analysing + code sections iteratively to help avoid making too-hasty reports.""" + + iteration_dependent_errors: IterationDependentErrors def __init__( self, errors: Errors, + iteration_dependent_errors: IterationDependentErrors, *, filter_errors: bool | Callable[[str, ErrorInfo], bool] = False, save_filtered_errors: bool = False, @@ -247,31 +269,71 @@ def __init__( save_filtered_errors=save_filtered_errors, filter_deprecated=filter_deprecated, ) - self.uselessness_errors = set() - self.unreachable_lines = set() - self.revealed_types = defaultdict(set) + self.iteration_dependent_errors = iteration_dependent_errors + iteration_dependent_errors.uselessness_errors.append(set()) + iteration_dependent_errors.unreachable_lines.append(set()) def on_error(self, file: str, info: ErrorInfo) -> bool: + """Filter out the "iteration-dependent" errors and notes and store their + information to handle them after iteration is completed.""" + + iter_errors = self.iteration_dependent_errors if info.code in (codes.UNREACHABLE, codes.REDUNDANT_EXPR, codes.REDUNDANT_CAST): - self.uselessness_errors.add( + iter_errors.uselessness_errors[-1].add( (info.code, info.message, info.line, info.column, info.end_line, info.end_column) ) if info.code == codes.UNREACHABLE: - self.unreachable_lines.update(range(info.line, info.end_line + 1)) + iter_errors.unreachable_lines[-1].update(range(info.line, info.end_line + 1)) return True if info.code == codes.MISC and info.message.startswith("Revealed type is "): key = info.function_or_member, info.line, info.column, info.end_line, info.end_column types = info.message.split('"')[1] if types.startswith("Union["): - self.revealed_types[key].update(types[6:-1].split(", ")) + iter_errors.revealed_types[key].update(types[6:-1].split(", ")) else: - self.revealed_types[key].add(types) + iter_errors.revealed_types[key].add(types) return True return super().on_error(file, info) + def yield_error_infos(self) -> Iterator[tuple[str, Context, ErrorCode]]: + """Report only those `unreachable`, `redundant-expr`, and `redundant-casts` + errors that could not be ruled out in any iteration step.""" + + persistent_uselessness_errors = set() + iter_errors = self.iteration_dependent_errors + for candidate in set(chain(*iter_errors.uselessness_errors)): + if all( + (candidate in errors) or (candidate[2] in lines) + for errors, lines in zip( + iter_errors.uselessness_errors, iter_errors.unreachable_lines + ) + ): + persistent_uselessness_errors.add(candidate) + for error_info in persistent_uselessness_errors: + context = Context(line=error_info[2], column=error_info[3]) + context.end_line = error_info[4] + context.end_column = error_info[5] + yield error_info[1], context, error_info[0] + + def yield_note_infos(self, options: Options) -> Iterator[tuple[str, Context]]: + """Yield all types revealed in at least one iteration step.""" + + for note_info, types in self.iteration_dependent_errors.revealed_types.items(): + sorted_ = sorted(types, key=lambda typ: typ.lower()) + if len(types) == 1: + revealed = sorted_[0] + elif options.use_or_syntax(): + revealed = " | ".join(sorted_) + else: + revealed = f"Union[{', '.join(sorted_)}]" + context = Context(line=note_info[1], column=note_info[2]) + context.end_line = note_info[3] + context.end_column = note_info[4] + yield f'Revealed type is "{revealed}"', context + class Errors: """Container for compile errors. diff --git a/test-data/unit/check-narrowing.test b/test-data/unit/check-narrowing.test index 0443bc8459070..7a053e1c5cab8 100644 --- a/test-data/unit/check-narrowing.test +++ b/test-data/unit/check-narrowing.test @@ -2446,6 +2446,25 @@ while x is not None and b(): x = f() [builtins fixtures/primitives.pyi] +[case testAvoidFalseUnreachableInFinally] +# flags: --allow-redefinition-new --local-partial-types --warn-unreachable +def f() -> None: + try: + x = 1 + if int(): + x = "" + return + if int(): + x = None + return + finally: + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str, None]" + if isinstance(x, str): + reveal_type(x) # N: Revealed type is "builtins.str" + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str, None]" + +[builtins fixtures/isinstancelist.pyi] + [case testNarrowingTypeVarMultiple] from typing import TypeVar diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 1062be6976c0c..924e665846697 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -791,8 +791,7 @@ def f3() -> None: x = "" return finally: - reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" \ - # N: Revealed type is "builtins.int" + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" reveal_type(x) # N: Revealed type is "builtins.int" def f4() -> None: diff --git a/test-data/unit/check-union-error-syntax.test b/test-data/unit/check-union-error-syntax.test index 3c541173a8916..d41281b774e1e 100644 --- a/test-data/unit/check-union-error-syntax.test +++ b/test-data/unit/check-union-error-syntax.test @@ -55,3 +55,24 @@ from typing import Literal, Union x : Union[Literal[1], None] x = 3 # E: Incompatible types in assignment (expression has type "Literal[3]", variable has type "Optional[Literal[1]]") [builtins fixtures/tuple.pyi] + +[case testUnionSyntaxRecombined] +# flags: --python-version 3.10 --force-union-syntax --allow-redefinition-new --local-partial-types +# The following revealed type is recombined because the finally body is visited twice. +try: + x = 1 + x = "" +finally: + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" +[builtins fixtures/isinstancelist.pyi] + +[case testOrSyntaxRecombined] +# flags: --python-version 3.10 --no-force-union-syntax --allow-redefinition-new --local-partial-types +# The following revealed type is recombined because the finally body is visited twice. +# ToDo: Improve this recombination logic, especially (but not only) for the "or syntax". +try: + x = 1 + x = "" +finally: + reveal_type(x) # N: Revealed type is "builtins.int | builtins.str | builtins.str" +[builtins fixtures/isinstancelist.pyi] From b18d3f82a300b28273a030e619f2fc2a8eb7a9a0 Mon Sep 17 00:00:00 2001 From: Robsdedude Date: Fri, 20 Jun 2025 17:44:48 +0000 Subject: [PATCH 020/424] Fix metaclass resolution algorithm (#17713) This PR fixes the algorithm for determining a classes metaclass. Fixes #14033 --- mypy/checker.py | 23 +++++++-------------- mypy/nodes.py | 28 +++++++++++++++++-------- test-data/unit/check-abstract.test | 8 ++++++-- test-data/unit/check-classes.test | 33 +++++++++++++++++++++++++++++- 4 files changed, 64 insertions(+), 28 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index e05523a1aa05c..70d3add74fd98 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -2933,23 +2933,14 @@ def check_metaclass_compatibility(self, typ: TypeInfo) -> None: ): return # Reasonable exceptions from this check - metaclasses = [ - entry.metaclass_type - for entry in typ.mro[1:-1] - if entry.metaclass_type - and not is_named_instance(entry.metaclass_type, "builtins.type") - ] - if not metaclasses: - return - if typ.metaclass_type is not None and all( - is_subtype(typ.metaclass_type, meta) for meta in metaclasses + if typ.metaclass_type is None and any( + base.type.metaclass_type is not None for base in typ.bases ): - return - self.fail( - "Metaclass conflict: the metaclass of a derived class must be " - "a (non-strict) subclass of the metaclasses of all its bases", - typ, - ) + self.fail( + "Metaclass conflict: the metaclass of a derived class must be " + "a (non-strict) subclass of the metaclasses of all its bases", + typ, + ) def visit_import_from(self, node: ImportFrom) -> None: for name, _ in node.names: diff --git a/mypy/nodes.py b/mypy/nodes.py index 1b6884f04bf57..d69ff10346c3e 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -3382,15 +3382,25 @@ def calculate_metaclass_type(self) -> mypy.types.Instance | None: return declared if self._fullname == "builtins.type": return mypy.types.Instance(self, []) - candidates = [ - s.declared_metaclass - for s in self.mro - if s.declared_metaclass is not None and s.declared_metaclass.type is not None - ] - for c in candidates: - if all(other.type in c.type.mro for other in candidates): - return c - return None + + winner = declared + for super_class in self.mro[1:]: + super_meta = super_class.declared_metaclass + if super_meta is None or super_meta.type is None: + continue + if winner is None: + winner = super_meta + continue + if winner.type.has_base(super_meta.type.fullname): + continue + if super_meta.type.has_base(winner.type.fullname): + winner = super_meta + continue + # metaclass conflict + winner = None + break + + return winner def is_metaclass(self, *, precise: bool = False) -> bool: return ( diff --git a/test-data/unit/check-abstract.test b/test-data/unit/check-abstract.test index 2fed3425c8d46..7507a31d115a9 100644 --- a/test-data/unit/check-abstract.test +++ b/test-data/unit/check-abstract.test @@ -571,8 +571,12 @@ from abc import abstractmethod, ABCMeta import typing class A(metaclass=ABCMeta): pass -class B(object, A): pass \ - # E: Cannot determine consistent method resolution order (MRO) for "B" +class B(object, A, metaclass=ABCMeta): # E: Cannot determine consistent method resolution order (MRO) for "B" + pass + +class C(object, A): # E: Cannot determine consistent method resolution order (MRO) for "C" \ + # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases + pass [case testOverloadedAbstractMethod] from foo import * diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index 1b3de53567d1f..7d2032ef25f00 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -7292,7 +7292,7 @@ class Conflict1(A1, B, E): ... # E: Metaclass conflict: the metaclass of a deri class Conflict2(A, B): ... # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases class Conflict3(B, A): ... # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases -class ChildOfConflict1(Conflict3): ... # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases +class ChildOfConflict1(Conflict3): ... class ChildOfConflict2(Conflict3, metaclass=CorrectMeta): ... class ConflictingMeta(MyMeta1, MyMeta3): ... @@ -7301,6 +7301,37 @@ class Conflict4(A1, B, E, metaclass=ConflictingMeta): ... # E: Metaclass confli class ChildOfCorrectButWrongMeta(CorrectSubclass1, metaclass=ConflictingMeta): # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases ... +[case testMetaClassConflictIssue14033] +class M1(type): pass +class M2(type): pass +class Mx(M1, M2): pass + +class A1(metaclass=M1): pass +class A2(A1): pass + +class B1(metaclass=M2): pass + +class C1(metaclass=Mx): pass + +class TestABC(A2, B1, C1): pass # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases +class TestBAC(B1, A2, C1): pass # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases + +# should not warn again for children +class ChildOfTestABC(TestABC): pass + +# no metaclass is assumed if super class has a metaclass conflict +class ChildOfTestABCMetaMx(TestABC, metaclass=Mx): pass +class ChildOfTestABCMetaM1(TestABC, metaclass=M1): pass + +class TestABCMx(A2, B1, C1, metaclass=Mx): pass +class TestBACMx(B1, A2, C1, metaclass=Mx): pass + +class TestACB(A2, C1, B1): pass +class TestBCA(B1, C1, A2): pass + +class TestCAB(C1, A2, B1): pass +class TestCBA(C1, B1, A2): pass + [case testGenericOverride] from typing import Generic, TypeVar, Any From f97a56e76d8dfcffea740babddc6bb9e060c0be2 Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Sat, 21 Jun 2025 01:27:43 +0200 Subject: [PATCH 021/424] Support running stubtest in non-UTF8 terminals (#19085) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #19071. I looked through the `open()` calls in the codebase, and only `reports.py` raises some concerns. Stubtest crashes due to this `print` call with incompatible encoding. I tested this on Linux with `LC_CTYPE=ru_RU.CP1251` (random non-utf8 locale I found in `/usr/share/i18n/SUPPORTED`) and confirmed that `stubtest` crashes without the patch and passes with it. Using a simple MRE (empty stub file and `A = "╙"` in a file, this symbol is `$'\u2559'`), I got this: ``` error: package.A is not present in stub Stub: in file /tmp/tmp.Cs4RioNSuR/demo/stub/package/__init__.pyi MISSING Runtime: '?' Found 1 error (checked 1 module) ``` Without the patch I get a crash - same as in the linked issue. --- mypy/stubtest.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index f9e6f7d337bee..8ea9d786be220 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -2062,7 +2062,7 @@ def warning_callback(msg: str) -> None: if args.generate_allowlist: generated_allowlist.add(error.object_desc) continue - print(error.get_description(concise=args.concise)) + safe_print(error.get_description(concise=args.concise)) error_count += 1 # Print unused allowlist entries @@ -2102,6 +2102,19 @@ def warning_callback(msg: str) -> None: return exit_code +def safe_print(text: str) -> None: + """Print a text replacing chars not representable in stdout encoding.""" + # If `sys.stdout` encoding is not the same as out (usually UTF8) string, + # if may cause painful crashes. I don't want to reconfigure `sys.stdout` + # to do `errors = "replace"` as that sounds scary. + out_encoding = sys.stdout.encoding + if out_encoding is not None: + # Can be None if stdout is replaced (including our own tests). This should be + # safe to omit if the actual stream doesn't care about encoding. + text = text.encode(out_encoding, errors="replace").decode(out_encoding, errors="replace") + print(text) + + def parse_options(args: list[str]) -> _Arguments: parser = argparse.ArgumentParser( description="Compares stubs to objects introspected from the runtime." From 0c26253fb16ca4fae89618e893a1ad5a2e553ed5 Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Sat, 21 Jun 2025 01:35:17 +0200 Subject: [PATCH 022/424] Ignore overload impl when checking __OP__ and __rOP__ compatibility (#18502) Fixes #18498 --- mypy/checker.py | 23 +++++++++++-- test-data/unit/check-classes.test | 54 +++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 2 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 70d3add74fd98..dbf2160d69886 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -107,6 +107,7 @@ OperatorAssignmentStmt, OpExpr, OverloadedFuncDef, + OverloadPart, PassStmt, PromoteExpr, RaiseStmt, @@ -407,6 +408,11 @@ def __init__( # argument through various `checker` and `checkmember` functions. self._is_final_def = False + # Track when we enter an overload implementation. Some checks should not be applied + # to the implementation signature when specific overloads are available. + # Use `enter_overload_impl` to modify. + self.overload_impl_stack: list[OverloadPart] = [] + # This flag is set when we run type-check or attribute access check for the purpose # of giving a note on possibly missing "await". It is used to avoid infinite recursion. self.checking_missing_await = False @@ -709,7 +715,8 @@ def _visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: if num_abstract not in (0, len(defn.items)): self.fail(message_registry.INCONSISTENT_ABSTRACT_OVERLOAD, defn) if defn.impl: - defn.impl.accept(self) + with self.enter_overload_impl(defn.impl): + defn.impl.accept(self) if not defn.is_property: self.check_overlapping_overloads(defn) if defn.type is None: @@ -752,6 +759,14 @@ def _visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: self.check_explicit_override_decorator(defn, found_method_base_classes, defn.impl) self.check_inplace_operator_method(defn) + @contextmanager + def enter_overload_impl(self, impl: OverloadPart) -> Iterator[None]: + self.overload_impl_stack.append(impl) + try: + yield + finally: + assert self.overload_impl_stack.pop() == impl + def extract_callable_type(self, inner_type: Type | None, ctx: Context) -> CallableType | None: """Get type as seen by an overload item caller.""" inner_type = get_proper_type(inner_type) @@ -1278,7 +1293,11 @@ def check_func_def( ) if name: # Special method names - if defn.info and self.is_reverse_op_method(name): + if ( + defn.info + and self.is_reverse_op_method(name) + and defn not in self.overload_impl_stack + ): self.check_reverse_op_method(item, typ, name, defn) elif name in ("__getattr__", "__getattribute__"): self.check_getattr_method(typ, defn, name) diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index 7d2032ef25f00..bf6c51e86446b 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -2487,6 +2487,60 @@ reveal_type(Num3() + Num1()) # N: Revealed type is "__main__.Num3" reveal_type(Num2() + Num3()) # N: Revealed type is "__main__.Num2" reveal_type(Num3() + Num2()) # N: Revealed type is "__main__.Num3" +[case testReverseOperatorWithOverloads3] +from typing import Union, overload + +class A: + def __mul__(self, value: A, /) -> A: ... + def __rmul__(self, value: A, /) -> A: ... + +class B: + @overload + def __mul__(self, other: B, /) -> B: ... + @overload + def __mul__(self, other: A, /) -> str: ... + def __mul__(self, other: Union[B, A], /) -> Union[B, str]: pass + + @overload + def __rmul__(self, other: B, /) -> B: ... + @overload + def __rmul__(self, other: A, /) -> str: ... + def __rmul__(self, other: Union[B, A], /) -> Union[B, str]: pass + +[case testReverseOperatorWithOverloadsNested] +from typing import Union, overload + +class A: + def __mul__(self, value: A, /) -> A: ... + def __rmul__(self, value: A, /) -> A: ... + +class B: + @overload + def __mul__(self, other: B, /) -> B: ... + @overload + def __mul__(self, other: A, /) -> str: ... + def __mul__(self, other: Union[B, A], /) -> Union[B, str]: pass + + @overload + def __rmul__(self, other: B, /) -> B: ... + @overload + def __rmul__(self, other: A, /) -> str: ... + def __rmul__(self, other: Union[B, A], /) -> Union[B, str]: + class A1: + def __add__(self, other: C1) -> int: ... + + class B1: + def __add__(self, other: C1) -> int: ... + + class C1: + @overload + def __radd__(self, other: A1) -> str: ... # E: Signatures of "__radd__" of "C1" and "__add__" of "A1" are unsafely overlapping + @overload + def __radd__(self, other: B1) -> str: ... # E: Signatures of "__radd__" of "C1" and "__add__" of "B1" are unsafely overlapping + def __radd__(self, other): pass + + return "" + [case testDivReverseOperator] # No error: __div__ has no special meaning in Python 3 class A1: From 0755a61b9528beca20c468e15e7c49e7b82671c8 Mon Sep 17 00:00:00 2001 From: Charlie Denton Date: Sat, 21 Jun 2025 01:08:46 +0100 Subject: [PATCH 023/424] Type ignore comments erroneously marked as unused by dmypy (#15043) There is currently a misbehaviour where "type: ignore" comments are erroneously marked as unused in re-runs of dmypy. There are also cases where errors disappear on the re-run. As far as I can tell, this only happens in modules which contain an import that we don't know how to type (such as a module which does not exist), and a submodule which is unused. There was a lot of commenting and investigation on this PR, but I hope that the committed tests and fixes illustrate and address the issue. Related to https://github.com/python/mypy/issues/9655 --------- Co-authored-by: David Seddon Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Co-authored-by: Ivan Levkivskyi Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- mypy/errors.py | 4 + mypy/server/update.py | 2 + test-data/unit/daemon.test | 137 +++++++++++++++++++++++++++++++ test-data/unit/fine-grained.test | 24 ++++++ 4 files changed, 167 insertions(+) diff --git a/mypy/errors.py b/mypy/errors.py index 41a4de6392366..5dd411c39e959 100644 --- a/mypy/errors.py +++ b/mypy/errors.py @@ -848,6 +848,8 @@ def generate_unused_ignore_errors(self, file: str) -> None: code=codes.UNUSED_IGNORE, blocker=False, only_once=False, + origin=(self.file, [line]), + target=self.target_module, ) self._add_error_info(file, info) @@ -899,6 +901,8 @@ def generate_ignore_without_code_errors( code=codes.IGNORE_WITHOUT_CODE, blocker=False, only_once=False, + origin=(self.file, [line]), + target=self.target_module, ) self._add_error_info(file, info) diff --git a/mypy/server/update.py b/mypy/server/update.py index 9891e2417b942..ea336154ae56e 100644 --- a/mypy/server/update.py +++ b/mypy/server/update.py @@ -668,6 +668,8 @@ def restore(ids: list[str]) -> None: state.type_check_first_pass() state.type_check_second_pass() state.detect_possibly_undefined_vars() + state.generate_unused_ignore_notes() + state.generate_ignore_without_code_notes() t2 = time.time() state.finish_passes() t3 = time.time() diff --git a/test-data/unit/daemon.test b/test-data/unit/daemon.test index ad3b51b27dfbd..295eb4000d812 100644 --- a/test-data/unit/daemon.test +++ b/test-data/unit/daemon.test @@ -648,6 +648,143 @@ from demo.test import a [file demo/test.py] a: int +[case testUnusedTypeIgnorePreservedOnRerun] +-- Regression test for https://github.com/python/mypy/issues/9655 +$ dmypy start -- --warn-unused-ignores --no-error-summary --hide-error-codes +Daemon started +$ dmypy check -- bar.py +bar.py:2: error: Unused "type: ignore" comment +== Return code: 1 +$ dmypy check -- bar.py +bar.py:2: error: Unused "type: ignore" comment +== Return code: 1 + +[file foo/__init__.py] +[file foo/empty.py] +[file bar.py] +from foo.empty import * +a = 1 # type: ignore + +[case testTypeIgnoreWithoutCodePreservedOnRerun] +-- Regression test for https://github.com/python/mypy/issues/9655 +$ dmypy start -- --enable-error-code ignore-without-code --no-error-summary +Daemon started +$ dmypy check -- bar.py +bar.py:2: error: "type: ignore" comment without error code [ignore-without-code] +== Return code: 1 +$ dmypy check -- bar.py +bar.py:2: error: "type: ignore" comment without error code [ignore-without-code] +== Return code: 1 + +[file foo/__init__.py] +[file foo/empty.py] +[file bar.py] +from foo.empty import * +a = 1 # type: ignore + +[case testPossiblyUndefinedVarsPreservedAfterRerun] +-- Regression test for https://github.com/python/mypy/issues/9655 +$ dmypy start -- --enable-error-code possibly-undefined --no-error-summary +Daemon started +$ dmypy check -- bar.py +bar.py:4: error: Name "a" may be undefined [possibly-undefined] +== Return code: 1 +$ dmypy check -- bar.py +bar.py:4: error: Name "a" may be undefined [possibly-undefined] +== Return code: 1 + +[file foo/__init__.py] +[file foo/empty.py] +[file bar.py] +from foo.empty import * +if False: + a = 1 +a + +[case testUnusedTypeIgnorePreservedOnRerunWithIgnoredMissingImports] +$ dmypy start -- --no-error-summary --ignore-missing-imports --warn-unused-ignores +Daemon started +$ dmypy check foo +foo/main.py:3: error: Unused "type: ignore" comment [unused-ignore] +== Return code: 1 +$ dmypy check foo +foo/main.py:3: error: Unused "type: ignore" comment [unused-ignore] +== Return code: 1 + +[file unused/__init__.py] +[file unused/submodule.py] +[file foo/empty.py] +[file foo/__init__.py] +from foo.main import * +from unused.submodule import * +[file foo/main.py] +from foo import empty +from foo.does_not_exist import * +a = 1 # type: ignore + +[case testModuleDoesNotExistPreservedOnRerun] +$ dmypy start -- --no-error-summary --ignore-missing-imports +Daemon started +$ dmypy check foo +foo/main.py:1: error: Module "foo" has no attribute "does_not_exist" [attr-defined] +== Return code: 1 +$ dmypy check foo +foo/main.py:1: error: Module "foo" has no attribute "does_not_exist" [attr-defined] +== Return code: 1 + +[file unused/__init__.py] +[file unused/submodule.py] +[file foo/__init__.py] +from foo.main import * +[file foo/main.py] +from foo import does_not_exist +from unused.submodule import * + +[case testReturnTypeIgnoreAfterUnknownImport] +-- Return type ignores after unknown imports and unused modules are respected on the second pass. +$ dmypy start -- --warn-unused-ignores --no-error-summary +Daemon started +$ dmypy check -- foo.py +foo.py:2: error: Cannot find implementation or library stub for module named "a_module_which_does_not_exist" [import-not-found] +foo.py:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== Return code: 1 +$ dmypy check -- foo.py +foo.py:2: error: Cannot find implementation or library stub for module named "a_module_which_does_not_exist" [import-not-found] +foo.py:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== Return code: 1 + +[file unused/__init__.py] +[file unused/empty.py] +[file foo.py] +from unused.empty import * +import a_module_which_does_not_exist +def is_foo() -> str: + return True # type: ignore + +[case testAttrsTypeIgnoreAfterUnknownImport] +$ dmypy start -- --warn-unused-ignores --no-error-summary +Daemon started +$ dmypy check -- foo.py +foo.py:3: error: Cannot find implementation or library stub for module named "a_module_which_does_not_exist" [import-not-found] +foo.py:3: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== Return code: 1 +$ dmypy check -- foo.py +foo.py:3: error: Cannot find implementation or library stub for module named "a_module_which_does_not_exist" [import-not-found] +foo.py:3: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== Return code: 1 + +[file unused/__init__.py] +[file unused/empty.py] +[file foo.py] +import attr +from unused.empty import * +import a_module_which_does_not_exist + +@attr.frozen +class A: + def __init__(self) -> None: + self.__attrs_init__() # type: ignore[attr-defined] + [case testDaemonImportAncestors] $ dmypy run test.py Daemon started diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index 7e34a2352dd60..222e38ea0280e 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -10540,6 +10540,30 @@ from pkg.sub import modb [out] == +[case testUnusedTypeIgnorePreservedAfterChange] +# flags: --warn-unused-ignores --no-error-summary +[file main.py] +a = 1 # type: ignore +[file main.py.2] +a = 1 # type: ignore +# Comment to trigger reload. +[out] +main.py:1: error: Unused "type: ignore" comment +== +main.py:1: error: Unused "type: ignore" comment + +[case testTypeIgnoreWithoutCodePreservedAfterChange] +# flags: --enable-error-code ignore-without-code --no-error-summary +[file main.py] +a = 1 # type: ignore +[file main.py.2] +a = 1 # type: ignore +# Comment to trigger reload. +[out] +main.py:1: error: "type: ignore" comment without error code +== +main.py:1: error: "type: ignore" comment without error code + [case testFineGrainedFunctoolsPartial] import m From fabe37f31e5bd7396e6a1183e3ed56b6e767561b Mon Sep 17 00:00:00 2001 From: Daniel Hnyk Date: Sat, 21 Jun 2025 08:43:28 +0200 Subject: [PATCH 024/424] Fix nit in documentation example --- docs/source/generics.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/generics.rst b/docs/source/generics.rst index 5d787d32b0056..4755c4f17ec82 100644 --- a/docs/source/generics.rst +++ b/docs/source/generics.rst @@ -630,7 +630,7 @@ Let us illustrate this by few simple examples: my_circles: list[Circle] = [] add_one(my_circles) # This may appear safe, but... - my_circles[-1].rotate() # ...this will fail, since my_circles[0] is now a Shape, not a Circle + my_circles[0].rotate() # ...this will fail, since my_circles[0] is now a Shape, not a Circle Another example of invariant type is ``dict``. Most mutable containers are invariant. From 34949c82274285b03423bde4def9beacd91c7d52 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 21 Jun 2025 14:12:35 +0100 Subject: [PATCH 025/424] Tweaks to perf_compare.py script to reduce RAM usage (#19321) Reduce default parallelism, since it could require many GBs of RAM. Add --multi-file flag which reduces RAM usage further, but performance might be slightly worse. --- misc/perf_compare.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/misc/perf_compare.py b/misc/perf_compare.py index 025d4065561e5..589912dd9826b 100644 --- a/misc/perf_compare.py +++ b/misc/perf_compare.py @@ -35,11 +35,13 @@ def heading(s: str) -> None: print() -def build_mypy(target_dir: str) -> None: +def build_mypy(target_dir: str, multi_file: bool) -> None: env = os.environ.copy() env["CC"] = "clang" env["MYPYC_OPT_LEVEL"] = "2" env["PYTHONHASHSEED"] = "1" + if multi_file: + env["MYPYC_MULTI_FILE"] = "1" cmd = [sys.executable, "setup.py", "--use-mypyc", "build_ext", "--inplace"] subprocess.run(cmd, env=env, check=True, cwd=target_dir) @@ -110,6 +112,12 @@ def main() -> None: action="store_true", help="measure incremental run (fully cached)", ) + parser.add_argument( + "--multi-file", + default=False, + action="store_true", + help="compile each mypy module to a separate C file (reduces RAM use)", + ) parser.add_argument( "--dont-setup", default=False, @@ -127,9 +135,9 @@ def main() -> None: parser.add_argument( "-j", metavar="N", - default=8, + default=4, type=int, - help="set maximum number of parallel builds (default=8)", + help="set maximum number of parallel builds (default=4) -- high numbers require a lot of RAM!", ) parser.add_argument( "-r", @@ -155,6 +163,7 @@ def main() -> None: args = parser.parse_args() incremental: bool = args.incremental dont_setup: bool = args.dont_setup + multi_file: bool = args.multi_file commits = args.commit num_runs: int = args.num_runs + 1 max_workers: int = args.j @@ -185,7 +194,9 @@ def main() -> None: print("(This will take a while...)") with ThreadPoolExecutor(max_workers=max_workers) as executor: - futures = [executor.submit(build_mypy, target_dir) for target_dir in target_dirs] + futures = [ + executor.submit(build_mypy, target_dir, multi_file) for target_dir in target_dirs + ] for future in as_completed(futures): future.result() From 6886b7a17f586e3da881be1b23c41e48916c57e4 Mon Sep 17 00:00:00 2001 From: Brian Schubert Date: Sat, 21 Jun 2025 14:57:46 -0400 Subject: [PATCH 026/424] Fix `TypeIs` negative narrowing of union of generics (#18193) Fixes #18009, fixes #19282, fixes #17181 Modelling the runtime behavior of `isinstance` (which erases generic type arguments) isn't applicable to `TypeIs`. This PR adds a flag so that we can skip that logic deep inside `conditional_types_with_intersection`. --------- Co-authored-by: Shantanu Jain --- mypy/checker.py | 44 ++++++++++++-- mypy/subtypes.py | 23 ++++++-- test-data/unit/check-typeis.test | 99 ++++++++++++++++++++++++++++++++ 3 files changed, 154 insertions(+), 12 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index dbf2160d69886..596564c98a40f 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -6185,6 +6185,7 @@ def find_isinstance_check_helper( self.lookup_type(expr), [TypeRange(node.callee.type_is, is_upper_bound=False)], expr, + consider_runtime_isinstance=False, ), ) elif isinstance(node, ComparisonExpr): @@ -7612,11 +7613,19 @@ def conditional_types_with_intersection( type_ranges: list[TypeRange] | None, ctx: Context, default: None = None, + *, + consider_runtime_isinstance: bool = True, ) -> tuple[Type | None, Type | None]: ... @overload def conditional_types_with_intersection( - self, expr_type: Type, type_ranges: list[TypeRange] | None, ctx: Context, default: Type + self, + expr_type: Type, + type_ranges: list[TypeRange] | None, + ctx: Context, + default: Type, + *, + consider_runtime_isinstance: bool = True, ) -> tuple[Type, Type]: ... def conditional_types_with_intersection( @@ -7625,8 +7634,15 @@ def conditional_types_with_intersection( type_ranges: list[TypeRange] | None, ctx: Context, default: Type | None = None, + *, + consider_runtime_isinstance: bool = True, ) -> tuple[Type | None, Type | None]: - initial_types = conditional_types(expr_type, type_ranges, default) + initial_types = conditional_types( + expr_type, + type_ranges, + default, + consider_runtime_isinstance=consider_runtime_isinstance, + ) # For some reason, doing "yes_map, no_map = conditional_types_to_typemaps(...)" # doesn't work: mypyc will decide that 'yes_map' is of type None if we try. yes_type: Type | None = initial_types[0] @@ -7938,18 +7954,30 @@ def visit_type_var(self, t: TypeVarType) -> None: @overload def conditional_types( - current_type: Type, proposed_type_ranges: list[TypeRange] | None, default: None = None + current_type: Type, + proposed_type_ranges: list[TypeRange] | None, + default: None = None, + *, + consider_runtime_isinstance: bool = True, ) -> tuple[Type | None, Type | None]: ... @overload def conditional_types( - current_type: Type, proposed_type_ranges: list[TypeRange] | None, default: Type + current_type: Type, + proposed_type_ranges: list[TypeRange] | None, + default: Type, + *, + consider_runtime_isinstance: bool = True, ) -> tuple[Type, Type]: ... def conditional_types( - current_type: Type, proposed_type_ranges: list[TypeRange] | None, default: Type | None = None + current_type: Type, + proposed_type_ranges: list[TypeRange] | None, + default: Type | None = None, + *, + consider_runtime_isinstance: bool = True, ) -> tuple[Type | None, Type | None]: """Takes in the current type and a proposed type of an expression. @@ -7991,7 +8019,11 @@ def conditional_types( if not type_range.is_upper_bound ] ) - remaining_type = restrict_subtype_away(current_type, proposed_precise_type) + remaining_type = restrict_subtype_away( + current_type, + proposed_precise_type, + consider_runtime_isinstance=consider_runtime_isinstance, + ) return proposed_type, remaining_type else: # An isinstance check, but we don't understand the type diff --git a/mypy/subtypes.py b/mypy/subtypes.py index a5e6938615e75..143d6783f43e9 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -2073,7 +2073,7 @@ def try_restrict_literal_union(t: UnionType, s: Type) -> list[Type] | None: return new_items -def restrict_subtype_away(t: Type, s: Type) -> Type: +def restrict_subtype_away(t: Type, s: Type, *, consider_runtime_isinstance: bool = True) -> Type: """Return t minus s for runtime type assertions. If we can't determine a precise result, return a supertype of the @@ -2087,16 +2087,27 @@ def restrict_subtype_away(t: Type, s: Type) -> Type: new_items = try_restrict_literal_union(p_t, s) if new_items is None: new_items = [ - restrict_subtype_away(item, s) + restrict_subtype_away( + item, s, consider_runtime_isinstance=consider_runtime_isinstance + ) for item in p_t.relevant_items() - if (isinstance(get_proper_type(item), AnyType) or not covers_at_runtime(item, s)) ] - return UnionType.make_union(new_items) + return UnionType.make_union( + [item for item in new_items if not isinstance(get_proper_type(item), UninhabitedType)] + ) elif isinstance(p_t, TypeVarType): return p_t.copy_modified(upper_bound=restrict_subtype_away(p_t.upper_bound, s)) - elif covers_at_runtime(t, s): - return UninhabitedType() + + if consider_runtime_isinstance: + if covers_at_runtime(t, s): + return UninhabitedType() + else: + return t else: + if is_proper_subtype(t, s, ignore_promotions=True): + return UninhabitedType() + if is_proper_subtype(t, s, ignore_promotions=True, erase_instances=True): + return UninhabitedType() return t diff --git a/test-data/unit/check-typeis.test b/test-data/unit/check-typeis.test index bb8beac72c3aa..997cc7474b916 100644 --- a/test-data/unit/check-typeis.test +++ b/test-data/unit/check-typeis.test @@ -134,6 +134,91 @@ def main(a: object) -> None: reveal_type(a) # N: Revealed type is "Union[builtins.int, builtins.str]" [builtins fixtures/tuple.pyi] +[case testTypeIsUnionWithGeneric] +from typing import Any, List, Sequence, Union +from typing_extensions import TypeIs + +def is_int_list(a: object) -> TypeIs[List[int]]: pass +def is_int_seq(a: object) -> TypeIs[Sequence[int]]: pass +def is_seq(a: object) -> TypeIs[Sequence[Any]]: pass + +def f1(a: Union[List[int], List[str]]) -> None: + if is_int_list(a): + reveal_type(a) # N: Revealed type is "builtins.list[builtins.int]" + else: + reveal_type(a) # N: Revealed type is "builtins.list[builtins.str]" + reveal_type(a) # N: Revealed type is "Union[builtins.list[builtins.int], builtins.list[builtins.str]]" + +def f2(a: Union[List[int], int]) -> None: + if is_int_list(a): + reveal_type(a) # N: Revealed type is "builtins.list[builtins.int]" + else: + reveal_type(a) # N: Revealed type is "builtins.int" + reveal_type(a) # N: Revealed type is "Union[builtins.list[builtins.int], builtins.int]" + +def f3(a: Union[List[bool], List[str]]) -> None: + if is_int_seq(a): + reveal_type(a) # N: Revealed type is "builtins.list[builtins.bool]" + else: + reveal_type(a) # N: Revealed type is "builtins.list[builtins.str]" + reveal_type(a) # N: Revealed type is "Union[builtins.list[builtins.bool], builtins.list[builtins.str]]" + +def f4(a: Union[List[int], int]) -> None: + if is_seq(a): + reveal_type(a) # N: Revealed type is "builtins.list[builtins.int]" + else: + reveal_type(a) # N: Revealed type is "builtins.int" + reveal_type(a) # N: Revealed type is "Union[builtins.list[builtins.int], builtins.int]" +[builtins fixtures/tuple.pyi] + +[case testTypeIsTupleGeneric] +# flags: --warn-unreachable +from __future__ import annotations +from typing_extensions import TypeIs, Unpack + +class A: ... +class B: ... + +def is_tuple_of_B(v: tuple[A | B, ...]) -> TypeIs[tuple[B, ...]]: ... + +def test1(t: tuple[A]) -> None: + if is_tuple_of_B(t): + reveal_type(t) # E: Statement is unreachable + else: + reveal_type(t) # N: Revealed type is "tuple[__main__.A]" + +def test2(t: tuple[B, A]) -> None: + if is_tuple_of_B(t): + reveal_type(t) # E: Statement is unreachable + else: + reveal_type(t) # N: Revealed type is "tuple[__main__.B, __main__.A]" + +def test3(t: tuple[A | B]) -> None: + if is_tuple_of_B(t): + reveal_type(t) # N: Revealed type is "tuple[__main__.B]" + else: + reveal_type(t) # N: Revealed type is "tuple[Union[__main__.A, __main__.B]]" + +def test4(t: tuple[A | B, A | B]) -> None: + if is_tuple_of_B(t): + reveal_type(t) # N: Revealed type is "tuple[__main__.B, __main__.B]" + else: + reveal_type(t) # N: Revealed type is "tuple[Union[__main__.A, __main__.B], Union[__main__.A, __main__.B]]" + +def test5(t: tuple[A | B, ...]) -> None: + if is_tuple_of_B(t): + reveal_type(t) # N: Revealed type is "builtins.tuple[__main__.B, ...]" + else: + reveal_type(t) # N: Revealed type is "builtins.tuple[Union[__main__.A, __main__.B], ...]" + +def test6(t: tuple[B, Unpack[tuple[A | B, ...]], B]) -> None: + if is_tuple_of_B(t): + # Should this be tuple[B, *tuple[B, ...], B] + reveal_type(t) # N: Revealed type is "tuple[__main__.B, Never, __main__.B]" + else: + reveal_type(t) # N: Revealed type is "tuple[__main__.B, Unpack[builtins.tuple[Union[__main__.A, __main__.B], ...]], __main__.B]" +[builtins fixtures/tuple.pyi] + [case testTypeIsNonzeroFloat] from typing_extensions import TypeIs def is_nonzero(a: object) -> TypeIs[float]: pass @@ -834,3 +919,17 @@ def handle(model: Model) -> None: if is_model_a(model): reveal_type(model) # N: Revealed type is "Literal[__main__.Model.A]" [builtins fixtures/tuple.pyi] + +[case testTypeIsAwaitableAny] +from __future__ import annotations +from typing import Any, Awaitable, Callable +from typing_extensions import TypeIs + +def is_async_callable(obj: Any) -> TypeIs[Callable[..., Awaitable[Any]]]: ... + +def main(f: Callable[[], int | Awaitable[int]]) -> None: + if is_async_callable(f): + reveal_type(f) # N: Revealed type is "def (*Any, **Any) -> typing.Awaitable[Any]" + else: + reveal_type(f) # N: Revealed type is "def () -> Union[builtins.int, typing.Awaitable[builtins.int]]" +[builtins fixtures/tuple.pyi] From be11ab84a2c03677c0e5484b8342b4b7a026c310 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 21 Jun 2025 19:58:44 +0100 Subject: [PATCH 027/424] Speed up type checking by caching argument inference context (#19323) This speeds up self check by about 1.5% on my computer, according to `perf_compare.py`: ``` ... === Results === master 4.264s (0.0%) | stdev 0.024s HEAD 4.201s (-1.5%) | stdev 0.030s ``` I noticed this bottleneck when I was looking at a CPU profile generated using the script from #19322. --- mypy/checkexpr.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 26cb2a35794b7..603acbe84f0f9 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -318,6 +318,8 @@ class ExpressionChecker(ExpressionVisitor[Type], ExpressionCheckerSharedApi): strfrm_checker: StringFormatterChecker plugin: Plugin + _arg_infer_context_cache: ArgumentInferContext | None + def __init__( self, chk: mypy.checker.TypeChecker, @@ -352,6 +354,8 @@ def __init__( self.is_callee = False type_state.infer_polymorphic = not self.chk.options.old_type_inference + self._arg_infer_context_cache = None + def reset(self) -> None: self.resolved_type = {} @@ -2277,9 +2281,11 @@ def infer_function_type_arguments_pass2( return callee_type, inferred_args def argument_infer_context(self) -> ArgumentInferContext: - return ArgumentInferContext( - self.chk.named_type("typing.Mapping"), self.chk.named_type("typing.Iterable") - ) + if self._arg_infer_context_cache is None: + self._arg_infer_context_cache = ArgumentInferContext( + self.chk.named_type("typing.Mapping"), self.chk.named_type("typing.Iterable") + ) + return self._arg_infer_context_cache def get_arg_infer_passes( self, From b678d9ff5b1b411a18ac6edc400f7f59961d2546 Mon Sep 17 00:00:00 2001 From: Brian Schubert Date: Sat, 21 Jun 2025 15:04:56 -0400 Subject: [PATCH 028/424] Fix incorrect signature suggestion from `dmypy suggest` when type name matches imported module name (#18937) Fixes #18935 --- mypy/suggestions.py | 2 +- test-data/unit/fine-grained-suggest.test | 30 ++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/mypy/suggestions.py b/mypy/suggestions.py index cfd7413860ec2..81eb20bd0ac32 100644 --- a/mypy/suggestions.py +++ b/mypy/suggestions.py @@ -852,7 +852,7 @@ def visit_instance(self, t: Instance) -> str: if self.module: parts = obj.split(".") # need to split the object part if it is a nested class tree = self.graph[self.module].tree - if tree and parts[0] in tree.names: + if tree and parts[0] in tree.names and mod not in tree.names: mod = self.module if (mod, obj) == ("builtins", "tuple"): diff --git a/test-data/unit/fine-grained-suggest.test b/test-data/unit/fine-grained-suggest.test index 3a696ce19c634..c2e544baf38bf 100644 --- a/test-data/unit/fine-grained-suggest.test +++ b/test-data/unit/fine-grained-suggest.test @@ -207,6 +207,36 @@ foo(B()) (baz.B) -> Tuple[foo.A, foo:A.C] == +[case testSuggestReexportNamingNameMatchesModule1] +# suggest: foo.foo +[file foo.py] +import bar +def foo(): + return bar.bar() + +[file bar.py] +class bar: ... # name matches module name + +[out] +() -> bar.bar +== + +[case testSuggestReexportNamingNameMatchesModule2] +# suggest: foo.foo +[file foo.py] +import bar +import qux +def foo(): + return qux.bar() + +[file bar.py] +[file qux.py] +class bar: ... # name matches another module name + +[out] +() -> qux.bar +== + [case testSuggestInferInit] # suggest: foo.Foo.__init__ [file foo.py] From b17027e1b3c034e0a39451d4bcbc4514d0a8429c Mon Sep 17 00:00:00 2001 From: Brian Schubert Date: Sun, 22 Jun 2025 01:26:15 -0400 Subject: [PATCH 029/424] Fix `TypeGuard`/`TypeIs` being forgotten when semanal defers (#19325) Fixes #19318 Don't re-analyze `type_is`/`type_guard` if the callable's type was successfully analyzed on a previous semanal pass. After the first successful pass, the return type will have been replaced by `builtins.bool`, so subsequent analysis can't detect `Type{Guard,Is}`. --- mypy/typeanal.py | 4 ++-- test-data/unit/check-typeguard.test | 17 +++++++++++++++++ test-data/unit/check-typeis.test | 17 +++++++++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index b0d11759303c4..204d3061c7349 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -1109,8 +1109,8 @@ def visit_callable_type( variables = t.variables else: variables, _ = self.bind_function_type_variables(t, t) - type_guard = self.anal_type_guard(t.ret_type) - type_is = self.anal_type_is(t.ret_type) + type_guard = self.anal_type_guard(t.ret_type) if t.type_guard is None else t.type_guard + type_is = self.anal_type_is(t.ret_type) if t.type_is is None else t.type_is arg_kinds = t.arg_kinds arg_types = [] diff --git a/test-data/unit/check-typeguard.test b/test-data/unit/check-typeguard.test index c43eead67876d..fdcfcc969adc1 100644 --- a/test-data/unit/check-typeguard.test +++ b/test-data/unit/check-typeguard.test @@ -813,3 +813,20 @@ raw_target: object if isinstance(raw_target, type) and is_xlike(raw_target): reveal_type(raw_target) # N: Revealed type is "type[__main__.X]" [builtins fixtures/tuple.pyi] + +[case testTypeGuardWithDefer] +from typing import Union +from typing_extensions import TypeGuard + +class A: ... +class B: ... + +def is_a(x: object) -> TypeGuard[A]: + return defer_not_defined() # E: Name "defer_not_defined" is not defined + +def main(x: Union[A, B]) -> None: + if is_a(x): + reveal_type(x) # N: Revealed type is "__main__.A" + else: + reveal_type(x) # N: Revealed type is "Union[__main__.A, __main__.B]" +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-typeis.test b/test-data/unit/check-typeis.test index 997cc7474b916..2f54ac5bf5dbf 100644 --- a/test-data/unit/check-typeis.test +++ b/test-data/unit/check-typeis.test @@ -933,3 +933,20 @@ def main(f: Callable[[], int | Awaitable[int]]) -> None: else: reveal_type(f) # N: Revealed type is "def () -> Union[builtins.int, typing.Awaitable[builtins.int]]" [builtins fixtures/tuple.pyi] + +[case testTypeIsWithDefer] +from typing import Union +from typing_extensions import TypeIs + +class A: ... +class B: ... + +def is_a(x: object) -> TypeIs[A]: + return defer_not_defined() # E: Name "defer_not_defined" is not defined + +def main(x: Union[A, B]) -> None: + if is_a(x): + reveal_type(x) # N: Revealed type is "__main__.A" + else: + reveal_type(x) # N: Revealed type is "__main__.B" +[builtins fixtures/tuple.pyi] From 24b831ab43db3671d75dd2fba4868735cadc8b9a Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sat, 21 Jun 2025 23:23:47 -0700 Subject: [PATCH 030/424] Avoid erasing type objects when checking runtime cover (#19320) In particular, avoid erasing CallableType that represent type objects Helps with #19159 --- mypy/subtypes.py | 3 ++- test-data/unit/check-python310.test | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 143d6783f43e9..9219bf1c544ba 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -2117,7 +2117,8 @@ def covers_at_runtime(item: Type, supertype: Type) -> bool: supertype = get_proper_type(supertype) # Since runtime type checks will ignore type arguments, erase the types. - supertype = erase_type(supertype) + if not (isinstance(supertype, CallableType) and supertype.is_type_obj()): + supertype = erase_type(supertype) if is_proper_subtype( erase_type(item), supertype, ignore_promotions=True, erase_instances=True ): diff --git a/test-data/unit/check-python310.test b/test-data/unit/check-python310.test index 0695bd0380cbb..bb8f038eb1eb2 100644 --- a/test-data/unit/check-python310.test +++ b/test-data/unit/check-python310.test @@ -2601,6 +2601,28 @@ def f(t: T) -> None: reveal_type(k) # N: Revealed type is "tuple[builtins.int, fallback=__main__.K]" [builtins fixtures/tuple.pyi] +[case testMatchTypeObjectTypeVar] +# flags: --warn-unreachable +from typing import TypeVar +import b + +T_Choice = TypeVar("T_Choice", bound=b.One | b.Two) + +def switch(choice: type[T_Choice]) -> None: + match choice: + case b.One: + reveal_type(choice) # N: Revealed type is "def () -> b.One" + case b.Two: + reveal_type(choice) # N: Revealed type is "def () -> b.Two" + case _: + reveal_type(choice) # N: Revealed type is "type[T_Choice`-1]" + +[file b.py] +class One: ... +class Two: ... + +[builtins fixtures/tuple.pyi] + [case testNewRedefineMatchBasics] # flags: --allow-redefinition-new --local-partial-types From dc031c9634e0fabaa1f7ff0a1c682b2e8ca1f863 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 22 Jun 2025 11:13:44 +0100 Subject: [PATCH 031/424] Add script for profiling self check (Linux only) (#19322) The script compiles mypy and profiles self check using the 'perf' profiler. Example of how to use this: ``` $ python misc/profile_self_check.py ... [will take several minutes] CPU profile collected. You can now analyze the profile: perf report -i mypy.profile.tmpdir/perf.data ``` --- misc/perf_compare.py | 2 +- misc/profile_self_check.py | 129 +++++++++++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+), 1 deletion(-) create mode 100644 misc/profile_self_check.py diff --git a/misc/perf_compare.py b/misc/perf_compare.py index 589912dd9826b..50543550b7ce3 100644 --- a/misc/perf_compare.py +++ b/misc/perf_compare.py @@ -35,7 +35,7 @@ def heading(s: str) -> None: print() -def build_mypy(target_dir: str, multi_file: bool) -> None: +def build_mypy(target_dir: str, multi_file: bool, *, cflags: str | None = None) -> None: env = os.environ.copy() env["CC"] = "clang" env["MYPYC_OPT_LEVEL"] = "2" diff --git a/misc/profile_self_check.py b/misc/profile_self_check.py new file mode 100644 index 0000000000000..eb853641d6d66 --- /dev/null +++ b/misc/profile_self_check.py @@ -0,0 +1,129 @@ +"""Compile mypy using mypyc and profile self-check using perf. + +Notes: + - Only Linux is supported for now (TODO: add support for other profilers) + - The profile is collected at C level + - It includes C functions compiled by mypyc and CPython runtime functions + - The names of mypy functions are mangled to C names, but usually it's clear what they mean + - Generally CPyDef_ prefix for native functions and CPyPy_ prefix for wrapper functions + - It's important to compile CPython using special flags (see below) to get good results + - Generally use the latest Python feature release (or the most recent beta if supported by mypyc) + - The tool prints a command that can be used to analyze the profile afterwards + +You may need to adjust kernel parameters temporarily, e.g. this (note that this has security +implications): + + sudo sysctl kernel.perf_event_paranoid=-1 + +This is the recommended way to configure CPython for profiling: + + ./configure \ + --enable-optimizations \ + --with-lto \ + CFLAGS="-O2 -g -fno-omit-frame-pointer" +""" + +import argparse +import glob +import os +import shutil +import subprocess +import sys +import time + +from perf_compare import build_mypy, clone + +# Use these C compiler flags when compiling mypy (important). Note that it's strongly recommended +# to also compile CPython using similar flags, but we don't enforce it in this script. +CFLAGS = "-O2 -fno-omit-frame-pointer -g" + +# Generated files, including binaries, go under this directory to avoid overwriting user state. +TARGET_DIR = "mypy.profile.tmpdir" + + +def _profile_self_check(target_dir: str) -> None: + cache_dir = os.path.join(target_dir, ".mypy_cache") + if os.path.exists(cache_dir): + shutil.rmtree(cache_dir) + files = [] + for pat in "mypy/*.py", "mypy/*/*.py", "mypyc/*.py", "mypyc/test/*.py": + files.extend(glob.glob(pat)) + self_check_cmd = ["python", "-m", "mypy", "--config-file", "mypy_self_check.ini"] + files + cmdline = ["perf", "record", "-g"] + self_check_cmd + t0 = time.time() + subprocess.run(cmdline, cwd=target_dir, check=True) + elapsed = time.time() - t0 + print(f"{elapsed:.2f}s elapsed") + + +def profile_self_check(target_dir: str) -> None: + try: + _profile_self_check(target_dir) + except subprocess.CalledProcessError: + print("\nProfiling failed! You may missing some permissions.") + print("\nThis may help (note that it has security implications):") + print(" sudo sysctl kernel.perf_event_paranoid=-1") + sys.exit(1) + + +def check_requirements() -> None: + if sys.platform != "linux": + # TODO: How to make this work on other platforms? + sys.exit("error: Only Linux is supported") + + try: + subprocess.run(["perf", "-h"], capture_output=True) + except (subprocess.CalledProcessError, FileNotFoundError): + print("error: The 'perf' profiler is not installed") + sys.exit(1) + + try: + subprocess.run(["clang", "--version"], capture_output=True) + except (subprocess.CalledProcessError, FileNotFoundError): + print("error: The clang compiler is not installed") + sys.exit(1) + + if not os.path.isfile("mypy_self_check.ini"): + print("error: Run this in the mypy repository root") + sys.exit(1) + + +def main() -> None: + check_requirements() + + parser = argparse.ArgumentParser( + description="Compile mypy and profile self checking using 'perf'." + ) + parser.add_argument( + "--multi-file", + action="store_true", + help="compile mypy into one C file per module (to reduce RAM use during compilation)", + ) + parser.add_argument( + "--skip-compile", action="store_true", help="use compiled mypy from previous run" + ) + args = parser.parse_args() + multi_file: bool = args.multi_file + skip_compile: bool = args.skip_compile + + target_dir = TARGET_DIR + + if not skip_compile: + clone(target_dir, "HEAD") + + print(f"Building mypy in {target_dir}...") + build_mypy(target_dir, multi_file, cflags=CFLAGS) + elif not os.path.isdir(target_dir): + sys.exit("error: Can't find compile mypy from previous run -- can't use --skip-compile") + + profile_self_check(target_dir) + + print() + print('NOTE: Compile CPython using CFLAGS="-O2 -g -fno-omit-frame-pointer" for good results') + print() + print("CPU profile collected. You can now analyze the profile:") + print(f" perf report -i {target_dir}/perf.data ") + + +if __name__ == "__main__": + main() From 16e99de5376464beaa2cf086c1cd3dc5d26a791a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 22 Jun 2025 12:34:46 +0100 Subject: [PATCH 032/424] Fix C compiler flags in the profile self check script (#19326) This was mistakenly omitted in the original PR. --- misc/perf_compare.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/misc/perf_compare.py b/misc/perf_compare.py index 50543550b7ce3..39dd22b313391 100644 --- a/misc/perf_compare.py +++ b/misc/perf_compare.py @@ -42,6 +42,8 @@ def build_mypy(target_dir: str, multi_file: bool, *, cflags: str | None = None) env["PYTHONHASHSEED"] = "1" if multi_file: env["MYPYC_MULTI_FILE"] = "1" + if cflags is not None: + env["CFLAGS"] = cflags cmd = [sys.executable, "setup.py", "--use-mypyc", "build_ext", "--inplace"] subprocess.run(cmd, env=env, check=True, cwd=target_dir) From 5e9d657e397cf3e4d43c491525d70144be35d0d8 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Wed, 25 Jun 2025 12:50:45 -0700 Subject: [PATCH 033/424] Fix for overloaded type object erasure (#19338) https://github.com/python/mypy/pull/19320#discussion_r2165179417 --- mypy/subtypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 9219bf1c544ba..86935d0613a27 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -2117,7 +2117,7 @@ def covers_at_runtime(item: Type, supertype: Type) -> bool: supertype = get_proper_type(supertype) # Since runtime type checks will ignore type arguments, erase the types. - if not (isinstance(supertype, CallableType) and supertype.is_type_obj()): + if not (isinstance(supertype, FunctionLike) and supertype.is_type_obj()): supertype = erase_type(supertype) if is_proper_subtype( erase_type(item), supertype, ignore_promotions=True, erase_instances=True From bca959f2815275a65f569ef078be028722032777 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 29 Jun 2025 21:25:57 +0100 Subject: [PATCH 034/424] Remove unnecessary workarounds from bind_self() (#19356) In my original PR I erroneously concluded that attribute access on `TypeVar` would result in `PyObject` attribute access after mypy is compiled, but this is actually no the case. I therefore remove some workarounds (and a bit of unused code). --- mypy/checkmember.py | 17 ++++++----------- mypy/typeops.py | 3 --- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/mypy/checkmember.py b/mypy/checkmember.py index 5b5580a648a8b..ef38cc3a0dcf0 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -1477,23 +1477,18 @@ def bind_self_fast(method: F, original_type: Type | None = None) -> F: items = [bind_self_fast(c, original_type) for c in method.items] return cast(F, Overloaded(items)) assert isinstance(method, CallableType) - func: CallableType = method - if not func.arg_types: + if not method.arg_types: # Invalid method, return something. return method - if func.arg_kinds[0] in (ARG_STAR, ARG_STAR2): + if method.arg_kinds[0] in (ARG_STAR, ARG_STAR2): # See typeops.py for details. return method - original_type = get_proper_type(original_type) - if isinstance(original_type, CallableType) and original_type.is_type_obj(): - original_type = TypeType.make_normalized(original_type.ret_type) - res = func.copy_modified( - arg_types=func.arg_types[1:], - arg_kinds=func.arg_kinds[1:], - arg_names=func.arg_names[1:], + return method.copy_modified( + arg_types=method.arg_types[1:], + arg_kinds=method.arg_kinds[1:], + arg_names=method.arg_names[1:], is_bound=True, ) - return cast(F, res) def has_operator(typ: Type, op_method: str, named_type: Callable[[str], Instance]) -> bool: diff --git a/mypy/typeops.py b/mypy/typeops.py index e84be19465cc7..9aa08b40a991d 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -472,9 +472,6 @@ class B(A): pass else: variables = func.variables - original_type = get_proper_type(original_type) - if isinstance(original_type, CallableType) and original_type.is_type_obj(): - original_type = TypeType.make_normalized(original_type.ret_type) res = func.copy_modified( arg_types=func.arg_types[1:], arg_kinds=func.arg_kinds[1:], From 9934278ec88b39057c774ae1181acc9bfeb9a00e Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Mon, 30 Jun 2025 03:00:50 +0200 Subject: [PATCH 035/424] Do not show protocol compatibility note against unpacked sequence or mapping (#19358) This was discovered in #19294 where an unrelated change produced a weird notice that should not be shown. Current behavior of the added testcase: ``` main.py:10: error: Argument 1 to "foo" has incompatible type "*list[object]"; expected "P" [arg-type] main.py:10: note: "list" is missing following "P" protocol member: main.py:10: note: arg main.py:11: error: Argument 1 to "foo" has incompatible type "**dict[str, object]"; expected "P" [arg-type] main.py:11: note: "dict" is missing following "P" protocol member: main.py:11: note: arg ``` https://mypy-play.net/?mypy=master&python=3.12&flags=strict&gist=d0228ba7d2802db8ac4457f374ccc148 --- mypy/checkexpr.py | 9 ++++++--- test-data/unit/check-functions.test | 16 ++++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 603acbe84f0f9..8223ccfe4ca08 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -2684,9 +2684,12 @@ def check_arg( context=context, outer_context=outer_context, ) - self.msg.incompatible_argument_note( - original_caller_type, callee_type, context, parent_error=error - ) + if not caller_kind.is_star(): + # For *args and **kwargs this note would be incorrect - we're comparing + # iterable/mapping type with union of relevant arg types. + self.msg.incompatible_argument_note( + original_caller_type, callee_type, context, parent_error=error + ) if not self.msg.prefer_simple_messages(): self.chk.check_possible_missing_await( caller_type, callee_type, context, error.code diff --git a/test-data/unit/check-functions.test b/test-data/unit/check-functions.test index ceb7af433dce2..07cfd09b25291 100644 --- a/test-data/unit/check-functions.test +++ b/test-data/unit/check-functions.test @@ -3694,3 +3694,19 @@ def defer() -> int: ... [out] main: note: In function "a": main:6: error: Unsupported operand types for + ("int" and "str") + +[case testNoExtraNoteForUnpacking] +from typing import Protocol + +class P(Protocol): + arg: int + # Something that list and dict also have + def __contains__(self, item: object) -> bool: ... + +def foo(x: P, y: P) -> None: ... + +args: list[object] +foo(*args) # E: Argument 1 to "foo" has incompatible type "*list[object]"; expected "P" +kwargs: dict[str, object] +foo(**kwargs) # E: Argument 1 to "foo" has incompatible type "**dict[str, object]"; expected "P" +[builtins fixtures/dict.pyi] From 657154b6748793f44be7b8238b7265c4e84c2e16 Mon Sep 17 00:00:00 2001 From: Robsdedude Date: Tue, 1 Jul 2025 01:36:43 +0000 Subject: [PATCH 036/424] Metaclass conflict check improvements (#17682) This PR fixes some points of #14033: * Give metaclass errors to their own error code (I chose `metaclass`). * Document shortcomings of and workarounds for mypy's metaclass handling. I didn't attempt to fix that mypy follows the logic for determining the metaclass as documented whereas it should follow what the interpreter is actually doing (https://github.com/python/mypy/issues/14033#issuecomment-1314025562). I think such a change is better kept as a separate PR, which is why I don't want to close the issue with this PR. Fixes: https://github.com/python/mypy/issues/14033 --------- Co-authored-by: hauntsaninja --- docs/source/error_code_list.rst | 29 ++++++++++++++++++++ docs/source/metaclasses.rst | 25 +++++++++++++++++ mypy/checker.py | 4 +++ mypy/errorcodes.py | 1 + mypy/nodes.py | 37 +++++++++++++++++++++++++ mypy/semanal.py | 17 +++++++++--- test-data/unit/check-classes.test | 38 ++++++++++++++++---------- test-data/unit/check-errorcodes.test | 41 ++++++++++++++++++++++++++++ test-data/unit/fine-grained.test | 10 +++++-- 9 files changed, 182 insertions(+), 20 deletions(-) diff --git a/docs/source/error_code_list.rst b/docs/source/error_code_list.rst index 49cb8a0c06c17..6deed549c2f17 100644 --- a/docs/source/error_code_list.rst +++ b/docs/source/error_code_list.rst @@ -215,6 +215,35 @@ You can use :py:class:`~collections.abc.Callable` as the type for callable objec for x in objs: f(x) +.. _code-metaclass: + +Check the validity of a class's metaclass [metaclass] +----------------------------------------------------- + +Mypy checks whether the metaclass of a class is valid. The metaclass +must be a subclass of ``type``. Further, the class hierarchy must yield +a consistent metaclass. For more details, see the +`Python documentation `_ + +Note that mypy's metaclass checking is limited and may produce false-positives. +See also :ref:`limitations`. + +Example with an error: + +.. code-block:: python + + class GoodMeta(type): + pass + + class BadMeta: + pass + + class A1(metaclass=GoodMeta): # OK + pass + + class A2(metaclass=BadMeta): # Error: Metaclasses not inheriting from "type" are not supported [metaclass] + pass + .. _code-var-annotated: Require annotation if variable type is unclear [var-annotated] diff --git a/docs/source/metaclasses.rst b/docs/source/metaclasses.rst index dd77a2f90ed8b..e30dfe80f9f95 100644 --- a/docs/source/metaclasses.rst +++ b/docs/source/metaclasses.rst @@ -90,3 +90,28 @@ so it's better not to combine metaclasses and class hierarchies: * ``Self`` is not allowed as annotation in metaclasses as per `PEP 673`_. .. _PEP 673: https://peps.python.org/pep-0673/#valid-locations-for-self + +For some builtin types, mypy may think their metaclass is :py:class:`abc.ABCMeta` +even if it is :py:class:`type` at runtime. In those cases, you can either: + +* use :py:class:`abc.ABCMeta` instead of :py:class:`type` as the + superclass of your metaclass if that works in your use-case +* mute the error with ``# type: ignore[metaclass]`` + +.. code-block:: python + + import abc + + assert type(tuple) is type # metaclass of tuple is type at runtime + + # The problem: + class M0(type): pass + class A0(tuple, metaclass=M0): pass # Mypy Error: metaclass conflict + + # Option 1: use ABCMeta instead of type + class M1(abc.ABCMeta): pass + class A1(tuple, metaclass=M1): pass + + # Option 2: mute the error + class M2(type): pass + class A2(tuple, metaclass=M2): pass # type: ignore[metaclass] diff --git a/mypy/checker.py b/mypy/checker.py index 596564c98a40f..7859934c1ef70 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -2959,7 +2959,11 @@ def check_metaclass_compatibility(self, typ: TypeInfo) -> None: "Metaclass conflict: the metaclass of a derived class must be " "a (non-strict) subclass of the metaclasses of all its bases", typ, + code=codes.METACLASS, ) + explanation = typ.explain_metaclass_conflict() + if explanation: + self.note(explanation, typ, code=codes.METACLASS) def visit_import_from(self, node: ImportFrom) -> None: for name, _ in node.names: diff --git a/mypy/errorcodes.py b/mypy/errorcodes.py index c22308e4a754a..8f85a6f6351a9 100644 --- a/mypy/errorcodes.py +++ b/mypy/errorcodes.py @@ -270,6 +270,7 @@ def __hash__(self) -> int: "General", default_enabled=False, ) +METACLASS: Final[ErrorCode] = ErrorCode("metaclass", "Ensure that metaclass is valid", "General") # Syntax errors are often blocking. SYNTAX: Final[ErrorCode] = ErrorCode("syntax", "Report syntax errors", "General") diff --git a/mypy/nodes.py b/mypy/nodes.py index d69ff10346c3e..fc2656ce2130e 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -3402,6 +3402,43 @@ def calculate_metaclass_type(self) -> mypy.types.Instance | None: return winner + def explain_metaclass_conflict(self) -> str | None: + # Compare to logic in calculate_metaclass_type + declared = self.declared_metaclass + if declared is not None and not declared.type.has_base("builtins.type"): + return None + if self._fullname == "builtins.type": + return None + + winner = declared + if declared is None: + resolution_steps = [] + else: + resolution_steps = [f'"{declared.type.fullname}" (metaclass of "{self.fullname}")'] + for super_class in self.mro[1:]: + super_meta = super_class.declared_metaclass + if super_meta is None or super_meta.type is None: + continue + if winner is None: + winner = super_meta + resolution_steps.append( + f'"{winner.type.fullname}" (metaclass of "{super_class.fullname}")' + ) + continue + if winner.type.has_base(super_meta.type.fullname): + continue + if super_meta.type.has_base(winner.type.fullname): + winner = super_meta + resolution_steps.append( + f'"{winner.type.fullname}" (metaclass of "{super_class.fullname}")' + ) + continue + # metaclass conflict + conflict = f'"{super_meta.type.fullname}" (metaclass of "{super_class.fullname}")' + return f"{' > '.join(resolution_steps)} conflicts with {conflict}" + + return None + def is_metaclass(self, *, precise: bool = False) -> bool: return ( self.has_base("builtins.type") diff --git a/mypy/semanal.py b/mypy/semanal.py index 87aef2595caf6..431c5ec04d3c3 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -2702,7 +2702,7 @@ def infer_metaclass_and_bases_from_compat_helpers(self, defn: ClassDef) -> None: if len(metas) == 0: return if len(metas) > 1: - self.fail("Multiple metaclass definitions", defn) + self.fail("Multiple metaclass definitions", defn, code=codes.METACLASS) return defn.metaclass = metas.pop() @@ -2758,7 +2758,11 @@ def get_declared_metaclass( elif isinstance(metaclass_expr, MemberExpr): metaclass_name = get_member_expr_fullname(metaclass_expr) if metaclass_name is None: - self.fail(f'Dynamic metaclass not supported for "{name}"', metaclass_expr) + self.fail( + f'Dynamic metaclass not supported for "{name}"', + metaclass_expr, + code=codes.METACLASS, + ) return None, False, True sym = self.lookup_qualified(metaclass_name, metaclass_expr) if sym is None: @@ -2769,6 +2773,7 @@ def get_declared_metaclass( self.fail( f'Class cannot use "{sym.node.name}" as a metaclass (has type "Any")', metaclass_expr, + code=codes.METACLASS, ) return None, False, True if isinstance(sym.node, PlaceholderNode): @@ -2786,11 +2791,15 @@ def get_declared_metaclass( metaclass_info = target.type if not isinstance(metaclass_info, TypeInfo) or metaclass_info.tuple_type is not None: - self.fail(f'Invalid metaclass "{metaclass_name}"', metaclass_expr) + self.fail( + f'Invalid metaclass "{metaclass_name}"', metaclass_expr, code=codes.METACLASS + ) return None, False, False if not metaclass_info.is_metaclass(): self.fail( - 'Metaclasses not inheriting from "type" are not supported', metaclass_expr + 'Metaclasses not inheriting from "type" are not supported', + metaclass_expr, + code=codes.METACLASS, ) return None, False, False inst = fill_typevars(metaclass_info) diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index bf6c51e86446b..173657620304c 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -4757,8 +4757,8 @@ class C(B): class X(type): pass class Y(type): pass class A(metaclass=X): pass -class B(A, metaclass=Y): pass # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases - +class B(A, metaclass=Y): pass # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases \ + # N: "__main__.Y" (metaclass of "__main__.B") conflicts with "__main__.X" (metaclass of "__main__.A") [case testMetaclassNoTypeReveal] class M: x = 0 # type: int @@ -5737,8 +5737,8 @@ def f() -> type: return M class C1(six.with_metaclass(M), object): pass # E: Unsupported dynamic base class "six.with_metaclass" class C2(C1, six.with_metaclass(M)): pass # E: Unsupported dynamic base class "six.with_metaclass" class C3(six.with_metaclass(A)): pass # E: Metaclasses not inheriting from "type" are not supported -@six.add_metaclass(A) # E: Metaclasses not inheriting from "type" are not supported \ - # E: Argument 1 to "add_metaclass" has incompatible type "type[A]"; expected "type[type]" +@six.add_metaclass(A) # E: Metaclasses not inheriting from "type" are not supported \ + # E: Argument 1 to "add_metaclass" has incompatible type "type[A]"; expected "type[type]" class D3(A): pass class C4(six.with_metaclass(M), metaclass=M): pass # E: Multiple metaclass definitions @@ -5754,8 +5754,10 @@ class CD(six.with_metaclass(M)): pass # E: Multiple metaclass definitions class M1(type): pass class Q1(metaclass=M1): pass @six.add_metaclass(M) -class CQA(Q1): pass # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases -class CQW(six.with_metaclass(M, Q1)): pass # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases +class CQA(Q1): pass # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases \ + # N: "__main__.M" (metaclass of "__main__.CQA") conflicts with "__main__.M1" (metaclass of "__main__.Q1") +class CQW(six.with_metaclass(M, Q1)): pass # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases \ + # N: "__main__.M" (metaclass of "__main__.CQW") conflicts with "__main__.M1" (metaclass of "__main__.Q1") [builtins fixtures/tuple.pyi] [case testSixMetaclassAny] @@ -5873,7 +5875,8 @@ class C5(future.utils.with_metaclass(f())): pass # E: Dynamic metaclass not sup class M1(type): pass class Q1(metaclass=M1): pass -class CQW(future.utils.with_metaclass(M, Q1)): pass # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases +class CQW(future.utils.with_metaclass(M, Q1)): pass # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases \ + # N: "__main__.M" (metaclass of "__main__.CQW") conflicts with "__main__.M1" (metaclass of "__main__.Q1") [builtins fixtures/tuple.pyi] [case testFutureMetaclassAny] @@ -7342,17 +7345,22 @@ class ChildOfCorrectSubclass1(CorrectSubclass1): ... class CorrectWithType1(C, A1): ... class CorrectWithType2(B, C): ... -class Conflict1(A1, B, E): ... # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases -class Conflict2(A, B): ... # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases -class Conflict3(B, A): ... # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases +class Conflict1(A1, B, E): ... # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases \ + # N: "__main__.MyMeta1" (metaclass of "__main__.A") conflicts with "__main__.MyMeta2" (metaclass of "__main__.B") +class Conflict2(A, B): ... # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases \ + # N: "__main__.MyMeta1" (metaclass of "__main__.A") conflicts with "__main__.MyMeta2" (metaclass of "__main__.B") +class Conflict3(B, A): ... # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases \ + # N: "__main__.MyMeta2" (metaclass of "__main__.B") conflicts with "__main__.MyMeta1" (metaclass of "__main__.A") class ChildOfConflict1(Conflict3): ... class ChildOfConflict2(Conflict3, metaclass=CorrectMeta): ... class ConflictingMeta(MyMeta1, MyMeta3): ... -class Conflict4(A1, B, E, metaclass=ConflictingMeta): ... # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases +class Conflict4(A1, B, E, metaclass=ConflictingMeta): ... # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases \ + # N: "__main__.ConflictingMeta" (metaclass of "__main__.Conflict4") conflicts with "__main__.MyMeta2" (metaclass of "__main__.B") -class ChildOfCorrectButWrongMeta(CorrectSubclass1, metaclass=ConflictingMeta): # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases +class ChildOfCorrectButWrongMeta(CorrectSubclass1, metaclass=ConflictingMeta): # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases \ + # N: "__main__.ConflictingMeta" (metaclass of "__main__.ChildOfCorrectButWrongMeta") conflicts with "__main__.CorrectMeta" (metaclass of "__main__.CorrectSubclass1") ... [case testMetaClassConflictIssue14033] @@ -7367,8 +7375,10 @@ class B1(metaclass=M2): pass class C1(metaclass=Mx): pass -class TestABC(A2, B1, C1): pass # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases -class TestBAC(B1, A2, C1): pass # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases +class TestABC(A2, B1, C1): pass # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases \ + # N: "__main__.M1" (metaclass of "__main__.A1") conflicts with "__main__.M2" (metaclass of "__main__.B1") +class TestBAC(B1, A2, C1): pass # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases \ + # N: "__main__.M2" (metaclass of "__main__.B1") conflicts with "__main__.M1" (metaclass of "__main__.A1") # should not warn again for children class ChildOfTestABC(TestABC): pass diff --git a/test-data/unit/check-errorcodes.test b/test-data/unit/check-errorcodes.test index d6e3366401dde..bb5f658ebb50b 100644 --- a/test-data/unit/check-errorcodes.test +++ b/test-data/unit/check-errorcodes.test @@ -1239,6 +1239,47 @@ def f(x: str) -> TypeIs[int]: # E: Narrowed type "int" is not a subtype of inpu [builtins fixtures/tuple.pyi] +[case testDynamicMetaclass] +class A(metaclass=type(tuple)): pass # E: Dynamic metaclass not supported for "A" [metaclass] +[builtins fixtures/tuple.pyi] + +[case testMetaclassOfTypeAny] +# mypy: disallow-subclassing-any=True +from typing import Any +foo: Any = ... +class A(metaclass=foo): pass # E: Class cannot use "foo" as a metaclass (has type "Any") [metaclass] + +[case testMetaclassOfWrongType] +class Foo: + bar = 1 +class A2(metaclass=Foo.bar): pass # E: Invalid metaclass "Foo.bar" [metaclass] + +[case testMetaclassNotTypeSubclass] +class M: pass +class A(metaclass=M): pass # E: Metaclasses not inheriting from "type" are not supported [metaclass] + +[case testMultipleMetaclasses] +import six +class M1(type): pass + +@six.add_metaclass(M1) +class A1(metaclass=M1): pass # E: Multiple metaclass definitions [metaclass] + +class A2(six.with_metaclass(M1), metaclass=M1): pass # E: Multiple metaclass definitions [metaclass] + +@six.add_metaclass(M1) +class A3(six.with_metaclass(M1)): pass # E: Multiple metaclass definitions [metaclass] +[builtins fixtures/tuple.pyi] + +[case testInvalidMetaclassStructure] +class X(type): pass +class Y(type): pass +class A(metaclass=X): pass +class B(A, metaclass=Y): pass # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases [metaclass] \ + # N: "__main__.Y" (metaclass of "__main__.B") conflicts with "__main__.X" (metaclass of "__main__.A") + + + [case testOverloadedFunctionSignature] from typing import overload, Union diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index 222e38ea0280e..503135d901f89 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -2936,10 +2936,12 @@ a.py:6: error: Argument 1 to "f" has incompatible type "type[B]"; expected "M" [case testFineMetaclassRecalculation] import a + [file a.py] from b import B class M2(type): pass class D(B, metaclass=M2): pass + [file b.py] import c class B: pass @@ -2949,27 +2951,31 @@ import c class B(metaclass=c.M): pass [file c.py] -class M(type): - pass +class M(type): pass [out] == a.py:3: error: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases +a.py:3: note: "a.M2" (metaclass of "a.D") conflicts with "c.M" (metaclass of "b.B") [case testFineMetaclassDeclaredUpdate] import a + [file a.py] import b class B(metaclass=b.M): pass class D(B, metaclass=b.M2): pass + [file b.py] class M(type): pass class M2(M): pass + [file b.py.2] class M(type): pass class M2(type): pass [out] == a.py:3: error: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases +a.py:3: note: "b.M2" (metaclass of "a.D") conflicts with "b.M" (metaclass of "a.B") [case testFineMetaclassRemoveFromClass] import a From 4cefd4643e25c14bc4748db365e2162a55fa3786 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 30 Jun 2025 18:37:15 -0700 Subject: [PATCH 037/424] Sync typeshed (#19364) Source commit: https://github.com/python/typeshed/commit/3f727b0cd6620b7fca45318dd34542b1e1c7dbfb --- mypy/typeshed/stdlib/VERSIONS | 1 + mypy/typeshed/stdlib/_csv.pyi | 2 + mypy/typeshed/stdlib/_zstd.pyi | 1 + mypy/typeshed/stdlib/ast.pyi | 97 +++++------ mypy/typeshed/stdlib/asyncio/tools.pyi | 41 +++++ .../stdlib/concurrent/futures/thread.pyi | 12 +- mypy/typeshed/stdlib/os/__init__.pyi | 2 +- mypy/typeshed/stdlib/sys/__init__.pyi | 11 +- mypy/typeshed/stdlib/tkinter/commondialog.pyi | 14 +- mypy/typeshed/stdlib/tkinter/filedialog.pyi | 42 ++--- mypy/typeshed/stdlib/tkinter/messagebox.pyi | 83 ++++++++-- mypy/typeshed/stdlib/token.pyi | 151 +++++++++--------- mypy/typeshed/stdlib/tokenize.pyi | 7 +- mypy/typeshed/stdlib/types.pyi | 2 +- 14 files changed, 292 insertions(+), 174 deletions(-) create mode 100644 mypy/typeshed/stdlib/asyncio/tools.pyi diff --git a/mypy/typeshed/stdlib/VERSIONS b/mypy/typeshed/stdlib/VERSIONS index c86bbb3146670..8baf207ad7b85 100644 --- a/mypy/typeshed/stdlib/VERSIONS +++ b/mypy/typeshed/stdlib/VERSIONS @@ -95,6 +95,7 @@ asyncio.staggered: 3.8- asyncio.taskgroups: 3.11- asyncio.threads: 3.9- asyncio.timeouts: 3.11- +asyncio.tools: 3.14- asyncio.trsock: 3.8- asyncore: 3.0-3.11 atexit: 3.0- diff --git a/mypy/typeshed/stdlib/_csv.pyi b/mypy/typeshed/stdlib/_csv.pyi index ecea4878907c4..efe9ad69bd31d 100644 --- a/mypy/typeshed/stdlib/_csv.pyi +++ b/mypy/typeshed/stdlib/_csv.pyi @@ -90,6 +90,7 @@ else: def writer( csvfile: SupportsWrite[str], + /, dialect: _DialectLike = "excel", *, delimiter: str = ",", @@ -103,6 +104,7 @@ def writer( ) -> _writer: ... def reader( csvfile: Iterable[str], + /, dialect: _DialectLike = "excel", *, delimiter: str = ",", diff --git a/mypy/typeshed/stdlib/_zstd.pyi b/mypy/typeshed/stdlib/_zstd.pyi index 0648d898448b6..2730232528fc2 100644 --- a/mypy/typeshed/stdlib/_zstd.pyi +++ b/mypy/typeshed/stdlib/_zstd.pyi @@ -52,6 +52,7 @@ class ZstdCompressor: self, /, data: ReadableBuffer, mode: _ZstdCompressorContinue | _ZstdCompressorFlushBlock | _ZstdCompressorFlushFrame = 0 ) -> bytes: ... def flush(self, /, mode: _ZstdCompressorFlushBlock | _ZstdCompressorFlushFrame = 2) -> bytes: ... + def set_pledged_input_size(self, size: int | None, /) -> None: ... @property def last_mode(self) -> _ZstdCompressorContinue | _ZstdCompressorFlushBlock | _ZstdCompressorFlushFrame: ... diff --git a/mypy/typeshed/stdlib/ast.pyi b/mypy/typeshed/stdlib/ast.pyi index af9d20d086b33..613940f5da6ab 100644 --- a/mypy/typeshed/stdlib/ast.pyi +++ b/mypy/typeshed/stdlib/ast.pyi @@ -1,3 +1,4 @@ +import ast import builtins import os import sys @@ -623,21 +624,6 @@ class AsyncWith(stmt): **kwargs: Unpack[_Attributes], ) -> Self: ... -if sys.version_info >= (3, 10): - class Match(stmt): - __match_args__ = ("subject", "cases") - subject: expr - cases: list[match_case] - if sys.version_info >= (3, 13): - def __init__(self, subject: expr, cases: list[match_case] = ..., **kwargs: Unpack[_Attributes]) -> None: ... - else: - def __init__(self, subject: expr, cases: list[match_case], **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, *, subject: expr = ..., cases: list[match_case] = ..., **kwargs: Unpack[_Attributes] - ) -> Self: ... - class Raise(stmt): if sys.version_info >= (3, 10): __match_args__ = ("exc", "cause") @@ -1076,13 +1062,13 @@ if sys.version_info >= (3, 14): value: expr str: builtins.str conversion: int - format_spec: builtins.str | None = None + format_spec: expr | None = None def __init__( self, value: expr = ..., str: builtins.str = ..., conversion: int = ..., - format_spec: builtins.str | None = ..., + format_spec: expr | None = ..., **kwargs: Unpack[_Attributes], ) -> None: ... def __replace__( @@ -1091,7 +1077,7 @@ if sys.version_info >= (3, 14): value: expr = ..., str: builtins.str = ..., conversion: int = ..., - format_spec: builtins.str | None = ..., + format_spec: expr | None = ..., **kwargs: Unpack[_Attributes], ) -> Self: ... @@ -1135,13 +1121,13 @@ class Subscript(expr): if sys.version_info >= (3, 10): __match_args__ = ("value", "slice", "ctx") value: expr - slice: _Slice + slice: expr ctx: expr_context # Not present in Python < 3.13 if not passed to `__init__` - def __init__(self, value: expr, slice: _Slice, ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> None: ... + def __init__(self, value: expr, slice: expr, ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> None: ... if sys.version_info >= (3, 14): def __replace__( - self, *, value: expr = ..., slice: _Slice = ..., ctx: expr_context = ..., **kwargs: Unpack[_Attributes] + self, *, value: expr = ..., slice: expr = ..., ctx: expr_context = ..., **kwargs: Unpack[_Attributes] ) -> Self: ... class Starred(expr): @@ -1194,36 +1180,28 @@ class Tuple(expr): @deprecated("Deprecated since Python 3.9.") class slice(AST): ... -_Slice: typing_extensions.TypeAlias = expr -_SliceAttributes: typing_extensions.TypeAlias = _Attributes - -class Slice(_Slice): +class Slice(expr): if sys.version_info >= (3, 10): __match_args__ = ("lower", "upper", "step") lower: expr | None upper: expr | None step: expr | None def __init__( - self, lower: expr | None = None, upper: expr | None = None, step: expr | None = None, **kwargs: Unpack[_SliceAttributes] + self, lower: expr | None = None, upper: expr | None = None, step: expr | None = None, **kwargs: Unpack[_Attributes] ) -> None: ... if sys.version_info >= (3, 14): def __replace__( - self, - *, - lower: expr | None = ..., - upper: expr | None = ..., - step: expr | None = ..., - **kwargs: Unpack[_SliceAttributes], + self, *, lower: expr | None = ..., upper: expr | None = ..., step: expr | None = ..., **kwargs: Unpack[_Attributes] ) -> Self: ... @deprecated("Deprecated since Python 3.9. Use ast.Tuple instead.") class ExtSlice(slice): - def __new__(cls, dims: Iterable[slice] = (), **kwargs: Unpack[_SliceAttributes]) -> Tuple: ... # type: ignore[misc] + def __new__(cls, dims: Iterable[slice] = (), **kwargs: Unpack[_Attributes]) -> Tuple: ... # type: ignore[misc] @deprecated("Deprecated since Python 3.9. Use the index value directly instead.") class Index(slice): - def __new__(cls, value: expr, **kwargs: Unpack[_SliceAttributes]) -> expr: ... # type: ignore[misc] + def __new__(cls, value: expr, **kwargs: Unpack[_Attributes]) -> expr: ... # type: ignore[misc] class expr_context(AST): ... @@ -1465,37 +1443,48 @@ class withitem(AST): def __replace__(self, *, context_expr: expr = ..., optional_vars: expr | None = ...) -> Self: ... if sys.version_info >= (3, 10): + class pattern(AST): + lineno: int + col_offset: int + end_lineno: int + end_col_offset: int + def __init__(self, **kwargs: Unpack[_Attributes[int]]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, lineno: int = ..., col_offset: int = ..., end_lineno: int = ..., end_col_offset: int = ... + ) -> Self: ... + class match_case(AST): __match_args__ = ("pattern", "guard", "body") - pattern: _Pattern + pattern: ast.pattern guard: expr | None body: list[stmt] if sys.version_info >= (3, 13): - def __init__(self, pattern: _Pattern, guard: expr | None = None, body: list[stmt] = ...) -> None: ... - else: + def __init__(self, pattern: ast.pattern, guard: expr | None = None, body: list[stmt] = ...) -> None: ... + elif sys.version_info >= (3, 10): @overload - def __init__(self, pattern: _Pattern, guard: expr | None, body: list[stmt]) -> None: ... + def __init__(self, pattern: ast.pattern, guard: expr | None, body: list[stmt]) -> None: ... @overload - def __init__(self, pattern: _Pattern, guard: expr | None = None, *, body: list[stmt]) -> None: ... + def __init__(self, pattern: ast.pattern, guard: expr | None = None, *, body: list[stmt]) -> None: ... if sys.version_info >= (3, 14): - def __replace__(self, *, pattern: _Pattern = ..., guard: expr | None = ..., body: list[stmt] = ...) -> Self: ... + def __replace__(self, *, pattern: ast.pattern = ..., guard: expr | None = ..., body: list[stmt] = ...) -> Self: ... - class pattern(AST): - lineno: int - col_offset: int - end_lineno: int - end_col_offset: int - def __init__(self, **kwargs: Unpack[_Attributes[int]]) -> None: ... + class Match(stmt): + __match_args__ = ("subject", "cases") + subject: expr + cases: list[match_case] + if sys.version_info >= (3, 13): + def __init__(self, subject: expr, cases: list[match_case] = ..., **kwargs: Unpack[_Attributes]) -> None: ... + else: + def __init__(self, subject: expr, cases: list[match_case], **kwargs: Unpack[_Attributes]) -> None: ... if sys.version_info >= (3, 14): def __replace__( - self, *, lineno: int = ..., col_offset: int = ..., end_lineno: int = ..., end_col_offset: int = ... + self, *, subject: expr = ..., cases: list[match_case] = ..., **kwargs: Unpack[_Attributes] ) -> Self: ... - # Without the alias, Pyright complains variables named pattern are recursively defined - _Pattern: typing_extensions.TypeAlias = pattern - class MatchValue(pattern): __match_args__ = ("value",) value: expr @@ -1590,22 +1579,22 @@ if sys.version_info >= (3, 10): class MatchStar(pattern): __match_args__ = ("name",) name: str | None - def __init__(self, name: str | None, **kwargs: Unpack[_Attributes[int]]) -> None: ... + def __init__(self, name: str | None = None, **kwargs: Unpack[_Attributes[int]]) -> None: ... if sys.version_info >= (3, 14): def __replace__(self, *, name: str | None = ..., **kwargs: Unpack[_Attributes[int]]) -> Self: ... class MatchAs(pattern): __match_args__ = ("pattern", "name") - pattern: _Pattern | None + pattern: ast.pattern | None name: str | None def __init__( - self, pattern: _Pattern | None = None, name: str | None = None, **kwargs: Unpack[_Attributes[int]] + self, pattern: ast.pattern | None = None, name: str | None = None, **kwargs: Unpack[_Attributes[int]] ) -> None: ... if sys.version_info >= (3, 14): def __replace__( - self, *, pattern: _Pattern | None = ..., name: str | None = ..., **kwargs: Unpack[_Attributes[int]] + self, *, pattern: ast.pattern | None = ..., name: str | None = ..., **kwargs: Unpack[_Attributes[int]] ) -> Self: ... class MatchOr(pattern): diff --git a/mypy/typeshed/stdlib/asyncio/tools.pyi b/mypy/typeshed/stdlib/asyncio/tools.pyi new file mode 100644 index 0000000000000..65c7f27e0b85e --- /dev/null +++ b/mypy/typeshed/stdlib/asyncio/tools.pyi @@ -0,0 +1,41 @@ +from collections.abc import Iterable +from enum import Enum +from typing import NamedTuple, SupportsIndex, type_check_only + +@type_check_only +class _AwaitedInfo(NamedTuple): # AwaitedInfo_Type from _remote_debugging + thread_id: int + awaited_by: list[_TaskInfo] + +@type_check_only +class _TaskInfo(NamedTuple): # TaskInfo_Type from _remote_debugging + task_id: int + task_name: str + coroutine_stack: list[_CoroInfo] + awaited_by: list[_CoroInfo] + +@type_check_only +class _CoroInfo(NamedTuple): # CoroInfo_Type from _remote_debugging + call_stack: list[_FrameInfo] + task_name: int | str + +@type_check_only +class _FrameInfo(NamedTuple): # FrameInfo_Type from _remote_debugging + filename: str + lineno: int + funcname: str + +class NodeType(Enum): + COROUTINE = 1 + TASK = 2 + +class CycleFoundException(Exception): + cycles: list[list[int]] + id2name: dict[int, str] + def __init__(self, cycles: list[list[int]], id2name: dict[int, str]) -> None: ... + +def get_all_awaited_by(pid: SupportsIndex) -> list[_AwaitedInfo]: ... +def build_async_tree(result: Iterable[_AwaitedInfo], task_emoji: str = "(T)", cor_emoji: str = "") -> list[list[str]]: ... +def build_task_table(result: Iterable[_AwaitedInfo]) -> list[list[int | str]]: ... +def display_awaited_by_tasks_table(pid: SupportsIndex) -> None: ... +def display_awaited_by_tasks_tree(pid: SupportsIndex) -> None: ... diff --git a/mypy/typeshed/stdlib/concurrent/futures/thread.pyi b/mypy/typeshed/stdlib/concurrent/futures/thread.pyi index 22df0dca5a3fd..50a6a9c6f43ea 100644 --- a/mypy/typeshed/stdlib/concurrent/futures/thread.pyi +++ b/mypy/typeshed/stdlib/concurrent/futures/thread.pyi @@ -91,8 +91,12 @@ class ThreadPoolExecutor(Executor): _shutdown: bool _shutdown_lock: Lock _thread_name_prefix: str | None - _initializer: Callable[..., None] | None - _initargs: tuple[Any, ...] + if sys.version_info >= (3, 14): + _create_worker_context: Callable[[], WorkerContext] + _resolve_work_item_task: _ResolveTaskFunc + else: + _initializer: Callable[..., None] | None + _initargs: tuple[Any, ...] _work_queue: queue.SimpleQueue[_WorkItem[Any]] if sys.version_info >= (3, 14): @@ -100,12 +104,12 @@ class ThreadPoolExecutor(Executor): @classmethod def prepare_context( cls, initializer: Callable[[], object], initargs: tuple[()] - ) -> tuple[Callable[[], Self], _ResolveTaskFunc]: ... + ) -> tuple[Callable[[], WorkerContext], _ResolveTaskFunc]: ... @overload @classmethod def prepare_context( cls, initializer: Callable[[Unpack[_Ts]], object], initargs: tuple[Unpack[_Ts]] - ) -> tuple[Callable[[], Self], _ResolveTaskFunc]: ... + ) -> tuple[Callable[[], WorkerContext], _ResolveTaskFunc]: ... @overload def __init__( diff --git a/mypy/typeshed/stdlib/os/__init__.pyi b/mypy/typeshed/stdlib/os/__init__.pyi index 5286c76d1b066..dd4479f9030a2 100644 --- a/mypy/typeshed/stdlib/os/__init__.pyi +++ b/mypy/typeshed/stdlib/os/__init__.pyi @@ -683,7 +683,7 @@ else: extsep: str pathsep: str defpath: str -linesep: str +linesep: Literal["\n", "\r\n"] devnull: str name: str diff --git a/mypy/typeshed/stdlib/sys/__init__.pyi b/mypy/typeshed/stdlib/sys/__init__.pyi index ce06551f975a0..054fe91b17c6f 100644 --- a/mypy/typeshed/stdlib/sys/__init__.pyi +++ b/mypy/typeshed/stdlib/sys/__init__.pyi @@ -6,7 +6,7 @@ from collections.abc import AsyncGenerator, Callable, Sequence from io import TextIOWrapper from types import FrameType, ModuleType, TracebackType from typing import Any, Final, Literal, NoReturn, Protocol, TextIO, TypeVar, final, type_check_only -from typing_extensions import LiteralString, TypeAlias +from typing_extensions import LiteralString, TypeAlias, deprecated _T = TypeVar("_T") @@ -335,7 +335,14 @@ class _version_info(_UninstantiableStructseq, tuple[int, int, int, _ReleaseLevel version_info: _version_info def call_tracing(func: Callable[..., _T], args: Any, /) -> _T: ... -def _clear_type_cache() -> None: ... + +if sys.version_info >= (3, 13): + @deprecated("Deprecated in Python 3.13; use _clear_internal_caches() instead.") + def _clear_type_cache() -> None: ... + +else: + def _clear_type_cache() -> None: ... + def _current_frames() -> dict[int, FrameType]: ... def _getframe(depth: int = 0, /) -> FrameType: ... def _debugmallocstats() -> None: ... diff --git a/mypy/typeshed/stdlib/tkinter/commondialog.pyi b/mypy/typeshed/stdlib/tkinter/commondialog.pyi index d5fc2f05ceec7..6dba6bd609284 100644 --- a/mypy/typeshed/stdlib/tkinter/commondialog.pyi +++ b/mypy/typeshed/stdlib/tkinter/commondialog.pyi @@ -1,12 +1,14 @@ -from _typeshed import Incomplete from collections.abc import Mapping -from typing import ClassVar +from tkinter import Misc +from typing import Any, ClassVar __all__ = ["Dialog"] class Dialog: command: ClassVar[str | None] - master: Incomplete | None - options: Mapping[str, Incomplete] - def __init__(self, master=None, **options) -> None: ... - def show(self, **options): ... + master: Misc | None + # Types of options are very dynamic. They depend on the command and are + # sometimes changed to a different type. + options: Mapping[str, Any] + def __init__(self, master: Misc | None = None, **options: Any) -> None: ... + def show(self, **options: Any) -> Any: ... diff --git a/mypy/typeshed/stdlib/tkinter/filedialog.pyi b/mypy/typeshed/stdlib/tkinter/filedialog.pyi index af033dae97c31..b6ef8f45d0350 100644 --- a/mypy/typeshed/stdlib/tkinter/filedialog.pyi +++ b/mypy/typeshed/stdlib/tkinter/filedialog.pyi @@ -1,6 +1,6 @@ -from _typeshed import Incomplete, StrOrBytesPath -from collections.abc import Iterable -from tkinter import Button, Entry, Frame, Listbox, Misc, Scrollbar, StringVar, Toplevel, commondialog +from _typeshed import Incomplete, StrOrBytesPath, StrPath +from collections.abc import Hashable, Iterable +from tkinter import Button, Entry, Event, Frame, Listbox, Misc, Scrollbar, StringVar, Toplevel, commondialog from typing import IO, ClassVar, Literal __all__ = [ @@ -19,12 +19,12 @@ __all__ = [ "askdirectory", ] -dialogstates: dict[Incomplete, tuple[Incomplete, Incomplete]] +dialogstates: dict[Hashable, tuple[str, str]] class FileDialog: title: str - master: Incomplete - directory: Incomplete | None + master: Misc + directory: str | None top: Toplevel botframe: Frame selection: Entry @@ -38,23 +38,23 @@ class FileDialog: filter_button: Button cancel_button: Button def __init__( - self, master, title=None + self, master: Misc, title: str | None = None ) -> None: ... # title is usually a str or None, but e.g. int doesn't raise en exception either - how: Incomplete | None - def go(self, dir_or_file=".", pattern: str = "*", default: str = "", key=None): ... - def quit(self, how=None) -> None: ... - def dirs_double_event(self, event) -> None: ... - def dirs_select_event(self, event) -> None: ... - def files_double_event(self, event) -> None: ... - def files_select_event(self, event) -> None: ... - def ok_event(self, event) -> None: ... + how: str | None + def go(self, dir_or_file: StrPath = ".", pattern: StrPath = "*", default: StrPath = "", key: Hashable | None = None): ... + def quit(self, how: str | None = None) -> None: ... + def dirs_double_event(self, event: Event) -> None: ... + def dirs_select_event(self, event: Event) -> None: ... + def files_double_event(self, event: Event) -> None: ... + def files_select_event(self, event: Event) -> None: ... + def ok_event(self, event: Event) -> None: ... def ok_command(self) -> None: ... - def filter_command(self, event=None) -> None: ... - def get_filter(self): ... - def get_selection(self): ... - def cancel_command(self, event=None) -> None: ... - def set_filter(self, dir, pat) -> None: ... - def set_selection(self, file) -> None: ... + def filter_command(self, event: Event | None = None) -> None: ... + def get_filter(self) -> tuple[str, str]: ... + def get_selection(self) -> str: ... + def cancel_command(self, event: Event | None = None) -> None: ... + def set_filter(self, dir: StrPath, pat: StrPath) -> None: ... + def set_selection(self, file: StrPath) -> None: ... class LoadFileDialog(FileDialog): title: str diff --git a/mypy/typeshed/stdlib/tkinter/messagebox.pyi b/mypy/typeshed/stdlib/tkinter/messagebox.pyi index 902fab62ac05a..8e5a88f92ea15 100644 --- a/mypy/typeshed/stdlib/tkinter/messagebox.pyi +++ b/mypy/typeshed/stdlib/tkinter/messagebox.pyi @@ -1,5 +1,6 @@ +from tkinter import Misc from tkinter.commondialog import Dialog -from typing import ClassVar, Final +from typing import ClassVar, Final, Literal __all__ = ["showinfo", "showwarning", "showerror", "askquestion", "askokcancel", "askyesno", "askyesnocancel", "askretrycancel"] @@ -23,11 +24,75 @@ NO: Final = "no" class Message(Dialog): command: ClassVar[str] -def showinfo(title: str | None = None, message: str | None = None, **options) -> str: ... -def showwarning(title: str | None = None, message: str | None = None, **options) -> str: ... -def showerror(title: str | None = None, message: str | None = None, **options) -> str: ... -def askquestion(title: str | None = None, message: str | None = None, **options) -> str: ... -def askokcancel(title: str | None = None, message: str | None = None, **options) -> bool: ... -def askyesno(title: str | None = None, message: str | None = None, **options) -> bool: ... -def askyesnocancel(title: str | None = None, message: str | None = None, **options) -> bool | None: ... -def askretrycancel(title: str | None = None, message: str | None = None, **options) -> bool: ... +def showinfo( + title: str | None = None, + message: str | None = None, + *, + detail: str = ..., + icon: Literal["error", "info", "question", "warning"] = ..., + default: Literal["ok"] = ..., + parent: Misc = ..., +) -> str: ... +def showwarning( + title: str | None = None, + message: str | None = None, + *, + detail: str = ..., + icon: Literal["error", "info", "question", "warning"] = ..., + default: Literal["ok"] = ..., + parent: Misc = ..., +) -> str: ... +def showerror( + title: str | None = None, + message: str | None = None, + *, + detail: str = ..., + icon: Literal["error", "info", "question", "warning"] = ..., + default: Literal["ok"] = ..., + parent: Misc = ..., +) -> str: ... +def askquestion( + title: str | None = None, + message: str | None = None, + *, + detail: str = ..., + icon: Literal["error", "info", "question", "warning"] = ..., + default: Literal["yes", "no"] = ..., + parent: Misc = ..., +) -> str: ... +def askokcancel( + title: str | None = None, + message: str | None = None, + *, + detail: str = ..., + icon: Literal["error", "info", "question", "warning"] = ..., + default: Literal["ok", "cancel"] = ..., + parent: Misc = ..., +) -> bool: ... +def askyesno( + title: str | None = None, + message: str | None = None, + *, + detail: str = ..., + icon: Literal["error", "info", "question", "warning"] = ..., + default: Literal["yes", "no"] = ..., + parent: Misc = ..., +) -> bool: ... +def askyesnocancel( + title: str | None = None, + message: str | None = None, + *, + detail: str = ..., + icon: Literal["error", "info", "question", "warning"] = ..., + default: Literal["cancel", "yes", "no"] = ..., + parent: Misc = ..., +) -> bool | None: ... +def askretrycancel( + title: str | None = None, + message: str | None = None, + *, + detail: str = ..., + icon: Literal["error", "info", "question", "warning"] = ..., + default: Literal["retry", "cancel"] = ..., + parent: Misc = ..., +) -> bool: ... diff --git a/mypy/typeshed/stdlib/token.pyi b/mypy/typeshed/stdlib/token.pyi index 7c13b15d95b79..fd1b10da1d12e 100644 --- a/mypy/typeshed/stdlib/token.pyi +++ b/mypy/typeshed/stdlib/token.pyi @@ -1,4 +1,5 @@ import sys +from typing import Final __all__ = [ "AMPER", @@ -81,87 +82,87 @@ if sys.version_info >= (3, 12): if sys.version_info >= (3, 14): __all__ += ["TSTRING_START", "TSTRING_MIDDLE", "TSTRING_END"] -ENDMARKER: int -NAME: int -NUMBER: int -STRING: int -NEWLINE: int -INDENT: int -DEDENT: int -LPAR: int -RPAR: int -LSQB: int -RSQB: int -COLON: int -COMMA: int -SEMI: int -PLUS: int -MINUS: int -STAR: int -SLASH: int -VBAR: int -AMPER: int -LESS: int -GREATER: int -EQUAL: int -DOT: int -PERCENT: int -LBRACE: int -RBRACE: int -EQEQUAL: int -NOTEQUAL: int -LESSEQUAL: int -GREATEREQUAL: int -TILDE: int -CIRCUMFLEX: int -LEFTSHIFT: int -RIGHTSHIFT: int -DOUBLESTAR: int -PLUSEQUAL: int -MINEQUAL: int -STAREQUAL: int -SLASHEQUAL: int -PERCENTEQUAL: int -AMPEREQUAL: int -VBAREQUAL: int -CIRCUMFLEXEQUAL: int -LEFTSHIFTEQUAL: int -RIGHTSHIFTEQUAL: int -DOUBLESTAREQUAL: int -DOUBLESLASH: int -DOUBLESLASHEQUAL: int -AT: int -RARROW: int -ELLIPSIS: int -ATEQUAL: int +ENDMARKER: Final[int] +NAME: Final[int] +NUMBER: Final[int] +STRING: Final[int] +NEWLINE: Final[int] +INDENT: Final[int] +DEDENT: Final[int] +LPAR: Final[int] +RPAR: Final[int] +LSQB: Final[int] +RSQB: Final[int] +COLON: Final[int] +COMMA: Final[int] +SEMI: Final[int] +PLUS: Final[int] +MINUS: Final[int] +STAR: Final[int] +SLASH: Final[int] +VBAR: Final[int] +AMPER: Final[int] +LESS: Final[int] +GREATER: Final[int] +EQUAL: Final[int] +DOT: Final[int] +PERCENT: Final[int] +LBRACE: Final[int] +RBRACE: Final[int] +EQEQUAL: Final[int] +NOTEQUAL: Final[int] +LESSEQUAL: Final[int] +GREATEREQUAL: Final[int] +TILDE: Final[int] +CIRCUMFLEX: Final[int] +LEFTSHIFT: Final[int] +RIGHTSHIFT: Final[int] +DOUBLESTAR: Final[int] +PLUSEQUAL: Final[int] +MINEQUAL: Final[int] +STAREQUAL: Final[int] +SLASHEQUAL: Final[int] +PERCENTEQUAL: Final[int] +AMPEREQUAL: Final[int] +VBAREQUAL: Final[int] +CIRCUMFLEXEQUAL: Final[int] +LEFTSHIFTEQUAL: Final[int] +RIGHTSHIFTEQUAL: Final[int] +DOUBLESTAREQUAL: Final[int] +DOUBLESLASH: Final[int] +DOUBLESLASHEQUAL: Final[int] +AT: Final[int] +RARROW: Final[int] +ELLIPSIS: Final[int] +ATEQUAL: Final[int] if sys.version_info < (3, 13): - AWAIT: int - ASYNC: int -OP: int -ERRORTOKEN: int -N_TOKENS: int -NT_OFFSET: int -tok_name: dict[int, str] -COMMENT: int -NL: int -ENCODING: int -TYPE_COMMENT: int -TYPE_IGNORE: int -COLONEQUAL: int -EXACT_TOKEN_TYPES: dict[str, int] + AWAIT: Final[int] + ASYNC: Final[int] +OP: Final[int] +ERRORTOKEN: Final[int] +N_TOKENS: Final[int] +NT_OFFSET: Final[int] +tok_name: Final[dict[int, str]] +COMMENT: Final[int] +NL: Final[int] +ENCODING: Final[int] +TYPE_COMMENT: Final[int] +TYPE_IGNORE: Final[int] +COLONEQUAL: Final[int] +EXACT_TOKEN_TYPES: Final[dict[str, int]] if sys.version_info >= (3, 10): - SOFT_KEYWORD: int + SOFT_KEYWORD: Final[int] if sys.version_info >= (3, 12): - EXCLAMATION: int - FSTRING_END: int - FSTRING_MIDDLE: int - FSTRING_START: int + EXCLAMATION: Final[int] + FSTRING_END: Final[int] + FSTRING_MIDDLE: Final[int] + FSTRING_START: Final[int] if sys.version_info >= (3, 14): - TSTRING_START: int - TSTRING_MIDDLE: int - TSTRING_END: int + TSTRING_START: Final[int] + TSTRING_MIDDLE: Final[int] + TSTRING_END: Final[int] def ISTERMINAL(x: int) -> bool: ... def ISNONTERMINAL(x: int) -> bool: ... diff --git a/mypy/typeshed/stdlib/tokenize.pyi b/mypy/typeshed/stdlib/tokenize.pyi index b658740a1ad7a..1a3a80937f22e 100644 --- a/mypy/typeshed/stdlib/tokenize.pyi +++ b/mypy/typeshed/stdlib/tokenize.pyi @@ -3,10 +3,15 @@ from _typeshed import FileDescriptorOrPath from collections.abc import Callable, Generator, Iterable, Sequence from re import Pattern from token import * -from token import EXACT_TOKEN_TYPES as EXACT_TOKEN_TYPES from typing import Any, NamedTuple, TextIO, type_check_only from typing_extensions import TypeAlias +if sys.version_info < (3, 12): + # Avoid double assignment to Final name by imports, which pyright objects to. + # EXACT_TOKEN_TYPES is already defined by 'from token import *' above + # in Python 3.12+. + from token import EXACT_TOKEN_TYPES as EXACT_TOKEN_TYPES + __all__ = [ "AMPER", "AMPEREQUAL", diff --git a/mypy/typeshed/stdlib/types.pyi b/mypy/typeshed/stdlib/types.pyi index d9f8e87568334..582cb653422f5 100644 --- a/mypy/typeshed/stdlib/types.pyi +++ b/mypy/typeshed/stdlib/types.pyi @@ -388,7 +388,7 @@ class GeneratorType(Generator[_YieldT_co, _SendT_contra, _ReturnT_co]): @property def gi_running(self) -> bool: ... @property - def gi_yieldfrom(self) -> GeneratorType[_YieldT_co, _SendT_contra, Any] | None: ... + def gi_yieldfrom(self) -> Iterator[_YieldT_co] | None: ... if sys.version_info >= (3, 11): @property def gi_suspended(self) -> bool: ... From 5b0ac327717ffc02e836d5456fb5785b75bba82a Mon Sep 17 00:00:00 2001 From: Jahongir Qurbonov <109198731+Jahongir-Qurbonov@users.noreply.github.com> Date: Tue, 1 Jul 2025 18:02:23 +0500 Subject: [PATCH 038/424] [mypyc] Implement `list.clear()` primitive (#19344) Add primitive for `list.clear`. Issue: [#1093](https://github.com/mypyc/mypyc/issues/1093) --- mypyc/lib-rt/CPy.h | 1 + mypyc/lib-rt/list_ops.c | 17 +++++++++++++++++ mypyc/primitives/list_ops.py | 9 +++++++++ mypyc/test-data/fixtures/ir.py | 1 + mypyc/test-data/irbuild-lists.test | 12 ++++++++++++ mypyc/test-data/run-lists.test | 26 ++++++++++++++++++++++++++ 6 files changed, 66 insertions(+) diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index 1f0cf4dd63d67..b8c39772dfae8 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -667,6 +667,7 @@ PyObject *CPySequence_Multiply(PyObject *seq, CPyTagged t_size); PyObject *CPySequence_RMultiply(CPyTagged t_size, PyObject *seq); PyObject *CPySequence_InPlaceMultiply(PyObject *seq, CPyTagged t_size); PyObject *CPyList_GetSlice(PyObject *obj, CPyTagged start, CPyTagged end); +char CPyList_Clear(PyObject *list); PyObject *CPyList_Copy(PyObject *list); int CPySequence_Check(PyObject *obj); diff --git a/mypyc/lib-rt/list_ops.c b/mypyc/lib-rt/list_ops.c index 4dddb2249f06b..03af8a769c0e5 100644 --- a/mypyc/lib-rt/list_ops.c +++ b/mypyc/lib-rt/list_ops.c @@ -29,6 +29,23 @@ PyObject *CPyList_Build(Py_ssize_t len, ...) { return res; } +char CPyList_Clear(PyObject *list) { + if (PyList_CheckExact(list)) { + PyList_Clear(list); + } else { + _Py_IDENTIFIER(clear); + PyObject *name = _PyUnicode_FromId(&PyId_clear); + if (name == NULL) { + return 0; + } + PyObject *res = PyObject_CallMethodNoArgs(list, name); + if (res == NULL) { + return 0; + } + } + return 1; +} + PyObject *CPyList_Copy(PyObject *list) { if(PyList_CheckExact(list)) { return PyList_GetSlice(list, 0, PyList_GET_SIZE(list)); diff --git a/mypyc/primitives/list_ops.py b/mypyc/primitives/list_ops.py index 99df6fe0dc9c6..d0e0af9f987fd 100644 --- a/mypyc/primitives/list_ops.py +++ b/mypyc/primitives/list_ops.py @@ -271,6 +271,15 @@ error_kind=ERR_MAGIC, ) +# list.clear() +method_op( + name="clear", + arg_types=[list_rprimitive], + return_type=bit_rprimitive, + c_function_name="CPyList_Clear", + error_kind=ERR_FALSE, +) + # list.copy() method_op( name="copy", diff --git a/mypyc/test-data/fixtures/ir.py b/mypyc/test-data/fixtures/ir.py index 1b92590a5fd49..532cbbc06177a 100644 --- a/mypyc/test-data/fixtures/ir.py +++ b/mypyc/test-data/fixtures/ir.py @@ -245,6 +245,7 @@ def sort(self) -> None: pass def reverse(self) -> None: pass def remove(self, o: _T) -> None: pass def index(self, o: _T) -> int: pass + def clear(self) -> None: pass def copy(self) -> List[_T]: pass class dict(Mapping[_K, _V]): diff --git a/mypyc/test-data/irbuild-lists.test b/mypyc/test-data/irbuild-lists.test index 2435b5aee350c..72caa5fad8d83 100644 --- a/mypyc/test-data/irbuild-lists.test +++ b/mypyc/test-data/irbuild-lists.test @@ -220,6 +220,18 @@ L0: r1 = r0 << 1 return r1 +[case testListClear] +from typing import List +def f(l: List[int]) -> None: + return l.clear() +[out] +def f(l): + l :: list + r0 :: bit +L0: + r0 = CPyList_Clear(l) + return 1 + [case testListCopy] from typing import List from typing import Any diff --git a/mypyc/test-data/run-lists.test b/mypyc/test-data/run-lists.test index 07c6d7735f101..85e0926027c5f 100644 --- a/mypyc/test-data/run-lists.test +++ b/mypyc/test-data/run-lists.test @@ -51,6 +51,32 @@ print(2, a) 1 [-1, 5] 2 [340282366920938463463374607431768211461, -170141183460469231731687303715884105736] +[case testListClear] +from typing import List, Any +from copysubclass import subc + +def test_list_clear() -> None: + l1 = [1, 2, 3, -4, 5] + l1.clear() + assert l1 == [] + l1.clear() + assert l1 == [] + l2: List[Any] = [] + l2.clear() + assert l2 == [] + l3 = [1, 2, 3, "abcdef"] + l3.clear() + assert l3 == [] + # subclass testing + l4: subc = subc([1, 2, 3]) + l4.clear() + assert l4 == [] + +[file copysubclass.py] +from typing import Any +class subc(list[Any]): + pass + [case testListCopy] from typing import List from copysubclass import subc From 3df3d796ee7821f04ee0443ec5011687c6af7f45 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Tue, 1 Jul 2025 11:03:27 -0400 Subject: [PATCH 039/424] feat: new mypyc primitives for str.count (#19264) This PR adds new mypyc primitives for all variations of `str.count` fixes https://github.com/mypyc/mypyc/issues/1096 --- mypyc/lib-rt/CPy.h | 2 + mypyc/lib-rt/str_ops.c | 24 +++++++++ mypyc/primitives/str_ops.py | 28 ++++++++++ mypyc/test-data/irbuild-str.test | 58 ++++++++++++++++++++ mypyc/test-data/run-strings.test | 91 ++++++++++++++++++++++++++++++++ 5 files changed, 203 insertions(+) diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index b8c39772dfae8..bdf3e0130a4cb 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -753,6 +753,8 @@ bool CPyStr_IsTrue(PyObject *obj); Py_ssize_t CPyStr_Size_size_t(PyObject *str); PyObject *CPy_Decode(PyObject *obj, PyObject *encoding, PyObject *errors); PyObject *CPy_Encode(PyObject *obj, PyObject *encoding, PyObject *errors); +Py_ssize_t CPyStr_Count(PyObject *unicode, PyObject *substring, CPyTagged start); +Py_ssize_t CPyStr_CountFull(PyObject *unicode, PyObject *substring, CPyTagged start, CPyTagged end); CPyTagged CPyStr_Ord(PyObject *obj); diff --git a/mypyc/lib-rt/str_ops.c b/mypyc/lib-rt/str_ops.c index 49fcbb8c68764..210172c57497f 100644 --- a/mypyc/lib-rt/str_ops.c +++ b/mypyc/lib-rt/str_ops.c @@ -511,6 +511,30 @@ PyObject *CPy_Encode(PyObject *obj, PyObject *encoding, PyObject *errors) { } } +Py_ssize_t CPyStr_Count(PyObject *unicode, PyObject *substring, CPyTagged start) { + Py_ssize_t temp_start = CPyTagged_AsSsize_t(start); + if (temp_start == -1 && PyErr_Occurred()) { + PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG); + return -1; + } + Py_ssize_t end = PyUnicode_GET_LENGTH(unicode); + return PyUnicode_Count(unicode, substring, temp_start, end); +} + +Py_ssize_t CPyStr_CountFull(PyObject *unicode, PyObject *substring, CPyTagged start, CPyTagged end) { + Py_ssize_t temp_start = CPyTagged_AsSsize_t(start); + if (temp_start == -1 && PyErr_Occurred()) { + PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG); + return -1; + } + Py_ssize_t temp_end = CPyTagged_AsSsize_t(end); + if (temp_end == -1 && PyErr_Occurred()) { + PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG); + return -1; + } + return PyUnicode_Count(unicode, substring, temp_start, temp_end); +} + CPyTagged CPyStr_Ord(PyObject *obj) { Py_ssize_t s = PyUnicode_GET_LENGTH(obj); diff --git a/mypyc/primitives/str_ops.py b/mypyc/primitives/str_ops.py index ded339b9672c3..9d46da9c35149 100644 --- a/mypyc/primitives/str_ops.py +++ b/mypyc/primitives/str_ops.py @@ -277,6 +277,34 @@ error_kind=ERR_MAGIC, ) +# str.count(substring) +method_op( + name="count", + arg_types=[str_rprimitive, str_rprimitive], + return_type=c_pyssize_t_rprimitive, + c_function_name="CPyStr_Count", + error_kind=ERR_NEG_INT, + extra_int_constants=[(0, c_pyssize_t_rprimitive)], +) + +# str.count(substring, start) +method_op( + name="count", + arg_types=[str_rprimitive, str_rprimitive, int_rprimitive], + return_type=c_pyssize_t_rprimitive, + c_function_name="CPyStr_Count", + error_kind=ERR_NEG_INT, +) + +# str.count(substring, start, end) +method_op( + name="count", + arg_types=[str_rprimitive, str_rprimitive, int_rprimitive, int_rprimitive], + return_type=c_pyssize_t_rprimitive, + c_function_name="CPyStr_CountFull", + error_kind=ERR_NEG_INT, +) + # str.replace(old, new) method_op( name="replace", diff --git a/mypyc/test-data/irbuild-str.test b/mypyc/test-data/irbuild-str.test index ad495dddcb151..2bf77a6cb5566 100644 --- a/mypyc/test-data/irbuild-str.test +++ b/mypyc/test-data/irbuild-str.test @@ -504,3 +504,61 @@ L0: r7 = CPyStr_Strip(s, 0) r8 = CPyStr_RStrip(s, 0) return 1 + +[case testCountAll] +def do_count(s: str) -> int: + return s.count("x") # type: ignore [attr-defined] +[out] +def do_count(s): + s, r0 :: str + r1 :: native_int + r2 :: bit + r3 :: object + r4 :: int +L0: + r0 = 'x' + r1 = CPyStr_Count(s, r0, 0) + r2 = r1 >= 0 :: signed + r3 = box(native_int, r1) + r4 = unbox(int, r3) + return r4 + +[case testCountStart] +def do_count(s: str, start: int) -> int: + return s.count("x", start) # type: ignore [attr-defined] +[out] +def do_count(s, start): + s :: str + start :: int + r0 :: str + r1 :: native_int + r2 :: bit + r3 :: object + r4 :: int +L0: + r0 = 'x' + r1 = CPyStr_Count(s, r0, start) + r2 = r1 >= 0 :: signed + r3 = box(native_int, r1) + r4 = unbox(int, r3) + return r4 + +[case testCountStartEnd] +def do_count(s: str, start: int, end: int) -> int: + return s.count("x", start, end) # type: ignore [attr-defined] +[out] +def do_count(s, start, end): + s :: str + start, end :: int + r0 :: str + r1 :: native_int + r2 :: bit + r3 :: object + r4 :: int +L0: + r0 = 'x' + r1 = CPyStr_CountFull(s, r0, start, end) + r2 = r1 >= 0 :: signed + r3 = box(native_int, r1) + r4 = unbox(int, r3) + return r4 diff --git a/mypyc/test-data/run-strings.test b/mypyc/test-data/run-strings.test index 9183b45b036a3..074e56f9068ac 100644 --- a/mypyc/test-data/run-strings.test +++ b/mypyc/test-data/run-strings.test @@ -815,3 +815,94 @@ def test_unicode_range() -> None: assert "\u2029 \U0010AAAA\U00104444B\u205F ".strip() == "\U0010AAAA\U00104444B" assert " \u3000\u205F ".strip() == "" assert "\u2029 \U00102865\u205F ".rstrip() == "\u2029 \U00102865" + +[case testCount] +# mypy: disable-error-code="attr-defined" +def test_count() -> None: + string = "abcbcb" + assert string.count("a") == 1 + assert string.count("b") == 3 + assert string.count("c") == 2 +def test_count_start() -> None: + string = "abcbcb" + assert string.count("a", 2) == string.count("a", -4) == 0, (string.count("a", 2), string.count("a", -4)) + assert string.count("b", 2) == string.count("b", -4) == 2, (string.count("b", 2), string.count("b", -4)) + assert string.count("c", 2) == string.count("c", -4) == 2, (string.count("c", 2), string.count("c", -4)) + # out of bounds + assert string.count("a", 8) == 0 + assert string.count("a", -8) == 1 + assert string.count("b", 8) == 0 + assert string.count("b", -8) == 3 + assert string.count("c", 8) == 0 + assert string.count("c", -8) == 2 +def test_count_start_end() -> None: + string = "abcbcb" + assert string.count("a", 0, 4) == 1, string.count("a", 0, 4) + assert string.count("b", 0, 4) == 2, string.count("b", 0, 4) + assert string.count("c", 0, 4) == 1, string.count("c", 0, 4) +def test_count_multi() -> None: + string = "aaabbbcccbbbcccbbb" + assert string.count("aaa") == 1, string.count("aaa") + assert string.count("bbb") == 3, string.count("bbb") + assert string.count("ccc") == 2, string.count("ccc") +def test_count_multi_start() -> None: + string = "aaabbbcccbbbcccbbb" + assert string.count("aaa", 6) == string.count("aaa", -12) == 0, (string.count("aaa", 6), string.count("aaa", -12)) + assert string.count("bbb", 6) == string.count("bbb", -12) == 2, (string.count("bbb", 6), string.count("bbb", -12)) + assert string.count("ccc", 6) == string.count("ccc", -12) == 2, (string.count("ccc", 6), string.count("ccc", -12)) + # out of bounds + assert string.count("aaa", 20) == 0 + assert string.count("aaa", -20) == 1 + assert string.count("bbb", 20) == 0 + assert string.count("bbb", -20) == 3 + assert string.count("ccc", 20) == 0 + assert string.count("ccc", -20) == 2 +def test_count_multi_start_end() -> None: + string = "aaabbbcccbbbcccbbb" + assert string.count("aaa", 0, 12) == 1, string.count("aaa", 0, 12) + assert string.count("bbb", 0, 12) == 2, string.count("bbb", 0, 12) + assert string.count("ccc", 0, 12) == 1, string.count("ccc", 0, 12) +def test_count_emoji() -> None: + string = "😴🚀ñ🚀ñ🚀" + assert string.count("😴") == 1, string.count("😴") + assert string.count("🚀") == 3, string.count("🚀") + assert string.count("ñ") == 2, string.count("ñ") +def test_count_start_emoji() -> None: + string = "😴🚀ñ🚀ñ🚀" + assert string.count("😴", 2) == string.count("😴", -4) == 0, (string.count("😴", 2), string.count("😴", -4)) + assert string.count("🚀", 2) == string.count("🚀", -4) == 2, (string.count("🚀", 2), string.count("🚀", -4)) + assert string.count("ñ", 2) == string.count("ñ", -4) == 2, (string.count("ñ", 2), string.count("ñ", -4)) + # Out of bounds + assert string.count("😴", 8) == 0, string.count("😴", 8) + assert string.count("😴", -8) == 1, string.count("😴", -8) + assert string.count("🚀", 8) == 0, string.count("🚀", 8) + assert string.count("🚀", -8) == 3, string.count("🚀", -8) + assert string.count("ñ", 8) == 0, string.count("ñ", 8) + assert string.count("ñ", -8) == 2, string.count("ñ", -8) +def test_count_start_end_emoji() -> None: + string = "😴🚀ñ🚀ñ🚀" + assert string.count("😴", 0, 4) == 1, string.count("😴", 0, 4) + assert string.count("🚀", 0, 4) == 2, string.count("🚀", 0, 4) + assert string.count("ñ", 0, 4) == 1, string.count("ñ", 0, 4) +def test_count_multi_emoji() -> None: + string = "😴😴😴🚀🚀🚀ñññ🚀🚀🚀ñññ🚀🚀🚀" + assert string.count("😴😴😴") == 1, string.count("😴😴😴") + assert string.count("🚀🚀🚀") == 3, string.count("🚀🚀🚀") + assert string.count("ñññ") == 2, string.count("ñññ") +def test_count_multi_start_emoji() -> None: + string = "😴😴😴🚀🚀🚀ñññ🚀🚀🚀ñññ🚀🚀🚀" + assert string.count("😴😴😴", 6) == string.count("😴😴😴", -12) == 0, (string.count("😴😴😴", 6), string.count("😴😴😴", -12)) + assert string.count("🚀🚀🚀", 6) == string.count("🚀🚀🚀", -12) == 2, (string.count("🚀🚀🚀", 6), string.count("🚀🚀🚀", -12)) + assert string.count("ñññ", 6) == string.count("ñññ", -12) == 2, (string.count("ñññ", 6), string.count("ñññ", -12)) + # Out of bounds + assert string.count("😴😴😴", 20) == 0, string.count("😴😴😴", 20) + assert string.count("😴😴😴", -20) == 1, string.count("😴😴😴", -20) + assert string.count("🚀🚀🚀", 20) == 0, string.count("🚀🚀🚀", 20) + assert string.count("🚀🚀🚀", -20) == 3, string.count("🚀🚀🚀", -20) + assert string.count("ñññ", 20) == 0, string.count("ñññ", 20) + assert string.count("ñññ", -20) == 2, string.count("ñññ", -20) +def test_count_multi_start_end_emoji() -> None: + string = "😴😴😴🚀🚀🚀ñññ🚀🚀🚀ñññ🚀🚀🚀" + assert string.count("😴😴😴", 0, 12) == 1, string.count("😴😴😴", 0, 12) + assert string.count("🚀🚀🚀", 0, 12) == 2, string.count("🚀🚀🚀", 0, 12) + assert string.count("ñññ", 0, 12) == 1, string.count("ñññ", 0, 12) From cbe28b23240f0091614594b05ef79708d01b5454 Mon Sep 17 00:00:00 2001 From: Chainfire Date: Wed, 2 Jul 2025 13:38:18 +0200 Subject: [PATCH 040/424] [mypyc] Fix AttributeError in async try/finally with mixed return paths (#19361) Async functions with try/finally blocks were raising AttributeError when: * Some paths in the try block return while others don't * The non-return path is executed at runtime * No further await calls are needed This occurred because mypyc's IR requires all control flow paths to assign to spill targets. The non-return path assigns NULL to maintain this invariant, but reading NULL attributes raises AttributeError in Python. Modified the GetAttr IR operation to support reading NULL attributes without raising AttributeError through a new allow_null parameter. This parameter is used specifically in try/finally resolution when reading spill targets. * Added allow_null: bool = False parameter to GetAttr.init in mypyc/ir/ops.py * When allow_null=True, sets error_kind=ERR_NEVER to prevent AttributeError * Modified read_nullable_attr in IRBuilder to create GetAttr with allow_null=True * Modified try_finally_resolve_control in statement.py to use read_nullable_attr only for spill targets (attributes starting with 'mypyc_temp') * Updated C code generation in emitfunc.py: * visit_get_attr checks for allow_null and delegates to get_attr_with_allow_null * get_attr_with_allow_null reads attributes without NULL checks and only increments reference count if not NULL Design decisions: * Targeted fix: Only applied to spill targets in try/finally resolution, not a general replacement for GetAttr. This minimizes risk and maintains existing behavior for all other attribute access. * No initialization changes: Initially tried initializing spill targets to Py_None instead of NULL, but this would incorrectly make try/finally blocks return None instead of falling through to subsequent code. Added two test cases to mypyc/test-data/run-async.test: * testAsyncTryFinallyMixedReturn: Tests the basic issue with async try/finally blocks containing mixed return/non-return paths. * testAsyncWithMixedReturn: Tests async with statements (which use try/finally under the hood) to ensure the fix works for this common pattern as well. Both tests verify that the AttributeError no longer occurs when taking the non-return path through the try block. See https://github.com/mypyc/mypyc/issues/1115 --- mypyc/codegen/emitfunc.py | 21 +++ mypyc/ir/ops.py | 9 +- mypyc/irbuild/builder.py | 9 + mypyc/irbuild/statement.py | 10 +- mypyc/test-data/run-async.test | 303 +++++++++++++++++++++++++++++++++ 5 files changed, 348 insertions(+), 4 deletions(-) diff --git a/mypyc/codegen/emitfunc.py b/mypyc/codegen/emitfunc.py index c854516825af1..00c7fd56b8999 100644 --- a/mypyc/codegen/emitfunc.py +++ b/mypyc/codegen/emitfunc.py @@ -358,6 +358,9 @@ def get_attr_expr(self, obj: str, op: GetAttr | SetAttr, decl_cl: ClassIR) -> st return f"({cast}{obj})->{self.emitter.attr(op.attr)}" def visit_get_attr(self, op: GetAttr) -> None: + if op.allow_null: + self.get_attr_with_allow_null(op) + return dest = self.reg(op) obj = self.reg(op.obj) rtype = op.class_type @@ -426,6 +429,24 @@ def visit_get_attr(self, op: GetAttr) -> None: elif not always_defined: self.emitter.emit_line("}") + def get_attr_with_allow_null(self, op: GetAttr) -> None: + """Handle GetAttr with allow_null=True which allows NULL without raising AttributeError.""" + dest = self.reg(op) + obj = self.reg(op.obj) + rtype = op.class_type + cl = rtype.class_ir + attr_rtype, decl_cl = cl.attr_details(op.attr) + + # Direct struct access without NULL check + attr_expr = self.get_attr_expr(obj, op, decl_cl) + self.emitter.emit_line(f"{dest} = {attr_expr};") + + # Only emit inc_ref if not NULL + if attr_rtype.is_refcounted and not op.is_borrowed: + self.emitter.emit_line(f"if ({dest} != NULL) {{") + self.emitter.emit_inc_ref(dest, attr_rtype) + self.emitter.emit_line("}") + def next_branch(self) -> Branch | None: if self.op_index + 1 < len(self.ops): next_op = self.ops[self.op_index + 1] diff --git a/mypyc/ir/ops.py b/mypyc/ir/ops.py index eec9c34a965e0..9dde658231d88 100644 --- a/mypyc/ir/ops.py +++ b/mypyc/ir/ops.py @@ -777,15 +777,20 @@ class GetAttr(RegisterOp): error_kind = ERR_MAGIC - def __init__(self, obj: Value, attr: str, line: int, *, borrow: bool = False) -> None: + def __init__( + self, obj: Value, attr: str, line: int, *, borrow: bool = False, allow_null: bool = False + ) -> None: super().__init__(line) self.obj = obj self.attr = attr + self.allow_null = allow_null assert isinstance(obj.type, RInstance), "Attribute access not supported: %s" % obj.type self.class_type = obj.type attr_type = obj.type.attr_type(attr) self.type = attr_type - if attr_type.error_overlap: + if allow_null: + self.error_kind = ERR_NEVER + elif attr_type.error_overlap: self.error_kind = ERR_MAGIC_OVERLAPPING self.is_borrowed = borrow and attr_type.is_refcounted diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index 75e059a5b5706..878c5e76df3d4 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -708,6 +708,15 @@ def read( assert False, "Unsupported lvalue: %r" % target + def read_nullable_attr(self, obj: Value, attr: str, line: int = -1) -> Value: + """Read an attribute that might be NULL without raising AttributeError. + + This is used for reading spill targets in try/finally blocks where NULL + indicates the non-return path was taken. + """ + assert isinstance(obj.type, RInstance) and obj.type.class_ir.is_ext_class + return self.add(GetAttr(obj, attr, line, allow_null=True)) + def assign(self, target: Register | AssignmentTarget, rvalue_reg: Value, line: int) -> None: if isinstance(target, Register): self.add(Assign(target, self.coerce_rvalue(rvalue_reg, target.type, line))) diff --git a/mypyc/irbuild/statement.py b/mypyc/irbuild/statement.py index 16a0483a87292..5c32d8f1a50c4 100644 --- a/mypyc/irbuild/statement.py +++ b/mypyc/irbuild/statement.py @@ -46,6 +46,7 @@ YieldExpr, YieldFromExpr, ) +from mypyc.common import TEMP_ATTR_NAME from mypyc.ir.ops import ( NAMESPACE_MODULE, NO_TRACEBACK_LINE_NO, @@ -653,10 +654,15 @@ def try_finally_resolve_control( if ret_reg: builder.activate_block(rest) return_block, rest = BasicBlock(), BasicBlock() - builder.add(Branch(builder.read(ret_reg), rest, return_block, Branch.IS_ERROR)) + # For spill targets in try/finally, use nullable read to avoid AttributeError + if isinstance(ret_reg, AssignmentTargetAttr) and ret_reg.attr.startswith(TEMP_ATTR_NAME): + ret_val = builder.read_nullable_attr(ret_reg.obj, ret_reg.attr, -1) + else: + ret_val = builder.read(ret_reg) + builder.add(Branch(ret_val, rest, return_block, Branch.IS_ERROR)) builder.activate_block(return_block) - builder.nonlocal_control[-1].gen_return(builder, builder.read(ret_reg), -1) + builder.nonlocal_control[-1].gen_return(builder, ret_val, -1) # TODO: handle break/continue builder.activate_block(rest) diff --git a/mypyc/test-data/run-async.test b/mypyc/test-data/run-async.test index 11ce67077270b..2dad720f99cd9 100644 --- a/mypyc/test-data/run-async.test +++ b/mypyc/test-data/run-async.test @@ -643,3 +643,306 @@ def test_async_def_contains_two_nested_functions() -> None: [file asyncio/__init__.pyi] def run(x: object) -> object: ... + +[case testAsyncTryFinallyMixedReturn] +# This used to raise an AttributeError, when: +# - the try block contains multiple paths +# - at least one of those explicitly returns +# - at least one of those does not explicitly return +# - the non-returning path is taken at runtime + +import asyncio + + +async def test_mixed_return(b: bool) -> bool: + try: + if b: + return b + finally: + pass + return b + + +async def test_run() -> None: + # Test return path + result1 = await test_mixed_return(True) + assert result1 == True + + # Test non-return path + result2 = await test_mixed_return(False) + assert result2 == False + + +def test_async_try_finally_mixed_return() -> None: + asyncio.run(test_run()) + +[file driver.py] +from native import test_async_try_finally_mixed_return +test_async_try_finally_mixed_return() + +[file asyncio/__init__.pyi] +def run(x: object) -> object: ... + +[case testAsyncWithMixedReturn] +# This used to raise an AttributeError, related to +# testAsyncTryFinallyMixedReturn, this is essentially +# a far more extensive version of that test surfacing +# more edge cases + +import asyncio +from typing import Optional, Type, Literal + + +class AsyncContextManager: + async def __aenter__(self) -> "AsyncContextManager": + return self + + async def __aexit__( + self, + t: Optional[Type[BaseException]], + v: Optional[BaseException], + tb: object, + ) -> Literal[False]: + return False + + +# Simple async functions (generator class) +async def test_gen_1(b: bool) -> bool: + async with AsyncContextManager(): + if b: + return b + return b + + +async def test_gen_2(b: bool) -> bool: + async with AsyncContextManager(): + if b: + return b + else: + return b + + +async def test_gen_3(b: bool) -> bool: + async with AsyncContextManager(): + if b: + return b + else: + pass + return b + + +async def test_gen_4(b: bool) -> bool: + ret: bool + async with AsyncContextManager(): + if b: + ret = b + else: + ret = b + return ret + + +async def test_gen_5(i: int) -> int: + async with AsyncContextManager(): + if i == 1: + return i + elif i == 2: + pass + elif i == 3: + return i + return i + + +async def test_gen_6(i: int) -> int: + async with AsyncContextManager(): + if i == 1: + return i + elif i == 2: + return i + elif i == 3: + return i + return i + + +async def test_gen_7(i: int) -> int: + async with AsyncContextManager(): + if i == 1: + return i + elif i == 2: + return i + elif i == 3: + return i + else: + return i + + +# Async functions with nested functions (environment class) +async def test_env_1(b: bool) -> bool: + def helper() -> bool: + return True + + async with AsyncContextManager(): + if b: + return helper() + return b + + +async def test_env_2(b: bool) -> bool: + def helper() -> bool: + return True + + async with AsyncContextManager(): + if b: + return helper() + else: + return b + + +async def test_env_3(b: bool) -> bool: + def helper() -> bool: + return True + + async with AsyncContextManager(): + if b: + return helper() + else: + pass + return b + + +async def test_env_4(b: bool) -> bool: + def helper() -> bool: + return True + + ret: bool + async with AsyncContextManager(): + if b: + ret = helper() + else: + ret = b + return ret + + +async def test_env_5(i: int) -> int: + def helper() -> int: + return 1 + + async with AsyncContextManager(): + if i == 1: + return helper() + elif i == 2: + pass + elif i == 3: + return i + return i + + +async def test_env_6(i: int) -> int: + def helper() -> int: + return 1 + + async with AsyncContextManager(): + if i == 1: + return helper() + elif i == 2: + return i + elif i == 3: + return i + return i + + +async def test_env_7(i: int) -> int: + def helper() -> int: + return 1 + + async with AsyncContextManager(): + if i == 1: + return helper() + elif i == 2: + return i + elif i == 3: + return i + else: + return i + + +async def run_all_tests() -> None: + # Test simple async functions (generator class) + # test_env_1: mixed return/no-return + assert await test_gen_1(True) is True + assert await test_gen_1(False) is False + + # test_gen_2: all branches return + assert await test_gen_2(True) is True + assert await test_gen_2(False) is False + + # test_gen_3: mixed return/pass + assert await test_gen_3(True) is True + assert await test_gen_3(False) is False + + # test_gen_4: no returns in async with + assert await test_gen_4(True) is True + assert await test_gen_4(False) is False + + # test_gen_5: multiple branches, some return + assert await test_gen_5(0) == 0 + assert await test_gen_5(1) == 1 + assert await test_gen_5(2) == 2 + assert await test_gen_5(3) == 3 + + # test_gen_6: all explicit branches return, implicit fallthrough + assert await test_gen_6(0) == 0 + assert await test_gen_6(1) == 1 + assert await test_gen_6(2) == 2 + assert await test_gen_6(3) == 3 + + # test_gen_7: all branches return including else + assert await test_gen_7(0) == 0 + assert await test_gen_7(1) == 1 + assert await test_gen_7(2) == 2 + assert await test_gen_7(3) == 3 + + # Test async functions with nested functions (environment class) + # test_env_1: mixed return/no-return + assert await test_env_1(True) is True + assert await test_env_1(False) is False + + # test_env_2: all branches return + assert await test_env_2(True) is True + assert await test_env_2(False) is False + + # test_env_3: mixed return/pass + assert await test_env_3(True) is True + assert await test_env_3(False) is False + + # test_env_4: no returns in async with + assert await test_env_4(True) is True + assert await test_env_4(False) is False + + # test_env_5: multiple branches, some return + assert await test_env_5(0) == 0 + assert await test_env_5(1) == 1 + assert await test_env_5(2) == 2 + assert await test_env_5(3) == 3 + + # test_env_6: all explicit branches return, implicit fallthrough + assert await test_env_6(0) == 0 + assert await test_env_6(1) == 1 + assert await test_env_6(2) == 2 + assert await test_env_6(3) == 3 + + # test_env_7: all branches return including else + assert await test_env_7(0) == 0 + assert await test_env_7(1) == 1 + assert await test_env_7(2) == 2 + assert await test_env_7(3) == 3 + + +def test_async_with_mixed_return() -> None: + asyncio.run(run_all_tests()) + +[file driver.py] +from native import test_async_with_mixed_return +test_async_with_mixed_return() + +[file asyncio/__init__.pyi] +def run(x: object) -> object: ... From eba2336467cd8fa9cff77dbf5821616691cfd76e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 3 Jul 2025 11:57:29 +0100 Subject: [PATCH 041/424] [mypyc] Speed up generator allocation by using a per-type freelist (#19316) Add support for per-type free "lists" that can cache up to one instance for quick allocation. If a free list is empty, fall back to regular object allocation. The per-type free list can be enabled for a class by setting a flag in ClassIR. Currently there is no way for users to control this, and these must be enabled based on heuristics. Use this free list for generator objects and coroutines, since they are often short-lived. Use a thread local variable for the free list so that each thread in a free threaded build has a separate free list. This way we need less synchronization, and the free list hit rate is higher for multithreaded workloads. This speeds up a microbenchmark that performs non-blocking calls of async functions in a loop by about 20%. The impact will become significantly bigger after some follow-up optimizations that I'm working on. This trades off memory use for performance, which is often good. This could use a lot of memory if many threads are calling async functions, but generally async functions are run on a single thread, so this case seems unlikely right now. Also, in my experience with large code bases only a small fraction of functions are async functions or generators, so the overall memory use impact shouldn't be too bad. We can later look into making this profile guided, so that only functions that are called frequently get the free list. Also we could add a compile-time flag to optimize for memory use, and it would turn this off. --- mypyc/codegen/emit.py | 25 ++++++ mypyc/codegen/emitclass.py | 98 +++++++++++++++++--- mypyc/codegen/emitmodule.py | 4 +- mypyc/ir/class_ir.py | 8 ++ mypyc/irbuild/generator.py | 1 + mypyc/lib-rt/mypyc_util.h | 25 ++++++ mypyc/test-data/run-generators.test | 134 +++++++++++++++++++++++++++- 7 files changed, 281 insertions(+), 14 deletions(-) diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index d7d7d9c7abda2..ba8b8307e1fdb 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -1115,6 +1115,31 @@ def emit_gc_clear(self, target: str, rtype: RType) -> None: else: assert False, "emit_gc_clear() not implemented for %s" % repr(rtype) + def emit_reuse_clear(self, target: str, rtype: RType) -> None: + """Emit attribute clear before object is added into freelist. + + Assume that 'target' represents a C expression that refers to a + struct member, such as 'self->x'. + + Unlike emit_gc_clear(), initialize attribute value to match a freshly + allocated object. + """ + if isinstance(rtype, RTuple): + for i, item_type in enumerate(rtype.types): + self.emit_reuse_clear(f"{target}.f{i}", item_type) + elif not rtype.is_refcounted: + self.emit_line(f"{target} = {rtype.c_undefined};") + elif isinstance(rtype, RPrimitive) and rtype.name == "builtins.int": + self.emit_line(f"if (CPyTagged_CheckLong({target})) {{") + self.emit_line(f"CPyTagged __tmp = {target};") + self.emit_line(f"{target} = {self.c_undefined_value(rtype)};") + self.emit_line("Py_XDECREF(CPyTagged_LongAsObject(__tmp));") + self.emit_line("} else {") + self.emit_line(f"{target} = {self.c_undefined_value(rtype)};") + self.emit_line("}") + else: + self.emit_gc_clear(target, rtype) + def emit_traceback( self, source_path: str, module_name: str, traceback_entry: tuple[str, int] ) -> None: diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index da3d14f9dafe4..576787424cbf9 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -186,6 +186,29 @@ def generate_class_type_decl( ) +def generate_class_reuse( + cl: ClassIR, c_emitter: Emitter, external_emitter: Emitter, emitter: Emitter +) -> None: + """Generate a definition of a single-object per-class free "list". + + This speeds up object allocation and freeing when there are many short-lived + objects. + + TODO: Generalize to support a free list with up to N objects. + """ + assert cl.reuse_freed_instance + + # The free list implementation doesn't support class hierarchies + assert cl.is_final_class or cl.children == [] + + context = c_emitter.context + name = cl.name_prefix(c_emitter.names) + "_free_instance" + struct_name = cl.struct_name(c_emitter.names) + context.declarations[name] = HeaderDeclaration( + f"CPyThreadLocal {struct_name} *{name};", needs_export=True + ) + + def generate_class(cl: ClassIR, module: str, emitter: Emitter) -> None: """Generate C code for a class. @@ -557,7 +580,22 @@ def generate_setup_for_class( emitter.emit_line("static PyObject *") emitter.emit_line(f"{func_name}(PyTypeObject *type)") emitter.emit_line("{") - emitter.emit_line(f"{cl.struct_name(emitter.names)} *self;") + struct_name = cl.struct_name(emitter.names) + emitter.emit_line(f"{struct_name} *self;") + + prefix = cl.name_prefix(emitter.names) + if cl.reuse_freed_instance: + # Attempt to use a per-type free list first (a free "list" with up to one object only). + emitter.emit_line(f"if ({prefix}_free_instance != NULL) {{") + emitter.emit_line(f"self = {prefix}_free_instance;") + emitter.emit_line(f"{prefix}_free_instance = NULL;") + emitter.emit_line("Py_SET_REFCNT(self, 1);") + emitter.emit_line("PyObject_GC_Track(self);") + if defaults_fn is not None: + emit_attr_defaults_func_call(defaults_fn, "self", emitter) + emitter.emit_line("return (PyObject *)self;") + emitter.emit_line("}") + emitter.emit_line(f"self = ({cl.struct_name(emitter.names)} *)type->tp_alloc(type, 0);") emitter.emit_line("if (self == NULL)") emitter.emit_line(" return NULL;") @@ -571,9 +609,7 @@ def generate_setup_for_class( else: emitter.emit_line(f"self->vtable = {vtable_name};") - for i in range(0, len(cl.bitmap_attrs), BITMAP_BITS): - field = emitter.bitmap_field(i) - emitter.emit_line(f"self->{field} = 0;") + emit_clear_bitmaps(cl, emitter) if cl.has_method("__call__"): name = cl.method_decl("__call__").cname(emitter.names) @@ -590,19 +626,34 @@ def generate_setup_for_class( # Initialize attributes to default values, if necessary if defaults_fn is not None: - emitter.emit_lines( - "if ({}{}((PyObject *)self) == 0) {{".format( - NATIVE_PREFIX, defaults_fn.cname(emitter.names) - ), - "Py_DECREF(self);", - "return NULL;", - "}", - ) + emit_attr_defaults_func_call(defaults_fn, "self", emitter) emitter.emit_line("return (PyObject *)self;") emitter.emit_line("}") +def emit_clear_bitmaps(cl: ClassIR, emitter: Emitter) -> None: + """Emit C code to clear bitmaps that track if attributes have an assigned value.""" + for i in range(0, len(cl.bitmap_attrs), BITMAP_BITS): + field = emitter.bitmap_field(i) + emitter.emit_line(f"self->{field} = 0;") + + +def emit_attr_defaults_func_call(defaults_fn: FuncIR, self_name: str, emitter: Emitter) -> None: + """Emit C code to initialize attribute defaults by calling defaults_fn. + + The code returns NULL on a raised exception. + """ + emitter.emit_lines( + "if ({}{}((PyObject *){}) == 0) {{".format( + NATIVE_PREFIX, defaults_fn.cname(emitter.names), self_name + ), + "Py_DECREF(self);", + "return NULL;", + "}", + ) + + def generate_constructor_for_class( cl: ClassIR, fn: FuncDecl, @@ -787,6 +838,8 @@ def generate_dealloc_for_class( emitter.emit_line("Py_TYPE(self)->tp_finalize((PyObject *)self);") emitter.emit_line("}") emitter.emit_line("PyObject_GC_UnTrack(self);") + if cl.reuse_freed_instance: + emit_reuse_dealloc(cl, emitter) # The trashcan is needed to handle deep recursive deallocations emitter.emit_line(f"CPy_TRASHCAN_BEGIN(self, {dealloc_func_name})") emitter.emit_line(f"{clear_func_name}(self);") @@ -795,6 +848,27 @@ def generate_dealloc_for_class( emitter.emit_line("}") +def emit_reuse_dealloc(cl: ClassIR, emitter: Emitter) -> None: + """Emit code to deallocate object by putting it to per-type free list. + + The free "list" currently can have up to one object. + """ + prefix = cl.name_prefix(emitter.names) + emitter.emit_line(f"if ({prefix}_free_instance == NULL) {{") + emitter.emit_line(f"{prefix}_free_instance = self;") + + # Clear attributes and free referenced objects. + + emit_clear_bitmaps(cl, emitter) + + for base in reversed(cl.base_mro): + for attr, rtype in base.attributes.items(): + emitter.emit_reuse_clear(f"self->{emitter.attr(attr)}", rtype) + + emitter.emit_line("return;") + emitter.emit_line("}") + + def generate_finalize_for_class( del_method: FuncIR, finalize_func_name: str, emitter: Emitter ) -> None: diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index f914bfd6345d5..e1b6a78572949 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -29,7 +29,7 @@ from mypy.util import hash_digest, json_dumps from mypyc.codegen.cstring import c_string_initializer from mypyc.codegen.emit import Emitter, EmitterContext, HeaderDeclaration, c_array_initializer -from mypyc.codegen.emitclass import generate_class, generate_class_type_decl +from mypyc.codegen.emitclass import generate_class, generate_class_reuse, generate_class_type_decl from mypyc.codegen.emitfunc import generate_native_function, native_function_header from mypyc.codegen.emitwrapper import ( generate_legacy_wrapper_function, @@ -609,6 +609,8 @@ def generate_c_for_modules(self) -> list[tuple[str, str]]: self.declare_finals(module_name, module.final_names, declarations) for cl in module.classes: generate_class_type_decl(cl, emitter, ext_declarations, declarations) + if cl.reuse_freed_instance: + generate_class_reuse(cl, emitter, ext_declarations, declarations) self.declare_type_vars(module_name, module.type_var_names, declarations) for fn in module.functions: generate_function_declaration(fn, declarations) diff --git a/mypyc/ir/class_ir.py b/mypyc/ir/class_ir.py index c88b9b0c7afc8..561dc9d438c43 100644 --- a/mypyc/ir/class_ir.py +++ b/mypyc/ir/class_ir.py @@ -204,6 +204,12 @@ def __init__( # If this is a generator environment class, what is the actual method for it self.env_user_function: FuncIR | None = None + # If True, keep one freed, cleared instance available for immediate reuse to + # speed up allocations. This helps if many objects are freed quickly, before + # other instances of the same class are allocated. This is effectively a + # per-type free "list" of up to length 1. + self.reuse_freed_instance = False + def __repr__(self) -> str: return ( "ClassIR(" @@ -403,6 +409,7 @@ def serialize(self) -> JsonDict: "_sometimes_initialized_attrs": sorted(self._sometimes_initialized_attrs), "init_self_leak": self.init_self_leak, "env_user_function": self.env_user_function.id if self.env_user_function else None, + "reuse_freed_instance": self.reuse_freed_instance, } @classmethod @@ -458,6 +465,7 @@ def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> ClassIR: ir.env_user_function = ( ctx.functions[data["env_user_function"]] if data["env_user_function"] else None ) + ir.reuse_freed_instance = data["reuse_freed_instance"] return ir diff --git a/mypyc/irbuild/generator.py b/mypyc/irbuild/generator.py index 782cb43197576..0e4b0e3e184a8 100644 --- a/mypyc/irbuild/generator.py +++ b/mypyc/irbuild/generator.py @@ -156,6 +156,7 @@ def setup_generator_class(builder: IRBuilder) -> ClassIR: name = f"{builder.fn_info.namespaced_name()}_gen" generator_class_ir = ClassIR(name, builder.module_name, is_generated=True, is_final_class=True) + generator_class_ir.reuse_freed_instance = True if builder.fn_info.can_merge_generator_and_env_classes(): builder.fn_info.env_class = generator_class_ir else: diff --git a/mypyc/lib-rt/mypyc_util.h b/mypyc/lib-rt/mypyc_util.h index 27a11ab9f581f..3d4eba3a3cdb4 100644 --- a/mypyc/lib-rt/mypyc_util.h +++ b/mypyc/lib-rt/mypyc_util.h @@ -23,6 +23,31 @@ #define CPy_NOINLINE #endif +#ifndef Py_GIL_DISABLED + +// Everything is running in the same thread, so no need for thread locals +#define CPyThreadLocal + +#else + +// 1. Use C11 standard thread_local storage, if available +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_THREADS__) +#define CPyThreadLocal _Thread_local + +// 2. Microsoft Visual Studio fallback +#elif defined(_MSC_VER) +#define CPyThreadLocal __declspec(thread) + +// 3. GNU thread local storage for GCC/Clang targets that still need it +#elif defined(__GNUC__) || defined(__clang__) +#define CPyThreadLocal __thread + +#else +#error "Can't define CPyThreadLocal for this compiler/target (consider using a non-free-threaded Python build)" +#endif + +#endif // Py_GIL_DISABLED + // INCREF and DECREF that assert the pointer is not NULL. // asserts are disabled in release builds so there shouldn't be a perf hit. // I'm honestly kind of surprised that this isn't done by default. diff --git a/mypyc/test-data/run-generators.test b/mypyc/test-data/run-generators.test index 2e55ded76f740..9c0b51d58e79f 100644 --- a/mypyc/test-data/run-generators.test +++ b/mypyc/test-data/run-generators.test @@ -681,7 +681,6 @@ def test_basic() -> None: assert context.x == 1 assert context.x == 0 - [case testYieldSpill] from typing import Generator from testutil import run_generator @@ -697,3 +696,136 @@ def test_basic() -> None: yields, val = x assert yields == ('foo',) assert val == 3, val + +[case testGeneratorReuse] +from typing import Iterator, Any + +def gen(x: list[int]) -> Iterator[list[int]]: + y = [9] + for z in x: + yield y + [z] + yield y + +def gen_range(n: int) -> Iterator[int]: + for x in range(n): + yield x + +def test_use_generator_multiple_times_one_at_a_time() -> None: + for i in range(100): + a = [] + for x in gen([2, i]): + a.append(x) + assert a == [[9, 2], [9, i], [9]] + +def test_use_multiple_generator_instances_at_same_time() -> None: + a = [] + for x in gen([2]): + a.append(x) + for y in gen([3, 4]): + a.append(y) + assert a == [[9, 2], [9, 3], [9, 4], [9], [9], [9, 3], [9, 4], [9]] + +def test_use_multiple_generator_instances_at_same_time_2() -> None: + a = [] + for x in gen_range(2): + a.append(x) + b = [] + for y in gen_range(3): + b.append(y) + c = [] + for z in gen_range(4): + c.append(z) + assert c == [0, 1, 2, 3] + assert b == [0, 1, 2] + assert a == [0, 1] + assert list(gen_range(5)) == list(range(5)) + +def gen_a(x: int) -> Iterator[int]: + yield x + 1 + +def gen_b(x: int) -> Iterator[int]: + yield x + 2 + +def test_generator_identities() -> None: + # Sanity check: two distinct live objects can't reuse the same memory location + g1 = gen_a(1) + g2 = gen_a(1) + assert g1 is not g2 + + # If two generators have non-overlapping lifetimes, they should reuse a memory location + g3 = gen_b(1) + id1 = id(g3) + g3 = gen_b(1) + assert id(g3) == id1 + + # More complex case of reuse: allocate other objects in between + g4: Any = gen_a(1) + id2 = id(g4) + g4 = gen_b(1) + g4 = [gen_b(n) for n in range(100)] + g4 = gen_a(1) + assert id(g4) == id2 + +[case testGeneratorReuseWithGilDisabled] +import sys +import threading +from typing import Iterator + +def gen() -> Iterator[int]: + yield 1 + +def is_gil_disabled() -> bool: + return hasattr(sys, "_is_gil_enabled") and not sys._is_gil_enabled() + +def test_each_thread_gets_separate_instance() -> None: + if not is_gil_disabled(): + # This only makes sense if GIL is disabled + return + + g = gen() + id1 = id(g) + + id2 = 0 + + def run() -> None: + nonlocal id2 + g = gen() + id2 = id(g) + + t = threading.Thread(target=run) + t.start() + t.join() + + # Each thread should get a separate reused instance + assert id1 != id2 + +[case testGeneratorWithUndefinedLocalInEnvironment] +from typing import Iterator + +from testutil import assertRaises + +def gen(set: bool) -> Iterator[float]: + if set: + y = float("-113.0") + yield 1.0 + yield y + +def test_bitmap_is_cleared_when_object_is_reused() -> None: + # This updates the bitmap of the shared instance. + list(gen(True)) + + # Ensure bitmap has been cleared. + with assertRaises(AttributeError): # TODO: Should be UnboundLocalError + list(gen(False)) + +def gen2(set: bool) -> Iterator[int]: + if set: + y = int("5") + yield 1 + yield y + +def test_undefined_int_in_environment() -> None: + list(gen2(True)) + + with assertRaises(AttributeError): # TODO: Should be UnboundLocalError + list(gen2(False)) From 1bf186cbf752401e8e85153f72c24514797d1be4 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 3 Jul 2025 11:59:20 +0100 Subject: [PATCH 042/424] [mypyc] Document some of our inheritance conventions (#19370) Our policy wasn't clear from just reading the code. --- mypyc/ir/ops.py | 65 +++++++++++++++++++++++++++++++++++++++++++--- mypyc/ir/rtypes.py | 24 ++++++++++++++++- 2 files changed, 84 insertions(+), 5 deletions(-) diff --git a/mypyc/ir/ops.py b/mypyc/ir/ops.py index 9dde658231d88..1cb3df916ac93 100644 --- a/mypyc/ir/ops.py +++ b/mypyc/ir/ops.py @@ -3,17 +3,31 @@ Opcodes operate on abstract values (Value) in a register machine. Each value has a type (RType). A value can hold various things, such as: -- local variables (Register) +- local variables or temporaries (Register) - intermediate values of expressions (RegisterOp subclasses) - condition flags (true/false) - literals (integer literals, True, False, etc.) + +NOTE: As a convention, we don't create subclasses of concrete Value/Op + subclasses (e.g. you shouldn't define a subclass of Integer, which + is a concrete class). + + If you want to introduce a variant of an existing class, you'd + typically add an attribute (e.g. a flag) to an existing concrete + class to enable the new behavior. Sometimes adding a new abstract + base class is also an option, or just creating a new subclass + without any inheritance relationship (some duplication of code + is preferred over introducing complex implementation inheritance). + + This makes it possible to use isinstance(x, ) checks without worrying about potential subclasses. """ from __future__ import annotations from abc import abstractmethod from collections.abc import Sequence -from typing import TYPE_CHECKING, Final, Generic, NamedTuple, TypeVar, Union +from typing import TYPE_CHECKING, Final, Generic, NamedTuple, TypeVar, Union, final from mypy_extensions import trait @@ -47,6 +61,7 @@ T = TypeVar("T") +@final class BasicBlock: """IR basic block. @@ -142,6 +157,7 @@ def is_void(self) -> bool: return isinstance(self.type, RVoid) +@final class Register(Value): """A Register holds a value of a specific type, and it can be read and mutated. @@ -168,6 +184,7 @@ def __repr__(self) -> str: return f"" +@final class Integer(Value): """Short integer literal. @@ -198,6 +215,7 @@ def numeric_value(self) -> int: return self.value +@final class Float(Value): """Float literal. @@ -257,13 +275,14 @@ def accept(self, visitor: OpVisitor[T]) -> T: class BaseAssign(Op): - """Base class for ops that assign to a register.""" + """Abstract base class for ops that assign to a register.""" def __init__(self, dest: Register, line: int = -1) -> None: super().__init__(line) self.dest = dest +@final class Assign(BaseAssign): """Assign a value to a Register (dest = src).""" @@ -286,6 +305,7 @@ def accept(self, visitor: OpVisitor[T]) -> T: return visitor.visit_assign(self) +@final class AssignMulti(BaseAssign): """Assign multiple values to a Register (dest = src1, src2, ...). @@ -320,7 +340,7 @@ def accept(self, visitor: OpVisitor[T]) -> T: class ControlOp(Op): - """Control flow operation.""" + """Abstract base class for control flow operations.""" def targets(self) -> Sequence[BasicBlock]: """Get all basic block targets of the control operation.""" @@ -331,6 +351,7 @@ def set_target(self, i: int, new: BasicBlock) -> None: raise AssertionError(f"Invalid set_target({self}, {i})") +@final class Goto(ControlOp): """Unconditional jump.""" @@ -360,6 +381,7 @@ def accept(self, visitor: OpVisitor[T]) -> T: return visitor.visit_goto(self) +@final class Branch(ControlOp): """Branch based on a value. @@ -426,6 +448,7 @@ def accept(self, visitor: OpVisitor[T]) -> T: return visitor.visit_branch(self) +@final class Return(ControlOp): """Return a value from a function.""" @@ -455,6 +478,7 @@ def accept(self, visitor: OpVisitor[T]) -> T: return visitor.visit_return(self) +@final class Unreachable(ControlOp): """Mark the end of basic block as unreachable. @@ -511,6 +535,7 @@ def can_raise(self) -> bool: return self.error_kind != ERR_NEVER +@final class IncRef(RegisterOp): """Increase reference count (inc_ref src).""" @@ -531,6 +556,7 @@ def accept(self, visitor: OpVisitor[T]) -> T: return visitor.visit_inc_ref(self) +@final class DecRef(RegisterOp): """Decrease reference count and free object if zero (dec_ref src). @@ -559,6 +585,7 @@ def accept(self, visitor: OpVisitor[T]) -> T: return visitor.visit_dec_ref(self) +@final class Call(RegisterOp): """Native call f(arg, ...). @@ -587,6 +614,7 @@ def accept(self, visitor: OpVisitor[T]) -> T: return visitor.visit_call(self) +@final class MethodCall(RegisterOp): """Native method call obj.method(arg, ...)""" @@ -618,6 +646,7 @@ def accept(self, visitor: OpVisitor[T]) -> T: return visitor.visit_method_call(self) +@final class PrimitiveDescription: """Description of a primitive op. @@ -670,6 +699,7 @@ def __repr__(self) -> str: return f"" +@final class PrimitiveOp(RegisterOp): """A higher-level primitive operation. @@ -707,6 +737,7 @@ def accept(self, visitor: OpVisitor[T]) -> T: return visitor.visit_primitive_op(self) +@final class LoadErrorValue(RegisterOp): """Load an error value. @@ -737,6 +768,7 @@ def accept(self, visitor: OpVisitor[T]) -> T: return visitor.visit_load_error_value(self) +@final class LoadLiteral(RegisterOp): """Load a Python literal object (dest = 'foo' / b'foo' / ...). @@ -772,6 +804,7 @@ def accept(self, visitor: OpVisitor[T]) -> T: return visitor.visit_load_literal(self) +@final class GetAttr(RegisterOp): """obj.attr (for a native object)""" @@ -804,6 +837,7 @@ def accept(self, visitor: OpVisitor[T]) -> T: return visitor.visit_get_attr(self) +@final class SetAttr(RegisterOp): """obj.attr = src (for a native object) @@ -855,6 +889,7 @@ def accept(self, visitor: OpVisitor[T]) -> T: NAMESPACE_TYPE_VAR: Final = "typevar" +@final class LoadStatic(RegisterOp): """Load a static name (name :: static). @@ -895,6 +930,7 @@ def accept(self, visitor: OpVisitor[T]) -> T: return visitor.visit_load_static(self) +@final class InitStatic(RegisterOp): """static = value :: static @@ -927,6 +963,7 @@ def accept(self, visitor: OpVisitor[T]) -> T: return visitor.visit_init_static(self) +@final class TupleSet(RegisterOp): """dest = (reg, ...) (for fixed-length tuple)""" @@ -959,6 +996,7 @@ def accept(self, visitor: OpVisitor[T]) -> T: return visitor.visit_tuple_set(self) +@final class TupleGet(RegisterOp): """Get item of a fixed-length tuple (src[index]).""" @@ -983,6 +1021,7 @@ def accept(self, visitor: OpVisitor[T]) -> T: return visitor.visit_tuple_get(self) +@final class Cast(RegisterOp): """cast(type, src) @@ -1014,6 +1053,7 @@ def accept(self, visitor: OpVisitor[T]) -> T: return visitor.visit_cast(self) +@final class Box(RegisterOp): """box(type, src) @@ -1048,6 +1088,7 @@ def accept(self, visitor: OpVisitor[T]) -> T: return visitor.visit_box(self) +@final class Unbox(RegisterOp): """unbox(type, src) @@ -1074,6 +1115,7 @@ def accept(self, visitor: OpVisitor[T]) -> T: return visitor.visit_unbox(self) +@final class RaiseStandardError(RegisterOp): """Raise built-in exception with an optional error string. @@ -1113,6 +1155,7 @@ def accept(self, visitor: OpVisitor[T]) -> T: StealsDescription = Union[bool, list[bool]] +@final class CallC(RegisterOp): """result = function(arg0, arg1, ...) @@ -1167,6 +1210,7 @@ def accept(self, visitor: OpVisitor[T]) -> T: return visitor.visit_call_c(self) +@final class Truncate(RegisterOp): """result = truncate src from src_type to dst_type @@ -1197,6 +1241,7 @@ def accept(self, visitor: OpVisitor[T]) -> T: return visitor.visit_truncate(self) +@final class Extend(RegisterOp): """result = extend src from src_type to dst_type @@ -1231,6 +1276,7 @@ def accept(self, visitor: OpVisitor[T]) -> T: return visitor.visit_extend(self) +@final class LoadGlobal(RegisterOp): """Load a low-level global variable/pointer. @@ -1258,6 +1304,7 @@ def accept(self, visitor: OpVisitor[T]) -> T: return visitor.visit_load_global(self) +@final class IntOp(RegisterOp): """Binary arithmetic or bitwise op on integer operands (e.g., r1 = r2 + r3). @@ -1322,6 +1369,7 @@ def accept(self, visitor: OpVisitor[T]) -> T: int_op_to_id: Final = {op: op_id for op_id, op in IntOp.op_str.items()} +@final class ComparisonOp(RegisterOp): """Low-level comparison op for integers and pointers. @@ -1383,6 +1431,7 @@ def accept(self, visitor: OpVisitor[T]) -> T: return visitor.visit_comparison_op(self) +@final class FloatOp(RegisterOp): """Binary float arithmetic op (e.g., r1 = r2 + r3). @@ -1424,6 +1473,7 @@ def accept(self, visitor: OpVisitor[T]) -> T: float_op_to_id: Final = {op: op_id for op_id, op in FloatOp.op_str.items()} +@final class FloatNeg(RegisterOp): """Float negation op (r1 = -r2).""" @@ -1444,6 +1494,7 @@ def accept(self, visitor: OpVisitor[T]) -> T: return visitor.visit_float_neg(self) +@final class FloatComparisonOp(RegisterOp): """Low-level comparison op for floats.""" @@ -1480,6 +1531,7 @@ def accept(self, visitor: OpVisitor[T]) -> T: float_comparison_op_to_id: Final = {op: op_id for op_id, op in FloatComparisonOp.op_str.items()} +@final class LoadMem(RegisterOp): """Read a memory location: result = *(type *)src. @@ -1509,6 +1561,7 @@ def accept(self, visitor: OpVisitor[T]) -> T: return visitor.visit_load_mem(self) +@final class SetMem(Op): """Write to a memory location: *(type *)dest = src @@ -1540,6 +1593,7 @@ def accept(self, visitor: OpVisitor[T]) -> T: return visitor.visit_set_mem(self) +@final class GetElementPtr(RegisterOp): """Get the address of a struct element. @@ -1566,6 +1620,7 @@ def accept(self, visitor: OpVisitor[T]) -> T: return visitor.visit_get_element_ptr(self) +@final class LoadAddress(RegisterOp): """Get the address of a value: result = (type)&src @@ -1600,6 +1655,7 @@ def accept(self, visitor: OpVisitor[T]) -> T: return visitor.visit_load_address(self) +@final class KeepAlive(RegisterOp): """A no-op operation that ensures source values aren't freed. @@ -1647,6 +1703,7 @@ def accept(self, visitor: OpVisitor[T]) -> T: return visitor.visit_keep_alive(self) +@final class Unborrow(RegisterOp): """A no-op op to create a regular reference from a borrowed one. diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 60a56065006f1..61aadce9b9d4e 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -18,12 +18,27 @@ mypyc.irbuild.mapper.Mapper.type_to_rtype converts mypy Types to mypyc RTypes. + +NOTE: As a convention, we don't create subclasses of concrete RType + subclasses (e.g. you shouldn't define a subclass of RTuple, which + is a concrete class). We prefer a flat class hierarchy. + + If you want to introduce a variant of an existing class, you'd + typically add an attribute (e.g. a flag) to an existing concrete + class to enable the new behavior. In rare cases, adding a new + abstract base class could also be an option. Adding a completely + separate class and sharing some functionality using module-level + helper functions may also be reasonable. + + This makes it possible to use isinstance(x, ) checks without worrying about potential subclasses + and avoids most trouble caused by implementation inheritance. """ from __future__ import annotations from abc import abstractmethod -from typing import TYPE_CHECKING, ClassVar, Final, Generic, TypeVar +from typing import TYPE_CHECKING, ClassVar, Final, Generic, TypeVar, final from typing_extensions import TypeGuard from mypyc.common import HAVE_IMMORTAL, IS_32_BIT_PLATFORM, PLATFORM_SIZE, JsonDict, short_name @@ -155,6 +170,7 @@ def visit_rvoid(self, typ: RVoid, /) -> T: raise NotImplementedError +@final class RVoid(RType): """The void type (no value). @@ -187,6 +203,7 @@ def __hash__(self) -> int: void_rtype: Final = RVoid() +@final class RPrimitive(RType): """Primitive type such as 'object' or 'int'. @@ -650,6 +667,7 @@ def visit_rvoid(self, t: RVoid) -> str: assert False, "rvoid in tuple?" +@final class RTuple(RType): """Fixed-length unboxed tuple (represented as a C struct). @@ -791,6 +809,7 @@ def compute_aligned_offsets_and_size(types: list[RType]) -> tuple[list[int], int return offsets, final_size +@final class RStruct(RType): """C struct type""" @@ -844,6 +863,7 @@ def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> RStruct: assert False +@final class RInstance(RType): """Instance of user-defined class (compiled to C extension class). @@ -904,6 +924,7 @@ def serialize(self) -> str: return self.name +@final class RUnion(RType): """union[x, ..., y]""" @@ -994,6 +1015,7 @@ def is_optional_type(rtype: RType) -> bool: return optional_value_type(rtype) is not None +@final class RArray(RType): """Fixed-length C array type (for example, int[5]). From 1a99ce22832ada034c8ca61337fd8e87678ab758 Mon Sep 17 00:00:00 2001 From: Piotr Sawicki Date: Fri, 4 Jul 2025 12:33:18 +0200 Subject: [PATCH 043/424] [mypyc] Fix comparison of tuples with different lengths (#19372) When comparing tuples, their lengths are now compared and their contents are only compared if their lengths are equal. Otherwise, when they have different lengths, the comparison always evaluates to `False`. --- mypyc/irbuild/ll_builder.py | 4 ++++ mypyc/test-data/run-tuples.test | 16 ++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index 6bc1eb9d04932..36b7c9241c711 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -1507,6 +1507,10 @@ def compare_tuples(self, lhs: Value, rhs: Value, op: str, line: int = -1) -> Val assert isinstance(lhs.type, RTuple) and isinstance(rhs.type, RTuple) equal = True if op == "==" else False result = Register(bool_rprimitive) + # tuples of different lengths + if len(lhs.type.types) != len(rhs.type.types): + self.add(Assign(result, self.false() if equal else self.true(), line)) + return result # empty tuples if len(lhs.type.types) == 0 and len(rhs.type.types) == 0: self.add(Assign(result, self.true() if equal else self.false(), line)) diff --git a/mypyc/test-data/run-tuples.test b/mypyc/test-data/run-tuples.test index 1437eaef2aa5d..fe9a8dff08c63 100644 --- a/mypyc/test-data/run-tuples.test +++ b/mypyc/test-data/run-tuples.test @@ -203,6 +203,22 @@ def f7(x: List[Tuple[int, int]]) -> int: def test_unbox_tuple() -> None: assert f7([(5, 6)]) == 11 +def test_comparison() -> None: + assert ('x','y') == ('x','y') + assert not(('x','y') != ('x','y')) + + assert ('x','y') != ('x','y',1) + assert not(('x','y') == ('x','y',1)) + + assert ('x','y',1) != ('x','y') + assert not(('x','y',1) == ('x','y')) + + assert ('x','y') != () + assert not(('x','y') == ()) + + assert () != ('x','y') + assert not(() == ('x','y')) + # Test that order is irrelevant to unions. Really I only care that this builds. class A: From 2de3f7766d972c6c0b2c4f9b5900b40f9244aab8 Mon Sep 17 00:00:00 2001 From: esarp <11684270+esarp@users.noreply.github.com> Date: Fri, 4 Jul 2025 11:12:53 -0500 Subject: [PATCH 044/424] Bump version to 1.18.0+dev (#19371) Release branch for 1.17.0: https://github.com/python/mypy/tree/release-1.17 Increase the dev version --- mypy/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/version.py b/mypy/version.py index 21d23758c6dcb..bb6a9582e74e5 100644 --- a/mypy/version.py +++ b/mypy/version.py @@ -8,7 +8,7 @@ # - Release versions have the form "1.2.3". # - Dev versions have the form "1.2.3+dev" (PLUS sign to conform to PEP 440). # - Before 1.0 we had the form "0.NNN". -__version__ = "1.17.0+dev" +__version__ = "1.18.0+dev" base_version = __version__ mypy_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) From 4980ae5b80e6e9b214d979a849e349dff6cad0ab Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 5 Jul 2025 12:16:54 +0100 Subject: [PATCH 045/424] Support type checking code fragment in profile script (#19379) Previously only self check was supported. Also rename the script since the old name was no longer suitable. --- ...profile_self_check.py => profile_check.py} | 38 +++++++++++++------ 1 file changed, 27 insertions(+), 11 deletions(-) rename misc/{profile_self_check.py => profile_check.py} (78%) diff --git a/misc/profile_self_check.py b/misc/profile_check.py similarity index 78% rename from misc/profile_self_check.py rename to misc/profile_check.py index eb853641d6d66..b29535020f0a7 100644 --- a/misc/profile_self_check.py +++ b/misc/profile_check.py @@ -1,4 +1,6 @@ -"""Compile mypy using mypyc and profile self-check using perf. +"""Compile mypy using mypyc and profile type checking using perf. + +By default does a self check. Notes: - Only Linux is supported for now (TODO: add support for other profilers) @@ -23,6 +25,8 @@ CFLAGS="-O2 -g -fno-omit-frame-pointer" """ +from __future__ import annotations + import argparse import glob import os @@ -41,24 +45,28 @@ TARGET_DIR = "mypy.profile.tmpdir" -def _profile_self_check(target_dir: str) -> None: +def _profile_type_check(target_dir: str, code: str | None) -> None: cache_dir = os.path.join(target_dir, ".mypy_cache") if os.path.exists(cache_dir): shutil.rmtree(cache_dir) - files = [] - for pat in "mypy/*.py", "mypy/*/*.py", "mypyc/*.py", "mypyc/test/*.py": - files.extend(glob.glob(pat)) - self_check_cmd = ["python", "-m", "mypy", "--config-file", "mypy_self_check.ini"] + files - cmdline = ["perf", "record", "-g"] + self_check_cmd + args = [] + if code is None: + args.extend(["--config-file", "mypy_self_check.ini"]) + for pat in "mypy/*.py", "mypy/*/*.py", "mypyc/*.py", "mypyc/test/*.py": + args.extend(glob.glob(pat)) + else: + args.extend(["-c", code]) + check_cmd = ["python", "-m", "mypy"] + args + cmdline = ["perf", "record", "-g"] + check_cmd t0 = time.time() subprocess.run(cmdline, cwd=target_dir, check=True) elapsed = time.time() - t0 print(f"{elapsed:.2f}s elapsed") -def profile_self_check(target_dir: str) -> None: +def profile_type_check(target_dir: str, code: str | None) -> None: try: - _profile_self_check(target_dir) + _profile_type_check(target_dir, code) except subprocess.CalledProcessError: print("\nProfiling failed! You may missing some permissions.") print("\nThis may help (note that it has security implications):") @@ -92,7 +100,7 @@ def main() -> None: check_requirements() parser = argparse.ArgumentParser( - description="Compile mypy and profile self checking using 'perf'." + description="Compile mypy and profile type checking using 'perf' (by default, self check)." ) parser.add_argument( "--multi-file", @@ -102,9 +110,17 @@ def main() -> None: parser.add_argument( "--skip-compile", action="store_true", help="use compiled mypy from previous run" ) + parser.add_argument( + "-c", + metavar="CODE", + default=None, + type=str, + help="profile type checking Python code fragment instead of mypy self-check", + ) args = parser.parse_args() multi_file: bool = args.multi_file skip_compile: bool = args.skip_compile + code: str | None = args.c target_dir = TARGET_DIR @@ -116,7 +132,7 @@ def main() -> None: elif not os.path.isdir(target_dir): sys.exit("error: Can't find compile mypy from previous run -- can't use --skip-compile") - profile_self_check(target_dir) + profile_type_check(target_dir, code) print() print('NOTE: Compile CPython using CFLAGS="-O2 -g -fno-omit-frame-pointer" for good results') From de23d08ff1784c763bdb2423d7caed0d421aa8d1 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 6 Jul 2025 17:57:40 +0100 Subject: [PATCH 046/424] Subtype checking micro-optimization (#19384) --- mypy/subtypes.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 86935d0613a27..05f34aaec8f13 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -155,15 +155,13 @@ def is_subtype( options=options, ) else: - assert not any( - { - ignore_type_params, - ignore_pos_arg_names, - ignore_declared_variance, - always_covariant, - ignore_promotions, - options, - } + assert ( + not ignore_type_params + and not ignore_pos_arg_names + and not ignore_declared_variance + and not always_covariant + and not ignore_promotions + and options is None ), "Don't pass both context and individual flags" if type_state.is_assumed_subtype(left, right): return True From 8df94d2b30383c230cdd51d449b4c46fdabd7be7 Mon Sep 17 00:00:00 2001 From: CoolCat467 <52022020+CoolCat467@users.noreply.github.com> Date: Mon, 7 Jul 2025 03:07:35 -0500 Subject: [PATCH 047/424] Lessen dmypy suggest path limitations for Windows machines (#19337) In this pull request, we allow dmypy suggest absolute paths to contain the drive letter colon for Windows machines. Fixes #19335. This is done by changing how `find_node` works slightly, allowing there to be at most two colon (`:`) characters in the passed key for windows machines instead of just one like on all other platforms, and then using `rsplit` and a split limit of 1 instead of just `split` like prior. --------- Co-authored-by: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> --- mypy/suggestions.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mypy/suggestions.py b/mypy/suggestions.py index 81eb20bd0ac32..45aa5ade47a4d 100644 --- a/mypy/suggestions.py +++ b/mypy/suggestions.py @@ -27,6 +27,7 @@ import itertools import json import os +import sys from collections.abc import Iterator from contextlib import contextmanager from typing import Callable, NamedTuple, TypedDict, TypeVar, cast @@ -549,12 +550,17 @@ def find_node(self, key: str) -> tuple[str, str, FuncDef]: # TODO: Also return OverloadedFuncDef -- currently these are ignored. node: SymbolNode | None = None if ":" in key: - if key.count(":") > 1: + # A colon might be part of a drive name on Windows (like `C:/foo/bar`) + # and is also used as a delimiter between file path and lineno. + # If a colon is there for any of those reasons, it must be a file+line + # reference. + platform_key_count = 2 if sys.platform == "win32" else 1 + if key.count(":") > platform_key_count: raise SuggestionFailure( "Malformed location for function: {}. Must be either" " package.module.Class.method or path/to/file.py:line".format(key) ) - file, line = key.split(":") + file, line = key.rsplit(":", 1) if not line.isdigit(): raise SuggestionFailure(f"Line number must be a number. Got {line}") line_number = int(line) From 3557e221f27b0bd15c62fc8162f54874598bbf0f Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 7 Jul 2025 10:09:29 +0100 Subject: [PATCH 048/424] Speed up the default plugin (#19385) Fix two kinds of inefficiency in the default plugin: * Pre-calculate various set objects, since set construction is pretty slow * Nested imports are pretty slow, so avoid doing them in the fast path I also had to refactor things a little in order to optimize the nested imports. This speeds up self check by about 2.2%. The default plugin is called a lot. --- mypy/plugins/constants.py | 20 +++++++++ mypy/plugins/default.py | 78 ++++++++++++++++++++++------------ mypy/plugins/enums.py | 10 +---- mypy/plugins/singledispatch.py | 19 ++------- 4 files changed, 76 insertions(+), 51 deletions(-) create mode 100644 mypy/plugins/constants.py diff --git a/mypy/plugins/constants.py b/mypy/plugins/constants.py new file mode 100644 index 0000000000000..9a09e89202de9 --- /dev/null +++ b/mypy/plugins/constants.py @@ -0,0 +1,20 @@ +"""Constant definitions for plugins kept here to help with import cycles.""" + +from typing import Final + +from mypy.semanal_enum import ENUM_BASES + +SINGLEDISPATCH_TYPE: Final = "functools._SingleDispatchCallable" +SINGLEDISPATCH_REGISTER_METHOD: Final = f"{SINGLEDISPATCH_TYPE}.register" +SINGLEDISPATCH_CALLABLE_CALL_METHOD: Final = f"{SINGLEDISPATCH_TYPE}.__call__" +SINGLEDISPATCH_REGISTER_RETURN_CLASS: Final = "_SingleDispatchRegisterCallable" +SINGLEDISPATCH_REGISTER_CALLABLE_CALL_METHOD: Final = ( + f"functools.{SINGLEDISPATCH_REGISTER_RETURN_CLASS}.__call__" +) + +ENUM_NAME_ACCESS: Final = {f"{prefix}.name" for prefix in ENUM_BASES} | { + f"{prefix}._name_" for prefix in ENUM_BASES +} +ENUM_VALUE_ACCESS: Final = {f"{prefix}.value" for prefix in ENUM_BASES} | { + f"{prefix}._value_" for prefix in ENUM_BASES +} diff --git a/mypy/plugins/default.py b/mypy/plugins/default.py index 2002a4f06093b..09f9795b593ed 100644 --- a/mypy/plugins/default.py +++ b/mypy/plugins/default.py @@ -15,6 +15,7 @@ MethodSigContext, Plugin, ) +from mypy.plugins import constants from mypy.plugins.common import try_getting_str_literals from mypy.subtypes import is_subtype from mypy.typeops import is_literal_type_like, make_simplified_union @@ -36,22 +37,36 @@ get_proper_types, ) +TD_SETDEFAULT_NAMES: Final = {n + ".setdefault" for n in TPDICT_FB_NAMES} +TD_POP_NAMES: Final = {n + ".pop" for n in TPDICT_FB_NAMES} + +TD_UPDATE_METHOD_NAMES: Final = ( + {n + ".update" for n in TPDICT_FB_NAMES} + | {n + ".__or__" for n in TPDICT_FB_NAMES} + | {n + ".__ror__" for n in TPDICT_FB_NAMES} + | {n + ".__ior__" for n in TPDICT_FB_NAMES} +) + class DefaultPlugin(Plugin): """Type checker plugin that is enabled by default.""" def get_function_hook(self, fullname: str) -> Callable[[FunctionContext], Type] | None: - from mypy.plugins import ctypes, enums, singledispatch - if fullname == "_ctypes.Array": + from mypy.plugins import ctypes + return ctypes.array_constructor_callback elif fullname == "functools.singledispatch": + from mypy.plugins import singledispatch + return singledispatch.create_singledispatch_function_callback elif fullname == "functools.partial": import mypy.plugins.functools return mypy.plugins.functools.partial_new_callback elif fullname == "enum.member": + from mypy.plugins import enums + return enums.enum_member_callback return None @@ -59,47 +74,42 @@ def get_function_hook(self, fullname: str) -> Callable[[FunctionContext], Type] def get_function_signature_hook( self, fullname: str ) -> Callable[[FunctionSigContext], FunctionLike] | None: - from mypy.plugins import attrs, dataclasses - if fullname in ("attr.evolve", "attrs.evolve", "attr.assoc", "attrs.assoc"): + from mypy.plugins import attrs + return attrs.evolve_function_sig_callback elif fullname in ("attr.fields", "attrs.fields"): + from mypy.plugins import attrs + return attrs.fields_function_sig_callback elif fullname == "dataclasses.replace": + from mypy.plugins import dataclasses + return dataclasses.replace_function_sig_callback return None def get_method_signature_hook( self, fullname: str ) -> Callable[[MethodSigContext], FunctionLike] | None: - from mypy.plugins import ctypes, singledispatch - if fullname == "typing.Mapping.get": return typed_dict_get_signature_callback - elif fullname in {n + ".setdefault" for n in TPDICT_FB_NAMES}: + elif fullname in TD_SETDEFAULT_NAMES: return typed_dict_setdefault_signature_callback - elif fullname in {n + ".pop" for n in TPDICT_FB_NAMES}: + elif fullname in TD_POP_NAMES: return typed_dict_pop_signature_callback elif fullname == "_ctypes.Array.__setitem__": - return ctypes.array_setitem_callback - elif fullname == singledispatch.SINGLEDISPATCH_CALLABLE_CALL_METHOD: - return singledispatch.call_singledispatch_function_callback + from mypy.plugins import ctypes - typed_dict_updates = set() - for n in TPDICT_FB_NAMES: - typed_dict_updates.add(n + ".update") - typed_dict_updates.add(n + ".__or__") - typed_dict_updates.add(n + ".__ror__") - typed_dict_updates.add(n + ".__ior__") + return ctypes.array_setitem_callback + elif fullname == constants.SINGLEDISPATCH_CALLABLE_CALL_METHOD: + from mypy.plugins import singledispatch - if fullname in typed_dict_updates: + return singledispatch.call_singledispatch_function_callback + elif fullname in TD_UPDATE_METHOD_NAMES: return typed_dict_update_signature_callback - return None def get_method_hook(self, fullname: str) -> Callable[[MethodContext], Type] | None: - from mypy.plugins import ctypes, singledispatch - if fullname == "typing.Mapping.get": return typed_dict_get_callback elif fullname == "builtins.int.__pow__": @@ -117,12 +127,20 @@ def get_method_hook(self, fullname: str) -> Callable[[MethodContext], Type] | No elif fullname in {n + ".__delitem__" for n in TPDICT_FB_NAMES}: return typed_dict_delitem_callback elif fullname == "_ctypes.Array.__getitem__": + from mypy.plugins import ctypes + return ctypes.array_getitem_callback elif fullname == "_ctypes.Array.__iter__": + from mypy.plugins import ctypes + return ctypes.array_iter_callback - elif fullname == singledispatch.SINGLEDISPATCH_REGISTER_METHOD: + elif fullname == constants.SINGLEDISPATCH_REGISTER_METHOD: + from mypy.plugins import singledispatch + return singledispatch.singledispatch_register_callback - elif fullname == singledispatch.REGISTER_CALLABLE_CALL_METHOD: + elif fullname == constants.SINGLEDISPATCH_REGISTER_CALLABLE_CALL_METHOD: + from mypy.plugins import singledispatch + return singledispatch.call_singledispatch_function_after_register_argument elif fullname == "functools.partial.__call__": import mypy.plugins.functools @@ -131,15 +149,21 @@ def get_method_hook(self, fullname: str) -> Callable[[MethodContext], Type] | No return None def get_attribute_hook(self, fullname: str) -> Callable[[AttributeContext], Type] | None: - from mypy.plugins import ctypes, enums - if fullname == "_ctypes.Array.value": + from mypy.plugins import ctypes + return ctypes.array_value_callback elif fullname == "_ctypes.Array.raw": + from mypy.plugins import ctypes + return ctypes.array_raw_callback - elif fullname in enums.ENUM_NAME_ACCESS: + elif fullname in constants.ENUM_NAME_ACCESS: + from mypy.plugins import enums + return enums.enum_name_callback - elif fullname in enums.ENUM_VALUE_ACCESS: + elif fullname in constants.ENUM_VALUE_ACCESS: + from mypy.plugins import enums + return enums.enum_value_callback return None diff --git a/mypy/plugins/enums.py b/mypy/plugins/enums.py index 8b7c5df6f51f5..dc58fc8110a5c 100644 --- a/mypy/plugins/enums.py +++ b/mypy/plugins/enums.py @@ -14,11 +14,10 @@ from __future__ import annotations from collections.abc import Iterable, Sequence -from typing import Final, TypeVar, cast +from typing import TypeVar, cast import mypy.plugin # To avoid circular imports. from mypy.nodes import TypeInfo -from mypy.semanal_enum import ENUM_BASES from mypy.subtypes import is_equivalent from mypy.typeops import fixup_partial_type, make_simplified_union from mypy.types import ( @@ -31,13 +30,6 @@ is_named_instance, ) -ENUM_NAME_ACCESS: Final = {f"{prefix}.name" for prefix in ENUM_BASES} | { - f"{prefix}._name_" for prefix in ENUM_BASES -} -ENUM_VALUE_ACCESS: Final = {f"{prefix}.value" for prefix in ENUM_BASES} | { - f"{prefix}._value_" for prefix in ENUM_BASES -} - def enum_name_callback(ctx: mypy.plugin.AttributeContext) -> Type: """This plugin refines the 'name' attribute in enums to act as if diff --git a/mypy/plugins/singledispatch.py b/mypy/plugins/singledispatch.py index be4b405ce6102..eb2bbe133bf09 100644 --- a/mypy/plugins/singledispatch.py +++ b/mypy/plugins/singledispatch.py @@ -1,7 +1,7 @@ from __future__ import annotations from collections.abc import Sequence -from typing import Final, NamedTuple, TypeVar, Union +from typing import NamedTuple, TypeVar, Union from typing_extensions import TypeAlias as _TypeAlias from mypy.messages import format_type @@ -9,6 +9,7 @@ from mypy.options import Options from mypy.plugin import CheckerPluginInterface, FunctionContext, MethodContext, MethodSigContext from mypy.plugins.common import add_method_to_class +from mypy.plugins.constants import SINGLEDISPATCH_REGISTER_RETURN_CLASS from mypy.subtypes import is_subtype from mypy.types import ( AnyType, @@ -33,13 +34,6 @@ class RegisterCallableInfo(NamedTuple): singledispatch_obj: Instance -SINGLEDISPATCH_TYPE: Final = "functools._SingleDispatchCallable" - -SINGLEDISPATCH_REGISTER_METHOD: Final = f"{SINGLEDISPATCH_TYPE}.register" - -SINGLEDISPATCH_CALLABLE_CALL_METHOD: Final = f"{SINGLEDISPATCH_TYPE}.__call__" - - def get_singledispatch_info(typ: Instance) -> SingledispatchTypeVars | None: if len(typ.args) == 2: return SingledispatchTypeVars(*typ.args) # type: ignore[arg-type] @@ -56,16 +50,11 @@ def get_first_arg(args: list[list[T]]) -> T | None: return None -REGISTER_RETURN_CLASS: Final = "_SingleDispatchRegisterCallable" - -REGISTER_CALLABLE_CALL_METHOD: Final = f"functools.{REGISTER_RETURN_CLASS}.__call__" - - def make_fake_register_class_instance( api: CheckerPluginInterface, type_args: Sequence[Type] ) -> Instance: - defn = ClassDef(REGISTER_RETURN_CLASS, Block([])) - defn.fullname = f"functools.{REGISTER_RETURN_CLASS}" + defn = ClassDef(SINGLEDISPATCH_REGISTER_RETURN_CLASS, Block([])) + defn.fullname = f"functools.{SINGLEDISPATCH_REGISTER_RETURN_CLASS}" info = TypeInfo(SymbolTable(), defn, "functools") obj_type = api.named_generic_type("builtins.object", []).type info.bases = [Instance(obj_type, [])] From 1fde1438f6c47963ecd6e10a8c410ac41ba05103 Mon Sep 17 00:00:00 2001 From: Christoph Tyralla Date: Mon, 7 Jul 2025 11:24:42 +0200 Subject: [PATCH 049/424] Combine the revealed types of multiple iteration steps in a more robust manner. (#19324) This PR fixes a regression introduced in #19118 and discussed in #19270. The combination of the revealed types of individual iteration steps now relies on collecting the original type objects instead of parts of preliminary `revealed_type` notes. As @JukkaL suspected, this approach is much more straightforward than introducing a sufficiently complete `revealed_type` note parser. Please note that I appended a commit that refactors already existing code. It is mainly code-moving, so I hope it does not complicate the review of this PR. --- mypy/checker.py | 19 ++-- mypy/errors.py | 103 +++++++++---------- mypy/messages.py | 34 +++++- test-data/unit/check-inference.test | 4 +- test-data/unit/check-narrowing.test | 2 +- test-data/unit/check-redefine2.test | 4 +- test-data/unit/check-typevar-tuple.test | 2 +- test-data/unit/check-union-error-syntax.test | 7 +- 8 files changed, 95 insertions(+), 80 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 7859934c1ef70..225a50c7e6461 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -618,7 +618,7 @@ def accept_loop( if on_enter_body is not None: on_enter_body() - with IterationErrorWatcher(self.msg.errors, iter_errors) as watcher: + with IterationErrorWatcher(self.msg.errors, iter_errors): self.accept(body) partials_new = sum(len(pts.map) for pts in self.partial_types) @@ -641,10 +641,7 @@ def accept_loop( if iter == 20: raise RuntimeError("Too many iterations when checking a loop") - for error_info in watcher.yield_error_infos(): - self.msg.fail(*error_info[:2], code=error_info[2]) - for note_info in watcher.yield_note_infos(self.options): - self.note(*note_info) + self.msg.iteration_dependent_errors(iter_errors) # If exit_condition is set, assume it must be False on exit from the loop: if exit_condition: @@ -3041,7 +3038,7 @@ def is_noop_for_reachability(self, s: Statement) -> bool: if isinstance(s.expr, EllipsisExpr): return True elif isinstance(s.expr, CallExpr): - with self.expr_checker.msg.filter_errors(): + with self.expr_checker.msg.filter_errors(filter_revealed_type=True): typ = get_proper_type( self.expr_checker.accept( s.expr, allow_none_return=True, always_allow_any=True @@ -4969,7 +4966,7 @@ def visit_try_stmt(self, s: TryStmt) -> None: if s.finally_body: # First we check finally_body is type safe on all abnormal exit paths iter_errors = IterationDependentErrors() - with IterationErrorWatcher(self.msg.errors, iter_errors) as watcher: + with IterationErrorWatcher(self.msg.errors, iter_errors): self.accept(s.finally_body) if s.finally_body: @@ -4986,13 +4983,9 @@ def visit_try_stmt(self, s: TryStmt) -> None: # that follows the try statement.) assert iter_errors is not None if not self.binder.is_unreachable(): - with IterationErrorWatcher(self.msg.errors, iter_errors) as watcher: + with IterationErrorWatcher(self.msg.errors, iter_errors): self.accept(s.finally_body) - - for error_info in watcher.yield_error_infos(): - self.msg.fail(*error_info[:2], code=error_info[2]) - for note_info in watcher.yield_note_infos(self.options): - self.msg.note(*note_info) + self.msg.iteration_dependent_errors(iter_errors) def visit_try_without_finally(self, s: TryStmt, try_frame: bool) -> None: """Type check a try statement, ignoring the finally block. diff --git a/mypy/errors.py b/mypy/errors.py index 5dd411c39e959..5c135146bcb71 100644 --- a/mypy/errors.py +++ b/mypy/errors.py @@ -15,6 +15,7 @@ from mypy.nodes import Context from mypy.options import Options from mypy.scope import Scope +from mypy.types import Type from mypy.util import DEFAULT_SOURCE_OFFSET, is_typeshed_file from mypy.version import __version__ as mypy_version @@ -166,6 +167,10 @@ class ErrorWatcher: out by one of the ErrorWatcher instances. """ + # public attribute for the special treatment of `reveal_type` by + # `MessageBuilder.reveal_type`: + filter_revealed_type: bool + def __init__( self, errors: Errors, @@ -173,11 +178,13 @@ def __init__( filter_errors: bool | Callable[[str, ErrorInfo], bool] = False, save_filtered_errors: bool = False, filter_deprecated: bool = False, + filter_revealed_type: bool = False, ) -> None: self.errors = errors self._has_new_errors = False self._filter = filter_errors self._filter_deprecated = filter_deprecated + self.filter_revealed_type = filter_revealed_type self._filtered: list[ErrorInfo] | None = [] if save_filtered_errors else None def __enter__(self) -> Self: @@ -236,15 +243,41 @@ class IterationDependentErrors: # the error report occurs but really all unreachable lines. unreachable_lines: list[set[int]] - # One set of revealed types for each `reveal_type` statement. Each created set can - # grow during the iteration. Meaning of the tuple items: function_or_member, line, - # column, end_line, end_column: - revealed_types: dict[tuple[str | None, int, int, int, int], set[str]] + # One list of revealed types for each `reveal_type` statement. Each created list + # can grow during the iteration. Meaning of the tuple items: line, column, + # end_line, end_column: + revealed_types: dict[tuple[int, int, int | None, int | None], list[Type]] def __init__(self) -> None: self.uselessness_errors = [] self.unreachable_lines = [] - self.revealed_types = defaultdict(set) + self.revealed_types = defaultdict(list) + + def yield_uselessness_error_infos(self) -> Iterator[tuple[str, Context, ErrorCode]]: + """Report only those `unreachable`, `redundant-expr`, and `redundant-casts` + errors that could not be ruled out in any iteration step.""" + + persistent_uselessness_errors = set() + for candidate in set(chain(*self.uselessness_errors)): + if all( + (candidate in errors) or (candidate[2] in lines) + for errors, lines in zip(self.uselessness_errors, self.unreachable_lines) + ): + persistent_uselessness_errors.add(candidate) + for error_info in persistent_uselessness_errors: + context = Context(line=error_info[2], column=error_info[3]) + context.end_line = error_info[4] + context.end_column = error_info[5] + yield error_info[1], context, error_info[0] + + def yield_revealed_type_infos(self) -> Iterator[tuple[list[Type], Context]]: + """Yield all types revealed in at least one iteration step.""" + + for note_info, types in self.revealed_types.items(): + context = Context(line=note_info[0], column=note_info[1]) + context.end_line = note_info[2] + context.end_column = note_info[3] + yield types, context class IterationErrorWatcher(ErrorWatcher): @@ -287,53 +320,8 @@ def on_error(self, file: str, info: ErrorInfo) -> bool: iter_errors.unreachable_lines[-1].update(range(info.line, info.end_line + 1)) return True - if info.code == codes.MISC and info.message.startswith("Revealed type is "): - key = info.function_or_member, info.line, info.column, info.end_line, info.end_column - types = info.message.split('"')[1] - if types.startswith("Union["): - iter_errors.revealed_types[key].update(types[6:-1].split(", ")) - else: - iter_errors.revealed_types[key].add(types) - return True - return super().on_error(file, info) - def yield_error_infos(self) -> Iterator[tuple[str, Context, ErrorCode]]: - """Report only those `unreachable`, `redundant-expr`, and `redundant-casts` - errors that could not be ruled out in any iteration step.""" - - persistent_uselessness_errors = set() - iter_errors = self.iteration_dependent_errors - for candidate in set(chain(*iter_errors.uselessness_errors)): - if all( - (candidate in errors) or (candidate[2] in lines) - for errors, lines in zip( - iter_errors.uselessness_errors, iter_errors.unreachable_lines - ) - ): - persistent_uselessness_errors.add(candidate) - for error_info in persistent_uselessness_errors: - context = Context(line=error_info[2], column=error_info[3]) - context.end_line = error_info[4] - context.end_column = error_info[5] - yield error_info[1], context, error_info[0] - - def yield_note_infos(self, options: Options) -> Iterator[tuple[str, Context]]: - """Yield all types revealed in at least one iteration step.""" - - for note_info, types in self.iteration_dependent_errors.revealed_types.items(): - sorted_ = sorted(types, key=lambda typ: typ.lower()) - if len(types) == 1: - revealed = sorted_[0] - elif options.use_or_syntax(): - revealed = " | ".join(sorted_) - else: - revealed = f"Union[{', '.join(sorted_)}]" - context = Context(line=note_info[1], column=note_info[2]) - context.end_line = note_info[3] - context.end_column = note_info[4] - yield f'Revealed type is "{revealed}"', context - class Errors: """Container for compile errors. @@ -596,18 +584,19 @@ def _add_error_info(self, file: str, info: ErrorInfo) -> None: if info.code in (IMPORT, IMPORT_UNTYPED, IMPORT_NOT_FOUND): self.seen_import_error = True + def get_watchers(self) -> Iterator[ErrorWatcher]: + """Yield the `ErrorWatcher` stack from top to bottom.""" + i = len(self._watchers) + while i > 0: + i -= 1 + yield self._watchers[i] + def _filter_error(self, file: str, info: ErrorInfo) -> bool: """ process ErrorWatcher stack from top to bottom, stopping early if error needs to be filtered out """ - i = len(self._watchers) - while i > 0: - i -= 1 - w = self._watchers[i] - if w.on_error(file, info): - return True - return False + return any(w.on_error(file, info) for w in self.get_watchers()) def add_error_info(self, info: ErrorInfo) -> None: file, lines = info.origin diff --git a/mypy/messages.py b/mypy/messages.py index 13a4facc82b03..44ed25a195179 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -23,7 +23,13 @@ from mypy import errorcodes as codes, message_registry from mypy.erasetype import erase_type from mypy.errorcodes import ErrorCode -from mypy.errors import ErrorInfo, Errors, ErrorWatcher +from mypy.errors import ( + ErrorInfo, + Errors, + ErrorWatcher, + IterationDependentErrors, + IterationErrorWatcher, +) from mypy.nodes import ( ARG_NAMED, ARG_NAMED_OPT, @@ -188,12 +194,14 @@ def filter_errors( filter_errors: bool | Callable[[str, ErrorInfo], bool] = True, save_filtered_errors: bool = False, filter_deprecated: bool = False, + filter_revealed_type: bool = False, ) -> ErrorWatcher: return ErrorWatcher( self.errors, filter_errors=filter_errors, save_filtered_errors=save_filtered_errors, filter_deprecated=filter_deprecated, + filter_revealed_type=filter_revealed_type, ) def add_errors(self, errors: list[ErrorInfo]) -> None: @@ -1738,6 +1746,24 @@ def invalid_signature_for_special_method( ) def reveal_type(self, typ: Type, context: Context) -> None: + + # Search for an error watcher that modifies the "normal" behaviour (we do not + # rely on the normal `ErrorWatcher` filtering approach because we might need to + # collect the original types for a later unionised response): + for watcher in self.errors.get_watchers(): + # The `reveal_type` statement should be ignored: + if watcher.filter_revealed_type: + return + # The `reveal_type` statement might be visited iteratively due to being + # placed in a loop or so. Hence, we collect the respective types of + # individual iterations so that we can report them all in one step later: + if isinstance(watcher, IterationErrorWatcher): + watcher.iteration_dependent_errors.revealed_types[ + (context.line, context.column, context.end_line, context.end_column) + ].append(typ) + return + + # Nothing special here; just create the note: visitor = TypeStrVisitor(options=self.options) self.note(f'Revealed type is "{typ.accept(visitor)}"', context) @@ -2481,6 +2507,12 @@ def match_statement_inexhaustive_match(self, typ: Type, context: Context) -> Non code=codes.EXHAUSTIVE_MATCH, ) + def iteration_dependent_errors(self, iter_errors: IterationDependentErrors) -> None: + for error_info in iter_errors.yield_uselessness_error_infos(): + self.fail(*error_info[:2], code=error_info[2]) + for types, context in iter_errors.yield_revealed_type_infos(): + self.reveal_type(mypy.typeops.make_simplified_union(types), context) + def quote_type_string(type_string: str) -> str: """Quotes a type representation for use in messages.""" diff --git a/test-data/unit/check-inference.test b/test-data/unit/check-inference.test index 90cb7d3799cf6..6564fb3192d04 100644 --- a/test-data/unit/check-inference.test +++ b/test-data/unit/check-inference.test @@ -343,7 +343,7 @@ for var2 in [g, h, i, j, k, l]: reveal_type(var2) # N: Revealed type is "Union[builtins.int, builtins.str]" for var3 in [m, n, o, p, q, r]: - reveal_type(var3) # N: Revealed type is "Union[Any, builtins.int]" + reveal_type(var3) # N: Revealed type is "Union[builtins.int, Any]" T = TypeVar("T", bound=Type[Foo]) @@ -1247,7 +1247,7 @@ class X(TypedDict): x: X for a in ("hourly", "daily"): - reveal_type(a) # N: Revealed type is "Union[Literal['daily']?, Literal['hourly']?]" + reveal_type(a) # N: Revealed type is "Union[Literal['hourly']?, Literal['daily']?]" reveal_type(x[a]) # N: Revealed type is "builtins.int" reveal_type(a.upper()) # N: Revealed type is "builtins.str" c = a diff --git a/test-data/unit/check-narrowing.test b/test-data/unit/check-narrowing.test index 7a053e1c5cab8..e322bd7a37b8a 100644 --- a/test-data/unit/check-narrowing.test +++ b/test-data/unit/check-narrowing.test @@ -2346,7 +2346,7 @@ def f() -> bool: ... y = None while f(): - reveal_type(y) # N: Revealed type is "Union[builtins.int, None]" + reveal_type(y) # N: Revealed type is "Union[None, builtins.int]" y = 1 reveal_type(y) # N: Revealed type is "Union[builtins.int, None]" diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 924e665846697..3523772611aab 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -628,7 +628,7 @@ def f1() -> None: def f2() -> None: x = None while int(): - reveal_type(x) # N: Revealed type is "Union[builtins.str, None]" + reveal_type(x) # N: Revealed type is "Union[None, builtins.str]" if int(): x = "" reveal_type(x) # N: Revealed type is "Union[None, builtins.str]" @@ -923,7 +923,7 @@ class X(TypedDict): x: X for a in ("hourly", "daily"): - reveal_type(a) # N: Revealed type is "Union[Literal['daily']?, Literal['hourly']?]" + reveal_type(a) # N: Revealed type is "Union[Literal['hourly']?, Literal['daily']?]" reveal_type(x[a]) # N: Revealed type is "builtins.int" reveal_type(a.upper()) # N: Revealed type is "builtins.str" c = a diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index f44758f7b51b6..db0e26ba2b364 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -989,7 +989,7 @@ from typing_extensions import Unpack def pipeline(*xs: Unpack[Tuple[int, Unpack[Tuple[float, ...]], bool]]) -> None: for x in xs: - reveal_type(x) # N: Revealed type is "Union[builtins.float, builtins.int]" + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.float]" [builtins fixtures/tuple.pyi] [case testFixedUnpackItemInInstanceArguments] diff --git a/test-data/unit/check-union-error-syntax.test b/test-data/unit/check-union-error-syntax.test index d41281b774e1e..e938598aaefe0 100644 --- a/test-data/unit/check-union-error-syntax.test +++ b/test-data/unit/check-union-error-syntax.test @@ -62,17 +62,18 @@ x = 3 # E: Incompatible types in assignment (expression has type "Literal[3]", v try: x = 1 x = "" + x = {1: ""} finally: - reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str, builtins.dict[builtins.int, builtins.str]]" [builtins fixtures/isinstancelist.pyi] [case testOrSyntaxRecombined] # flags: --python-version 3.10 --no-force-union-syntax --allow-redefinition-new --local-partial-types # The following revealed type is recombined because the finally body is visited twice. -# ToDo: Improve this recombination logic, especially (but not only) for the "or syntax". try: x = 1 x = "" + x = {1: ""} finally: - reveal_type(x) # N: Revealed type is "builtins.int | builtins.str | builtins.str" + reveal_type(x) # N: Revealed type is "builtins.int | builtins.str | builtins.dict[builtins.int, builtins.str]" [builtins fixtures/isinstancelist.pyi] From b4d52e1fdb8709f5cfa8bc9e61ea3be386564df7 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 7 Jul 2025 13:00:22 +0100 Subject: [PATCH 050/424] Remove all nested imports from default plugin (#19388) It looks like these nested imports are not needed anymore. --- mypy/plugins/default.py | 173 ++++++++++++++++++++-------------------- 1 file changed, 86 insertions(+), 87 deletions(-) diff --git a/mypy/plugins/default.py b/mypy/plugins/default.py index 09f9795b593ed..3d27ca99302ff 100644 --- a/mypy/plugins/default.py +++ b/mypy/plugins/default.py @@ -15,8 +15,51 @@ MethodSigContext, Plugin, ) -from mypy.plugins import constants +from mypy.plugins.attrs import ( + attr_class_maker_callback, + attr_class_makers, + attr_dataclass_makers, + attr_define_makers, + attr_frozen_makers, + attr_tag_callback, + evolve_function_sig_callback, + fields_function_sig_callback, +) from mypy.plugins.common import try_getting_str_literals +from mypy.plugins.constants import ( + ENUM_NAME_ACCESS, + ENUM_VALUE_ACCESS, + SINGLEDISPATCH_CALLABLE_CALL_METHOD, + SINGLEDISPATCH_REGISTER_CALLABLE_CALL_METHOD, + SINGLEDISPATCH_REGISTER_METHOD, +) +from mypy.plugins.ctypes import ( + array_constructor_callback, + array_getitem_callback, + array_iter_callback, + array_raw_callback, + array_setitem_callback, + array_value_callback, +) +from mypy.plugins.dataclasses import ( + dataclass_class_maker_callback, + dataclass_makers, + dataclass_tag_callback, + replace_function_sig_callback, +) +from mypy.plugins.enums import enum_member_callback, enum_name_callback, enum_value_callback +from mypy.plugins.functools import ( + functools_total_ordering_maker_callback, + functools_total_ordering_makers, + partial_call_callback, + partial_new_callback, +) +from mypy.plugins.singledispatch import ( + call_singledispatch_function_after_register_argument, + call_singledispatch_function_callback, + create_singledispatch_function_callback, + singledispatch_register_callback, +) from mypy.subtypes import is_subtype from mypy.typeops import is_literal_type_like, make_simplified_union from mypy.types import ( @@ -53,39 +96,24 @@ class DefaultPlugin(Plugin): def get_function_hook(self, fullname: str) -> Callable[[FunctionContext], Type] | None: if fullname == "_ctypes.Array": - from mypy.plugins import ctypes - - return ctypes.array_constructor_callback + return array_constructor_callback elif fullname == "functools.singledispatch": - from mypy.plugins import singledispatch - - return singledispatch.create_singledispatch_function_callback + return create_singledispatch_function_callback elif fullname == "functools.partial": - import mypy.plugins.functools - - return mypy.plugins.functools.partial_new_callback + return partial_new_callback elif fullname == "enum.member": - from mypy.plugins import enums - - return enums.enum_member_callback - + return enum_member_callback return None def get_function_signature_hook( self, fullname: str ) -> Callable[[FunctionSigContext], FunctionLike] | None: if fullname in ("attr.evolve", "attrs.evolve", "attr.assoc", "attrs.assoc"): - from mypy.plugins import attrs - - return attrs.evolve_function_sig_callback + return evolve_function_sig_callback elif fullname in ("attr.fields", "attrs.fields"): - from mypy.plugins import attrs - - return attrs.fields_function_sig_callback + return fields_function_sig_callback elif fullname == "dataclasses.replace": - from mypy.plugins import dataclasses - - return dataclasses.replace_function_sig_callback + return replace_function_sig_callback return None def get_method_signature_hook( @@ -98,13 +126,9 @@ def get_method_signature_hook( elif fullname in TD_POP_NAMES: return typed_dict_pop_signature_callback elif fullname == "_ctypes.Array.__setitem__": - from mypy.plugins import ctypes - - return ctypes.array_setitem_callback - elif fullname == constants.SINGLEDISPATCH_CALLABLE_CALL_METHOD: - from mypy.plugins import singledispatch - - return singledispatch.call_singledispatch_function_callback + return array_setitem_callback + elif fullname == SINGLEDISPATCH_CALLABLE_CALL_METHOD: + return call_singledispatch_function_callback elif fullname in TD_UPDATE_METHOD_NAMES: return typed_dict_update_signature_callback return None @@ -127,88 +151,63 @@ def get_method_hook(self, fullname: str) -> Callable[[MethodContext], Type] | No elif fullname in {n + ".__delitem__" for n in TPDICT_FB_NAMES}: return typed_dict_delitem_callback elif fullname == "_ctypes.Array.__getitem__": - from mypy.plugins import ctypes - - return ctypes.array_getitem_callback + return array_getitem_callback elif fullname == "_ctypes.Array.__iter__": - from mypy.plugins import ctypes - - return ctypes.array_iter_callback - elif fullname == constants.SINGLEDISPATCH_REGISTER_METHOD: - from mypy.plugins import singledispatch - - return singledispatch.singledispatch_register_callback - elif fullname == constants.SINGLEDISPATCH_REGISTER_CALLABLE_CALL_METHOD: - from mypy.plugins import singledispatch - - return singledispatch.call_singledispatch_function_after_register_argument + return array_iter_callback + elif fullname == SINGLEDISPATCH_REGISTER_METHOD: + return singledispatch_register_callback + elif fullname == SINGLEDISPATCH_REGISTER_CALLABLE_CALL_METHOD: + return call_singledispatch_function_after_register_argument elif fullname == "functools.partial.__call__": - import mypy.plugins.functools - - return mypy.plugins.functools.partial_call_callback + return partial_call_callback return None def get_attribute_hook(self, fullname: str) -> Callable[[AttributeContext], Type] | None: if fullname == "_ctypes.Array.value": - from mypy.plugins import ctypes - - return ctypes.array_value_callback + return array_value_callback elif fullname == "_ctypes.Array.raw": - from mypy.plugins import ctypes - - return ctypes.array_raw_callback - elif fullname in constants.ENUM_NAME_ACCESS: - from mypy.plugins import enums - - return enums.enum_name_callback - elif fullname in constants.ENUM_VALUE_ACCESS: - from mypy.plugins import enums - - return enums.enum_value_callback + return array_raw_callback + elif fullname in ENUM_NAME_ACCESS: + return enum_name_callback + elif fullname in ENUM_VALUE_ACCESS: + return enum_value_callback return None def get_class_decorator_hook(self, fullname: str) -> Callable[[ClassDefContext], None] | None: - from mypy.plugins import attrs, dataclasses - # These dataclass and attrs hooks run in the main semantic analysis pass # and only tag known dataclasses/attrs classes, so that the second # hooks (in get_class_decorator_hook_2) can detect dataclasses/attrs classes # in the MRO. - if fullname in dataclasses.dataclass_makers: - return dataclasses.dataclass_tag_callback + if fullname in dataclass_makers: + return dataclass_tag_callback if ( - fullname in attrs.attr_class_makers - or fullname in attrs.attr_dataclass_makers - or fullname in attrs.attr_frozen_makers - or fullname in attrs.attr_define_makers + fullname in attr_class_makers + or fullname in attr_dataclass_makers + or fullname in attr_frozen_makers + or fullname in attr_define_makers ): - return attrs.attr_tag_callback - + return attr_tag_callback return None def get_class_decorator_hook_2( self, fullname: str ) -> Callable[[ClassDefContext], bool] | None: - import mypy.plugins.functools - from mypy.plugins import attrs, dataclasses - - if fullname in dataclasses.dataclass_makers: - return dataclasses.dataclass_class_maker_callback - elif fullname in mypy.plugins.functools.functools_total_ordering_makers: - return mypy.plugins.functools.functools_total_ordering_maker_callback - elif fullname in attrs.attr_class_makers: - return attrs.attr_class_maker_callback - elif fullname in attrs.attr_dataclass_makers: - return partial(attrs.attr_class_maker_callback, auto_attribs_default=True) - elif fullname in attrs.attr_frozen_makers: + if fullname in dataclass_makers: + return dataclass_class_maker_callback + elif fullname in functools_total_ordering_makers: + return functools_total_ordering_maker_callback + elif fullname in attr_class_makers: + return attr_class_maker_callback + elif fullname in attr_dataclass_makers: + return partial(attr_class_maker_callback, auto_attribs_default=True) + elif fullname in attr_frozen_makers: return partial( - attrs.attr_class_maker_callback, auto_attribs_default=None, frozen_default=True + attr_class_maker_callback, auto_attribs_default=None, frozen_default=True ) - elif fullname in attrs.attr_define_makers: + elif fullname in attr_define_makers: return partial( - attrs.attr_class_maker_callback, auto_attribs_default=None, slots_default=True + attr_class_maker_callback, auto_attribs_default=None, slots_default=True ) - return None From 0b7afdaf281f23fd2982b79d736fa2ab054bc76a Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Mon, 7 Jul 2025 14:29:38 +0200 Subject: [PATCH 051/424] Check property decorators stricter (#19313) Fixes #19312. Fixes #18327. Only accept `@current_prop_name.{setter,deleter}` as property-related decorators. --- mypy/semanal.py | 31 ++++++++--- test-data/unit/check-classes.test | 80 +++++++++++++++++++++++++++-- test-data/unit/check-functions.test | 4 +- test-data/unit/semanal-errors.test | 2 +- 4 files changed, 103 insertions(+), 14 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 431c5ec04d3c3..435c1e682e359 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1527,33 +1527,33 @@ def analyze_property_with_multi_part_definition( assert isinstance(first_item, Decorator) deleted_items = [] bare_setter_type = None + func_name = first_item.func.name for i, item in enumerate(items[1:]): if isinstance(item, Decorator): item.func.accept(self) if item.decorators: first_node = item.decorators[0] - if isinstance(first_node, MemberExpr): + if self._is_valid_property_decorator(first_node, func_name): + # Get abstractness from the original definition. + item.func.abstract_status = first_item.func.abstract_status if first_node.name == "setter": # The first item represents the entire property. first_item.var.is_settable_property = True - # Get abstractness from the original definition. - item.func.abstract_status = first_item.func.abstract_status setter_func_type = function_type( item.func, self.named_type("builtins.function") ) assert isinstance(setter_func_type, CallableType) bare_setter_type = setter_func_type defn.setter_index = i + 1 - if first_node.name == "deleter": - item.func.abstract_status = first_item.func.abstract_status for other_node in item.decorators[1:]: other_node.accept(self) else: self.fail( - f"Only supported top decorator is @{first_item.func.name}.setter", item + f'Only supported top decorators are "@{func_name}.setter" and "@{func_name}.deleter"', + first_node, ) else: - self.fail(f'Unexpected definition for property "{first_item.func.name}"', item) + self.fail(f'Unexpected definition for property "{func_name}"', item) deleted_items.append(i + 1) for i in reversed(deleted_items): del items[i] @@ -1567,6 +1567,23 @@ def analyze_property_with_multi_part_definition( ) return bare_setter_type + def _is_valid_property_decorator( + self, deco: Expression, property_name: str + ) -> TypeGuard[MemberExpr]: + if not isinstance(deco, MemberExpr): + return False + if not isinstance(deco.expr, NameExpr) or deco.expr.name != property_name: + return False + if deco.name not in {"setter", "deleter"}: + # This intentionally excludes getter. While `@prop.getter` is valid at + # runtime, that would mean replacing the already processed getter type. + # Such usage is almost definitely a mistake (except for overrides in + # subclasses but we don't support them anyway) and might be a typo + # (only one letter away from `setter`), it's likely almost never used, + # so supporting it properly won't pay off. + return False + return True + def add_function_to_symbol_table(self, func: FuncDef | OverloadedFuncDef) -> None: if self.is_class_scope(): assert self.type is not None diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index 173657620304c..ae91815d1e9ef 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -772,7 +772,7 @@ class A: class B(A): @property def f(self) -> Callable[[object], None]: pass - @func.setter + @f.setter def f(self, x: object) -> None: pass [builtins fixtures/property.pyi] @@ -786,7 +786,7 @@ class A: class B(A): @property def f(self) -> Callable[[object], None]: pass - @func.setter + @f.setter def f(self, x: object) -> None: pass [builtins fixtures/property.pyi] @@ -1622,7 +1622,81 @@ class A: self.x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") return '' [builtins fixtures/property.pyi] + +[case testPropertyNameIsChecked] +class A: + @property + def f(self) -> str: ... + @not_f.setter # E: Only supported top decorators are "@f.setter" and "@f.deleter" + def f(self, val: str) -> None: ... + +a = A() +reveal_type(a.f) # N: Revealed type is "builtins.str" +a.f = '' # E: Property "f" defined in "A" is read-only + +class B: + @property + def f(self) -> str: ... + @not_f.deleter # E: Only supported top decorators are "@f.setter" and "@f.deleter" + def f(self) -> None: ... + +class C: + @property + def f(self) -> str: ... + @not_f.setter # E: Only supported top decorators are "@f.setter" and "@f.deleter" + def f(self, val: str) -> None: ... + @not_f.deleter # E: Only supported top decorators are "@f.setter" and "@f.deleter" + def f(self) -> None: ... +[builtins fixtures/property.pyi] + +[case testPropertyAttributeIsChecked] +class A: + @property + def f(self) -> str: ... + @f.unknown # E: Only supported top decorators are "@f.setter" and "@f.deleter" + def f(self, val: str) -> None: ... + @f.bad.setter # E: Only supported top decorators are "@f.setter" and "@f.deleter" + def f(self, val: str) -> None: ... + @f # E: Only supported top decorators are "@f.setter" and "@f.deleter" + def f(self, val: str) -> None: ... + @int # E: Only supported top decorators are "@f.setter" and "@f.deleter" + def f(self, val: str) -> None: ... +[builtins fixtures/property.pyi] + +[case testPropertyNameAndAttributeIsCheckedPretty] +# flags: --pretty +class A: + @property + def f(self) -> str: ... + @not_f.setter + def f(self, val: str) -> None: ... + @not_f.deleter + def f(self) -> None: ... + +class B: + @property + def f(self) -> str: ... + @f.unknown + def f(self, val: str) -> None: ... +[builtins fixtures/property.pyi] [out] +main:5: error: Only supported top decorators are "@f.setter" and "@f.deleter" + @not_f.setter + ^~~~~~~~~~~~ +main:7: error: Only supported top decorators are "@f.setter" and "@f.deleter" + @not_f.deleter + ^~~~~~~~~~~~~ +main:13: error: Only supported top decorators are "@f.setter" and "@f.deleter" + @f.unknown + ^~~~~~~~~ + +[case testPropertyGetterDecoratorIsRejected] +class A: + @property + def f(self) -> str: ... + @f.getter # E: Only supported top decorators are "@f.setter" and "@f.deleter" + def f(self, val: str) -> None: ... +[builtins fixtures/property.pyi] [case testDynamicallyTypedProperty] import typing @@ -7739,7 +7813,7 @@ class A: def y(self) -> int: ... @y.setter def y(self, value: int) -> None: ... - @dec # E: Only supported top decorator is @y.setter + @dec # E: Only supported top decorators are "@y.setter" and "@y.deleter" def y(self) -> None: ... reveal_type(A().y) # N: Revealed type is "builtins.int" diff --git a/test-data/unit/check-functions.test b/test-data/unit/check-functions.test index 07cfd09b25291..7fa34a398ea05 100644 --- a/test-data/unit/check-functions.test +++ b/test-data/unit/check-functions.test @@ -2752,7 +2752,7 @@ class B: @property @dec def f(self) -> int: pass - @dec # E: Only supported top decorator is @f.setter + @dec # E: Only supported top decorators are "@f.setter" and "@f.deleter" @f.setter def f(self, v: int) -> None: pass @@ -2764,7 +2764,6 @@ class C: @dec def f(self, v: int) -> None: pass [builtins fixtures/property.pyi] -[out] [case testInvalidArgCountForProperty] from typing import Callable, TypeVar @@ -2783,7 +2782,6 @@ class A: @property def h(self, *args, **kwargs) -> int: pass # OK [builtins fixtures/property.pyi] -[out] [case testSubtypingUnionGenericBounds] from typing import Callable, TypeVar, Union, Sequence diff --git a/test-data/unit/semanal-errors.test b/test-data/unit/semanal-errors.test index 1e760799828a8..2d381644629b3 100644 --- a/test-data/unit/semanal-errors.test +++ b/test-data/unit/semanal-errors.test @@ -1236,7 +1236,7 @@ class A: @overload # E: Decorators on top of @property are not supported @property def f(self) -> int: pass - @property # E: Only supported top decorator is @f.setter + @property # E: Only supported top decorators are "@f.setter" and "@f.deleter" @overload def f(self) -> int: pass [builtins fixtures/property.pyi] From fa5d94284600a8e6e48dcc37f1aa050e7a593617 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 7 Jul 2025 14:16:08 +0100 Subject: [PATCH 052/424] [mypyc] Fix error value check for GetAttr that allows nullable values (#19378) Also rename `allow_null` to `allow_error_value` to make it clear that this works with non-pointer types such as tuples. --- mypyc/codegen/emitfunc.py | 33 ++++++++++++++++++++------------- mypyc/ir/ops.py | 12 +++++++++--- mypyc/irbuild/builder.py | 8 ++------ mypyc/test/test_emitfunc.py | 12 ++++++++++++ 4 files changed, 43 insertions(+), 22 deletions(-) diff --git a/mypyc/codegen/emitfunc.py b/mypyc/codegen/emitfunc.py index 00c7fd56b8999..e81b119b24d6f 100644 --- a/mypyc/codegen/emitfunc.py +++ b/mypyc/codegen/emitfunc.py @@ -209,6 +209,16 @@ def visit_goto(self, op: Goto) -> None: if op.label is not self.next_block: self.emit_line("goto %s;" % self.label(op.label)) + def error_value_check(self, value: Value, compare: str) -> str: + typ = value.type + if isinstance(typ, RTuple): + # TODO: What about empty tuple? + return self.emitter.tuple_undefined_check_cond( + typ, self.reg(value), self.c_error_value, compare + ) + else: + return f"{self.reg(value)} {compare} {self.c_error_value(typ)}" + def visit_branch(self, op: Branch) -> None: true, false = op.true, op.false negated = op.negated @@ -225,15 +235,8 @@ def visit_branch(self, op: Branch) -> None: expr_result = self.reg(op.value) cond = f"{neg}{expr_result}" elif op.op == Branch.IS_ERROR: - typ = op.value.type compare = "!=" if negated else "==" - if isinstance(typ, RTuple): - # TODO: What about empty tuple? - cond = self.emitter.tuple_undefined_check_cond( - typ, self.reg(op.value), self.c_error_value, compare - ) - else: - cond = f"{self.reg(op.value)} {compare} {self.c_error_value(typ)}" + cond = self.error_value_check(op.value, compare) else: assert False, "Invalid branch" @@ -358,8 +361,8 @@ def get_attr_expr(self, obj: str, op: GetAttr | SetAttr, decl_cl: ClassIR) -> st return f"({cast}{obj})->{self.emitter.attr(op.attr)}" def visit_get_attr(self, op: GetAttr) -> None: - if op.allow_null: - self.get_attr_with_allow_null(op) + if op.allow_error_value: + self.get_attr_with_allow_error_value(op) return dest = self.reg(op) obj = self.reg(op.obj) @@ -429,8 +432,11 @@ def visit_get_attr(self, op: GetAttr) -> None: elif not always_defined: self.emitter.emit_line("}") - def get_attr_with_allow_null(self, op: GetAttr) -> None: - """Handle GetAttr with allow_null=True which allows NULL without raising AttributeError.""" + def get_attr_with_allow_error_value(self, op: GetAttr) -> None: + """Handle GetAttr with allow_error_value=True. + + This allows NULL or other error value without raising AttributeError. + """ dest = self.reg(op) obj = self.reg(op.obj) rtype = op.class_type @@ -443,7 +449,8 @@ def get_attr_with_allow_null(self, op: GetAttr) -> None: # Only emit inc_ref if not NULL if attr_rtype.is_refcounted and not op.is_borrowed: - self.emitter.emit_line(f"if ({dest} != NULL) {{") + check = self.error_value_check(op, "!=") + self.emitter.emit_line(f"if ({check}) {{") self.emitter.emit_inc_ref(dest, attr_rtype) self.emitter.emit_line("}") diff --git a/mypyc/ir/ops.py b/mypyc/ir/ops.py index 1cb3df916ac93..668e3097ce30e 100644 --- a/mypyc/ir/ops.py +++ b/mypyc/ir/ops.py @@ -811,17 +811,23 @@ class GetAttr(RegisterOp): error_kind = ERR_MAGIC def __init__( - self, obj: Value, attr: str, line: int, *, borrow: bool = False, allow_null: bool = False + self, + obj: Value, + attr: str, + line: int, + *, + borrow: bool = False, + allow_error_value: bool = False, ) -> None: super().__init__(line) self.obj = obj self.attr = attr - self.allow_null = allow_null + self.allow_error_value = allow_error_value assert isinstance(obj.type, RInstance), "Attribute access not supported: %s" % obj.type self.class_type = obj.type attr_type = obj.type.attr_type(attr) self.type = attr_type - if allow_null: + if allow_error_value: self.error_kind = ERR_NEVER elif attr_type.error_overlap: self.error_kind = ERR_MAGIC_OVERLAPPING diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index 878c5e76df3d4..323450f7c340b 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -709,13 +709,9 @@ def read( assert False, "Unsupported lvalue: %r" % target def read_nullable_attr(self, obj: Value, attr: str, line: int = -1) -> Value: - """Read an attribute that might be NULL without raising AttributeError. - - This is used for reading spill targets in try/finally blocks where NULL - indicates the non-return path was taken. - """ + """Read an attribute that might have an error value without raising AttributeError.""" assert isinstance(obj.type, RInstance) and obj.type.class_ir.is_ext_class - return self.add(GetAttr(obj, attr, line, allow_null=True)) + return self.add(GetAttr(obj, attr, line, allow_error_value=True)) def assign(self, target: Register | AssignmentTarget, rvalue_reg: Value, line: int) -> None: if isinstance(target, Register): diff --git a/mypyc/test/test_emitfunc.py b/mypyc/test/test_emitfunc.py index 275e8c383a4b5..0a696e9e33800 100644 --- a/mypyc/test/test_emitfunc.py +++ b/mypyc/test/test_emitfunc.py @@ -111,6 +111,7 @@ def add_local(name: str, rtype: RType) -> Register: "y": int_rprimitive, "i1": int64_rprimitive, "i2": int32_rprimitive, + "t": RTuple([object_rprimitive, object_rprimitive]), } ir.bitmap_attrs = ["i1", "i2"] compute_vtable(ir) @@ -418,6 +419,17 @@ def test_get_attr_with_bitmap(self) -> None: """, ) + def test_get_attr_nullable_with_tuple(self) -> None: + self.assert_emit( + GetAttr(self.r, "t", 1, allow_error_value=True), + """cpy_r_r0 = ((mod___AObject *)cpy_r_r)->_t; + if (cpy_r_r0.f0 != NULL) { + CPy_INCREF(cpy_r_r0.f0); + CPy_INCREF(cpy_r_r0.f1); + } + """, + ) + def test_set_attr(self) -> None: self.assert_emit( SetAttr(self.r, "y", self.m, 1), From b8dd6f3556f361ae2cd312b9edcc604eb6b1f465 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 7 Jul 2025 14:16:42 +0100 Subject: [PATCH 053/424] [mypyc] Add support for C string literals in the IR (#19383) Previously only Python str and bytes literals were supported, but sometimes we want zero-terminated C string literals instead. They don't need to be allocated from the heap and are usually stored in a read-only data section, so they are more efficient in some use cases. These will be useful for a feature I'm working on. --- mypyc/codegen/emitfunc.py | 30 ++++++++++++++++++++++++++++++ mypyc/ir/ops.py | 15 +++++++++++++++ mypyc/ir/pprint.py | 3 +++ mypyc/ir/rtypes.py | 10 ++++++---- mypyc/test/test_emitfunc.py | 26 ++++++++++++++++++++++++++ 5 files changed, 80 insertions(+), 4 deletions(-) diff --git a/mypyc/codegen/emitfunc.py b/mypyc/codegen/emitfunc.py index e81b119b24d6f..625ec0643df07 100644 --- a/mypyc/codegen/emitfunc.py +++ b/mypyc/codegen/emitfunc.py @@ -33,6 +33,7 @@ Cast, ComparisonOp, ControlOp, + CString, DecRef, Extend, Float, @@ -850,6 +851,8 @@ def reg(self, reg: Value) -> str: elif r == "nan": return "NAN" return r + elif isinstance(reg, CString): + return '"' + encode_c_string_literal(reg.value) + '"' else: return self.emitter.reg(reg) @@ -911,3 +914,30 @@ def emit_unsigned_int_cast(self, type: RType) -> str: return "(uint64_t)" else: return "" + + +_translation_table: Final[dict[int, str]] = {} + + +def encode_c_string_literal(b: bytes) -> str: + """Convert bytestring to the C string literal syntax (with necessary escaping). + + For example, b'foo\n' gets converted to 'foo\\n' (note that double quotes are not added). + """ + if not _translation_table: + # Initialize the translation table on the first call. + d = { + ord("\n"): "\\n", + ord("\r"): "\\r", + ord("\t"): "\\t", + ord('"'): '\\"', + ord("\\"): "\\\\", + } + for i in range(256): + if i not in d: + if i < 32 or i >= 127: + d[i] = "\\x%.2x" % i + else: + d[i] = chr(i) + _translation_table.update(str.maketrans(d)) + return b.decode("latin1").translate(_translation_table) diff --git a/mypyc/ir/ops.py b/mypyc/ir/ops.py index 668e3097ce30e..e15d494c2c579 100644 --- a/mypyc/ir/ops.py +++ b/mypyc/ir/ops.py @@ -39,6 +39,7 @@ class to enable the new behavior. Sometimes adding a new abstract RVoid, bit_rprimitive, bool_rprimitive, + cstring_rprimitive, float_rprimitive, int_rprimitive, is_bit_rprimitive, @@ -230,6 +231,20 @@ def __init__(self, value: float, line: int = -1) -> None: self.line = line +@final +class CString(Value): + """C string literal (zero-terminated). + + You can also include zero values in the value, but then you'll need to track + the length of the string separately. + """ + + def __init__(self, value: bytes, line: int = -1) -> None: + self.value = value + self.type = cstring_rprimitive + self.line = line + + class Op(Value): """Abstract base class for all IR operations. diff --git a/mypyc/ir/pprint.py b/mypyc/ir/pprint.py index 6c96a21e473bf..5bb11cc231ccc 100644 --- a/mypyc/ir/pprint.py +++ b/mypyc/ir/pprint.py @@ -21,6 +21,7 @@ Cast, ComparisonOp, ControlOp, + CString, DecRef, Extend, Float, @@ -327,6 +328,8 @@ def format(self, fmt: str, *args: Any) -> str: result.append(str(arg.value)) elif isinstance(arg, Float): result.append(repr(arg.value)) + elif isinstance(arg, CString): + result.append(f"CString({arg.value!r})") else: result.append(self.names[arg]) elif typespec == "d": diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 61aadce9b9d4e..8dc7d5c9c9497 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -254,13 +254,11 @@ def __init__( elif ctype == "CPyPtr": # TODO: Invent an overlapping error value? self.c_undefined = "0" - elif ctype == "PyObject *": - # Boxed types use the null pointer as the error value. + elif ctype.endswith("*"): + # Boxed and pointer types use the null pointer as the error value. self.c_undefined = "NULL" elif ctype == "char": self.c_undefined = "2" - elif ctype in ("PyObject **", "void *"): - self.c_undefined = "NULL" elif ctype == "double": self.c_undefined = "-113.0" elif ctype in ("uint8_t", "uint16_t", "uint32_t", "uint64_t"): @@ -445,6 +443,10 @@ def __hash__(self) -> int: "c_ptr", is_unboxed=False, is_refcounted=False, ctype="void *" ) +cstring_rprimitive: Final = RPrimitive( + "cstring", is_unboxed=True, is_refcounted=False, ctype="const char *" +) + # The type corresponding to mypyc.common.BITMAP_TYPE bitmap_rprimitive: Final = uint32_rprimitive diff --git a/mypyc/test/test_emitfunc.py b/mypyc/test/test_emitfunc.py index 0a696e9e33800..6be4875dafa12 100644 --- a/mypyc/test/test_emitfunc.py +++ b/mypyc/test/test_emitfunc.py @@ -19,6 +19,7 @@ CallC, Cast, ComparisonOp, + CString, DecRef, Extend, GetAttr, @@ -49,6 +50,7 @@ RType, bool_rprimitive, c_int_rprimitive, + cstring_rprimitive, dict_rprimitive, int32_rprimitive, int64_rprimitive, @@ -836,6 +838,30 @@ def test_inc_ref_int_literal(self) -> None: b = LoadLiteral(x, object_rprimitive) self.assert_emit([b, IncRef(b)], "CPy_INCREF(cpy_r_r0);") + def test_c_string(self) -> None: + s = Register(cstring_rprimitive, "s") + self.assert_emit(Assign(s, CString(b"foo")), """cpy_r_s = "foo";""") + self.assert_emit(Assign(s, CString(b'foo "o')), r"""cpy_r_s = "foo \"o";""") + self.assert_emit(Assign(s, CString(b"\x00")), r"""cpy_r_s = "\x00";""") + self.assert_emit(Assign(s, CString(b"\\")), r"""cpy_r_s = "\\";""") + for i in range(256): + b = bytes([i]) + if b == b"\n": + target = "\\n" + elif b == b"\r": + target = "\\r" + elif b == b"\t": + target = "\\t" + elif b == b'"': + target = '\\"' + elif b == b"\\": + target = "\\\\" + elif i < 32 or i >= 127: + target = "\\x%.2x" % i + else: + target = b.decode("ascii") + self.assert_emit(Assign(s, CString(b)), f'cpy_r_s = "{target}";') + def assert_emit( self, op: Op | list[Op], From 526fec3f8a4b2b575a7beb8b9d4a87720f57686e Mon Sep 17 00:00:00 2001 From: Brian Schubert Date: Mon, 7 Jul 2025 09:24:33 -0400 Subject: [PATCH 054/424] [mypyc] Generate introspection signatures for compiled functions (#19307) Refs https://github.com/mypyc/mypyc/issues/838 This PR populates `__text_signature__` for compiled functions, making runtime signature introspection possible (i.e. `inspect.signature(func)`). While `__text_signature__` is an undocumented CPython implementation detail, other extension module generators are using it in practice. For example, PyO3 and Cython both support it. I think it would be reasonable for mypyc to support it too. Some function signatures can't be represented by `__text_signature__` (namely, those with complex default arguments). In those cases, no signatures are generated (same as the current behavior). --- mypyc/codegen/emitclass.py | 29 +++- mypyc/codegen/emitfunc.py | 18 ++- mypyc/codegen/emitmodule.py | 13 +- mypyc/doc/differences_from_python.rst | 3 +- mypyc/ir/func_ir.py | 96 +++++++++++- mypyc/test-data/run-signatures.test | 207 ++++++++++++++++++++++++++ mypyc/test/test_run.py | 1 + 7 files changed, 358 insertions(+), 9 deletions(-) create mode 100644 mypyc/test-data/run-signatures.test diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 576787424cbf9..0c2d470104d03 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -5,8 +5,9 @@ from collections.abc import Mapping from typing import Callable +from mypyc.codegen.cstring import c_string_initializer from mypyc.codegen.emit import Emitter, HeaderDeclaration, ReturnHandler -from mypyc.codegen.emitfunc import native_function_header +from mypyc.codegen.emitfunc import native_function_doc_initializer, native_function_header from mypyc.codegen.emitwrapper import ( generate_bin_op_wrapper, generate_bool_wrapper, @@ -21,7 +22,13 @@ ) from mypyc.common import BITMAP_BITS, BITMAP_TYPE, NATIVE_PREFIX, PREFIX, REG_PREFIX from mypyc.ir.class_ir import ClassIR, VTableEntries -from mypyc.ir.func_ir import FUNC_CLASSMETHOD, FUNC_STATICMETHOD, FuncDecl, FuncIR +from mypyc.ir.func_ir import ( + FUNC_CLASSMETHOD, + FUNC_STATICMETHOD, + FuncDecl, + FuncIR, + get_text_signature, +) from mypyc.ir.rtypes import RTuple, RType, object_rprimitive from mypyc.namegen import NameGenerator from mypyc.sametype import is_same_type @@ -368,6 +375,8 @@ def emit_line() -> None: flags.append("Py_TPFLAGS_MANAGED_DICT") fields["tp_flags"] = " | ".join(flags) + fields["tp_doc"] = native_class_doc_initializer(cl) + emitter.emit_line(f"static PyTypeObject {emitter.type_struct_name(cl)}_template_ = {{") emitter.emit_line("PyVarObject_HEAD_INIT(NULL, 0)") for field, value in fields.items(): @@ -915,7 +924,8 @@ def generate_methods_table(cl: ClassIR, name: str, emitter: Emitter) -> None: elif fn.decl.kind == FUNC_CLASSMETHOD: flags.append("METH_CLASS") - emitter.emit_line(" {}, NULL}},".format(" | ".join(flags))) + doc = native_function_doc_initializer(fn) + emitter.emit_line(" {}, {}}},".format(" | ".join(flags), doc)) # Provide a default __getstate__ and __setstate__ if not cl.has_method("__setstate__") and not cl.has_method("__getstate__"): @@ -1173,3 +1183,16 @@ def has_managed_dict(cl: ClassIR, emitter: Emitter) -> bool: and cl.has_dict and cl.builtin_base != "PyBaseExceptionObject" ) + + +def native_class_doc_initializer(cl: ClassIR) -> str: + init_fn = cl.get_method("__init__") + if init_fn is not None: + text_sig = get_text_signature(init_fn, bound=True) + if text_sig is None: + return "NULL" + text_sig = text_sig.replace("__init__", cl.name, 1) + else: + text_sig = f"{cl.name}()" + docstring = f"{text_sig}\n--\n\n" + return c_string_initializer(docstring.encode("ascii", errors="backslashreplace")) diff --git a/mypyc/codegen/emitfunc.py b/mypyc/codegen/emitfunc.py index 625ec0643df07..4b618f3c67db3 100644 --- a/mypyc/codegen/emitfunc.py +++ b/mypyc/codegen/emitfunc.py @@ -5,6 +5,7 @@ from typing import Final from mypyc.analysis.blockfreq import frequently_executed_blocks +from mypyc.codegen.cstring import c_string_initializer from mypyc.codegen.emit import DEBUG_ERRORS, Emitter, TracebackAndGotoHandler, c_array_initializer from mypyc.common import ( HAVE_IMMORTAL, @@ -16,7 +17,14 @@ TYPE_VAR_PREFIX, ) from mypyc.ir.class_ir import ClassIR -from mypyc.ir.func_ir import FUNC_CLASSMETHOD, FUNC_STATICMETHOD, FuncDecl, FuncIR, all_values +from mypyc.ir.func_ir import ( + FUNC_CLASSMETHOD, + FUNC_STATICMETHOD, + FuncDecl, + FuncIR, + all_values, + get_text_signature, +) from mypyc.ir.ops import ( ERR_FALSE, NAMESPACE_MODULE, @@ -106,6 +114,14 @@ def native_function_header(fn: FuncDecl, emitter: Emitter) -> str: ) +def native_function_doc_initializer(func: FuncIR) -> str: + text_sig = get_text_signature(func) + if text_sig is None: + return "NULL" + docstring = f"{text_sig}\n--\n\n" + return c_string_initializer(docstring.encode("ascii", errors="backslashreplace")) + + def generate_native_function( fn: FuncIR, emitter: Emitter, source_path: str, module_name: str ) -> None: diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index e1b6a78572949..2a6f17cea5e2e 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -30,7 +30,11 @@ from mypyc.codegen.cstring import c_string_initializer from mypyc.codegen.emit import Emitter, EmitterContext, HeaderDeclaration, c_array_initializer from mypyc.codegen.emitclass import generate_class, generate_class_reuse, generate_class_type_decl -from mypyc.codegen.emitfunc import generate_native_function, native_function_header +from mypyc.codegen.emitfunc import ( + generate_native_function, + native_function_doc_initializer, + native_function_header, +) from mypyc.codegen.emitwrapper import ( generate_legacy_wrapper_function, generate_wrapper_function, @@ -917,11 +921,14 @@ def emit_module_methods( flag = "METH_FASTCALL" else: flag = "METH_VARARGS" + doc = native_function_doc_initializer(fn) emitter.emit_line( ( '{{"{name}", (PyCFunction){prefix}{cname}, {flag} | METH_KEYWORDS, ' - "NULL /* docstring */}}," - ).format(name=name, cname=fn.cname(emitter.names), prefix=PREFIX, flag=flag) + "{doc} /* docstring */}}," + ).format( + name=name, cname=fn.cname(emitter.names), prefix=PREFIX, flag=flag, doc=doc + ) ) emitter.emit_line("{NULL, NULL, 0, NULL}") emitter.emit_line("};") diff --git a/mypyc/doc/differences_from_python.rst b/mypyc/doc/differences_from_python.rst index 5a230bd984c22..b910e3b3c9290 100644 --- a/mypyc/doc/differences_from_python.rst +++ b/mypyc/doc/differences_from_python.rst @@ -316,7 +316,8 @@ non-exhaustive list of what won't work: - Instance ``__annotations__`` is usually not kept - Frames of compiled functions can't be inspected using ``inspect`` - Compiled methods aren't considered methods by ``inspect.ismethod`` -- ``inspect.signature`` chokes on compiled functions +- ``inspect.signature`` chokes on compiled functions with default arguments that + are not simple literals - ``inspect.iscoroutinefunction`` and ``asyncio.iscoroutinefunction`` will always return False for compiled functions, even those defined with `async def` Profiling hooks and tracing diff --git a/mypyc/ir/func_ir.py b/mypyc/ir/func_ir.py index beef8def7f43d..881ac5939c275 100644 --- a/mypyc/ir/func_ir.py +++ b/mypyc/ir/func_ir.py @@ -2,6 +2,7 @@ from __future__ import annotations +import inspect from collections.abc import Sequence from typing import Final @@ -11,13 +12,24 @@ Assign, AssignMulti, BasicBlock, + Box, ControlOp, DeserMaps, + Float, + Integer, LoadAddress, + LoadLiteral, Register, + TupleSet, Value, ) -from mypyc.ir.rtypes import RType, bitmap_rprimitive, deserialize_type +from mypyc.ir.rtypes import ( + RType, + bitmap_rprimitive, + deserialize_type, + is_bool_rprimitive, + is_none_rprimitive, +) from mypyc.namegen import NameGenerator @@ -379,3 +391,85 @@ def all_values_full(args: list[Register], blocks: list[BasicBlock]) -> list[Valu values.append(op) return values + + +_ARG_KIND_TO_INSPECT: Final = { + ArgKind.ARG_POS: inspect.Parameter.POSITIONAL_OR_KEYWORD, + ArgKind.ARG_OPT: inspect.Parameter.POSITIONAL_OR_KEYWORD, + ArgKind.ARG_STAR: inspect.Parameter.VAR_POSITIONAL, + ArgKind.ARG_NAMED: inspect.Parameter.KEYWORD_ONLY, + ArgKind.ARG_STAR2: inspect.Parameter.VAR_KEYWORD, + ArgKind.ARG_NAMED_OPT: inspect.Parameter.KEYWORD_ONLY, +} + +# Sentinel indicating a value that cannot be represented in a text signature. +_NOT_REPRESENTABLE = object() + + +def get_text_signature(fn: FuncIR, *, bound: bool = False) -> str | None: + """Return a text signature in CPython's internal doc format, or None + if the function's signature cannot be represented. + """ + parameters = [] + mark_self = (fn.class_name is not None) and (fn.decl.kind != FUNC_STATICMETHOD) and not bound + sig = fn.decl.bound_sig if bound and fn.decl.bound_sig is not None else fn.decl.sig + # Pre-scan for end of positional-only parameters. + # This is needed to handle signatures like 'def foo(self, __x)', where mypy + # currently sees 'self' as being positional-or-keyword and '__x' as positional-only. + pos_only_idx = -1 + for idx, arg in enumerate(sig.args): + if arg.pos_only and arg.kind in (ArgKind.ARG_POS, ArgKind.ARG_OPT): + pos_only_idx = idx + for idx, arg in enumerate(sig.args): + if arg.name.startswith(("__bitmap", "__mypyc")): + continue + kind = ( + inspect.Parameter.POSITIONAL_ONLY + if idx <= pos_only_idx + else _ARG_KIND_TO_INSPECT[arg.kind] + ) + default: object = inspect.Parameter.empty + if arg.optional: + default = _find_default_argument(arg.name, fn.blocks) + if default is _NOT_REPRESENTABLE: + # This default argument cannot be represented in a __text_signature__ + return None + + curr_param = inspect.Parameter(arg.name, kind, default=default) + parameters.append(curr_param) + if mark_self: + # Parameter.__init__/Parameter.replace do not accept $ + curr_param._name = f"${arg.name}" # type: ignore[attr-defined] + mark_self = False + return f"{fn.name}{inspect.Signature(parameters)}" + + +def _find_default_argument(name: str, blocks: list[BasicBlock]) -> object: + # Find assignment inserted by gen_arg_defaults. Assumed to be the first assignment. + for block in blocks: + for op in block.ops: + if isinstance(op, Assign) and op.dest.name == name: + return _extract_python_literal(op.src) + return _NOT_REPRESENTABLE + + +def _extract_python_literal(value: Value) -> object: + if isinstance(value, Integer): + if is_none_rprimitive(value.type): + return None + val = value.numeric_value() + if is_bool_rprimitive(value.type): + return bool(val) + return val + elif isinstance(value, Float): + return value.value + elif isinstance(value, LoadLiteral): + return value.value + elif isinstance(value, Box): + return _extract_python_literal(value.src) + elif isinstance(value, TupleSet): + items = tuple(_extract_python_literal(item) for item in value.items) + if any(itm is _NOT_REPRESENTABLE for itm in items): + return _NOT_REPRESENTABLE + return items + return _NOT_REPRESENTABLE diff --git a/mypyc/test-data/run-signatures.test b/mypyc/test-data/run-signatures.test new file mode 100644 index 0000000000000..a2de7076f5ef4 --- /dev/null +++ b/mypyc/test-data/run-signatures.test @@ -0,0 +1,207 @@ +[case testSignaturesBasic] +def f1(): pass +def f2(x): pass +def f3(x, /): pass +def f4(*, x): pass +def f5(*x): pass +def f6(**x): pass +def f7(x=None): pass +def f8(x=None, /): pass +def f9(*, x=None): pass +def f10(a, /, b, c=None, *args, d=None, **h): pass + +[file driver.py] +import inspect +from native import * + +assert str(inspect.signature(f1)) == "()" +assert str(inspect.signature(f2)) == "(x)" +assert str(inspect.signature(f3)) == "(x, /)" +assert str(inspect.signature(f4)) == "(*, x)" +assert str(inspect.signature(f5)) == "(*x)" +assert str(inspect.signature(f6)) == "(**x)" +assert str(inspect.signature(f7)) == "(x=None)" +assert str(inspect.signature(f8)) == "(x=None, /)" +assert str(inspect.signature(f9)) == "(*, x=None)" +assert str(inspect.signature(f10)) == "(a, /, b, c=None, *args, d=None, **h)" + +for fn in [f1, f2, f3, f4, f5, f6, f7, f8, f9, f10]: + assert getattr(fn, "__doc__") is None + +[case testSignaturesValidDefaults] +from typing import Final +A: Final = 1 + +def default_int(x=1): pass +def default_str(x="a"): pass +def default_float(x=1.0): pass +def default_true(x=True): pass +def default_false(x=False): pass +def default_none(x=None): pass +def default_tuple_empty(x=()): pass +def default_tuple_literals(x=(1, "a", 1.0, False, True, None, (), (1,2,(3,4)))): pass +def default_tuple_singleton(x=(1,)): pass +def default_named_constant(x=A): pass + +[file driver.py] +import inspect +from native import * + +assert str(inspect.signature(default_int)) == "(x=1)" +assert str(inspect.signature(default_str)) == "(x='a')" +assert str(inspect.signature(default_float)) == "(x=1.0)" +assert str(inspect.signature(default_true)) == "(x=True)" +assert str(inspect.signature(default_false)) == "(x=False)" +assert str(inspect.signature(default_none)) == "(x=None)" +assert str(inspect.signature(default_tuple_empty)) == "(x=())" +assert str(inspect.signature(default_tuple_literals)) == "(x=(1, 'a', 1.0, False, True, None, (), (1, 2, (3, 4))))" +assert str(inspect.signature(default_named_constant)) == "(x=1)" + +# Check __text_signature__ directly since inspect.signature produces +# an incorrect signature for 1-tuple default arguments prior to +# Python 3.12 (cpython#102379). +# assert str(inspect.signature(default_tuple_singleton)) == "(x=(1,))" +assert getattr(default_tuple_singleton, "__text_signature__") == "(x=(1,))" + +[case testSignaturesStringDefaults] +def f1(x="'foo"): pass +def f2(x='"foo'): pass +def f3(x=""""Isn\'t," they said."""): pass +def f4(x="\\ \a \b \f \n \r \t \v \x00"): pass +def f5(x="\N{BANANA}sv"): pass + +[file driver.py] +import inspect +from native import * + +assert str(inspect.signature(f1)) == """(x="'foo")""" +assert str(inspect.signature(f2)) == """(x='"foo')""" +assert str(inspect.signature(f3)) == r"""(x='"Isn\'t," they said.')""" +assert str(inspect.signature(f4)) == r"""(x='\\ \x07 \x08 \x0c \n \r \t \x0b \x00')""" +assert str(inspect.signature(f5)) == """(x='\N{BANANA}sv')""" + +[case testSignaturesIrrepresentableDefaults] +import enum +class Color(enum.Enum): + RED = 1 +misc = object() + +# Default arguments that cannot be represented in a __text_signature__ +def bad_object(x=misc): pass +def bad_list_nonliteral(x=[misc]): pass +def bad_dict_nonliteral(x={'a': misc}): pass +def bad_set_nonliteral(x={misc}): pass +def bad_set_empty(x=set()): pass # supported by ast.literal_eval, but not by inspect._signature_fromstr +def bad_nan(x=float("nan")): pass +def bad_enum(x=Color.RED): pass + +# TODO: Default arguments that could potentially be represented in a +# __text_signature__, but which are not currently supported. +# See 'inspect._signature_fromstr' for what default values are supported at runtime. +def bad_complex(x=1+2j): pass +def bad_list_empty(x=[]): pass +def bad_list_literals(x=[1, 2, 3]): pass +def bad_dict_empty(x={}): pass +def bad_dict_literals(x={'a': 1}): pass +def bad_set_literals(x={1, 2, 3}): pass +def bad_tuple_literals(x=([1, 2, 3], {'a': 1}, {1, 2, 3})): pass +def bad_ellipsis(x=...): pass +def bad_literal_fold(x=1+2): pass + +[file driver.py] +import inspect +from testutil import assertRaises +import native + +all_bad = [fn for name, fn in vars(native).items() if name.startswith("bad_")] +assert all_bad + +for bad in all_bad: + assert bad.__text_signature__ is None, f"{bad.__name__} has unexpected __text_signature__" + with assertRaises(ValueError, "no signature found for builtin"): + inspect.signature(bad) + +[case testSignaturesMethods] +class Foo: + def f1(self, x): pass + @classmethod + def f2(cls, x): pass + @staticmethod + def f3(x): pass + def __eq__(self, x: object): pass + +[file driver.py] +import inspect +from native import * + +assert str(inspect.signature(Foo.f1)) == "(self, /, x)" +assert str(inspect.signature(Foo().f1)) == "(x)" + +assert str(inspect.signature(Foo.f2)) == "(x)" +assert str(inspect.signature(Foo().f2)) == "(x)" + +assert str(inspect.signature(Foo.f3)) == "(x)" +assert str(inspect.signature(Foo().f3)) == "(x)" + +assert str(inspect.signature(Foo.__eq__)) == "(self, value, /)" +assert str(inspect.signature(Foo().__eq__)) == "(value, /)" + +[case testSignaturesConstructors] +class Empty: pass + +class HasInit: + def __init__(self, x) -> None: pass + +class InheritedInit(HasInit): pass + +class HasInitBad: + def __init__(self, x=[]) -> None: pass + +[file driver.py] +import inspect +from testutil import assertRaises +from native import * + +assert str(inspect.signature(Empty)) == "()" +assert str(inspect.signature(Empty.__init__)) == "(self, /, *args, **kwargs)" + +assert str(inspect.signature(HasInit)) == "(x)" +assert str(inspect.signature(HasInit.__init__)) == "(self, /, *args, **kwargs)" + +assert str(inspect.signature(InheritedInit)) == "(x)" +assert str(inspect.signature(InheritedInit.__init__)) == "(self, /, *args, **kwargs)" + +assert getattr(HasInitBad, "__text_signature__") is None +with assertRaises(ValueError, "no signature found for builtin"): + inspect.signature(HasInitBad) + +# CPython detail note: type objects whose tp_doc contains only a text signature behave +# differently from method objects whose ml_doc contains only a test signature: type +# objects will have __doc__="" whereas method objects will have __doc__=None. This +# difference stems from the former using _PyType_GetDocFromInternalDoc(...) and the +# latter using PyUnicode_FromString(_PyType_DocWithoutSignature(...)). +for cls in [Empty, HasInit, InheritedInit]: + assert getattr(cls, "__doc__") == "" +assert getattr(HasInitBad, "__doc__") is None + +[case testSignaturesHistoricalPositionalOnly] +import inspect + +def f1(__x): pass +def f2(__x, y): pass +def f3(*, __y): pass +def f4(x, *, __y): pass +def f5(__x, *, __y): pass + +class A: + def func(self, __x): pass + +def test_historical_positional_only() -> None: + assert str(inspect.signature(f1)) == "(__x, /)" + assert str(inspect.signature(f2)) == "(__x, /, y)" + assert str(inspect.signature(f3)) == "(*, __y)" + assert str(inspect.signature(f4)) == "(x, *, __y)" + assert str(inspect.signature(f5)) == "(__x, /, *, __y)" + + assert str(inspect.signature(A.func)) == "(self, __x, /)" + assert str(inspect.signature(A().func)) == "(__x, /)" diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py index b96c4241f30d8..fcc24403df8ec 100644 --- a/mypyc/test/test_run.py +++ b/mypyc/test/test_run.py @@ -69,6 +69,7 @@ "run-dunders-special.test", "run-singledispatch.test", "run-attrs.test", + "run-signatures.test", "run-python37.test", "run-python38.test", ] From ada0d2a3bb227a23ef8955f2590c581f97cbcfaf Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 7 Jul 2025 15:43:02 +0100 Subject: [PATCH 055/424] [mypyc] Call generator helper method directly in await expression (#19376) Previously calls like `await foo()` were compiled to code that included code like this (in Python-like pseudocode): ``` a = foo() ... b = get_coro(a) ... c = next(b) ``` In the above code, `get_coro(a)` just returns `a` if `foo` is a native async function, so we now optimize this call away. Also `next(b)` calls `b.__next__()`, which calls the generated generator helper method `__mypyc_generator_helper__`. Now we call the helper method directly, which saves some unnecessary calls. More importantly, in a follow-up PR I can easily change the way `__mypyc_generator_helper__` is called, since we now call it directly. This makes it possible to avoid raising a `StopIteration` exception in many await expressions. The goal of this PR is to prepare for the latter optimization. This PR doesn't help performance significantly by itself. In order to call the helper method directly, I had to generate the declaration of this method and the generated generator class before the main irbuild pass, since otherwise a call site could be processed before we have processed the called generator. I also improved test coverage of related functionality. We don't have an IR test for async calls, since the IR is very verbose. I manually inspected the generated IR to verify that the new code path works both when calling a top-level function and when calling a method. I'll later add a mypyc benchmark to ensure that we will notice if the performance of async calls is degraded. --- mypyc/irbuild/function.py | 2 +- mypyc/irbuild/generator.py | 64 ++++++-------------- mypyc/irbuild/main.py | 17 +++++- mypyc/irbuild/mapper.py | 11 +++- mypyc/irbuild/prepare.py | 47 ++++++++++++++- mypyc/irbuild/statement.py | 39 ++++++++++-- mypyc/test-data/run-async.test | 93 ++++++++++++++++++++++++++++- mypyc/test-data/run-generators.test | 48 ++++++++++++--- 8 files changed, 251 insertions(+), 70 deletions(-) diff --git a/mypyc/irbuild/function.py b/mypyc/irbuild/function.py index dbebc350bb6c1..dcc5a306bcde5 100644 --- a/mypyc/irbuild/function.py +++ b/mypyc/irbuild/function.py @@ -261,7 +261,7 @@ def c() -> None: ) # Re-enter the FuncItem and visit the body of the function this time. - gen_generator_func_body(builder, fn_info, sig, func_reg) + gen_generator_func_body(builder, fn_info, func_reg) else: func_ir, func_reg = gen_func_body(builder, sig, cdef, is_singledispatch) diff --git a/mypyc/irbuild/generator.py b/mypyc/irbuild/generator.py index 0e4b0e3e184a8..eec27e1cfb84d 100644 --- a/mypyc/irbuild/generator.py +++ b/mypyc/irbuild/generator.py @@ -13,9 +13,9 @@ from typing import Callable from mypy.nodes import ARG_OPT, FuncDef, Var -from mypyc.common import ENV_ATTR_NAME, NEXT_LABEL_ATTR_NAME, SELF_NAME +from mypyc.common import ENV_ATTR_NAME, NEXT_LABEL_ATTR_NAME from mypyc.ir.class_ir import ClassIR -from mypyc.ir.func_ir import FuncDecl, FuncIR, FuncSignature, RuntimeArg +from mypyc.ir.func_ir import FuncDecl, FuncIR from mypyc.ir.ops import ( NO_TRACEBACK_LINE_NO, BasicBlock, @@ -78,9 +78,7 @@ def gen_generator_func( return func_ir, func_reg -def gen_generator_func_body( - builder: IRBuilder, fn_info: FuncInfo, sig: FuncSignature, func_reg: Value | None -) -> None: +def gen_generator_func_body(builder: IRBuilder, fn_info: FuncInfo, func_reg: Value | None) -> None: """Generate IR based on the body of a generator function. Add "__next__", "__iter__" and other generator methods to the generator @@ -88,7 +86,7 @@ class that implements the function (each function gets a separate class). Return the symbol table for the body. """ - builder.enter(fn_info, ret_type=sig.ret_type) + builder.enter(fn_info, ret_type=object_rprimitive) setup_env_for_generator_class(builder) load_outer_envs(builder, builder.fn_info.generator_class) @@ -117,7 +115,7 @@ class that implements the function (each function gets a separate class). args, _, blocks, ret_type, fn_info = builder.leave() - add_methods_to_generator_class(builder, fn_info, sig, args, blocks, fitem.is_coroutine) + add_methods_to_generator_class(builder, fn_info, args, blocks, fitem.is_coroutine) # Evaluate argument defaults in the surrounding scope, since we # calculate them *once* when the function definition is evaluated. @@ -153,10 +151,9 @@ def instantiate_generator_class(builder: IRBuilder) -> Value: def setup_generator_class(builder: IRBuilder) -> ClassIR: - name = f"{builder.fn_info.namespaced_name()}_gen" - - generator_class_ir = ClassIR(name, builder.module_name, is_generated=True, is_final_class=True) - generator_class_ir.reuse_freed_instance = True + mapper = builder.mapper + assert isinstance(builder.fn_info.fitem, FuncDef) + generator_class_ir = mapper.fdef_to_generator[builder.fn_info.fitem] if builder.fn_info.can_merge_generator_and_env_classes(): builder.fn_info.env_class = generator_class_ir else: @@ -216,46 +213,25 @@ def add_raise_exception_blocks_to_generator_class(builder: IRBuilder, line: int) def add_methods_to_generator_class( builder: IRBuilder, fn_info: FuncInfo, - sig: FuncSignature, arg_regs: list[Register], blocks: list[BasicBlock], is_coroutine: bool, ) -> None: - helper_fn_decl = add_helper_to_generator_class(builder, arg_regs, blocks, sig, fn_info) - add_next_to_generator_class(builder, fn_info, helper_fn_decl, sig) - add_send_to_generator_class(builder, fn_info, helper_fn_decl, sig) + helper_fn_decl = add_helper_to_generator_class(builder, arg_regs, blocks, fn_info) + add_next_to_generator_class(builder, fn_info, helper_fn_decl) + add_send_to_generator_class(builder, fn_info, helper_fn_decl) add_iter_to_generator_class(builder, fn_info) - add_throw_to_generator_class(builder, fn_info, helper_fn_decl, sig) + add_throw_to_generator_class(builder, fn_info, helper_fn_decl) add_close_to_generator_class(builder, fn_info) if is_coroutine: add_await_to_generator_class(builder, fn_info) def add_helper_to_generator_class( - builder: IRBuilder, - arg_regs: list[Register], - blocks: list[BasicBlock], - sig: FuncSignature, - fn_info: FuncInfo, + builder: IRBuilder, arg_regs: list[Register], blocks: list[BasicBlock], fn_info: FuncInfo ) -> FuncDecl: """Generates a helper method for a generator class, called by '__next__' and 'throw'.""" - sig = FuncSignature( - ( - RuntimeArg(SELF_NAME, object_rprimitive), - RuntimeArg("type", object_rprimitive), - RuntimeArg("value", object_rprimitive), - RuntimeArg("traceback", object_rprimitive), - RuntimeArg("arg", object_rprimitive), - ), - sig.ret_type, - ) - helper_fn_decl = FuncDecl( - "__mypyc_generator_helper__", - fn_info.generator_class.ir.name, - builder.module_name, - sig, - internal=True, - ) + helper_fn_decl = fn_info.generator_class.ir.method_decls["__mypyc_generator_helper__"] helper_fn_ir = FuncIR( helper_fn_decl, arg_regs, blocks, fn_info.fitem.line, traceback_name=fn_info.fitem.name ) @@ -272,9 +248,7 @@ def add_iter_to_generator_class(builder: IRBuilder, fn_info: FuncInfo) -> None: builder.add(Return(builder.self())) -def add_next_to_generator_class( - builder: IRBuilder, fn_info: FuncInfo, fn_decl: FuncDecl, sig: FuncSignature -) -> None: +def add_next_to_generator_class(builder: IRBuilder, fn_info: FuncInfo, fn_decl: FuncDecl) -> None: """Generates the '__next__' method for a generator class.""" with builder.enter_method(fn_info.generator_class.ir, "__next__", object_rprimitive, fn_info): none_reg = builder.none_object() @@ -289,9 +263,7 @@ def add_next_to_generator_class( builder.add(Return(result)) -def add_send_to_generator_class( - builder: IRBuilder, fn_info: FuncInfo, fn_decl: FuncDecl, sig: FuncSignature -) -> None: +def add_send_to_generator_class(builder: IRBuilder, fn_info: FuncInfo, fn_decl: FuncDecl) -> None: """Generates the 'send' method for a generator class.""" with builder.enter_method(fn_info.generator_class.ir, "send", object_rprimitive, fn_info): arg = builder.add_argument("arg", object_rprimitive) @@ -307,9 +279,7 @@ def add_send_to_generator_class( builder.add(Return(result)) -def add_throw_to_generator_class( - builder: IRBuilder, fn_info: FuncInfo, fn_decl: FuncDecl, sig: FuncSignature -) -> None: +def add_throw_to_generator_class(builder: IRBuilder, fn_info: FuncInfo, fn_decl: FuncDecl) -> None: """Generates the 'throw' method for a generator class.""" with builder.enter_method(fn_info.generator_class.ir, "throw", object_rprimitive, fn_info): typ = builder.add_argument("type", object_rprimitive) diff --git a/mypyc/irbuild/main.py b/mypyc/irbuild/main.py index 7cdc6b6867785..894e8f277723d 100644 --- a/mypyc/irbuild/main.py +++ b/mypyc/irbuild/main.py @@ -25,7 +25,7 @@ def f(x: int) -> int: from typing import Any, Callable, TypeVar, cast from mypy.build import Graph -from mypy.nodes import ClassDef, Expression, MypyFile +from mypy.nodes import ClassDef, Expression, FuncDef, MypyFile from mypy.state import state from mypy.types import Type from mypyc.analysis.attrdefined import analyze_always_defined_attrs @@ -37,7 +37,11 @@ def f(x: int) -> int: from mypyc.irbuild.builder import IRBuilder from mypyc.irbuild.mapper import Mapper from mypyc.irbuild.prebuildvisitor import PreBuildVisitor -from mypyc.irbuild.prepare import build_type_map, find_singledispatch_register_impls +from mypyc.irbuild.prepare import ( + build_type_map, + create_generator_class_if_needed, + find_singledispatch_register_impls, +) from mypyc.irbuild.visitor import IRBuilderVisitor from mypyc.irbuild.vtable import compute_vtable from mypyc.options import CompilerOptions @@ -76,6 +80,15 @@ def build_ir( pbv = PreBuildVisitor(errors, module, singledispatch_info.decorators_to_remove, types) module.accept(pbv) + # Declare generator classes for nested async functions and generators. + for fdef in pbv.nested_funcs: + if isinstance(fdef, FuncDef): + # Make generator class name sufficiently unique. + suffix = f"___{fdef.line}" + create_generator_class_if_needed( + module.fullname, None, fdef, mapper, name_suffix=suffix + ) + # Construct and configure builder objects (cyclic runtime dependency). visitor = IRBuilderVisitor() builder = IRBuilder( diff --git a/mypyc/irbuild/mapper.py b/mypyc/irbuild/mapper.py index 7c6e03d0037cd..4a01255e2d5d5 100644 --- a/mypyc/irbuild/mapper.py +++ b/mypyc/irbuild/mapper.py @@ -64,6 +64,8 @@ def __init__(self, group_map: dict[str, str | None]) -> None: self.type_to_ir: dict[TypeInfo, ClassIR] = {} self.func_to_decl: dict[SymbolNode, FuncDecl] = {} self.symbol_fullnames: set[str] = set() + # The corresponding generator class that implements a generator/async function + self.fdef_to_generator: dict[FuncDef, ClassIR] = {} def type_to_rtype(self, typ: Type | None) -> RType: if typ is None: @@ -171,7 +173,14 @@ def fdef_to_sig(self, fdef: FuncDef, strict_dunders_typing: bool) -> FuncSignatu for typ, kind in zip(fdef.type.arg_types, fdef.type.arg_kinds) ] arg_pos_onlys = [name is None for name in fdef.type.arg_names] - ret = self.type_to_rtype(fdef.type.ret_type) + # TODO: We could probably support decorators sometimes (static and class method?) + if (fdef.is_coroutine or fdef.is_generator) and not fdef.is_decorated: + # Give a more precise type for generators, so that we can optimize + # code that uses them. They return a generator object, which has a + # specific class. Without this, the type would have to be 'object'. + ret: RType = RInstance(self.fdef_to_generator[fdef]) + else: + ret = self.type_to_rtype(fdef.type.ret_type) else: # Handle unannotated functions arg_types = [object_rprimitive for _ in fdef.arguments] diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index 65951999dcf99..147392585b25e 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -38,7 +38,7 @@ from mypy.semanal import refers_to_fullname from mypy.traverser import TraverserVisitor from mypy.types import Instance, Type, get_proper_type -from mypyc.common import PROPSET_PREFIX, get_id_from_name +from mypyc.common import PROPSET_PREFIX, SELF_NAME, get_id_from_name from mypyc.crash import catch_errors from mypyc.errors import Errors from mypyc.ir.class_ir import ClassIR @@ -51,7 +51,14 @@ RuntimeArg, ) from mypyc.ir.ops import DeserMaps -from mypyc.ir.rtypes import RInstance, RType, dict_rprimitive, none_rprimitive, tuple_rprimitive +from mypyc.ir.rtypes import ( + RInstance, + RType, + dict_rprimitive, + none_rprimitive, + object_rprimitive, + tuple_rprimitive, +) from mypyc.irbuild.mapper import Mapper from mypyc.irbuild.util import ( get_func_def, @@ -115,7 +122,7 @@ def build_type_map( # Collect all the functions also. We collect from the symbol table # so that we can easily pick out the right copy of a function that - # is conditionally defined. + # is conditionally defined. This doesn't include nested functions! for module in modules: for func in get_module_func_defs(module): prepare_func_def(module.fullname, None, func, mapper, options) @@ -179,6 +186,8 @@ def prepare_func_def( mapper: Mapper, options: CompilerOptions, ) -> FuncDecl: + create_generator_class_if_needed(module_name, class_name, fdef, mapper) + kind = ( FUNC_STATICMETHOD if fdef.is_static @@ -190,6 +199,38 @@ def prepare_func_def( return decl +def create_generator_class_if_needed( + module_name: str, class_name: str | None, fdef: FuncDef, mapper: Mapper, name_suffix: str = "" +) -> None: + """If function is a generator/async function, declare a generator class. + + Each generator and async function gets a dedicated class that implements the + generator protocol with generated methods. + """ + if fdef.is_coroutine or fdef.is_generator: + name = "_".join(x for x in [fdef.name, class_name] if x) + "_gen" + name_suffix + cir = ClassIR(name, module_name, is_generated=True, is_final_class=True) + cir.reuse_freed_instance = True + mapper.fdef_to_generator[fdef] = cir + + helper_sig = FuncSignature( + ( + RuntimeArg(SELF_NAME, object_rprimitive), + RuntimeArg("type", object_rprimitive), + RuntimeArg("value", object_rprimitive), + RuntimeArg("traceback", object_rprimitive), + RuntimeArg("arg", object_rprimitive), + ), + object_rprimitive, + ) + + # The implementation of most generator functionality is behind this magic method. + helper_fn_decl = FuncDecl( + "__mypyc_generator_helper__", name, module_name, helper_sig, internal=True + ) + cir.method_decls[helper_fn_decl.name] = helper_fn_decl + + def prepare_method_def( ir: ClassIR, module_name: str, diff --git a/mypyc/irbuild/statement.py b/mypyc/irbuild/statement.py index 5c32d8f1a50c4..9c7ffb6a3adf9 100644 --- a/mypyc/irbuild/statement.py +++ b/mypyc/irbuild/statement.py @@ -48,11 +48,13 @@ ) from mypyc.common import TEMP_ATTR_NAME from mypyc.ir.ops import ( + ERR_NEVER, NAMESPACE_MODULE, NO_TRACEBACK_LINE_NO, Assign, BasicBlock, Branch, + Call, InitStatic, Integer, LoadAddress, @@ -930,16 +932,41 @@ def emit_yield_from_or_await( to_yield_reg = Register(object_rprimitive) received_reg = Register(object_rprimitive) - get_op = coro_op if is_await else iter_op - if isinstance(get_op, PrimitiveDescription): - iter_val = builder.primitive_op(get_op, [val], line) + helper_method = "__mypyc_generator_helper__" + if ( + isinstance(val, (Call, MethodCall)) + and isinstance(val.type, RInstance) + and val.type.class_ir.has_method(helper_method) + ): + # This is a generated native generator class, and we can use a fast path. + # This allows two optimizations: + # 1) No need to call CPy_GetCoro() or iter() since for native generators + # it just returns the generator object (implemented here). + # 2) Instead of calling next(), call generator helper method directly, + # since next() just calls __next__ which calls the helper method. + iter_val: Value = val else: - iter_val = builder.call_c(get_op, [val], line) + get_op = coro_op if is_await else iter_op + if isinstance(get_op, PrimitiveDescription): + iter_val = builder.primitive_op(get_op, [val], line) + else: + iter_val = builder.call_c(get_op, [val], line) iter_reg = builder.maybe_spill_assignable(iter_val) stop_block, main_block, done_block = BasicBlock(), BasicBlock(), BasicBlock() - _y_init = builder.call_c(next_raw_op, [builder.read(iter_reg)], line) + + if isinstance(iter_reg.type, RInstance) and iter_reg.type.class_ir.has_method(helper_method): + # Second fast path optimization: call helper directly (see also comment above). + obj = builder.read(iter_reg) + nn = builder.none_object() + m = MethodCall(obj, helper_method, [nn, nn, nn, nn], line) + # Generators have custom error handling, so disable normal error handling. + m.error_kind = ERR_NEVER + _y_init = builder.add(m) + else: + _y_init = builder.call_c(next_raw_op, [builder.read(iter_reg)], line) + builder.add(Branch(_y_init, stop_block, main_block, Branch.IS_ERROR)) # Try extracting a return value from a StopIteration and return it. @@ -948,7 +975,7 @@ def emit_yield_from_or_await( builder.assign(result, builder.call_c(check_stop_op, [], line), line) # Clear the spilled iterator/coroutine so that it will be freed. # Otherwise, the freeing of the spilled register would likely be delayed. - err = builder.add(LoadErrorValue(object_rprimitive)) + err = builder.add(LoadErrorValue(iter_reg.type)) builder.assign(iter_reg, err, line) builder.goto(done_block) diff --git a/mypyc/test-data/run-async.test b/mypyc/test-data/run-async.test index 2dad720f99cd9..b8c4c22daf719 100644 --- a/mypyc/test-data/run-async.test +++ b/mypyc/test-data/run-async.test @@ -2,6 +2,7 @@ [case testRunAsyncBasics] import asyncio +from typing import Callable, Awaitable from testutil import assertRaises @@ -72,6 +73,63 @@ def test_exception() -> None: assert asyncio.run(exc5()) == 3 assert asyncio.run(exc6()) == 3 +async def indirect_call(x: int, c: Callable[[int], Awaitable[int]]) -> int: + return await c(x) + +async def indirect_call_2(a: Awaitable[None]) -> None: + await a + +async def indirect_call_3(a: Awaitable[float]) -> float: + return (await a) + 1.0 + +async def inc(x: int) -> int: + await asyncio.sleep(0) + return x + 1 + +async def ident(x: float, err: bool = False) -> float: + await asyncio.sleep(0.0) + if err: + raise MyError() + return x + float("0.0") + +def test_indirect_call() -> None: + assert asyncio.run(indirect_call(3, inc)) == 4 + + with assertRaises(MyError): + asyncio.run(indirect_call_2(exc1())) + + assert asyncio.run(indirect_call_3(ident(2.0))) == 3.0 + assert asyncio.run(indirect_call_3(ident(-113.0))) == -112.0 + assert asyncio.run(indirect_call_3(ident(-114.0))) == -113.0 + + with assertRaises(MyError): + asyncio.run(indirect_call_3(ident(1.0, True))) + with assertRaises(MyError): + asyncio.run(indirect_call_3(ident(-113.0, True))) + +class C: + def __init__(self, n: int) -> None: + self.n = n + + async def add(self, x: int, err: bool = False) -> int: + await asyncio.sleep(0) + if err: + raise MyError() + return x + self.n + +async def method_call(x: int) -> int: + c = C(5) + return await c.add(x) + +async def method_call_exception() -> int: + c = C(5) + return await c.add(3, err=True) + +def test_async_method_call() -> None: + assert asyncio.run(method_call(3)) == 8 + with assertRaises(MyError): + asyncio.run(method_call_exception()) + [file asyncio/__init__.pyi] async def sleep(t: float) -> None: ... # eh, we could use the real type but it doesn't seem important @@ -563,8 +621,10 @@ def test_bool() -> None: def run(x: object) -> object: ... [case testRunAsyncNestedFunctions] +from __future__ import annotations + import asyncio -from typing import cast, Iterator +from typing import cast, Iterator, overload, Awaitable, Any, TypeVar from testutil import assertRaises @@ -641,6 +701,37 @@ def test_async_def_contains_two_nested_functions() -> None: (5 + 3 + 1), (7 + 4 + 10 + 1) ) +async def async_def_contains_overloaded_async_def(n: int) -> int: + @overload + async def f(x: int) -> int: ... + + @overload + async def f(x: str) -> str: ... + + async def f(x: int | str) -> Any: + return x + + return (await f(n)) + 1 + + +def test_async_def_contains_overloaded_async_def() -> None: + assert asyncio.run(async_def_contains_overloaded_async_def(5)) == 6 + +T = TypeVar("T") + +def deco(f: T) -> T: + return f + +async def async_def_contains_decorated_async_def(n: int) -> int: + @deco + async def f(x: int) -> int: + return x + 2 + + return (await f(n)) + 1 + + +def test_async_def_contains_decorated_async_def() -> None: + assert asyncio.run(async_def_contains_decorated_async_def(7)) == 10 [file asyncio/__init__.pyi] def run(x: object) -> object: ... diff --git a/mypyc/test-data/run-generators.test b/mypyc/test-data/run-generators.test index 9c0b51d58e79f..a43aff27dd45e 100644 --- a/mypyc/test-data/run-generators.test +++ b/mypyc/test-data/run-generators.test @@ -190,7 +190,9 @@ exit! a exception! ((1,), 'exception!') [case testYieldNested] -from typing import Callable, Generator +from typing import Callable, Generator, Iterator, TypeVar, overload + +from testutil import run_generator def normal(a: int, b: float) -> Callable: def generator(x: int, y: str) -> Generator: @@ -235,15 +237,43 @@ def outer() -> Generator: yield i return recursive(10) -[file driver.py] -from native import normal, generator, triple, another_triple, outer -from testutil import run_generator +def test_return_nested_generator() -> None: + assert run_generator(normal(1, 2.0)(3, '4.00')) == ((1, 2.0, 3, '4.00'), None) + assert run_generator(generator(1)) == ((1, 2, 3), None) + assert run_generator(triple()()) == ((1, 2, 3), None) + assert run_generator(another_triple()()) == ((1,), None) + assert run_generator(outer()) == ((0, 1, 2, 3, 4), None) + +def call_nested(x: int) -> list[int]: + def generator() -> Iterator[int]: + n = int() + 2 + yield x + yield n * x + + a = [] + for x in generator(): + a.append(x) + return a + +T = TypeVar("T") + +def deco(f: T) -> T: + return f + +def call_nested_decorated(x: int) -> list[int]: + @deco + def generator() -> Iterator[int]: + n = int() + 3 + yield x + yield n * x + + a = [] + for x in generator(): + a.append(x) + return a -assert run_generator(normal(1, 2.0)(3, '4.00')) == ((1, 2.0, 3, '4.00'), None) -assert run_generator(generator(1)) == ((1, 2, 3), None) -assert run_generator(triple()()) == ((1, 2, 3), None) -assert run_generator(another_triple()()) == ((1,), None) -assert run_generator(outer()) == ((0, 1, 2, 3, 4), None) +def test_call_nested_generator_in_function() -> None: + assert call_nested_decorated(5) == [5, 15] [case testYieldThrow] from typing import Generator, Iterable, Any, Union From 6600073b0abb76d9dfb3cbcee01667c3c0a54b31 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 7 Jul 2025 17:12:48 +0100 Subject: [PATCH 056/424] [mypyc] Use per-type free "lists" for nested functions (#19390) Often at most one instance is allocated at any given time, so this improves performance quite significantly in common cases. Uses the freelist feature introduced in #19316. This speeds up self check by about 0.5% (measured 100 samples). Speeds up this microbenchmark that gets close to the maximal benefit by about 90%: ``` def f(x: int) -> int: def inc(y: int) -> int: return y + 1 return inc(x) def bench(n: int) -> None: for x in range(n): f(x) from time import time bench(1000) t0 = time() bench(50 * 1000 * 1000) print(time() - t0) ``` --- mypyc/irbuild/callable_class.py | 1 + mypyc/irbuild/env_class.py | 1 + 2 files changed, 2 insertions(+) diff --git a/mypyc/irbuild/callable_class.py b/mypyc/irbuild/callable_class.py index c7c3c7677cda7..bbd1b909afb65 100644 --- a/mypyc/irbuild/callable_class.py +++ b/mypyc/irbuild/callable_class.py @@ -56,6 +56,7 @@ class for the nested function. # environment to point at the previously defined environment # class. callable_class_ir = ClassIR(name, builder.module_name, is_generated=True, is_final_class=True) + callable_class_ir.reuse_freed_instance = True # The functools @wraps decorator attempts to call setattr on # nested functions, so we create a dict for these nested diff --git a/mypyc/irbuild/env_class.py b/mypyc/irbuild/env_class.py index 9e72f7efcf940..51c854a4a2b2d 100644 --- a/mypyc/irbuild/env_class.py +++ b/mypyc/irbuild/env_class.py @@ -48,6 +48,7 @@ class is generated, the function environment has not yet been is_generated=True, is_final_class=True, ) + env_class.reuse_freed_instance = True env_class.attributes[SELF_NAME] = RInstance(env_class) if builder.fn_info.is_nested: # If the function is nested, its environment class must contain an environment From cb3bddd41149405e2dfaa615cfc4c6f109eb80f3 Mon Sep 17 00:00:00 2001 From: Christoph Tyralla Date: Mon, 7 Jul 2025 21:42:26 +0200 Subject: [PATCH 057/424] Improve the support for promotions inside unions. (#19245) Fixes #14987 I was puzzled as to why my previous attempts to avoid false `unreachable` warnings for loops failed for issue #14987. After some debugging, I realised that the underlying problem is that type narrowing does not work with promotions if both the declared type and the constraining type are unions: ```python x: float | None y: int | None x = y reveal_type(x) # None !!! ``` The fix seems straightforward (but let's see what the Mypy primer says) and is checked by the test cases `testNarrowPromotionsInsideUnions1` and `testNarrowPromotionsInsideUnions2`. --------- Co-authored-by: Ivan Levkivskyi --- mypy/meet.py | 18 ++++++++++++---- test-data/unit/check-narrowing.test | 33 +++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/mypy/meet.py b/mypy/meet.py index 7a44feabc10c8..2e238be7765e2 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -128,18 +128,28 @@ def narrow_declared_type(declared: Type, narrowed: Type) -> Type: if declared == narrowed: return original_declared if isinstance(declared, UnionType): + declared_items = declared.relevant_items() + if isinstance(narrowed, UnionType): + narrowed_items = narrowed.relevant_items() + else: + narrowed_items = [narrowed] return make_simplified_union( [ - narrow_declared_type(x, narrowed) - for x in declared.relevant_items() + narrow_declared_type(d, n) + for d in declared_items + for n in narrowed_items # This (ugly) special-casing is needed to support checking # branches like this: # x: Union[float, complex] # if isinstance(x, int): # ... + # And assignments like this: + # x: float | None + # y: int | None + # x = y if ( - is_overlapping_types(x, narrowed, ignore_promotions=True) - or is_subtype(narrowed, x, ignore_promotions=False) + is_overlapping_types(d, n, ignore_promotions=True) + or is_subtype(n, d, ignore_promotions=False) ) ] ) diff --git a/test-data/unit/check-narrowing.test b/test-data/unit/check-narrowing.test index e322bd7a37b8a..7fffd3ce94e54 100644 --- a/test-data/unit/check-narrowing.test +++ b/test-data/unit/check-narrowing.test @@ -2446,6 +2446,39 @@ while x is not None and b(): x = f() [builtins fixtures/primitives.pyi] +[case testNarrowPromotionsInsideUnions1] + +from typing import Union + +x: Union[str, float, None] +y: Union[int, str] +x = y +reveal_type(x) # N: Revealed type is "Union[builtins.str, builtins.int]" +z: Union[complex, str] +z = x +reveal_type(z) # N: Revealed type is "Union[builtins.int, builtins.str]" + +[builtins fixtures/primitives.pyi] + +[case testNarrowPromotionsInsideUnions2] +# flags: --warn-unreachable + +from typing import Optional + +def b() -> bool: ... +def i() -> int: ... +x: Optional[float] + +while b(): + x = None + while b(): + reveal_type(x) # N: Revealed type is "Union[None, builtins.int]" + if x is None or b(): + x = i() + reveal_type(x) # N: Revealed type is "builtins.int" + +[builtins fixtures/bool.pyi] + [case testAvoidFalseUnreachableInFinally] # flags: --allow-redefinition-new --local-partial-types --warn-unreachable def f() -> None: From afd5a382dc95532da61e6ebc7002323fbca75548 Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Tue, 8 Jul 2025 01:53:05 +0200 Subject: [PATCH 058/424] perf: add `__slots__` to `SubtypeVisitor` (#19394) We construct quite a lot of them. This made a selfcheck benchmark 0.4-0.7% faster when run on my local PC. --- mypy/subtypes.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 05f34aaec8f13..428e6dec6749e 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -390,6 +390,15 @@ def check_type_parameter( class SubtypeVisitor(TypeVisitor[bool]): + __slots__ = ( + "right", + "orig_right", + "proper_subtype", + "subtype_context", + "options", + "_subtype_kind", + ) + def __init__(self, right: Type, subtype_context: SubtypeContext, proper_subtype: bool) -> None: self.right = get_proper_type(right) self.orig_right = right From f9fe33147a9137b2579b6695dbd81980b1cca7f5 Mon Sep 17 00:00:00 2001 From: Piotr Sawicki Date: Tue, 8 Jul 2025 12:02:54 +0200 Subject: [PATCH 059/424] [mypyc] Simplify comparison of tuple elements (#19396) Got rid of unnecessary operations when comparing tuple elements returns a bit primitive. Example code: ``` def f(x: tuple[float], y: tuple[float]) -> bool: return x == y ``` IR before: ``` def f(x, y): x, y :: tuple[float] r0, r1 :: float r2 :: bit r3 :: object r4 :: i32 r5 :: bit r6, r7, r8 :: bool L0: r0 = x[0] r1 = y[0] r2 = r0 == r1 r3 = box(bit, r2) r4 = PyObject_IsTrue(r3) r5 = r4 >= 0 :: signed if not r5 goto L5 (error at f:2) else goto L1 :: bool L1: r6 = truncate r4: i32 to builtins.bool if not r6 goto L2 else goto L3 :: bool L2: r7 = 0 goto L4 L3: r7 = 1 L4: return r7 L5: r8 = :: bool return r8 ``` IR after: ``` def f(x, y): x, y :: tuple[float] r0, r1 :: float r2 :: bit r3 :: bool L0: r0 = x[0] r1 = y[0] r2 = r0 == r1 if not r2 goto L1 else goto L2 :: bool L1: r3 = 0 goto L3 L2: r3 = 1 L3: return r3 ``` Tested using the following benchmark: ``` def f(x: tuple[float,float], y: tuple[float,float]) -> bool: return x == y def bench(n: int) -> None: for x in range(n): lhs = (float(x), float(x * 2)) rhs = (float(x), float(x * 3)) f(lhs, rhs) from time import time bench(1000) t0 = time() bench(50 * 1000 * 1000) print(time() - t0) ``` Execution time goes from ~315ms to ~150ms on my machine. --- mypyc/irbuild/ll_builder.py | 2 +- mypyc/test-data/irbuild-tuple.test | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index 36b7c9241c711..be4178a4a71ae 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -1534,7 +1534,7 @@ def compare_tuples(self, lhs: Value, rhs: Value, op: str, line: int = -1) -> Val compare = self.binary_op(lhs_item, rhs_item, op, line) # Cast to bool if necessary since most types uses comparison returning a object type # See generic_ops.py for more information - if not is_bool_rprimitive(compare.type): + if not (is_bool_rprimitive(compare.type) or is_bit_rprimitive(compare.type)): compare = self.primitive_op(bool_op, [compare], line) if i < len(lhs.type.types) - 1: branch = Branch(compare, early_stop, check_blocks[i + 1], Branch.BOOL) diff --git a/mypyc/test-data/irbuild-tuple.test b/mypyc/test-data/irbuild-tuple.test index 582391ff6f984..2220217510805 100644 --- a/mypyc/test-data/irbuild-tuple.test +++ b/mypyc/test-data/irbuild-tuple.test @@ -453,3 +453,26 @@ L0: r0 = CPySequence_Multiply(a, 4) b = r0 return 1 + +[case testTupleFloatElementComparison] +def f(x: tuple[float], y: tuple[float]) -> bool: + return x == y + +[out] +def f(x, y): + x, y :: tuple[float] + r0, r1 :: float + r2 :: bit + r3 :: bool +L0: + r0 = x[0] + r1 = y[0] + r2 = r0 == r1 + if not r2 goto L1 else goto L2 :: bool +L1: + r3 = 0 + goto L3 +L2: + r3 = 1 +L3: + return r3 From d503edfb906814a815bf048437280ad74cd46c92 Mon Sep 17 00:00:00 2001 From: Piotr Sawicki Date: Tue, 8 Jul 2025 12:05:25 +0200 Subject: [PATCH 060/424] [mypyc] Raise NameError on undefined names (#19395) Fixes https://github.com/mypyc/mypyc/issues/879 Changed the type of exception that is raised on undefined names from `RuntimeError` to `NameError`. Moved the runtime error that refers to unexpected execution of unreachable code to `shortcircuit_expr` to keep existing behavior and generate the exception in blocks that are unreachable due to short-circuiting. In the future this could probably be optimized to not generate any code at all and just constant-fold the boolean expression. --- mypyc/irbuild/builder.py | 18 +++++++++++++----- mypyc/irbuild/expression.py | 4 +--- mypyc/test-data/irbuild-basic.test | 15 +++++++++++++++ mypyc/test-data/irbuild-unreachable.test | 15 +++------------ 4 files changed, 32 insertions(+), 20 deletions(-) diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index 323450f7c340b..daed97cb896df 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -1141,12 +1141,20 @@ def call_refexpr_with_args( ) def shortcircuit_expr(self, expr: OpExpr) -> Value: + def handle_right() -> Value: + if expr.right_unreachable: + self.builder.add( + RaiseStandardError( + RaiseStandardError.RUNTIME_ERROR, + "mypyc internal error: should be unreachable", + expr.right.line, + ) + ) + return self.builder.none() + return self.accept(expr.right) + return self.builder.shortcircuit_helper( - expr.op, - self.node_type(expr), - lambda: self.accept(expr.left), - lambda: self.accept(expr.right), - expr.line, + expr.op, self.node_type(expr), lambda: self.accept(expr.left), handle_right, expr.line ) # Basic helpers diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index c8c67cae309b2..c4a3f5f38ce38 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -117,9 +117,7 @@ def transform_name_expr(builder: IRBuilder, expr: NameExpr) -> Value: if expr.node is None: builder.add( RaiseStandardError( - RaiseStandardError.RUNTIME_ERROR, - "mypyc internal error: should be unreachable", - expr.line, + RaiseStandardError.NAME_ERROR, f'name "{expr.name}" is not defined', expr.line ) ) return builder.none() diff --git a/mypyc/test-data/irbuild-basic.test b/mypyc/test-data/irbuild-basic.test index 6e5267fc34dd3..d652cb9c9a149 100644 --- a/mypyc/test-data/irbuild-basic.test +++ b/mypyc/test-data/irbuild-basic.test @@ -3556,3 +3556,18 @@ L3: s = arg r3 = CPyObject_Size(s) return r3 + +[case testUndefinedFunction] +def f(): + non_existent_function() + +[out] +def f(): + r0 :: bool + r1, r2, r3 :: object +L0: + r0 = raise NameError('name "non_existent_function" is not defined') + r1 = box(None, 1) + r2 = PyObject_Vectorcall(r1, 0, 0, 0) + r3 = box(None, 1) + return r3 diff --git a/mypyc/test-data/irbuild-unreachable.test b/mypyc/test-data/irbuild-unreachable.test index 7209c00ce75d8..cebd4582923bb 100644 --- a/mypyc/test-data/irbuild-unreachable.test +++ b/mypyc/test-data/irbuild-unreachable.test @@ -17,11 +17,7 @@ def f(): r8, r9, r10 :: bit r11, r12 :: bool r13 :: object - r14 :: str - r15 :: object - r16 :: tuple[int, int] - r17, r18 :: object - r19, y :: bool + r14, y :: bool L0: r0 = sys :: module r1 = 'platform' @@ -46,13 +42,8 @@ L4: L5: r12 = raise RuntimeError('mypyc internal error: should be unreachable') r13 = box(None, 1) - r14 = 'version_info' - r15 = CPyObject_GetAttr(r13, r14) - r16 = (6, 10) - r17 = box(tuple[int, int], r16) - r18 = PyObject_RichCompare(r15, r17, 4) - r19 = unbox(bool, r18) - r11 = r19 + r14 = unbox(bool, r13) + r11 = r14 L6: y = r11 return 1 From 503f5bdd780c1895467c872d3f27f74d756cddec Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 8 Jul 2025 14:00:25 +0100 Subject: [PATCH 061/424] [mypyc] Add tests for string equality (#19401) This is in preparation for adding a faster str equality primitive. --- mypyc/test-data/run-strings.test | 58 ++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/mypyc/test-data/run-strings.test b/mypyc/test-data/run-strings.test index 074e56f9068ac..c726c4c70896c 100644 --- a/mypyc/test-data/run-strings.test +++ b/mypyc/test-data/run-strings.test @@ -92,6 +92,64 @@ assert remove_prefix_suffix('', '') == ('', '') assert remove_prefix_suffix('abc', 'a') == ('bc', 'abc') assert remove_prefix_suffix('abc', 'c') == ('abc', 'ab') +[case testStringEquality] +def eq(a: str, b: str) -> bool: + return a == b +def ne(a: str, b: str) -> bool: + return a != b + +def test_basic() -> None: + xy = "xy" + xy2 = str().join(["x", "y"]) + xx = "xx" + yy = "yy" + xxx = "xxx" + + assert eq("", str()) + assert not ne("", str()) + + assert eq("x", "x" + str()) + assert ne("x", "y") + + assert eq(xy, xy) + assert eq(xy, xy2) + assert not eq(xy, yy) + assert ne(xy, xx) + assert not ne(xy, xy) + assert not ne(xy, xy2) + + assert ne(xx, xxx) + assert ne(xxx, xx) + assert ne("x", "") + assert ne("", "x") + + assert ne("XX", xx) + assert ne(yy, xy) + +def test_unicode() -> None: + assert eq(chr(200), chr(200) + str()) + assert ne(chr(200), chr(201)) + + assert eq(chr(1234), chr(1234) + str()) + assert ne(chr(1234), chr(1235)) + + assert eq("\U0001f4a9", "\U0001f4a9" + str()) + assert eq("\U0001f4a9", "\U0001F4A9" + str()) + assert ne("\U0001f4a9", "\U0002f4a9" + str()) + assert ne("\U0001f4a9", "\U0001f5a9" + str()) + assert ne("\U0001f4a9", "\U0001f4a8" + str()) + + assert eq("foobar\u1234", "foobar\u1234" + str()) + assert eq("\u1234foobar", "\u1234foobar" + str()) + assert ne("foobar\uf234", "foobar\uf235") + assert ne("foobar\uf234", "foobar\uf334") + assert ne("foobar\u1234", "Foobar\u1234" + str()) + + assert eq("foo\U0001f4a9", "foo\U0001f4a9" + str()) + assert eq("\U0001f4a9foo", "\U0001f4a9foo" + str()) + assert ne("foo\U0001f4a9", "foo\U0001f4a8" + str()) + assert ne("\U0001f4a9foo", "\U0001f4a8foo" + str()) + [case testStringOps] from typing import List, Optional, Tuple from testutil import assertRaises From 4a427e9f1ac8a007d5c9bea946cfeb9e163cfa89 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 8 Jul 2025 17:46:00 +0100 Subject: [PATCH 062/424] [mypyc] Speed up native-to-native calls using await (#19398) When calling a native async function using `await`, e.g. `await foo()`, avoid raising `StopIteration` to pass the return value, since this is expensive. Instead, pass an extra `PyObject **` argument to the generator helper method and use that to return the return value. This is mostly helpful when there are many calls using await that don't block (e.g. there is a fast path that is usually taken that doesn't block). When awaiting from non-compiled code, the slow path is still taken. This builds on top of #19376. This PR makes this microbenchmark about 3x faster, which is about the ideal scenario for this optimization: ``` import asyncio from time import time async def inc(x: int) -> int: return x + 1 async def bench(n: int) -> int: x = 0 for i in range(n): x = await inc(x) return x asyncio.run(bench(1000)) t0 = time() asyncio.run(bench(1000 * 1000 * 200)) print(time() - t0) ``` --- mypyc/irbuild/context.py | 5 ++++ mypyc/irbuild/generator.py | 41 ++++++++++++++++++++++++++++---- mypyc/irbuild/nonlocalcontrol.py | 19 +++++++++++++++ mypyc/irbuild/prepare.py | 3 +++ mypyc/irbuild/statement.py | 22 +++++++++++++---- mypyc/lower/misc_ops.py | 10 ++++++-- mypyc/primitives/exc_ops.py | 12 +++++++++- 7 files changed, 101 insertions(+), 11 deletions(-) diff --git a/mypyc/irbuild/context.py b/mypyc/irbuild/context.py index 8d35c0ce25997..8d2e55ed96fb9 100644 --- a/mypyc/irbuild/context.py +++ b/mypyc/irbuild/context.py @@ -167,6 +167,11 @@ def __init__(self, ir: ClassIR) -> None: # Holds the arg passed to send self.send_arg_reg: Value | None = None + # Holds the PyObject ** pointer through which return value can be passed + # instead of raising StopIteration(ret_value) (only if not NULL). This + # is used for faster native-to-native calls. + self.stop_iter_value_reg: Value | None = None + # The switch block is used to decide which instruction to go using the value held in the # next-label register. self.switch_block = BasicBlock() diff --git a/mypyc/irbuild/generator.py b/mypyc/irbuild/generator.py index eec27e1cfb84d..ae45aed2fc67c 100644 --- a/mypyc/irbuild/generator.py +++ b/mypyc/irbuild/generator.py @@ -32,7 +32,12 @@ Unreachable, Value, ) -from mypyc.ir.rtypes import RInstance, int32_rprimitive, object_rprimitive +from mypyc.ir.rtypes import ( + RInstance, + int32_rprimitive, + object_pointer_rprimitive, + object_rprimitive, +) from mypyc.irbuild.builder import IRBuilder, calculate_arg_defaults, gen_arg_defaults from mypyc.irbuild.context import FuncInfo, GeneratorClass from mypyc.irbuild.env_class import ( @@ -256,7 +261,14 @@ def add_next_to_generator_class(builder: IRBuilder, fn_info: FuncInfo, fn_decl: result = builder.add( Call( fn_decl, - [builder.self(), none_reg, none_reg, none_reg, none_reg], + [ + builder.self(), + none_reg, + none_reg, + none_reg, + none_reg, + Integer(0, object_pointer_rprimitive), + ], fn_info.fitem.line, ) ) @@ -272,7 +284,14 @@ def add_send_to_generator_class(builder: IRBuilder, fn_info: FuncInfo, fn_decl: result = builder.add( Call( fn_decl, - [builder.self(), none_reg, none_reg, none_reg, builder.read(arg)], + [ + builder.self(), + none_reg, + none_reg, + none_reg, + builder.read(arg), + Integer(0, object_pointer_rprimitive), + ], fn_info.fitem.line, ) ) @@ -297,7 +316,14 @@ def add_throw_to_generator_class(builder: IRBuilder, fn_info: FuncInfo, fn_decl: result = builder.add( Call( fn_decl, - [builder.self(), builder.read(typ), builder.read(val), builder.read(tb), none_reg], + [ + builder.self(), + builder.read(typ), + builder.read(val), + builder.read(tb), + none_reg, + Integer(0, object_pointer_rprimitive), + ], fn_info.fitem.line, ) ) @@ -377,8 +403,15 @@ def setup_env_for_generator_class(builder: IRBuilder) -> None: # TODO: Use the right type here instead of object? exc_arg = builder.add_local(Var("arg"), object_rprimitive, is_arg=True) + # Parameter that can used to pass a pointer which can used instead of + # raising StopIteration(value). If the value is NULL, this won't be used. + stop_iter_value_arg = builder.add_local( + Var("stop_iter_ptr"), object_pointer_rprimitive, is_arg=True + ) + cls.exc_regs = (exc_type, exc_val, exc_tb) cls.send_arg_reg = exc_arg + cls.stop_iter_value_reg = stop_iter_value_arg cls.self_reg = builder.read(self_target, fitem.line) if builder.fn_info.can_merge_generator_and_env_classes(): diff --git a/mypyc/irbuild/nonlocalcontrol.py b/mypyc/irbuild/nonlocalcontrol.py index 0ac9bd3cee31b..887f6786718df 100644 --- a/mypyc/irbuild/nonlocalcontrol.py +++ b/mypyc/irbuild/nonlocalcontrol.py @@ -16,9 +16,11 @@ Integer, Register, Return, + SetMem, Unreachable, Value, ) +from mypyc.ir.rtypes import object_rprimitive from mypyc.irbuild.targets import AssignmentTarget from mypyc.primitives.exc_ops import restore_exc_info_op, set_stop_iteration_value @@ -108,10 +110,27 @@ def gen_return(self, builder: IRBuilder, value: Value, line: int) -> None: # StopIteration instead of using RaiseStandardError because # the obvious thing doesn't work if the value is a tuple # (???). + + true, false = BasicBlock(), BasicBlock() + stop_iter_reg = builder.fn_info.generator_class.stop_iter_value_reg + assert stop_iter_reg is not None + + builder.add(Branch(stop_iter_reg, true, false, Branch.IS_ERROR)) + + builder.activate_block(true) + # The default/slow path is to raise a StopIteration exception with + # return value. builder.call_c(set_stop_iteration_value, [value], NO_TRACEBACK_LINE_NO) builder.add(Unreachable()) builder.builder.pop_error_handler() + builder.activate_block(false) + # The fast path is to store return value via caller-provided pointer + # instead of raising an exception. This can only be used when the + # caller is a native function. + builder.add(SetMem(object_rprimitive, stop_iter_reg, value)) + builder.add(Return(Integer(0, object_rprimitive))) + class CleanupNonlocalControl(NonlocalControl): """Abstract nonlocal control that runs some cleanup code.""" diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index 147392585b25e..d4ec814372cda 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -56,6 +56,7 @@ RType, dict_rprimitive, none_rprimitive, + object_pointer_rprimitive, object_rprimitive, tuple_rprimitive, ) @@ -220,6 +221,8 @@ def create_generator_class_if_needed( RuntimeArg("value", object_rprimitive), RuntimeArg("traceback", object_rprimitive), RuntimeArg("arg", object_rprimitive), + # If non-NULL, used to store return value instead of raising StopIteration(retv) + RuntimeArg("stop_iter_ptr", object_pointer_rprimitive), ), object_rprimitive, ) diff --git a/mypyc/irbuild/statement.py b/mypyc/irbuild/statement.py index 9c7ffb6a3adf9..5f75a60d8d0a6 100644 --- a/mypyc/irbuild/statement.py +++ b/mypyc/irbuild/statement.py @@ -103,6 +103,7 @@ get_exc_info_op, get_exc_value_op, keep_propagating_op, + propagate_if_error_op, raise_exception_op, reraise_exception_op, restore_exc_info_op, @@ -958,21 +959,34 @@ def emit_yield_from_or_await( if isinstance(iter_reg.type, RInstance) and iter_reg.type.class_ir.has_method(helper_method): # Second fast path optimization: call helper directly (see also comment above). + # + # Calling a generated generator, so avoid raising StopIteration by passing + # an extra PyObject ** argument to helper where the stop iteration value is stored. + fast_path = True obj = builder.read(iter_reg) nn = builder.none_object() - m = MethodCall(obj, helper_method, [nn, nn, nn, nn], line) + stop_iter_val = Register(object_rprimitive) + err = builder.add(LoadErrorValue(object_rprimitive, undefines=True)) + builder.assign(stop_iter_val, err, line) + ptr = builder.add(LoadAddress(object_pointer_rprimitive, stop_iter_val)) + m = MethodCall(obj, helper_method, [nn, nn, nn, nn, ptr], line) # Generators have custom error handling, so disable normal error handling. m.error_kind = ERR_NEVER _y_init = builder.add(m) else: + fast_path = False _y_init = builder.call_c(next_raw_op, [builder.read(iter_reg)], line) builder.add(Branch(_y_init, stop_block, main_block, Branch.IS_ERROR)) - # Try extracting a return value from a StopIteration and return it. - # If it wasn't, this reraises the exception. builder.activate_block(stop_block) - builder.assign(result, builder.call_c(check_stop_op, [], line), line) + if fast_path: + builder.primitive_op(propagate_if_error_op, [stop_iter_val], line) + builder.assign(result, stop_iter_val, line) + else: + # Try extracting a return value from a StopIteration and return it. + # If it wasn't, this reraises the exception. + builder.assign(result, builder.call_c(check_stop_op, [], line), line) # Clear the spilled iterator/coroutine so that it will be freed. # Otherwise, the freeing of the spilled register would likely be delayed. err = builder.add(LoadErrorValue(iter_reg.type)) diff --git a/mypyc/lower/misc_ops.py b/mypyc/lower/misc_ops.py index 1effcd4f42ac4..3c42257c0dbe3 100644 --- a/mypyc/lower/misc_ops.py +++ b/mypyc/lower/misc_ops.py @@ -1,7 +1,7 @@ from __future__ import annotations -from mypyc.ir.ops import GetElementPtr, LoadMem, Value -from mypyc.ir.rtypes import PyVarObject, c_pyssize_t_rprimitive +from mypyc.ir.ops import ComparisonOp, GetElementPtr, Integer, LoadMem, Value +from mypyc.ir.rtypes import PyVarObject, c_pyssize_t_rprimitive, object_rprimitive from mypyc.irbuild.ll_builder import LowLevelIRBuilder from mypyc.lower.registry import lower_primitive_op @@ -10,3 +10,9 @@ def var_object_size(builder: LowLevelIRBuilder, args: list[Value], line: int) -> Value: elem_address = builder.add(GetElementPtr(args[0], PyVarObject, "ob_size")) return builder.add(LoadMem(c_pyssize_t_rprimitive, elem_address)) + + +@lower_primitive_op("propagate_if_error") +def propagate_if_error_op(builder: LowLevelIRBuilder, args: list[Value], line: int) -> Value: + # Return False on NULL. The primitive uses ERR_FALSE, so this is an error. + return builder.add(ComparisonOp(args[0], Integer(0, object_rprimitive), ComparisonOp.NEQ)) diff --git a/mypyc/primitives/exc_ops.py b/mypyc/primitives/exc_ops.py index 9a5f6392a9171..e1234f807afae 100644 --- a/mypyc/primitives/exc_ops.py +++ b/mypyc/primitives/exc_ops.py @@ -4,7 +4,7 @@ from mypyc.ir.ops import ERR_ALWAYS, ERR_FALSE, ERR_NEVER from mypyc.ir.rtypes import bit_rprimitive, exc_rtuple, object_rprimitive, void_rtype -from mypyc.primitives.registry import custom_op +from mypyc.primitives.registry import custom_op, custom_primitive_op # If the argument is a class, raise an instance of the class. Otherwise, assume # that the argument is an exception object, and raise it. @@ -62,6 +62,16 @@ error_kind=ERR_FALSE, ) +# If argument is NULL, propagate currently raised exception (in this case +# an exception must have been raised). If this can be used, it's faster +# than using PyErr_Occurred(). +propagate_if_error_op = custom_primitive_op( + "propagate_if_error", + arg_types=[object_rprimitive], + return_type=bit_rprimitive, + error_kind=ERR_FALSE, +) + # Catches a propagating exception and makes it the "currently # handled exception" (by sticking it into sys.exc_info()). Returns the # exception that was previously being handled, which must be restored From 10f95e666d12c0060a638b39838df4c6474648f2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 9 Jul 2025 10:25:52 +0100 Subject: [PATCH 063/424] [mypyc] Add faster primitive for string equality (#19402) This speeds up self check by ~1.4%. String equality is one of the top five most common primitive function calls in self check. We previously used a string comparison primitive that calculated the relative order of two strings. Usually we only care about equality, which we can do quicker since we can fast path using a length check, for example. I checked the CPython implementation of string equality in 3.9 (lowest supported Python version) and 3.13, and both of them had a fast path based on string object kind, and equality checks overall had the same semantics. Current CPython implementation: https://github.com/python/cpython/blob/main/Objects/stringlib/eq.h Tests for this were added in #19401. --- mypyc/irbuild/ll_builder.py | 12 +++- mypyc/lib-rt/CPy.h | 1 + mypyc/lib-rt/str_ops.c | 16 +++++ mypyc/primitives/str_ops.py | 10 ++++ mypyc/test-data/irbuild-dict.test | 31 +++------- mypyc/test-data/irbuild-str.test | 38 +++--------- mypyc/test-data/irbuild-unreachable.test | 76 ++++++++---------------- 7 files changed, 78 insertions(+), 106 deletions(-) diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index be4178a4a71ae..e25079c1146bb 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -175,7 +175,12 @@ unary_ops, ) from mypyc.primitives.set_ops import new_set_op -from mypyc.primitives.str_ops import str_check_if_true, str_ssize_t_size_op, unicode_compare +from mypyc.primitives.str_ops import ( + str_check_if_true, + str_eq, + str_ssize_t_size_op, + unicode_compare, +) from mypyc.primitives.tuple_ops import list_tuple_op, new_tuple_op, new_tuple_with_length_op from mypyc.rt_subtype import is_runtime_subtype from mypyc.sametype import is_same_type @@ -1471,6 +1476,11 @@ def check_tagged_short_int(self, val: Value, line: int, negated: bool = False) - def compare_strings(self, lhs: Value, rhs: Value, op: str, line: int) -> Value: """Compare two strings""" + if op == "==": + return self.primitive_op(str_eq, [lhs, rhs], line) + elif op == "!=": + eq = self.primitive_op(str_eq, [lhs, rhs], line) + return self.add(ComparisonOp(eq, self.false(), ComparisonOp.EQ, line)) compare_result = self.call_c(unicode_compare, [lhs, rhs], line) error_constant = Integer(-1, c_int_rprimitive, line) compare_error_check = self.add( diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index bdf3e0130a4cb..a0f1b06cc0d5b 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -726,6 +726,7 @@ static inline char CPyDict_CheckSize(PyObject *dict, CPyTagged size) { #define RIGHTSTRIP 1 #define BOTHSTRIP 2 +char CPyStr_Equal(PyObject *str1, PyObject *str2); PyObject *CPyStr_Build(Py_ssize_t len, ...); PyObject *CPyStr_GetItem(PyObject *str, CPyTagged index); CPyTagged CPyStr_Find(PyObject *str, PyObject *substr, CPyTagged start, int direction); diff --git a/mypyc/lib-rt/str_ops.c b/mypyc/lib-rt/str_ops.c index 210172c57497f..5fd376f21cfae 100644 --- a/mypyc/lib-rt/str_ops.c +++ b/mypyc/lib-rt/str_ops.c @@ -64,6 +64,22 @@ make_bloom_mask(int kind, const void* ptr, Py_ssize_t len) #undef BLOOM_UPDATE } +// Adapted from CPython 3.13.1 (_PyUnicode_Equal) +char CPyStr_Equal(PyObject *str1, PyObject *str2) { + if (str1 == str2) { + return 1; + } + Py_ssize_t len = PyUnicode_GET_LENGTH(str1); + if (PyUnicode_GET_LENGTH(str2) != len) + return 0; + int kind = PyUnicode_KIND(str1); + if (PyUnicode_KIND(str2) != kind) + return 0; + const void *data1 = PyUnicode_DATA(str1); + const void *data2 = PyUnicode_DATA(str2); + return memcmp(data1, data2, len * kind) == 0; +} + PyObject *CPyStr_GetItem(PyObject *str, CPyTagged index) { if (PyUnicode_READY(str) != -1) { if (CPyTagged_CheckShort(index)) { diff --git a/mypyc/primitives/str_ops.py b/mypyc/primitives/str_ops.py index 9d46da9c35149..37dbdf21bb5d1 100644 --- a/mypyc/primitives/str_ops.py +++ b/mypyc/primitives/str_ops.py @@ -21,6 +21,7 @@ ERR_NEG_INT, binary_op, custom_op, + custom_primitive_op, function_op, load_address_op, method_op, @@ -69,6 +70,15 @@ steals=[True, False], ) +# str1 == str2 (very common operation, so we provide our own) +str_eq = custom_primitive_op( + name="str_eq", + c_function_name="CPyStr_Equal", + arg_types=[str_rprimitive, str_rprimitive], + return_type=bool_rprimitive, + error_kind=ERR_NEVER, +) + unicode_compare = custom_op( arg_types=[str_rprimitive, str_rprimitive], return_type=c_int_rprimitive, diff --git a/mypyc/test-data/irbuild-dict.test b/mypyc/test-data/irbuild-dict.test index a71f5aa2d8a26..cacb14dae2736 100644 --- a/mypyc/test-data/irbuild-dict.test +++ b/mypyc/test-data/irbuild-dict.test @@ -399,12 +399,9 @@ def typeddict(d): r9, k :: str v :: object r10 :: str - r11 :: i32 - r12 :: bit - r13 :: object - r14, r15, r16 :: bit + r11 :: bool name :: object - r17, r18 :: bit + r12, r13 :: bit L0: r0 = 0 r1 = PyDict_Size(d) @@ -415,7 +412,7 @@ L1: r5 = r4[1] r0 = r5 r6 = r4[0] - if r6 goto L2 else goto L9 :: bool + if r6 goto L2 else goto L6 :: bool L2: r7 = r4[2] r8 = r4[3] @@ -423,27 +420,17 @@ L2: k = r9 v = r8 r10 = 'name' - r11 = PyUnicode_Compare(k, r10) - r12 = r11 == -1 - if r12 goto L3 else goto L5 :: bool + r11 = CPyStr_Equal(k, r10) + if r11 goto L3 else goto L4 :: bool L3: - r13 = PyErr_Occurred() - r14 = r13 != 0 - if r14 goto L4 else goto L5 :: bool + name = v L4: - r15 = CPy_KeepPropagating() L5: - r16 = r11 == 0 - if r16 goto L6 else goto L7 :: bool + r12 = CPyDict_CheckSize(d, r2) + goto L1 L6: - name = v + r13 = CPy_NoErrOccurred() L7: -L8: - r17 = CPyDict_CheckSize(d, r2) - goto L1 -L9: - r18 = CPy_NoErrOccurred() -L10: return 1 [case testDictLoadAddress] diff --git a/mypyc/test-data/irbuild-str.test b/mypyc/test-data/irbuild-str.test index 2bf77a6cb5566..4a4992d41a5d0 100644 --- a/mypyc/test-data/irbuild-str.test +++ b/mypyc/test-data/irbuild-str.test @@ -65,42 +65,18 @@ def neq(x: str, y: str) -> bool: [out] def eq(x, y): x, y :: str - r0 :: i32 - r1 :: bit - r2 :: object - r3, r4, r5 :: bit + r0 :: bool L0: - r0 = PyUnicode_Compare(x, y) - r1 = r0 == -1 - if r1 goto L1 else goto L3 :: bool -L1: - r2 = PyErr_Occurred() - r3 = r2 != 0 - if r3 goto L2 else goto L3 :: bool -L2: - r4 = CPy_KeepPropagating() -L3: - r5 = r0 == 0 - return r5 + r0 = CPyStr_Equal(x, y) + return r0 def neq(x, y): x, y :: str - r0 :: i32 + r0 :: bool r1 :: bit - r2 :: object - r3, r4, r5 :: bit L0: - r0 = PyUnicode_Compare(x, y) - r1 = r0 == -1 - if r1 goto L1 else goto L3 :: bool -L1: - r2 = PyErr_Occurred() - r3 = r2 != 0 - if r3 goto L2 else goto L3 :: bool -L2: - r4 = CPy_KeepPropagating() -L3: - r5 = r0 != 0 - return r5 + r0 = CPyStr_Equal(x, y) + r1 = r0 == 0 + return r1 [case testStrReplace] from typing import Optional diff --git a/mypyc/test-data/irbuild-unreachable.test b/mypyc/test-data/irbuild-unreachable.test index cebd4582923bb..a4f1ef8c7dba4 100644 --- a/mypyc/test-data/irbuild-unreachable.test +++ b/mypyc/test-data/irbuild-unreachable.test @@ -11,41 +11,27 @@ def f(): r1 :: str r2 :: object r3, r4 :: str - r5 :: i32 - r6 :: bit - r7 :: object - r8, r9, r10 :: bit - r11, r12 :: bool - r13 :: object - r14, y :: bool + r5, r6, r7 :: bool + r8 :: object + r9, y :: bool L0: r0 = sys :: module r1 = 'platform' r2 = CPyObject_GetAttr(r0, r1) r3 = cast(str, r2) r4 = 'x' - r5 = PyUnicode_Compare(r3, r4) - r6 = r5 == -1 - if r6 goto L1 else goto L3 :: bool + r5 = CPyStr_Equal(r3, r4) + if r5 goto L2 else goto L1 :: bool L1: - r7 = PyErr_Occurred() - r8 = r7 != 0 - if r8 goto L2 else goto L3 :: bool + r6 = r5 + goto L3 L2: - r9 = CPy_KeepPropagating() + r7 = raise RuntimeError('mypyc internal error: should be unreachable') + r8 = box(None, 1) + r9 = unbox(bool, r8) + r6 = r9 L3: - r10 = r5 == 0 - if r10 goto L5 else goto L4 :: bool -L4: - r11 = r10 - goto L6 -L5: - r12 = raise RuntimeError('mypyc internal error: should be unreachable') - r13 = box(None, 1) - r14 = unbox(bool, r13) - r11 = r14 -L6: - y = r11 + y = r6 return 1 [case testUnreachableNameExpr] @@ -59,41 +45,27 @@ def f(): r1 :: str r2 :: object r3, r4 :: str - r5 :: i32 - r6 :: bit - r7 :: object - r8, r9, r10 :: bit - r11, r12 :: bool - r13 :: object - r14, y :: bool + r5, r6, r7 :: bool + r8 :: object + r9, y :: bool L0: r0 = sys :: module r1 = 'platform' r2 = CPyObject_GetAttr(r0, r1) r3 = cast(str, r2) r4 = 'x' - r5 = PyUnicode_Compare(r3, r4) - r6 = r5 == -1 - if r6 goto L1 else goto L3 :: bool + r5 = CPyStr_Equal(r3, r4) + if r5 goto L2 else goto L1 :: bool L1: - r7 = PyErr_Occurred() - r8 = r7 != 0 - if r8 goto L2 else goto L3 :: bool + r6 = r5 + goto L3 L2: - r9 = CPy_KeepPropagating() + r7 = raise RuntimeError('mypyc internal error: should be unreachable') + r8 = box(None, 1) + r9 = unbox(bool, r8) + r6 = r9 L3: - r10 = r5 == 0 - if r10 goto L5 else goto L4 :: bool -L4: - r11 = r10 - goto L6 -L5: - r12 = raise RuntimeError('mypyc internal error: should be unreachable') - r13 = box(None, 1) - r14 = unbox(bool, r13) - r11 = r14 -L6: - y = r11 + y = r6 return 1 [case testUnreachableStatementAfterReturn] From 930a379f05a74745302eebdbcdd9670f8f323373 Mon Sep 17 00:00:00 2001 From: Piotr Sawicki Date: Wed, 9 Jul 2025 11:28:47 +0200 Subject: [PATCH 064/424] [mypyc] Add is_bool_or_bit_rprimitive (#19406) Added a wrapper to check if a type is either a bool or bit primitive as these two checks are often done together. The wrapper should help in preventing suboptimal code generation if one forgets to check for the bit primitive in cases when it can be trivially expanded to bool. One such case was in translation of binary ops, which is fixed in this PR. Example code: ``` def f(a: float, b: float, c: float) -> bool: return (a == b) & (a == c) ``` IR before: ``` def f(a, b, c): a, b, c :: float r0, r1 :: bit r2 :: bool r3 :: int r4 :: bool r5, r6 :: int r7 :: object r8, r9 :: bool L0: r0 = a == b r1 = a == c r2 = r0 << 1 r3 = extend r2: builtins.bool to builtins.int r4 = r1 << 1 r5 = extend r4: builtins.bool to builtins.int r6 = CPyTagged_And(r3, r5) dec_ref r3 :: int dec_ref r5 :: int r7 = box(int, r6) r8 = unbox(bool, r7) dec_ref r7 if is_error(r8) goto L2 (error at f:2) else goto L1 L1: return r8 L2: r9 = :: bool return r9 ``` IR after: ``` def f(a, b, c): a, b, c :: float r0, r1 :: bit r2 :: bool L0: r0 = a == b r1 = a == c r2 = r0 & r1 return r2 ``` --- mypyc/codegen/emit.py | 12 +++----- mypyc/ir/ops.py | 9 ++---- mypyc/ir/rtypes.py | 4 +++ mypyc/irbuild/ll_builder.py | 33 ++++++++++---------- mypyc/test-data/irbuild-bool.test | 51 +++++++++++++++++++++++++++++++ 5 files changed, 78 insertions(+), 31 deletions(-) diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index ba8b8307e1fdb..f27a8668142d7 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -28,8 +28,7 @@ RType, RUnion, int_rprimitive, - is_bit_rprimitive, - is_bool_rprimitive, + is_bool_or_bit_rprimitive, is_bytes_rprimitive, is_dict_rprimitive, is_fixed_width_rtype, @@ -615,8 +614,7 @@ def emit_cast( or is_range_rprimitive(typ) or is_float_rprimitive(typ) or is_int_rprimitive(typ) - or is_bool_rprimitive(typ) - or is_bit_rprimitive(typ) + or is_bool_or_bit_rprimitive(typ) or is_fixed_width_rtype(typ) ): if declare_dest: @@ -638,7 +636,7 @@ def emit_cast( elif is_int_rprimitive(typ) or is_fixed_width_rtype(typ): # TODO: Range check for fixed-width types? prefix = "PyLong" - elif is_bool_rprimitive(typ) or is_bit_rprimitive(typ): + elif is_bool_or_bit_rprimitive(typ): prefix = "PyBool" else: assert False, f"unexpected primitive type: {typ}" @@ -889,7 +887,7 @@ def emit_unbox( self.emit_line("else {") self.emit_line(failure) self.emit_line("}") - elif is_bool_rprimitive(typ) or is_bit_rprimitive(typ): + elif is_bool_or_bit_rprimitive(typ): # Whether we are borrowing or not makes no difference. if declare_dest: self.emit_line(f"char {dest};") @@ -1015,7 +1013,7 @@ def emit_box( if is_int_rprimitive(typ) or is_short_int_rprimitive(typ): # Steal the existing reference if it exists. self.emit_line(f"{declaration}{dest} = CPyTagged_StealAsObject({src});") - elif is_bool_rprimitive(typ) or is_bit_rprimitive(typ): + elif is_bool_or_bit_rprimitive(typ): # N.B: bool is special cased to produce a borrowed value # after boxing, so we don't need to increment the refcount # when this comes directly from a Box op. diff --git a/mypyc/ir/ops.py b/mypyc/ir/ops.py index e15d494c2c579..f362b0cca1979 100644 --- a/mypyc/ir/ops.py +++ b/mypyc/ir/ops.py @@ -42,8 +42,7 @@ class to enable the new behavior. Sometimes adding a new abstract cstring_rprimitive, float_rprimitive, int_rprimitive, - is_bit_rprimitive, - is_bool_rprimitive, + is_bool_or_bit_rprimitive, is_int_rprimitive, is_none_rprimitive, is_pointer_rprimitive, @@ -1089,11 +1088,7 @@ def __init__(self, src: Value, line: int = -1) -> None: self.src = src self.type = object_rprimitive # When we box None and bool values, we produce a borrowed result - if ( - is_none_rprimitive(self.src.type) - or is_bool_rprimitive(self.src.type) - or is_bit_rprimitive(self.src.type) - ): + if is_none_rprimitive(self.src.type) or is_bool_or_bit_rprimitive(self.src.type): self.is_borrowed = True def sources(self) -> list[Value]: diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 8dc7d5c9c9497..c0871bba258c4 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -582,6 +582,10 @@ def is_bit_rprimitive(rtype: RType) -> bool: return isinstance(rtype, RPrimitive) and rtype.name == "bit" +def is_bool_or_bit_rprimitive(rtype: RType) -> bool: + return is_bool_rprimitive(rtype) or is_bit_rprimitive(rtype) + + def is_object_rprimitive(rtype: RType) -> bool: return isinstance(rtype, RPrimitive) and rtype.name == "builtins.object" diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index e25079c1146bb..a97bc52bc7e4d 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -93,8 +93,7 @@ dict_rprimitive, float_rprimitive, int_rprimitive, - is_bit_rprimitive, - is_bool_rprimitive, + is_bool_or_bit_rprimitive, is_bytes_rprimitive, is_dict_rprimitive, is_fixed_width_rtype, @@ -381,16 +380,12 @@ def coerce( ): # Equivalent types return src - elif (is_bool_rprimitive(src_type) or is_bit_rprimitive(src_type)) and is_tagged( - target_type - ): + elif is_bool_or_bit_rprimitive(src_type) and is_tagged(target_type): shifted = self.int_op( bool_rprimitive, src, Integer(1, bool_rprimitive), IntOp.LEFT_SHIFT ) return self.add(Extend(shifted, target_type, signed=False)) - elif ( - is_bool_rprimitive(src_type) or is_bit_rprimitive(src_type) - ) and is_fixed_width_rtype(target_type): + elif is_bool_or_bit_rprimitive(src_type) and is_fixed_width_rtype(target_type): return self.add(Extend(src, target_type, signed=False)) elif isinstance(src, Integer) and is_float_rprimitive(target_type): if is_tagged(src_type): @@ -1341,7 +1336,11 @@ def binary_op(self, lreg: Value, rreg: Value, op: str, line: int) -> Value: return self.compare_strings(lreg, rreg, op, line) if is_bytes_rprimitive(ltype) and is_bytes_rprimitive(rtype) and op in ("==", "!="): return self.compare_bytes(lreg, rreg, op, line) - if is_bool_rprimitive(ltype) and is_bool_rprimitive(rtype) and op in BOOL_BINARY_OPS: + if ( + is_bool_or_bit_rprimitive(ltype) + and is_bool_or_bit_rprimitive(rtype) + and op in BOOL_BINARY_OPS + ): if op in ComparisonOp.signed_ops: return self.bool_comparison_op(lreg, rreg, op, line) else: @@ -1355,7 +1354,7 @@ def binary_op(self, lreg: Value, rreg: Value, op: str, line: int) -> Value: op_id = int_op_to_id[op] else: op_id = IntOp.DIV - if is_bool_rprimitive(rtype) or is_bit_rprimitive(rtype): + if is_bool_or_bit_rprimitive(rtype): rreg = self.coerce(rreg, ltype, line) rtype = ltype if is_fixed_width_rtype(rtype) or is_tagged(rtype): @@ -1367,7 +1366,7 @@ def binary_op(self, lreg: Value, rreg: Value, op: str, line: int) -> Value: elif op in ComparisonOp.signed_ops: if is_int_rprimitive(rtype): rreg = self.coerce_int_to_fixed_width(rreg, ltype, line) - elif is_bool_rprimitive(rtype) or is_bit_rprimitive(rtype): + elif is_bool_or_bit_rprimitive(rtype): rreg = self.coerce(rreg, ltype, line) op_id = ComparisonOp.signed_ops[op] if is_fixed_width_rtype(rreg.type): @@ -1387,13 +1386,13 @@ def binary_op(self, lreg: Value, rreg: Value, op: str, line: int) -> Value: ) if is_tagged(ltype): return self.fixed_width_int_op(rtype, lreg, rreg, op_id, line) - if is_bool_rprimitive(ltype) or is_bit_rprimitive(ltype): + if is_bool_or_bit_rprimitive(ltype): lreg = self.coerce(lreg, rtype, line) return self.fixed_width_int_op(rtype, lreg, rreg, op_id, line) elif op in ComparisonOp.signed_ops: if is_int_rprimitive(ltype): lreg = self.coerce_int_to_fixed_width(lreg, rtype, line) - elif is_bool_rprimitive(ltype) or is_bit_rprimitive(ltype): + elif is_bool_or_bit_rprimitive(ltype): lreg = self.coerce(lreg, rtype, line) op_id = ComparisonOp.signed_ops[op] if isinstance(lreg, Integer): @@ -1544,7 +1543,7 @@ def compare_tuples(self, lhs: Value, rhs: Value, op: str, line: int = -1) -> Val compare = self.binary_op(lhs_item, rhs_item, op, line) # Cast to bool if necessary since most types uses comparison returning a object type # See generic_ops.py for more information - if not (is_bool_rprimitive(compare.type) or is_bit_rprimitive(compare.type)): + if not is_bool_or_bit_rprimitive(compare.type): compare = self.primitive_op(bool_op, [compare], line) if i < len(lhs.type.types) - 1: branch = Branch(compare, early_stop, check_blocks[i + 1], Branch.BOOL) @@ -1563,7 +1562,7 @@ def compare_tuples(self, lhs: Value, rhs: Value, op: str, line: int = -1) -> Val def translate_instance_contains(self, inst: Value, item: Value, op: str, line: int) -> Value: res = self.gen_method_call(inst, "__contains__", [item], None, line) - if not is_bool_rprimitive(res.type): + if not is_bool_or_bit_rprimitive(res.type): res = self.primitive_op(bool_op, [res], line) if op == "not in": res = self.bool_bitwise_op(res, Integer(1, rtype=bool_rprimitive), "^", line) @@ -1590,7 +1589,7 @@ def unary_not(self, value: Value, line: int) -> Value: def unary_op(self, value: Value, expr_op: str, line: int) -> Value: typ = value.type - if is_bool_rprimitive(typ) or is_bit_rprimitive(typ): + if is_bool_or_bit_rprimitive(typ): if expr_op == "not": return self.unary_not(value, line) if expr_op == "+": @@ -1748,7 +1747,7 @@ def bool_value(self, value: Value) -> Value: The result type can be bit_rprimitive or bool_rprimitive. """ - if is_bool_rprimitive(value.type) or is_bit_rprimitive(value.type): + if is_bool_or_bit_rprimitive(value.type): result = value elif is_runtime_subtype(value.type, int_rprimitive): zero = Integer(0, short_int_rprimitive) diff --git a/mypyc/test-data/irbuild-bool.test b/mypyc/test-data/irbuild-bool.test index 128266e6b1d75..9810daf487fae 100644 --- a/mypyc/test-data/irbuild-bool.test +++ b/mypyc/test-data/irbuild-bool.test @@ -422,3 +422,54 @@ L0: r1 = extend r0: builtins.bool to builtins.int x = r1 return x + +[case testBitToBoolPromotion] +def bitand(x: float, y: float, z: float) -> bool: + b = (x == y) & (x == z) + return b +def bitor(x: float, y: float, z: float) -> bool: + b = (x == y) | (x == z) + return b +def bitxor(x: float, y: float, z: float) -> bool: + b = (x == y) ^ (x == z) + return b +def invert(x: float, y: float) -> bool: + return not(x == y) +[out] +def bitand(x, y, z): + x, y, z :: float + r0, r1 :: bit + r2, b :: bool +L0: + r0 = x == y + r1 = x == z + r2 = r0 & r1 + b = r2 + return b +def bitor(x, y, z): + x, y, z :: float + r0, r1 :: bit + r2, b :: bool +L0: + r0 = x == y + r1 = x == z + r2 = r0 | r1 + b = r2 + return b +def bitxor(x, y, z): + x, y, z :: float + r0, r1 :: bit + r2, b :: bool +L0: + r0 = x == y + r1 = x == z + r2 = r0 ^ r1 + b = r2 + return b +def invert(x, y): + x, y :: float + r0, r1 :: bit +L0: + r0 = x == y + r1 = r0 ^ 1 + return r1 From 35d8c697f7021a1dc5677fe9d507962f4f00ec84 Mon Sep 17 00:00:00 2001 From: Piotr Sawicki Date: Wed, 9 Jul 2025 11:30:10 +0200 Subject: [PATCH 065/424] [mypyc] Report error on reserved method name (#19407) Changed mypyc to report an error if a method of a compiled class is called `__mypyc_generator_helper__`. The name should be reserved because it might clash with an auto-generated method. --- mypyc/irbuild/classdef.py | 9 +++++++++ mypyc/irbuild/generator.py | 5 +++-- mypyc/irbuild/prepare.py | 4 +++- mypyc/irbuild/statement.py | 3 ++- mypyc/test-data/irbuild-classes.test | 25 +++++++++++++++++++++++++ 5 files changed, 42 insertions(+), 4 deletions(-) diff --git a/mypyc/irbuild/classdef.py b/mypyc/irbuild/classdef.py index 13121707773ad..30020aabb3100 100644 --- a/mypyc/irbuild/classdef.py +++ b/mypyc/irbuild/classdef.py @@ -64,6 +64,7 @@ handle_non_ext_method, load_type, ) +from mypyc.irbuild.prepare import GENERATOR_HELPER_NAME from mypyc.irbuild.util import dataclass_type, get_func_def, is_constant, is_dataclass_decorator from mypyc.primitives.dict_ops import dict_new_op, dict_set_item_op from mypyc.primitives.generic_ops import ( @@ -135,6 +136,14 @@ def transform_class_def(builder: IRBuilder, cdef: ClassDef) -> None: cls_builder = NonExtClassBuilder(builder, cdef) for stmt in cdef.defs.body: + if ( + isinstance(stmt, (FuncDef, Decorator, OverloadedFuncDef)) + and stmt.name == GENERATOR_HELPER_NAME + ): + builder.error( + f'Method name "{stmt.name}" is reserved for mypyc internal use', stmt.line + ) + if isinstance(stmt, OverloadedFuncDef) and stmt.is_property: if isinstance(cls_builder, NonExtClassBuilder): # properties with both getters and setters in non_extension diff --git a/mypyc/irbuild/generator.py b/mypyc/irbuild/generator.py index ae45aed2fc67c..545a9daf956bf 100644 --- a/mypyc/irbuild/generator.py +++ b/mypyc/irbuild/generator.py @@ -50,6 +50,7 @@ setup_func_for_recursive_call, ) from mypyc.irbuild.nonlocalcontrol import ExceptNonlocalControl +from mypyc.irbuild.prepare import GENERATOR_HELPER_NAME from mypyc.primitives.exc_ops import ( error_catch_op, exc_matches_op, @@ -236,11 +237,11 @@ def add_helper_to_generator_class( builder: IRBuilder, arg_regs: list[Register], blocks: list[BasicBlock], fn_info: FuncInfo ) -> FuncDecl: """Generates a helper method for a generator class, called by '__next__' and 'throw'.""" - helper_fn_decl = fn_info.generator_class.ir.method_decls["__mypyc_generator_helper__"] + helper_fn_decl = fn_info.generator_class.ir.method_decls[GENERATOR_HELPER_NAME] helper_fn_ir = FuncIR( helper_fn_decl, arg_regs, blocks, fn_info.fitem.line, traceback_name=fn_info.fitem.name ) - fn_info.generator_class.ir.methods["__mypyc_generator_helper__"] = helper_fn_ir + fn_info.generator_class.ir.methods[GENERATOR_HELPER_NAME] = helper_fn_ir builder.functions.append(helper_fn_ir) fn_info.env_class.env_user_function = helper_fn_ir diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index d4ec814372cda..c22101ac193d9 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -71,6 +71,8 @@ from mypyc.options import CompilerOptions from mypyc.sametype import is_same_type +GENERATOR_HELPER_NAME = "__mypyc_generator_helper__" + def build_type_map( mapper: Mapper, @@ -229,7 +231,7 @@ def create_generator_class_if_needed( # The implementation of most generator functionality is behind this magic method. helper_fn_decl = FuncDecl( - "__mypyc_generator_helper__", name, module_name, helper_sig, internal=True + GENERATOR_HELPER_NAME, name, module_name, helper_sig, internal=True ) cir.method_decls[helper_fn_decl.name] = helper_fn_decl diff --git a/mypyc/irbuild/statement.py b/mypyc/irbuild/statement.py index 5f75a60d8d0a6..c8091c0313d02 100644 --- a/mypyc/irbuild/statement.py +++ b/mypyc/irbuild/statement.py @@ -90,6 +90,7 @@ FinallyNonlocalControl, TryFinallyNonlocalControl, ) +from mypyc.irbuild.prepare import GENERATOR_HELPER_NAME from mypyc.irbuild.targets import ( AssignmentTarget, AssignmentTargetAttr, @@ -933,7 +934,7 @@ def emit_yield_from_or_await( to_yield_reg = Register(object_rprimitive) received_reg = Register(object_rprimitive) - helper_method = "__mypyc_generator_helper__" + helper_method = GENERATOR_HELPER_NAME if ( isinstance(val, (Call, MethodCall)) and isinstance(val.type, RInstance) diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index fa4708f02e0bf..1543568fccad4 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -1383,3 +1383,28 @@ class M(type): # E: Inheriting from most builtin types is unimplemented \ @mypyc_attr(native_class=True) class A(metaclass=M): # E: Class is marked as native_class=True but it can't be a native class. Classes with a metaclass other than ABCMeta, TypingMeta or GenericMeta can't be native classes. pass + +[case testReservedName] +from typing import Any, overload + +def decorator(cls): + return cls + +class TestMethod: + def __mypyc_generator_helper__(self) -> None: # E: Method name "__mypyc_generator_helper__" is reserved for mypyc internal use + pass + +class TestDecorator: + @decorator # E: Method name "__mypyc_generator_helper__" is reserved for mypyc internal use + def __mypyc_generator_helper__(self) -> None: + pass + +class TestOverload: + @overload # E: Method name "__mypyc_generator_helper__" is reserved for mypyc internal use + def __mypyc_generator_helper__(self, x: int) -> int: ... + + @overload + def __mypyc_generator_helper__(self, x: str) -> str: ... + + def __mypyc_generator_helper__(self, x: Any) -> Any: + return x From a79e85e3958a1ee751a2287a6a8aaa4e77845679 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Wed, 9 Jul 2025 11:27:01 -0400 Subject: [PATCH 066/424] feat: add helpful info to internal AssertionError excs (#19404) I added some contextual information to various assert statements throughout the repo. This information is helpful for me while debugging [Issue 1116](https://github.com/mypyc/mypyc/issues/1116) and should be similarly useful in many other debugging contexts. --- mypy/checker.py | 4 ++-- mypy/checkpattern.py | 4 ++-- mypy/plugins/attrs.py | 2 +- mypy/plugins/dataclasses.py | 2 +- mypy/semanal.py | 2 +- mypy/semanal_main.py | 2 +- mypyc/analysis/attrdefined.py | 4 ++-- mypyc/analysis/selfleaks.py | 2 +- mypyc/codegen/emit.py | 4 ++-- mypyc/codegen/emitfunc.py | 8 ++++---- mypyc/irbuild/builder.py | 10 +++++----- mypyc/irbuild/classdef.py | 8 ++++---- mypyc/irbuild/expression.py | 4 ++-- mypyc/irbuild/for_helpers.py | 2 +- mypyc/irbuild/function.py | 19 ++++++++----------- mypyc/irbuild/generator.py | 2 +- mypyc/irbuild/ll_builder.py | 9 +++++---- mypyc/irbuild/match.py | 4 ++-- mypyc/irbuild/nonlocalcontrol.py | 2 +- mypyc/irbuild/statement.py | 2 +- mypyc/lower/list_ops.py | 2 +- mypyc/transform/ir_transform.py | 2 +- mypyc/transform/refcount.py | 2 +- 23 files changed, 50 insertions(+), 52 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 225a50c7e6461..edd9519da4f89 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -7273,7 +7273,7 @@ def named_type(self, name: str) -> Instance: if isinstance(node, TypeAlias): assert isinstance(node.target, Instance) # type: ignore[misc] node = node.target.type - assert isinstance(node, TypeInfo) + assert isinstance(node, TypeInfo), node any_type = AnyType(TypeOfAny.from_omitted_generics) return Instance(node, [any_type] * len(node.defn.type_vars)) @@ -7292,7 +7292,7 @@ def lookup_typeinfo(self, fullname: str) -> TypeInfo: # Assume that the name refers to a class. sym = self.lookup_qualified(fullname) node = sym.node - assert isinstance(node, TypeInfo) + assert isinstance(node, TypeInfo), node return node def type_type(self) -> Instance: diff --git a/mypy/checkpattern.py b/mypy/checkpattern.py index 4cf7c1ca78626..48840466f0d89 100644 --- a/mypy/checkpattern.py +++ b/mypy/checkpattern.py @@ -796,9 +796,9 @@ def get_var(expr: Expression) -> Var: Warning: this in only true for expressions captured by a match statement. Don't call it from anywhere else """ - assert isinstance(expr, NameExpr) + assert isinstance(expr, NameExpr), expr node = expr.node - assert isinstance(node, Var) + assert isinstance(node, Var), node return node diff --git a/mypy/plugins/attrs.py b/mypy/plugins/attrs.py index b7b3821576eab..47c6ad9f305a2 100644 --- a/mypy/plugins/attrs.py +++ b/mypy/plugins/attrs.py @@ -458,7 +458,7 @@ def _analyze_class( if isinstance(node, PlaceholderNode): # This node is not ready yet. continue - assert isinstance(node, Var) + assert isinstance(node, Var), node node.is_initialized_in_class = False # Traverse the MRO and collect attributes from the parents. diff --git a/mypy/plugins/dataclasses.py b/mypy/plugins/dataclasses.py index 99d4ef56a540c..ee6f8889b8946 100644 --- a/mypy/plugins/dataclasses.py +++ b/mypy/plugins/dataclasses.py @@ -610,7 +610,7 @@ def collect_attributes(self) -> list[DataclassAttribute] | None: # We will issue an error later. continue - assert isinstance(node, Var) + assert isinstance(node, Var), node # x: ClassVar[int] is ignored by dataclasses. if node.is_classvar: diff --git a/mypy/semanal.py b/mypy/semanal.py index 435c1e682e359..01b7f4989d800 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -6630,7 +6630,7 @@ def named_type(self, fullname: str, args: list[Type] | None = None) -> Instance: sym = self.lookup_fully_qualified(fullname) assert sym, "Internal error: attempted to construct unknown type" node = sym.node - assert isinstance(node, TypeInfo) + assert isinstance(node, TypeInfo), node if args: # TODO: assert len(args) == len(node.defn.type_vars) return Instance(node, args) diff --git a/mypy/semanal_main.py b/mypy/semanal_main.py index 00d795c64e447..7301e9f9b9b3e 100644 --- a/mypy/semanal_main.py +++ b/mypy/semanal_main.py @@ -290,7 +290,7 @@ def process_functions(graph: Graph, scc: list[str], patches: Patches) -> None: for module, target, node, active_type in order_by_subclassing(all_targets): analyzer = graph[module].manager.semantic_analyzer - assert isinstance(node, (FuncDef, OverloadedFuncDef, Decorator)) + assert isinstance(node, (FuncDef, OverloadedFuncDef, Decorator)), node process_top_level_function( analyzer, graph[module], module, target, node, active_type, patches ) diff --git a/mypyc/analysis/attrdefined.py b/mypyc/analysis/attrdefined.py index 896527bdcf141..4fd0017257a01 100644 --- a/mypyc/analysis/attrdefined.py +++ b/mypyc/analysis/attrdefined.py @@ -285,7 +285,7 @@ def mark_attr_initialization_ops( def attributes_initialized_by_init_call(op: Call) -> set[str]: """Calculate attributes that are always initialized by a super().__init__ call.""" self_type = op.fn.sig.args[0].type - assert isinstance(self_type, RInstance) + assert isinstance(self_type, RInstance), self_type cl = self_type.class_ir return {a for base in cl.mro for a in base.attributes if base.is_always_defined(a)} @@ -293,7 +293,7 @@ def attributes_initialized_by_init_call(op: Call) -> set[str]: def attributes_maybe_initialized_by_init_call(op: Call) -> set[str]: """Calculate attributes that may be initialized by a super().__init__ call.""" self_type = op.fn.sig.args[0].type - assert isinstance(self_type, RInstance) + assert isinstance(self_type, RInstance), self_type cl = self_type.class_ir return attributes_initialized_by_init_call(op) | cl._sometimes_initialized_attrs diff --git a/mypyc/analysis/selfleaks.py b/mypyc/analysis/selfleaks.py index 4d3a7c87c5d15..9f7e00db78d27 100644 --- a/mypyc/analysis/selfleaks.py +++ b/mypyc/analysis/selfleaks.py @@ -92,7 +92,7 @@ def visit_call(self, op: Call) -> GenAndKill: fn = op.fn if fn.class_name and fn.name == "__init__": self_type = op.fn.sig.args[0].type - assert isinstance(self_type, RInstance) + assert isinstance(self_type, RInstance), self_type cl = self_type.class_ir if not cl.init_self_leak: return CLEAN diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index f27a8668142d7..8c4a69cfa3cbb 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -742,7 +742,7 @@ def emit_cast_error_handler( self.emit_traceback(error.source_path, error.module_name, error.traceback_entry) self.emit_line("goto %s;" % error.label) else: - assert isinstance(error, ReturnHandler) + assert isinstance(error, ReturnHandler), error self.emit_line("return %s;" % error.value) def emit_union_cast( @@ -871,7 +871,7 @@ def emit_unbox( elif isinstance(error, GotoHandler): failure = "goto %s;" % error.label else: - assert isinstance(error, ReturnHandler) + assert isinstance(error, ReturnHandler), error failure = "return %s;" % error.value if raise_exception: raise_exc = f'CPy_TypeError("{self.pretty_name(typ)}", {src}); ' diff --git a/mypyc/codegen/emitfunc.py b/mypyc/codegen/emitfunc.py index 4b618f3c67db3..3fdd08037d1af 100644 --- a/mypyc/codegen/emitfunc.py +++ b/mypyc/codegen/emitfunc.py @@ -160,7 +160,7 @@ def generate_native_function( # eliminated during code generation. for block in fn.blocks: terminator = block.terminator - assert isinstance(terminator, ControlOp) + assert isinstance(terminator, ControlOp), terminator for target in terminator.targets(): is_next_block = target.label == block.label + 1 @@ -309,7 +309,7 @@ def visit_assign(self, op: Assign) -> None: def visit_assign_multi(self, op: AssignMulti) -> None: typ = op.dest.type - assert isinstance(typ, RArray) + assert isinstance(typ, RArray), typ dest = self.reg(op.dest) # RArray values can only be assigned to once, so we can always # declare them on initialization. @@ -586,7 +586,7 @@ def visit_method_call(self, op: MethodCall) -> None: def emit_method_call(self, dest: str, op_obj: Value, name: str, op_args: list[Value]) -> None: obj = self.reg(op_obj) rtype = op_obj.type - assert isinstance(rtype, RInstance) + assert isinstance(rtype, RInstance), rtype class_ir = rtype.class_ir method = rtype.class_ir.get_method(name) assert method is not None @@ -805,7 +805,7 @@ def visit_get_element_ptr(self, op: GetElementPtr) -> None: dest = self.reg(op) src = self.reg(op.src) # TODO: support tuple type - assert isinstance(op.src_type, RStruct) + assert isinstance(op.src_type, RStruct), op.src_type assert op.field in op.src_type.names, "Invalid field name." self.emit_line( "{} = ({})&(({} *){})->{};".format( diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index daed97cb896df..28ebcf2075fb2 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -551,8 +551,8 @@ def init_final_static( *, type_override: RType | None = None, ) -> None: - assert isinstance(lvalue, NameExpr) - assert isinstance(lvalue.node, Var) + assert isinstance(lvalue, NameExpr), lvalue + assert isinstance(lvalue.node, Var), lvalue.node if lvalue.node.final_value is None: if class_name is None: name = lvalue.name @@ -1271,7 +1271,7 @@ def add_local(self, symbol: SymbolNode, typ: RType, is_arg: bool = False) -> Reg Args: is_arg: is this a function argument """ - assert isinstance(symbol, SymbolNode) + assert isinstance(symbol, SymbolNode), symbol reg = Register( typ, remangle_redefinition_name(symbol.name), is_arg=is_arg, line=symbol.line ) @@ -1286,7 +1286,7 @@ def add_local_reg( """Like add_local, but return an assignment target instead of value.""" self.add_local(symbol, typ, is_arg) target = self.symtables[-1][symbol] - assert isinstance(target, AssignmentTargetRegister) + assert isinstance(target, AssignmentTargetRegister), target return target def add_self_to_env(self, cls: ClassIR) -> AssignmentTargetRegister: @@ -1446,7 +1446,7 @@ def get_default() -> Value: GetAttr(builder.fn_info.callable_class.self_reg, name, arg.line) ) - assert isinstance(target, AssignmentTargetRegister) + assert isinstance(target, AssignmentTargetRegister), target reg = target.register if not reg.type.error_overlap: builder.assign_if_null(target.register, get_default, arg.initializer.line) diff --git a/mypyc/irbuild/classdef.py b/mypyc/irbuild/classdef.py index 30020aabb3100..6b59750c7dec5 100644 --- a/mypyc/irbuild/classdef.py +++ b/mypyc/irbuild/classdef.py @@ -424,7 +424,7 @@ def get_type_annotation(self, stmt: AssignmentStmt) -> TypeInfo | None: type_name = stmt.rvalue.args[index] if isinstance(type_name, NameExpr) and isinstance(type_name.node, TypeInfo): lvalue = stmt.lvalues[0] - assert isinstance(lvalue, NameExpr) + assert isinstance(lvalue, NameExpr), lvalue return type_name.node return None @@ -765,7 +765,7 @@ def generate_attr_defaults_init( self_var = builder.self() for stmt in default_assignments: lvalue = stmt.lvalues[0] - assert isinstance(lvalue, NameExpr) + assert isinstance(lvalue, NameExpr), lvalue if not stmt.is_final_def and not is_constant(stmt.rvalue): builder.warning("Unsupported default attribute value", stmt.rvalue.line) @@ -871,7 +871,7 @@ def load_decorated_class(builder: IRBuilder, cdef: ClassDef, type_obj: Value) -> dec_class = type_obj for d in reversed(decorators): decorator = d.accept(builder.visitor) - assert isinstance(decorator, Value) + assert isinstance(decorator, Value), decorator dec_class = builder.py_call(decorator, [dec_class], dec_class.line) return dec_class @@ -882,7 +882,7 @@ def cache_class_attrs( """Add class attributes to be cached to the global cache.""" typ = builder.load_native_type_object(cdef.info.fullname) for lval, rtype in attrs_to_cache: - assert isinstance(lval, NameExpr) + assert isinstance(lval, NameExpr), lval rval = builder.py_get_attr(typ, lval.name, cdef.line) builder.init_final_static(lval, rval, cdef.name, type_override=rtype) diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index c4a3f5f38ce38..990c904dc4473 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -359,7 +359,7 @@ def translate_method_call(builder: IRBuilder, expr: CallExpr, callee: MemberExpr and all(kind in (ARG_POS, ARG_NAMED) for kind in expr.arg_kinds) ): # Call a method via the *class* - assert isinstance(callee.expr.node, TypeInfo) + assert isinstance(callee.expr.node, TypeInfo), callee.expr.node ir = builder.mapper.type_to_ir[callee.expr.node] return call_classmethod(builder, ir, expr, callee) elif builder.is_module_member_expr(callee): @@ -722,7 +722,7 @@ def transform_comparison_expr(builder: IRBuilder, e: ComparisonExpr) -> Value: mypy_file = builder.graph["builtins"].tree assert mypy_file is not None info = mypy_file.names["bool"].node - assert isinstance(info, TypeInfo) + assert isinstance(info, TypeInfo), info bool_type = Instance(info, []) exprs = [] for item in items: diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index c5b1d1273bef7..6066aa615e1b9 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -902,7 +902,7 @@ def begin_body(self) -> None: value = builder.add(TupleGet(self.next_tuple, 3, line)) # Coerce just in case e.g. key is itself a tuple to be unpacked. - assert isinstance(self.target_type, RTuple) + assert isinstance(self.target_type, RTuple), self.target_type key = builder.coerce(key, self.target_type.types[0], line) value = builder.coerce(value, self.target_type.types[1], line) diff --git a/mypyc/irbuild/function.py b/mypyc/irbuild/function.py index dcc5a306bcde5..90506adde672c 100644 --- a/mypyc/irbuild/function.py +++ b/mypyc/irbuild/function.py @@ -136,7 +136,7 @@ def transform_decorator(builder: IRBuilder, dec: Decorator) -> None: def transform_lambda_expr(builder: IRBuilder, expr: LambdaExpr) -> Value: typ = get_proper_type(builder.types[expr]) - assert isinstance(typ, CallableType) + assert isinstance(typ, CallableType), typ runtime_args = [] for arg, arg_type in zip(expr.arguments, typ.arg_types): @@ -269,7 +269,7 @@ def c() -> None: # add the generated main singledispatch function builder.functions.append(func_ir) # create the dispatch function - assert isinstance(fitem, FuncDef) + assert isinstance(fitem, FuncDef), fitem return gen_dispatch_func_ir(builder, fitem, fn_info.name, name, sig) return func_ir, func_reg @@ -336,8 +336,9 @@ def gen_func_ir( add_get_to_callable_class(builder, fn_info) func_reg = instantiate_callable_class(builder, fn_info) else: - assert isinstance(fn_info.fitem, FuncDef) - func_decl = builder.mapper.func_to_decl[fn_info.fitem] + fitem = fn_info.fitem + assert isinstance(fitem, FuncDef), fitem + func_decl = builder.mapper.func_to_decl[fitem] if fn_info.is_decorated or is_singledispatch_main_func: class_name = None if cdef is None else cdef.name func_decl = FuncDecl( @@ -349,13 +350,9 @@ def gen_func_ir( func_decl.is_prop_getter, func_decl.is_prop_setter, ) - func_ir = FuncIR( - func_decl, args, blocks, fn_info.fitem.line, traceback_name=fn_info.fitem.name - ) + func_ir = FuncIR(func_decl, args, blocks, fitem.line, traceback_name=fitem.name) else: - func_ir = FuncIR( - func_decl, args, blocks, fn_info.fitem.line, traceback_name=fn_info.fitem.name - ) + func_ir = FuncIR(func_decl, args, blocks, fitem.line, traceback_name=fitem.name) return (func_ir, func_reg) @@ -483,7 +480,7 @@ def load_decorated_func(builder: IRBuilder, fdef: FuncDef, orig_func_reg: Value) func_reg = orig_func_reg for d in reversed(decorators): decorator = d.accept(builder.visitor) - assert isinstance(decorator, Value) + assert isinstance(decorator, Value), decorator func_reg = builder.py_call(decorator, [func_reg], func_reg.line) return func_reg diff --git a/mypyc/irbuild/generator.py b/mypyc/irbuild/generator.py index 545a9daf956bf..c858946f33c40 100644 --- a/mypyc/irbuild/generator.py +++ b/mypyc/irbuild/generator.py @@ -158,7 +158,7 @@ def instantiate_generator_class(builder: IRBuilder) -> Value: def setup_generator_class(builder: IRBuilder) -> ClassIR: mapper = builder.mapper - assert isinstance(builder.fn_info.fitem, FuncDef) + assert isinstance(builder.fn_info.fitem, FuncDef), builder.fn_info.fitem generator_class_ir = mapper.fdef_to_generator[builder.fn_info.fitem] if builder.fn_info.can_merge_generator_and_env_classes(): builder.fn_info.env_class = generator_class_ir diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index a97bc52bc7e4d..40f1d40b478b2 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -432,7 +432,7 @@ def coerce( def coerce_int_to_fixed_width(self, src: Value, target_type: RType, line: int) -> Value: assert is_fixed_width_rtype(target_type), target_type - assert isinstance(target_type, RPrimitive) + assert isinstance(target_type, RPrimitive), target_type res = Register(target_type) @@ -538,10 +538,11 @@ def coerce_fixed_width_to_int(self, src: Value, line: int) -> Value: line, ) - assert is_fixed_width_rtype(src.type) - assert isinstance(src.type, RPrimitive) src_type = src.type + assert is_fixed_width_rtype(src_type), src_type + assert isinstance(src_type, RPrimitive), src_type + res = Register(int_rprimitive) fast, fast2, slow, end = BasicBlock(), BasicBlock(), BasicBlock(), BasicBlock() @@ -1513,7 +1514,7 @@ def compare_bytes(self, lhs: Value, rhs: Value, op: str, line: int) -> Value: def compare_tuples(self, lhs: Value, rhs: Value, op: str, line: int = -1) -> Value: """Compare two tuples item by item""" # type cast to pass mypy check - assert isinstance(lhs.type, RTuple) and isinstance(rhs.type, RTuple) + assert isinstance(lhs.type, RTuple) and isinstance(rhs.type, RTuple), (lhs.type, rhs.type) equal = True if op == "==" else False result = Register(bool_rprimitive) # tuples of different lengths diff --git a/mypyc/irbuild/match.py b/mypyc/irbuild/match.py index d7bf9e0b94def..c2ca9cfd32ff7 100644 --- a/mypyc/irbuild/match.py +++ b/mypyc/irbuild/match.py @@ -151,7 +151,7 @@ def visit_class_pattern(self, pattern: ClassPattern) -> None: return node = pattern.class_ref.node - assert isinstance(node, TypeInfo) + assert isinstance(node, TypeInfo), node match_args = extract_dunder_match_args_names(node) for i, expr in enumerate(pattern.positionals): @@ -345,7 +345,7 @@ def extract_dunder_match_args_names(info: TypeInfo) -> list[str]: ty = info.names.get("__match_args__") assert ty match_args_type = get_proper_type(ty.type) - assert isinstance(match_args_type, TupleType) + assert isinstance(match_args_type, TupleType), match_args_type match_args: list[str] = [] for item in match_args_type.items: diff --git a/mypyc/irbuild/nonlocalcontrol.py b/mypyc/irbuild/nonlocalcontrol.py index 887f6786718df..4a7136fbd18d5 100644 --- a/mypyc/irbuild/nonlocalcontrol.py +++ b/mypyc/irbuild/nonlocalcontrol.py @@ -175,7 +175,7 @@ def gen_return(self, builder: IRBuilder, value: Value, line: int) -> None: self.ret_reg = Register(builder.ret_types[-1]) # assert needed because of apparent mypy bug... it loses track of the union # and infers the type as object - assert isinstance(self.ret_reg, (Register, AssignmentTarget)) + assert isinstance(self.ret_reg, (Register, AssignmentTarget)), self.ret_reg builder.assign(self.ret_reg, value, line) builder.add(Goto(self.target)) diff --git a/mypyc/irbuild/statement.py b/mypyc/irbuild/statement.py index c8091c0313d02..4362f42b41d29 100644 --- a/mypyc/irbuild/statement.py +++ b/mypyc/irbuild/statement.py @@ -785,7 +785,7 @@ def maybe_natively_call_exit(exc_info: bool) -> Value: args = [none, none, none] if is_native: - assert isinstance(mgr_v.type, RInstance) + assert isinstance(mgr_v.type, RInstance), mgr_v.type exit_val = builder.gen_method_call( builder.read(mgr), f"__{al}exit__", diff --git a/mypyc/lower/list_ops.py b/mypyc/lower/list_ops.py index f719a9fcd23dc..63a1ecca8d114 100644 --- a/mypyc/lower/list_ops.py +++ b/mypyc/lower/list_ops.py @@ -22,7 +22,7 @@ def buf_init_item(builder: LowLevelIRBuilder, args: list[Value], line: int) -> V base = args[0] index_value = args[1] value = args[2] - assert isinstance(index_value, Integer) + assert isinstance(index_value, Integer), index_value index = index_value.numeric_value() if index == 0: ptr = base diff --git a/mypyc/transform/ir_transform.py b/mypyc/transform/ir_transform.py index 326a5baca1e74..7834fed394657 100644 --- a/mypyc/transform/ir_transform.py +++ b/mypyc/transform/ir_transform.py @@ -357,7 +357,7 @@ def visit_get_element_ptr(self, op: GetElementPtr) -> None: def visit_load_address(self, op: LoadAddress) -> None: if isinstance(op.src, LoadStatic): new = self.fix_op(op.src) - assert isinstance(new, LoadStatic) + assert isinstance(new, LoadStatic), new op.src = new def visit_keep_alive(self, op: KeepAlive) -> None: diff --git a/mypyc/transform/refcount.py b/mypyc/transform/refcount.py index b2ca03d446305..c589918986f07 100644 --- a/mypyc/transform/refcount.py +++ b/mypyc/transform/refcount.py @@ -127,7 +127,7 @@ def transform_block( # For assignments to registers that were already live, # decref the old value. if dest not in pre_borrow[key] and dest in pre_live[key]: - assert isinstance(op, Assign) + assert isinstance(op, Assign), op maybe_append_dec_ref(ops, dest, post_must_defined, key) # Strip KeepAlive. Its only purpose is to help with this transform. From 6f23e476b83eafa9dd6f63b02cc1467bbce0ca5b Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 9 Jul 2025 17:04:38 +0100 Subject: [PATCH 067/424] [mypyc] Speed up for loop over native generator (#19415) Call the generator helper method directly instead of calling `PyIter_Next` when calling a native generator from a native function. This way we can avoid raising StopIteration when the generator is exhausted. The approach is similar to what I used to speed up calls using await in #19398. Refer to that PR for a more detailed explanation. This helps mostly when a generator produces a small number of values, which is quite common. This PR improves the performance of this microbenchmark, which is a close to the ideal use case, by about 2.6x (now 5.7x faster than interpreted): ``` from typing import Iterator def foo(x: int) -> Iterator[int]: for a in range(x): yield a def bench(n: int) -> None: for i in range(n): for a in foo(1): pass from time import time bench(1000 * 1000) t0 = time() bench(50 * 1000 * 1000) print(time() - t0) ``` --- mypyc/irbuild/for_helpers.py | 76 +++++++++++++++++++++++++++++++++++- mypyc/irbuild/prepare.py | 4 +- 2 files changed, 76 insertions(+), 4 deletions(-) diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index 6066aa615e1b9..ab90a8c86b281 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -24,12 +24,15 @@ TypeAlias, ) from mypyc.ir.ops import ( + ERR_NEVER, BasicBlock, Branch, Integer, IntOp, LoadAddress, + LoadErrorValue, LoadMem, + MethodCall, RaiseStandardError, Register, TupleGet, @@ -37,6 +40,7 @@ Value, ) from mypyc.ir.rtypes import ( + RInstance, RTuple, RType, bool_rprimitive, @@ -48,10 +52,13 @@ is_short_int_rprimitive, is_str_rprimitive, is_tuple_rprimitive, + object_pointer_rprimitive, + object_rprimitive, pointer_rprimitive, short_int_rprimitive, ) from mypyc.irbuild.builder import IRBuilder +from mypyc.irbuild.prepare import GENERATOR_HELPER_NAME from mypyc.irbuild.targets import AssignmentTarget, AssignmentTargetTuple from mypyc.primitives.dict_ops import ( dict_check_size_op, @@ -62,7 +69,7 @@ dict_next_value_op, dict_value_iter_op, ) -from mypyc.primitives.exc_ops import no_err_occurred_op +from mypyc.primitives.exc_ops import no_err_occurred_op, propagate_if_error_op from mypyc.primitives.generic_ops import aiter_op, anext_op, iter_op, next_op from mypyc.primitives.list_ops import list_append_op, list_get_item_unsafe_op, new_list_set_item_op from mypyc.primitives.misc_ops import stop_async_iteration_op @@ -511,7 +518,15 @@ def make_for_loop_generator( # Default to a generic for loop. if iterable_expr_reg is None: iterable_expr_reg = builder.accept(expr) - for_obj = ForIterable(builder, index, body_block, loop_exit, line, nested) + + it = iterable_expr_reg.type + for_obj: ForNativeGenerator | ForIterable + if isinstance(it, RInstance) and it.class_ir.has_method(GENERATOR_HELPER_NAME): + # Directly call generator object methods if iterating over a native generator. + for_obj = ForNativeGenerator(builder, index, body_block, loop_exit, line, nested) + else: + # Generic implementation that works of arbitrary iterables. + for_obj = ForIterable(builder, index, body_block, loop_exit, line, nested) item_type = builder._analyze_iterable_item_type(expr) item_rtype = builder.type_to_rtype(item_type) for_obj.init(iterable_expr_reg, item_rtype) @@ -623,6 +638,63 @@ def gen_cleanup(self) -> None: self.builder.call_c(no_err_occurred_op, [], self.line) +class ForNativeGenerator(ForGenerator): + """Generate IR for a for loop over a native generator.""" + + def need_cleanup(self) -> bool: + # Create a new cleanup block for when the loop is finished. + return True + + def init(self, expr_reg: Value, target_type: RType) -> None: + # Define target to contains the generator expression. It's also the iterator. + # If we are inside a generator function, spill these into the environment class. + builder = self.builder + self.iter_target = builder.maybe_spill(expr_reg) + self.target_type = target_type + + def gen_condition(self) -> None: + builder = self.builder + line = self.line + self.return_value = Register(object_rprimitive) + err = builder.add(LoadErrorValue(object_rprimitive, undefines=True)) + builder.assign(self.return_value, err, line) + + # Call generated generator helper method, passing a PyObject ** as the final + # argument that will be used to store the return value in the return value + # register. We ignore the return value but the presence of a return value + # indicates that the generator has finished. This is faster than raising + # and catching StopIteration, which is the non-native way of doing this. + ptr = builder.add(LoadAddress(object_pointer_rprimitive, self.return_value)) + nn = builder.none_object() + helper_call = MethodCall( + builder.read(self.iter_target), GENERATOR_HELPER_NAME, [nn, nn, nn, nn, ptr], line + ) + # We provide custom handling for error values. + helper_call.error_kind = ERR_NEVER + + self.next_reg = builder.add(helper_call) + builder.add(Branch(self.next_reg, self.loop_exit, self.body_block, Branch.IS_ERROR)) + + def begin_body(self) -> None: + # Assign the value obtained from the generator helper method to the + # lvalue so that it can be referenced by code in the body of the loop. + builder = self.builder + line = self.line + # We unbox here so that iterating with tuple unpacking generates a tuple based + # unpack instead of an iterator based one. + next_reg = builder.coerce(self.next_reg, self.target_type, line) + builder.assign(builder.get_assignment_target(self.index), next_reg, line) + + def gen_step(self) -> None: + # Nothing to do here, since we get the next item as part of gen_condition(). + pass + + def gen_cleanup(self) -> None: + # If return value is NULL (it wasn't assigned to by the generator helper method), + # an exception was raised that we need to propagate. + self.builder.primitive_op(propagate_if_error_op, [self.return_value], self.line) + + class ForAsyncIterable(ForGenerator): """Generate IR for an async for loop.""" diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index c22101ac193d9..4eff90f90b7d9 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -15,7 +15,7 @@ from collections import defaultdict from collections.abc import Iterable -from typing import NamedTuple +from typing import Final, NamedTuple from mypy.build import Graph from mypy.nodes import ( @@ -71,7 +71,7 @@ from mypyc.options import CompilerOptions from mypyc.sametype import is_same_type -GENERATOR_HELPER_NAME = "__mypyc_generator_helper__" +GENERATOR_HELPER_NAME: Final = "__mypyc_generator_helper__" def build_type_map( From 40277a14740ac020373a9afc25dcd4c1e9325ab7 Mon Sep 17 00:00:00 2001 From: Randolf Scholz Date: Wed, 9 Jul 2025 18:38:21 +0200 Subject: [PATCH 068/424] More shards for `mypy-primer` (#19405) Increases the number of shards by 1 which helps with https://github.com/python/mypy/issues/19403. --- .github/workflows/mypy_primer.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mypy_primer.yml b/.github/workflows/mypy_primer.yml index ee868484751e1..1ff984247fb65 100644 --- a/.github/workflows/mypy_primer.yml +++ b/.github/workflows/mypy_primer.yml @@ -28,7 +28,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - shard-index: [0, 1, 2, 3, 4] + shard-index: [0, 1, 2, 3, 4, 5] fail-fast: false timeout-minutes: 60 steps: @@ -63,7 +63,7 @@ jobs: mypy_primer \ --repo mypy_to_test \ --new $GITHUB_SHA --old base_commit \ - --num-shards 5 --shard-index ${{ matrix.shard-index }} \ + --num-shards 6 --shard-index ${{ matrix.shard-index }} \ --debug \ --additional-flags="--debug-serialize" \ --output concise \ From 5d59e4340446a5bd187ae9388bb1cb6c706bbb0a Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Wed, 9 Jul 2025 18:52:27 +0200 Subject: [PATCH 069/424] Support `_value_` as a fallback for ellipsis Enum members (#19352) Fixes #19334. This does not affect enums with explicit values different from ellipsis and is limited to enums defined in stub files. --- mypy/plugins/enums.py | 22 +++++- mypy/stubtest.py | 2 +- mypy/types.py | 4 +- test-data/unit/check-enum.test | 122 +++++++++++++++++++++++++++---- test-data/unit/lib-stub/enum.pyi | 5 ++ 5 files changed, 139 insertions(+), 16 deletions(-) diff --git a/mypy/plugins/enums.py b/mypy/plugins/enums.py index dc58fc8110a5c..860c56c635702 100644 --- a/mypy/plugins/enums.py +++ b/mypy/plugins/enums.py @@ -17,10 +17,12 @@ from typing import TypeVar, cast import mypy.plugin # To avoid circular imports. -from mypy.nodes import TypeInfo +from mypy.checker import TypeChecker +from mypy.nodes import TypeInfo, Var from mypy.subtypes import is_equivalent from mypy.typeops import fixup_partial_type, make_simplified_union from mypy.types import ( + ELLIPSIS_TYPE_NAMES, CallableType, Instance, LiteralType, @@ -79,6 +81,19 @@ def _infer_value_type_with_auto_fallback( if proper_type is None: return None proper_type = get_proper_type(fixup_partial_type(proper_type)) + # Enums in stubs may have ... instead of actual values. If `_value_` is annotated + # (manually or inherited from IntEnum, for example), it is a more reasonable guess + # than literal ellipsis type. + if ( + _is_defined_in_stub(ctx) + and isinstance(proper_type, Instance) + and proper_type.type.fullname in ELLIPSIS_TYPE_NAMES + and isinstance(ctx.type, Instance) + ): + value_type = ctx.type.type.get("_value_") + if value_type is not None and isinstance(var := value_type.node, Var): + return var.type + return proper_type if not (isinstance(proper_type, Instance) and proper_type.type.fullname == "enum.auto"): if is_named_instance(proper_type, "enum.member") and proper_type.args: return proper_type.args[0] @@ -106,6 +121,11 @@ def _infer_value_type_with_auto_fallback( return ctx.default_attr_type +def _is_defined_in_stub(ctx: mypy.plugin.AttributeContext) -> bool: + assert isinstance(ctx.api, TypeChecker) + return isinstance(ctx.type, Instance) and ctx.api.modules[ctx.type.type.module_name].is_stub + + def _implements_new(info: TypeInfo) -> bool: """Check whether __new__ comes from enum.Enum or was implemented in a subclass. In the latter case, we must infer Any as long as mypy can't infer diff --git a/mypy/stubtest.py b/mypy/stubtest.py index 8ea9d786be220..d16e491fb1ab1 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -1149,7 +1149,7 @@ def verify_var( proper_type = mypy.types.get_proper_type(stub.type) if ( isinstance(proper_type, mypy.types.Instance) - and proper_type.type.fullname == "builtins.ellipsis" + and proper_type.type.fullname in mypy.types.ELLIPSIS_TYPE_NAMES ): should_error = False diff --git a/mypy/types.py b/mypy/types.py index 8ecd2ccf52d98..05b02acc68c00 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -181,6 +181,8 @@ # Supported @override decorator names. OVERRIDE_DECORATOR_NAMES: Final = ("typing.override", "typing_extensions.override") +ELLIPSIS_TYPE_NAMES: Final = ("builtins.ellipsis", "types.EllipsisType") + # A placeholder used for Bogus[...] parameters _dummy: Final[Any] = object() @@ -1574,7 +1576,7 @@ def is_singleton_type(self) -> bool: return ( self.type.is_enum and len(self.type.enum_members) == 1 - or self.type.fullname in {"builtins.ellipsis", "types.EllipsisType"} + or self.type.fullname in ELLIPSIS_TYPE_NAMES ) diff --git a/test-data/unit/check-enum.test b/test-data/unit/check-enum.test index 1ab8109eda75e..d034fe1a6f5f7 100644 --- a/test-data/unit/check-enum.test +++ b/test-data/unit/check-enum.test @@ -618,9 +618,8 @@ reveal_type(B.a) # N: Revealed type is "Literal[__main__.B.a]?" reveal_type(A.x.name) # N: Revealed type is "Literal['x']?" reveal_type(B.a.name) # N: Revealed type is "Literal['a']?" -# TODO: The revealed type should be 'int' here -reveal_type(A.x.value) # N: Revealed type is "Any" -reveal_type(B.a.value) # N: Revealed type is "Any" +reveal_type(A.x.value) # N: Revealed type is "builtins.int" +reveal_type(B.a.value) # N: Revealed type is "builtins.int" [builtins fixtures/enum.pyi] [case testAnonymousFunctionalEnum] @@ -755,12 +754,10 @@ class B2(IntEnum): class B3(IntEnum): x = 1 -# TODO: getting B1.x._value_ and B2.x._value_ to have type 'int' requires a typeshed change - is_x(reveal_type(B1.x.name)) # N: Revealed type is "Literal['x']" is_x(reveal_type(B1.x._name_)) # N: Revealed type is "Literal['x']" reveal_type(B1.x.value) # N: Revealed type is "builtins.int" -reveal_type(B1.x._value_) # N: Revealed type is "Any" +reveal_type(B1.x._value_) # N: Revealed type is "builtins.int" is_x(reveal_type(B2.x.name)) # N: Revealed type is "Literal['x']" is_x(reveal_type(B2.x._name_)) # N: Revealed type is "Literal['x']" reveal_type(B2.x.value) # N: Revealed type is "builtins.int" @@ -770,9 +767,6 @@ is_x(reveal_type(B3.x._name_)) # N: Revealed type is "Literal['x']" reveal_type(B3.x.value) # N: Revealed type is "Literal[1]?" reveal_type(B3.x._value_) # N: Revealed type is "Literal[1]?" -# TODO: C1.x.value and C2.x.value should also be of type 'int' -# This requires either a typeshed change or a plugin refinement - C1 = IntFlag('C1', 'x') class C2(IntFlag): x = auto() @@ -781,8 +775,8 @@ class C3(IntFlag): is_x(reveal_type(C1.x.name)) # N: Revealed type is "Literal['x']" is_x(reveal_type(C1.x._name_)) # N: Revealed type is "Literal['x']" -reveal_type(C1.x.value) # N: Revealed type is "Any" -reveal_type(C1.x._value_) # N: Revealed type is "Any" +reveal_type(C1.x.value) # N: Revealed type is "builtins.int" +reveal_type(C1.x._value_) # N: Revealed type is "builtins.int" is_x(reveal_type(C2.x.name)) # N: Revealed type is "Literal['x']" is_x(reveal_type(C2.x._name_)) # N: Revealed type is "Literal['x']" reveal_type(C2.x.value) # N: Revealed type is "builtins.int" @@ -800,8 +794,8 @@ class D3(Flag): is_x(reveal_type(D1.x.name)) # N: Revealed type is "Literal['x']" is_x(reveal_type(D1.x._name_)) # N: Revealed type is "Literal['x']" -reveal_type(D1.x.value) # N: Revealed type is "Any" -reveal_type(D1.x._value_) # N: Revealed type is "Any" +reveal_type(D1.x.value) # N: Revealed type is "builtins.int" +reveal_type(D1.x._value_) # N: Revealed type is "builtins.int" is_x(reveal_type(D2.x.name)) # N: Revealed type is "Literal['x']" is_x(reveal_type(D2.x._name_)) # N: Revealed type is "Literal['x']" reveal_type(D2.x.value) # N: Revealed type is "builtins.int" @@ -2539,3 +2533,105 @@ def check(thing: Things) -> None: return None return None # E: Statement is unreachable [builtins fixtures/enum.pyi] + +[case testSunderValueTypeEllipsis] +from foo.bar import ( + Basic, FromStub, InheritedInt, InheritedStr, InheritedFlag, + InheritedIntFlag, Wrapper +) + +reveal_type(Basic.FOO) # N: Revealed type is "Literal[foo.bar.Basic.FOO]?" +reveal_type(Basic.FOO.value) # N: Revealed type is "Literal[1]?" +reveal_type(Basic.FOO._value_) # N: Revealed type is "builtins.int" + +reveal_type(FromStub.FOO) # N: Revealed type is "Literal[foo.bar.FromStub.FOO]?" +reveal_type(FromStub.FOO.value) # N: Revealed type is "builtins.int" +reveal_type(FromStub.FOO._value_) # N: Revealed type is "builtins.int" + +reveal_type(Wrapper.Nested.FOO) # N: Revealed type is "Literal[foo.bar.Wrapper.Nested.FOO]?" +reveal_type(Wrapper.Nested.FOO.value) # N: Revealed type is "builtins.int" +reveal_type(Wrapper.Nested.FOO._value_) # N: Revealed type is "builtins.int" + +reveal_type(InheritedInt.FOO) # N: Revealed type is "Literal[foo.bar.InheritedInt.FOO]?" +reveal_type(InheritedInt.FOO.value) # N: Revealed type is "builtins.int" +reveal_type(InheritedInt.FOO._value_) # N: Revealed type is "builtins.int" + +reveal_type(InheritedStr.FOO) # N: Revealed type is "Literal[foo.bar.InheritedStr.FOO]?" +reveal_type(InheritedStr.FOO.value) # N: Revealed type is "builtins.str" +reveal_type(InheritedStr.FOO._value_) # N: Revealed type is "builtins.str" + +reveal_type(InheritedFlag.FOO) # N: Revealed type is "Literal[foo.bar.InheritedFlag.FOO]?" +reveal_type(InheritedFlag.FOO.value) # N: Revealed type is "builtins.int" +reveal_type(InheritedFlag.FOO._value_) # N: Revealed type is "builtins.int" + +reveal_type(InheritedIntFlag.FOO) # N: Revealed type is "Literal[foo.bar.InheritedIntFlag.FOO]?" +reveal_type(InheritedIntFlag.FOO.value) # N: Revealed type is "builtins.int" +reveal_type(InheritedIntFlag.FOO._value_) # N: Revealed type is "builtins.int" + +[file foo/__init__.pyi] +[file foo/bar/__init__.pyi] +from enum import Enum, IntEnum, StrEnum, Flag, IntFlag + +class Basic(Enum): + _value_: int + FOO = 1 + +class FromStub(Enum): + _value_: int + FOO = ... + +class Wrapper: + class Nested(Enum): + _value_: int + FOO = ... + +class InheritedInt(IntEnum): + FOO = ... + +class InheritedStr(StrEnum): + FOO = ... + +class InheritedFlag(Flag): + FOO = ... + +class InheritedIntFlag(IntFlag): + FOO = ... +[builtins fixtures/enum.pyi] + +[case testSunderValueTypeEllipsisNonStub] +from enum import Enum, StrEnum + +class Basic(Enum): + _value_: int + FOO = 1 + +reveal_type(Basic.FOO) # N: Revealed type is "Literal[__main__.Basic.FOO]?" +reveal_type(Basic.FOO.value) # N: Revealed type is "Literal[1]?" +reveal_type(Basic.FOO._value_) # N: Revealed type is "builtins.int" + +# TODO: this and below should produce diagnostics, Ellipsis is not assignable to int +# Now we do not check members against _value_ at all. + +class FromStub(Enum): + _value_: int + FOO = ... + +reveal_type(FromStub.FOO) # N: Revealed type is "Literal[__main__.FromStub.FOO]?" +reveal_type(FromStub.FOO.value) # N: Revealed type is "builtins.ellipsis" +reveal_type(FromStub.FOO._value_) # N: Revealed type is "builtins.int" + +class InheritedStr(StrEnum): + FOO = ... + +reveal_type(InheritedStr.FOO) # N: Revealed type is "Literal[__main__.InheritedStr.FOO]?" +reveal_type(InheritedStr.FOO.value) # N: Revealed type is "builtins.ellipsis" +reveal_type(InheritedStr.FOO._value_) # N: Revealed type is "builtins.ellipsis" + +class Wrapper: + class Nested(StrEnum): + FOO = ... + +reveal_type(Wrapper.Nested.FOO) # N: Revealed type is "Literal[__main__.Wrapper.Nested.FOO]?" +reveal_type(Wrapper.Nested.FOO.value) # N: Revealed type is "builtins.ellipsis" +reveal_type(Wrapper.Nested.FOO._value_) # N: Revealed type is "builtins.ellipsis" +[builtins fixtures/enum.pyi] diff --git a/test-data/unit/lib-stub/enum.pyi b/test-data/unit/lib-stub/enum.pyi index ccb3818b9d25d..5047f7083804a 100644 --- a/test-data/unit/lib-stub/enum.pyi +++ b/test-data/unit/lib-stub/enum.pyi @@ -29,6 +29,7 @@ class Enum(metaclass=EnumMeta): class IntEnum(int, Enum): value: int + _value_: int def __new__(cls: Type[_T], value: Union[int, _T]) -> _T: ... def unique(enumeration: _T) -> _T: pass @@ -36,6 +37,8 @@ def unique(enumeration: _T) -> _T: pass # In reality Flag and IntFlag are 3.6 only class Flag(Enum): + value: int + _value_: int def __or__(self: _T, other: Union[int, _T]) -> _T: pass @@ -49,6 +52,8 @@ class auto(IntFlag): # It is python-3.11+ only: class StrEnum(str, Enum): + _value_: str + value: str def __new__(cls: Type[_T], value: str | _T) -> _T: ... # It is python-3.11+ only: From 1091321d4ae4e0316928d5832de9eb97b60f496b Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Wed, 9 Jul 2025 18:54:28 +0200 Subject: [PATCH 070/424] Infer empty list without annotation for `__slots__` and module `__all__` (#19348) Fixes #10870, fixes #10103. This adds a fake `Iterable[str]` context when checking the following: * `__all__ = []` at top level * `__slots__ = []` at class level (also works for sets but not for dicts) Additionally, this fixes a bug with `__slots__` being mistakenly checked in other contexts (at top level or in function bodies), so e.g. the following is now accepted: ```python def foo() -> None: __slots__ = 1 ``` --- mypy/checker.py | 8 +++++++- mypy/checker_shared.py | 4 ++++ test-data/unit/check-modules.test | 30 +++++++++++++++++++++++++++++- test-data/unit/check-slots.test | 23 +++++++++++++++++++++++ 4 files changed, 63 insertions(+), 2 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index edd9519da4f89..64bcc0871c385 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -3138,7 +3138,7 @@ def check_assignment( else: self.check_getattr_method(signature, lvalue, name) - if name == "__slots__": + if name == "__slots__" and self.scope.active_class() is not None: typ = lvalue_type or self.expr_checker.accept(rvalue) self.check_slots_definition(typ, lvalue) if name == "__match_args__" and inferred is not None: @@ -3317,6 +3317,12 @@ def get_variable_type_context(self, inferred: Var, rvalue: Expression) -> Type | type_contexts.append(base_type) # Use most derived supertype as type context if available. if not type_contexts: + if inferred.name == "__slots__" and self.scope.active_class() is not None: + str_type = self.named_type("builtins.str") + return self.named_generic_type("typing.Iterable", [str_type]) + if inferred.name == "__all__" and self.scope.is_top_level(): + str_type = self.named_type("builtins.str") + return self.named_generic_type("typing.Sequence", [str_type]) return None candidate = type_contexts[0] for other in type_contexts: diff --git a/mypy/checker_shared.py b/mypy/checker_shared.py index 2ab4548edfafb..a9cbae643dca4 100644 --- a/mypy/checker_shared.py +++ b/mypy/checker_shared.py @@ -334,6 +334,10 @@ def current_self_type(self) -> Instance | TupleType | None: return fill_typevars(item) return None + def is_top_level(self) -> bool: + """Is current scope top-level (no classes or functions)?""" + return len(self.stack) == 1 + @contextmanager def push_function(self, item: FuncItem) -> Iterator[None]: self.stack.append(item) diff --git a/test-data/unit/check-modules.test b/test-data/unit/check-modules.test index 858024e7daf25..862cd8ea39055 100644 --- a/test-data/unit/check-modules.test +++ b/test-data/unit/check-modules.test @@ -423,7 +423,35 @@ import typing __all__ = [1, 2, 3] [builtins fixtures/module_all.pyi] [out] -main:2: error: Type of __all__ must be "Sequence[str]", not "list[int]" +main:2: error: List item 0 has incompatible type "int"; expected "str" +main:2: error: List item 1 has incompatible type "int"; expected "str" +main:2: error: List item 2 has incompatible type "int"; expected "str" + +[case testAllMustBeSequenceStr2] +import typing +__all__ = 1 # E: Type of __all__ must be "Sequence[str]", not "int" +reveal_type(__all__) # N: Revealed type is "builtins.int" +[builtins fixtures/module_all.pyi] + +[case testAllMustBeSequenceStr3] +import typing +__all__ = set() # E: Need type annotation for "__all__" (hint: "__all__: set[] = ...") \ + # E: Type of __all__ must be "Sequence[str]", not "set[Any]" +reveal_type(__all__) # N: Revealed type is "builtins.set[Any]" +[builtins fixtures/set.pyi] + +[case testModuleAllEmptyList] +__all__ = [] +reveal_type(__all__) # N: Revealed type is "builtins.list[builtins.str]" +[builtins fixtures/module_all.pyi] + +[case testDunderAllNotGlobal] +class A: + __all__ = 1 + +def foo() -> None: + __all__ = 1 +[builtins fixtures/module_all.pyi] [case testUnderscoreExportedValuesInImportAll] import typing diff --git a/test-data/unit/check-slots.test b/test-data/unit/check-slots.test index b7ce5e596101a..e924ac9e5f57e 100644 --- a/test-data/unit/check-slots.test +++ b/test-data/unit/check-slots.test @@ -496,6 +496,29 @@ class A: self.missing = 3 [builtins fixtures/dict.pyi] +[case testSlotsNotInClass] +# Shouldn't be triggered +__slots__ = [1, 2] +reveal_type(__slots__) # N: Revealed type is "builtins.list[builtins.int]" + +def foo() -> None: + __slots__ = 1 + reveal_type(__slots__) # N: Revealed type is "builtins.int" + +[case testSlotsEmptyList] +class A: + __slots__ = [] + reveal_type(__slots__) # N: Revealed type is "builtins.list[builtins.str]" + +reveal_type(A.__slots__) # N: Revealed type is "builtins.list[builtins.str]" + +[case testSlotsEmptySet] +class A: + __slots__ = set() + reveal_type(__slots__) # N: Revealed type is "builtins.set[builtins.str]" + +reveal_type(A.__slots__) # N: Revealed type is "builtins.set[builtins.str]" +[builtins fixtures/set.pyi] [case testSlotsWithAny] from typing import Any From 02c97661281443483a73f6ca9247c7faf63213e6 Mon Sep 17 00:00:00 2001 From: Piotr Sawicki Date: Wed, 9 Jul 2025 19:13:14 +0200 Subject: [PATCH 071/424] [mypyc] Use PyList_Check for isinstance(obj, list) (#19416) Added a primitive to translate `isinstance(obj, list)` to `PyList_Check(obj)` instead of `PyObject_IsInstance(obj, type)`, as the former is faster. Similar primitives can be added for other built-in types, which I plan to do in next PRs. ``` def f(x) -> bool: return isinstance(x, list) def bench(n: int) -> None: for x in range(n): if x % 2 == 0: f(x) else: f([x]) from time import time bench(1000) t0 = time() bench(50 * 1000 * 1000) print(time() - t0) ``` Using the above benchmark, execution time goes from ~1.4s to ~0.97s on my machine. --- mypyc/irbuild/specialize.py | 25 +++++++++++++------ mypyc/primitives/list_ops.py | 9 +++++++ mypyc/test-data/irbuild-lists.test | 32 ++++++++++-------------- mypyc/test-data/run-lists.test | 40 ++++++++++++++++++++++++------ 4 files changed, 73 insertions(+), 33 deletions(-) diff --git a/mypyc/irbuild/specialize.py b/mypyc/irbuild/specialize.py index f652449f52891..b490c2a52e574 100644 --- a/mypyc/irbuild/specialize.py +++ b/mypyc/irbuild/specialize.py @@ -14,7 +14,7 @@ from __future__ import annotations -from typing import Callable, Optional +from typing import Callable, Final, Optional from mypy.nodes import ( ARG_NAMED, @@ -89,7 +89,7 @@ dict_setdefault_spec_init_op, dict_values_op, ) -from mypyc.primitives.list_ops import new_list_set_item_op +from mypyc.primitives.list_ops import isinstance_list, new_list_set_item_op from mypyc.primitives.str_ops import ( str_encode_ascii_strict, str_encode_latin1_strict, @@ -546,6 +546,9 @@ def gen_inner_stmts() -> None: return retval +isinstance_primitives: Final = {"builtins.list": isinstance_list} + + @specialize_function("builtins.isinstance") def translate_isinstance(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: """Special case for builtins.isinstance. @@ -554,11 +557,10 @@ def translate_isinstance(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> there is no need to coerce something to a new type before checking what type it is, and the coercion could lead to bugs. """ - if ( - len(expr.args) == 2 - and expr.arg_kinds == [ARG_POS, ARG_POS] - and isinstance(expr.args[1], (RefExpr, TupleExpr)) - ): + if not (len(expr.args) == 2 and expr.arg_kinds == [ARG_POS, ARG_POS]): + return None + + if isinstance(expr.args[1], (RefExpr, TupleExpr)): builder.types[expr.args[0]] = AnyType(TypeOfAny.from_error) irs = builder.flatten_classes(expr.args[1]) @@ -569,6 +571,15 @@ def translate_isinstance(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> ) obj = builder.accept(expr.args[0], can_borrow=can_borrow) return builder.builder.isinstance_helper(obj, irs, expr.line) + + if isinstance(expr.args[1], RefExpr): + node = expr.args[1].node + if node: + desc = isinstance_primitives.get(node.fullname) + if desc: + obj = builder.accept(expr.args[0]) + return builder.primitive_op(desc, [obj], expr.line) + return None diff --git a/mypyc/primitives/list_ops.py b/mypyc/primitives/list_ops.py index d0e0af9f987fd..7442e31c91182 100644 --- a/mypyc/primitives/list_ops.py +++ b/mypyc/primitives/list_ops.py @@ -55,6 +55,15 @@ extra_int_constants=[(0, int_rprimitive)], ) +# isinstance(obj, list) +isinstance_list = function_op( + name="builtins.isinstance", + arg_types=[object_rprimitive], + return_type=bit_rprimitive, + c_function_name="PyList_Check", + error_kind=ERR_NEVER, +) + new_list_op = custom_op( arg_types=[c_pyssize_t_rprimitive], return_type=list_rprimitive, diff --git a/mypyc/test-data/irbuild-lists.test b/mypyc/test-data/irbuild-lists.test index 72caa5fad8d83..efd38870974d9 100644 --- a/mypyc/test-data/irbuild-lists.test +++ b/mypyc/test-data/irbuild-lists.test @@ -498,29 +498,23 @@ def nested_union(a: Union[List[str], List[Optional[str]]]) -> None: [out] def narrow(a): a :: union[list, int] - r0 :: object - r1 :: i32 - r2 :: bit - r3 :: bool - r4 :: list - r5 :: native_int - r6 :: short_int - r7 :: int + r0 :: bit + r1 :: list + r2 :: native_int + r3 :: short_int + r4 :: int L0: - r0 = load_address PyList_Type - r1 = PyObject_IsInstance(a, r0) - r2 = r1 >= 0 :: signed - r3 = truncate r1: i32 to builtins.bool - if r3 goto L1 else goto L2 :: bool + r0 = PyList_Check(a) + if r0 goto L1 else goto L2 :: bool L1: - r4 = borrow cast(list, a) - r5 = var_object_size r4 - r6 = r5 << 1 + r1 = borrow cast(list, a) + r2 = var_object_size r1 + r3 = r2 << 1 keep_alive a - return r6 + return r3 L2: - r7 = unbox(int, a) - return r7 + r4 = unbox(int, a) + return r4 def loop(a): a :: list r0 :: short_int diff --git a/mypyc/test-data/run-lists.test b/mypyc/test-data/run-lists.test index 85e0926027c5f..ee1bd27e6352e 100644 --- a/mypyc/test-data/run-lists.test +++ b/mypyc/test-data/run-lists.test @@ -466,7 +466,7 @@ assert not list_in_mixed(object) assert list_in_mixed(type) [case testListBuiltFromGenerator] -def test() -> None: +def test_from_gen() -> None: source_a = ["a", "b", "c"] a = list(x + "f2" for x in source_a) assert a == ["af2", "bf2", "cf2"] @@ -486,12 +486,6 @@ def test() -> None: f = list("str:" + x for x in source_str) assert f == ["str:a", "str:b", "str:c", "str:d"] -[case testNextBug] -from typing import List, Optional - -def test(x: List[int]) -> None: - res = next((i for i in x), None) - [case testListGetItemWithBorrow] from typing import List @@ -537,3 +531,35 @@ def test_sorted() -> None: assert sorted((2, 1, 3)) == res assert sorted({2, 1, 3}) == res assert sorted({2: "", 1: "", 3: ""}) == res + +[case testIsInstance] +from copysubclass import subc +def test_built_in() -> None: + assert isinstance([], list) + assert isinstance([1,2,3], list) + assert isinstance(['a','b'], list) + assert isinstance(subc(), list) + assert isinstance(subc([1,2,3]), list) + assert isinstance(subc(['a','b']), list) + + assert not isinstance({}, list) + assert not isinstance((), list) + assert not isinstance((1,2,3), list) + assert not isinstance(('a','b'), list) + assert not isinstance(1, list) + assert not isinstance('a', list) + +def test_user_defined() -> None: + from userdefinedlist import list + + assert isinstance(list(), list) + assert not isinstance([list()], list) + +[file copysubclass.py] +from typing import Any +class subc(list[Any]): + pass + +[file userdefinedlist.py] +class list: + pass From 7ea925d38657f8c2a81975f8cfc71eb8455ade61 Mon Sep 17 00:00:00 2001 From: Chainfire Date: Thu, 10 Jul 2025 11:58:57 +0200 Subject: [PATCH 072/424] [mypyc] Fix exception swallowing in async try/finally blocks with await (#19353) When a try/finally block in an async function contains an await statement in the finally block, exceptions raised in the try block are silently swallowed if a context switch occurs. This happens because mypyc stores exception information in registers that don't survive across await points. The Problem: - mypyc's transform_try_finally_stmt uses error_catch_op to save exceptions - to a register, then reraise_exception_op to restore from that register - When await causes a context switch, register values are lost - The exception information is gone, causing silent exception swallowing The Solution: - Add new transform_try_finally_stmt_async for async-aware exception handling - Use sys.exc_info() to preserve exceptions across context switches instead - of registers - Check error indicator first to handle new exceptions raised in finally - Route to async version when finally block contains await expressions Implementation Details: - transform_try_finally_stmt_async uses get_exc_info_op/restore_exc_info_op - which work with sys.exc_info() that survives context switches - Proper exception priority: new exceptions in finally replace originals - Added has_await_in_block helper to detect await expressions Test Coverage: Added comprehensive async exception handling tests: - testAsyncTryExceptFinallyAwait: 8 test cases covering various scenarios - Simple try/finally with exception and await - Exception caught but not re-raised - Exception caught and re-raised - Different exception raised in except - Try/except inside finally block - Try/finally inside finally block - Control case without await - Normal flow without exceptions - testAsyncContextManagerExceptionHandling: Verifies async with still works - Basic exception propagation - Exception in **aexit** replacing original See mypyc/mypyc#1114. --- mypyc/irbuild/statement.py | 137 ++++++++++++++++++++- mypyc/test-data/run-async.test | 211 +++++++++++++++++++++++++++++++++ 2 files changed, 346 insertions(+), 2 deletions(-) diff --git a/mypyc/irbuild/statement.py b/mypyc/irbuild/statement.py index 4362f42b41d29..eeeb40ac672fa 100644 --- a/mypyc/irbuild/statement.py +++ b/mypyc/irbuild/statement.py @@ -12,6 +12,7 @@ from collections.abc import Sequence from typing import Callable +import mypy.nodes from mypy.nodes import ( ARG_NAMED, ARG_POS, @@ -104,6 +105,7 @@ get_exc_info_op, get_exc_value_op, keep_propagating_op, + no_err_occurred_op, propagate_if_error_op, raise_exception_op, reraise_exception_op, @@ -683,7 +685,7 @@ def try_finally_resolve_control( def transform_try_finally_stmt( - builder: IRBuilder, try_body: GenFunc, finally_body: GenFunc + builder: IRBuilder, try_body: GenFunc, finally_body: GenFunc, line: int = -1 ) -> None: """Generalized try/finally handling that takes functions to gen the bodies. @@ -719,6 +721,118 @@ def transform_try_finally_stmt( builder.activate_block(out_block) +def transform_try_finally_stmt_async( + builder: IRBuilder, try_body: GenFunc, finally_body: GenFunc, line: int = -1 +) -> None: + """Async-aware try/finally handling for when finally contains await. + + This version uses a modified approach that preserves exceptions across await.""" + + # We need to handle returns properly, so we'll use TryFinallyNonlocalControl + # to track return values, similar to the regular try/finally implementation + + err_handler, main_entry, return_entry, finally_entry = ( + BasicBlock(), + BasicBlock(), + BasicBlock(), + BasicBlock(), + ) + + # Track if we're returning from the try block + control = TryFinallyNonlocalControl(return_entry) + builder.builder.push_error_handler(err_handler) + builder.nonlocal_control.append(control) + builder.goto_and_activate(BasicBlock()) + try_body() + builder.goto(main_entry) + builder.nonlocal_control.pop() + builder.builder.pop_error_handler() + ret_reg = control.ret_reg + + # Normal case - no exception or return + builder.activate_block(main_entry) + builder.goto(finally_entry) + + # Return case + builder.activate_block(return_entry) + builder.goto(finally_entry) + + # Exception case - need to catch to clear the error indicator + builder.activate_block(err_handler) + # Catch the error to clear Python's error indicator + builder.call_c(error_catch_op, [], line) + # We're not going to use old_exc since it won't survive await + # The exception is now in sys.exc_info() + builder.goto(finally_entry) + + # Finally block + builder.activate_block(finally_entry) + + # Execute finally body + finally_body() + + # After finally, we need to handle exceptions carefully: + # 1. If finally raised a new exception, it's in the error indicator - let it propagate + # 2. If finally didn't raise, check if we need to reraise the original from sys.exc_info() + # 3. If there was a return, return that value + # 4. Otherwise, normal exit + + # First, check if there's a current exception in the error indicator + # (this would be from the finally block) + no_current_exc = builder.call_c(no_err_occurred_op, [], line) + finally_raised = BasicBlock() + check_original = BasicBlock() + builder.add(Branch(no_current_exc, check_original, finally_raised, Branch.BOOL)) + + # Finally raised an exception - let it propagate naturally + builder.activate_block(finally_raised) + builder.call_c(keep_propagating_op, [], NO_TRACEBACK_LINE_NO) + builder.add(Unreachable()) + + # No exception from finally, check if we need to handle return or original exception + builder.activate_block(check_original) + + # Check if we have a return value + if ret_reg: + return_block, check_old_exc = BasicBlock(), BasicBlock() + builder.add(Branch(builder.read(ret_reg), check_old_exc, return_block, Branch.IS_ERROR)) + + builder.activate_block(return_block) + builder.nonlocal_control[-1].gen_return(builder, builder.read(ret_reg), -1) + + builder.activate_block(check_old_exc) + + # Check if we need to reraise the original exception from sys.exc_info + exc_info = builder.call_c(get_exc_info_op, [], line) + exc_type = builder.add(TupleGet(exc_info, 0, line)) + + # Check if exc_type is None + none_obj = builder.none_object() + has_exc = builder.binary_op(exc_type, none_obj, "is not", line) + + reraise_block, exit_block = BasicBlock(), BasicBlock() + builder.add(Branch(has_exc, reraise_block, exit_block, Branch.BOOL)) + + # Reraise the original exception + builder.activate_block(reraise_block) + builder.call_c(reraise_exception_op, [], NO_TRACEBACK_LINE_NO) + builder.add(Unreachable()) + + # Normal exit + builder.activate_block(exit_block) + + +# A simple visitor to detect await expressions +class AwaitDetector(mypy.traverser.TraverserVisitor): + def __init__(self) -> None: + super().__init__() + self.has_await = False + + def visit_await_expr(self, o: mypy.nodes.AwaitExpr) -> None: + self.has_await = True + super().visit_await_expr(o) + + def transform_try_stmt(builder: IRBuilder, t: TryStmt) -> None: # Our compilation strategy for try/except/else/finally is to # treat try/except/else and try/finally as separate language @@ -727,6 +841,17 @@ def transform_try_stmt(builder: IRBuilder, t: TryStmt) -> None: # body of a try/finally block. if t.is_star: builder.error("Exception groups and except* cannot be compiled yet", t.line) + + # Check if we're in an async function with a finally block that contains await + use_async_version = False + if t.finally_body and builder.fn_info.is_coroutine: + detector = AwaitDetector() + t.finally_body.accept(detector) + + if detector.has_await: + # Use the async version that handles exceptions correctly + use_async_version = True + if t.finally_body: def transform_try_body() -> None: @@ -737,7 +862,14 @@ def transform_try_body() -> None: body = t.finally_body - transform_try_finally_stmt(builder, transform_try_body, lambda: builder.accept(body)) + if use_async_version: + transform_try_finally_stmt_async( + builder, transform_try_body, lambda: builder.accept(body), t.line + ) + else: + transform_try_finally_stmt( + builder, transform_try_body, lambda: builder.accept(body), t.line + ) else: transform_try_except_stmt(builder, t) @@ -828,6 +960,7 @@ def finally_body() -> None: builder, lambda: transform_try_except(builder, try_body, [(None, None, except_body)], None, line), finally_body, + line, ) diff --git a/mypyc/test-data/run-async.test b/mypyc/test-data/run-async.test index b8c4c22daf719..f1ec7e8f85e08 100644 --- a/mypyc/test-data/run-async.test +++ b/mypyc/test-data/run-async.test @@ -1037,3 +1037,214 @@ test_async_with_mixed_return() [file asyncio/__init__.pyi] def run(x: object) -> object: ... + +[case testAsyncTryExceptFinallyAwait] +import asyncio +from testutil import assertRaises + +class TestError(Exception): + pass + +# Test 0: Simplest case - just try/finally with raise and await +async def simple_try_finally_await() -> None: + try: + raise ValueError("simple error") + finally: + await asyncio.sleep(0) + +# Test 1: Raise inside try, catch in except, don't re-raise +async def async_try_except_no_reraise() -> int: + try: + raise ValueError("test error") + return 1 # Never reached + except ValueError: + return 2 # Should return this + finally: + await asyncio.sleep(0) + return 3 # Should not reach this + +# Test 2: Raise inside try, catch in except, re-raise +async def async_try_except_reraise() -> int: + try: + raise ValueError("test error") + return 1 # Never reached + except ValueError: + raise # Re-raise the exception + finally: + await asyncio.sleep(0) + return 2 # Should not reach this + +# Test 3: Raise inside try, catch in except, raise different error +async def async_try_except_raise_different() -> int: + try: + raise ValueError("original error") + return 1 # Never reached + except ValueError: + raise RuntimeError("different error") + finally: + await asyncio.sleep(0) + return 2 # Should not reach this + +# Test 4: Another try/except block inside finally +async def async_try_except_inside_finally() -> int: + try: + raise ValueError("outer error") + return 1 # Never reached + finally: + await asyncio.sleep(0) + try: + raise RuntimeError("inner error") + except RuntimeError: + pass # Catch inner error + return 2 # What happens after finally with inner exception handled? + +# Test 5: Another try/finally block inside finally +async def async_try_finally_inside_finally() -> int: + try: + raise ValueError("outer error") + return 1 # Never reached + finally: + await asyncio.sleep(0) + try: + raise RuntimeError("inner error") + finally: + await asyncio.sleep(0) + return 2 # Should not reach this + +# Control case: No await in finally - should work correctly +async def async_exception_no_await_in_finally() -> None: + """Control case: This works correctly - exception propagates""" + try: + raise TestError("This exception will propagate!") + finally: + pass # No await here + +# Test function with no exception to check normal flow +async def async_no_exception_with_await_in_finally() -> int: + try: + return 1 # Normal return + finally: + await asyncio.sleep(0) + return 2 # Should not reach this + +def test_async_try_except_finally_await() -> None: + # Test 0: Simplest case - just try/finally with exception + # Expected: ValueError propagates + with assertRaises(ValueError): + asyncio.run(simple_try_finally_await()) + + # Test 1: Exception caught, not re-raised + # Expected: return 2 (from except block) + result = asyncio.run(async_try_except_no_reraise()) + assert result == 2, f"Expected 2, got {result}" + + # Test 2: Exception caught and re-raised + # Expected: ValueError propagates + with assertRaises(ValueError): + asyncio.run(async_try_except_reraise()) + + # Test 3: Exception caught, different exception raised + # Expected: RuntimeError propagates + with assertRaises(RuntimeError): + asyncio.run(async_try_except_raise_different()) + + # Test 4: Try/except inside finally + # Expected: ValueError propagates (outer exception) + with assertRaises(ValueError): + asyncio.run(async_try_except_inside_finally()) + + # Test 5: Try/finally inside finally + # Expected: RuntimeError propagates (inner error) + with assertRaises(RuntimeError): + asyncio.run(async_try_finally_inside_finally()) + + # Control case: No await in finally (should work correctly) + with assertRaises(TestError): + asyncio.run(async_exception_no_await_in_finally()) + + # Test normal flow (no exception) + # Expected: return 1 + result = asyncio.run(async_no_exception_with_await_in_finally()) + assert result == 1, f"Expected 1, got {result}" + +[file asyncio/__init__.pyi] +async def sleep(t: float) -> None: ... +def run(x: object) -> object: ... + +[case testAsyncContextManagerExceptionHandling] +import asyncio +from typing import Optional, Type +from testutil import assertRaises + +# Test 1: Basic async context manager that doesn't suppress exceptions +class AsyncContextManager: + async def __aenter__(self) -> 'AsyncContextManager': + return self + + async def __aexit__(self, exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: object) -> None: + # This await in __aexit__ is like await in finally + await asyncio.sleep(0) + # Don't suppress the exception (return None/False) + +async def func_with_async_context_manager() -> str: + async with AsyncContextManager(): + raise ValueError("Exception inside async with") + return "should not reach" # Never reached + return "should not reach either" # Never reached + +async def test_basic_exception() -> str: + try: + await func_with_async_context_manager() + return "func_a returned normally - bug!" + except ValueError: + return "caught ValueError - correct!" + except Exception as e: + return f"caught different exception: {type(e).__name__}" + +# Test 2: Async context manager that raises a different exception in __aexit__ +class AsyncContextManagerRaisesInExit: + async def __aenter__(self) -> 'AsyncContextManagerRaisesInExit': + return self + + async def __aexit__(self, exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: object) -> None: + # This await in __aexit__ is like await in finally + await asyncio.sleep(0) + # Raise a different exception - this should replace the original exception + raise RuntimeError("Exception in __aexit__") + +async def func_with_raising_context_manager() -> str: + async with AsyncContextManagerRaisesInExit(): + raise ValueError("Original exception") + return "should not reach" # Never reached + return "should not reach either" # Never reached + +async def test_exception_in_aexit() -> str: + try: + await func_with_raising_context_manager() + return "func returned normally - unexpected!" + except RuntimeError: + return "caught RuntimeError - correct!" + except ValueError: + return "caught ValueError - original exception not replaced!" + except Exception as e: + return f"caught different exception: {type(e).__name__}" + +def test_async_context_manager_exception_handling() -> None: + # Test 1: Basic exception propagation + result = asyncio.run(test_basic_exception()) + # Expected: "caught ValueError - correct!" + assert result == "caught ValueError - correct!", f"Expected exception to propagate, got: {result}" + + # Test 2: Exception raised in __aexit__ replaces original exception + result = asyncio.run(test_exception_in_aexit()) + # Expected: "caught RuntimeError - correct!" + # (The RuntimeError from __aexit__ should replace the ValueError) + assert result == "caught RuntimeError - correct!", f"Expected RuntimeError from __aexit__, got: {result}" + +[file asyncio/__init__.pyi] +async def sleep(t: float) -> None: ... +def run(x: object) -> object: ... From a794ae3eebfb1c36efd1fa5207cf072a1725b8d0 Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Thu, 10 Jul 2025 12:10:06 +0200 Subject: [PATCH 073/424] Move `is_defined_in_stub` to shared checker API to break import cycle (#19417) Follow-up after #19352 as requested by @JukkaL --- mypy/checker.py | 3 +++ mypy/checker_shared.py | 4 ++++ mypy/plugins/enums.py | 6 +++--- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 64bcc0871c385..159569849061c 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -7889,6 +7889,9 @@ def has_valid_attribute(self, typ: Type, name: str) -> bool: def get_expression_type(self, node: Expression, type_context: Type | None = None) -> Type: return self.expr_checker.accept(node, type_context=type_context) + def is_defined_in_stub(self, typ: Instance, /) -> bool: + return self.modules[typ.type.module_name].is_stub + def check_deprecated(self, node: Node | None, context: Context) -> None: """Warn if deprecated and not directly imported with a `from` statement.""" if isinstance(node, Decorator): diff --git a/mypy/checker_shared.py b/mypy/checker_shared.py index a9cbae643dca4..65cec41d52023 100644 --- a/mypy/checker_shared.py +++ b/mypy/checker_shared.py @@ -277,6 +277,10 @@ def checking_await_set(self) -> Iterator[None]: def get_precise_awaitable_type(self, typ: Type, local_errors: ErrorWatcher) -> Type | None: raise NotImplementedError + @abstractmethod + def is_defined_in_stub(self, typ: Instance, /) -> bool: + raise NotImplementedError + class CheckerScope: # We keep two stacks combined, to maintain the relative order diff --git a/mypy/plugins/enums.py b/mypy/plugins/enums.py index 860c56c635702..d21b21fb39f84 100644 --- a/mypy/plugins/enums.py +++ b/mypy/plugins/enums.py @@ -17,7 +17,7 @@ from typing import TypeVar, cast import mypy.plugin # To avoid circular imports. -from mypy.checker import TypeChecker +from mypy.checker_shared import TypeCheckerSharedApi from mypy.nodes import TypeInfo, Var from mypy.subtypes import is_equivalent from mypy.typeops import fixup_partial_type, make_simplified_union @@ -122,8 +122,8 @@ def _infer_value_type_with_auto_fallback( def _is_defined_in_stub(ctx: mypy.plugin.AttributeContext) -> bool: - assert isinstance(ctx.api, TypeChecker) - return isinstance(ctx.type, Instance) and ctx.api.modules[ctx.type.type.module_name].is_stub + assert isinstance(ctx.api, TypeCheckerSharedApi) + return isinstance(ctx.type, Instance) and ctx.api.is_defined_in_stub(ctx.type) def _implements_new(info: TypeInfo) -> bool: From 77ce6464c52db1201fd0f17c5b34952da8121ca5 Mon Sep 17 00:00:00 2001 From: Piotr Sawicki Date: Fri, 11 Jul 2025 11:35:31 +0200 Subject: [PATCH 074/424] [mypyc] Fail run test if default driver does not find test cases (#19420) If a `run-` test doesn't define a custom `driver.py`, the default one looks for functions prefixed `test_` and calls them. If the prefix is missing or misspelled, or there is some other issue that causes the test to have no test cases, the test succeeds without running the cases. To try to prevent this, the default driver will now fail if it doesn't find any test cases. Existing tests are changed to conform to this requirement in this PR. For most tests that meant simply moving the statements from top level to a function, but some were revealed to have been broken and weren't actually run, for example because `[typing fixtures/typing-full.pyi]` was at the top of the test instead of at the bottom, which made the test setup code ignore all of the test case. --- mypyc/test-data/driver/driver.py | 4 ++ mypyc/test-data/fixtures/ir.py | 7 ++- mypyc/test-data/run-bools.test | 5 +- mypyc/test-data/run-bytes.test | 2 +- mypyc/test-data/run-classes.test | 79 +++++++++++++----------- mypyc/test-data/run-dunders-special.test | 3 +- mypyc/test-data/run-functions.test | 17 ++--- mypyc/test-data/run-generators.test | 16 ++++- mypyc/test-data/run-generics.test | 32 +++++----- mypyc/test-data/run-imports.test | 32 +++++----- mypyc/test-data/run-misc.test | 53 +++++++++------- mypyc/test-data/run-python38.test | 11 ++-- mypyc/test-data/run-singledispatch.test | 22 ++++--- mypyc/test-data/run-strings.test | 2 +- 14 files changed, 167 insertions(+), 118 deletions(-) diff --git a/mypyc/test-data/driver/driver.py b/mypyc/test-data/driver/driver.py index c9d179224a30d..1ec1c48dfb751 100644 --- a/mypyc/test-data/driver/driver.py +++ b/mypyc/test-data/driver/driver.py @@ -11,10 +11,12 @@ import native failures = [] +tests_run = 0 for name in dir(native): if name.startswith('test_'): test_func = getattr(native, name) + tests_run += 1 try: test_func() except Exception as e: @@ -46,3 +48,5 @@ def extract_line(tb): print(f'<< {failures[-1][0]} >>') sys.stdout.flush() raise failures[-1][1][1] + +assert tests_run > 0, 'Default test driver did not find any functions prefixed "test_" to run.' diff --git a/mypyc/test-data/fixtures/ir.py b/mypyc/test-data/fixtures/ir.py index 532cbbc06177a..3776a3dcc79a1 100644 --- a/mypyc/test-data/fixtures/ir.py +++ b/mypyc/test-data/fixtures/ir.py @@ -350,7 +350,12 @@ class GeneratorExit(BaseException): pass def any(i: Iterable[_T]) -> bool: pass def all(i: Iterable[_T]) -> bool: pass -def sum(i: Iterable[_T]) -> int: pass +@overload +def sum(i: Iterable[bool]) -> int: pass +@overload +def sum(i: Iterable[_T]) -> _T: pass +@overload +def sum(i: Iterable[_T], start: _T) -> _T: pass def reversed(object: Sequence[_T]) -> Iterator[_T]: ... def id(o: object) -> int: pass # This type is obviously wrong but the test stubs don't have Sized anymore diff --git a/mypyc/test-data/run-bools.test b/mypyc/test-data/run-bools.test index a0b8ea31ebc0d..3409665bfb377 100644 --- a/mypyc/test-data/run-bools.test +++ b/mypyc/test-data/run-bools.test @@ -223,7 +223,8 @@ def test_mixed_comparisons_i64() -> None: assert gt_mixed_i64(n, x) == (n > int(x)) [case testBoolMixInt] -y = False -print((y or 0) and True) +def test_mix() -> None: + y = False + print((y or 0) and True) [out] 0 diff --git a/mypyc/test-data/run-bytes.test b/mypyc/test-data/run-bytes.test index fa63c46a67983..bee6b6fe9f76a 100644 --- a/mypyc/test-data/run-bytes.test +++ b/mypyc/test-data/run-bytes.test @@ -277,7 +277,6 @@ class bytes_subclass(bytes): return b'spook' [case testBytesFormatting] -[typing fixtures/typing-full.pyi] from testutil import assertRaises # https://www.python.org/dev/peps/pep-0461/ @@ -314,6 +313,7 @@ def test_bytes_formatting_2() -> None: aa = b'\xe4\xbd\xa0\xe5\xa5\xbd%b' % b'\xe4\xbd\xa0\xe5\xa5\xbd' assert aa == b'\xe4\xbd\xa0\xe5\xa5\xbd\xe4\xbd\xa0\xe5\xa5\xbd' assert aa.decode() == '你好你好' +[typing fixtures/typing-full.pyi] class A: diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index fd486980ef16a..54f5343bc7bb8 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -1278,9 +1278,10 @@ class Bar(Foo): def f(self, *args: int, **kwargs: int) -> None: print("stuff", args, kwargs) -z: Foo = Bar() -z.f(1, z=50) -z.f() +def test_override() -> None: + z: Foo = Bar() + z.f(1, z=50) + z.f() [out] stuff (1,) {'z': 50} @@ -1300,18 +1301,19 @@ class Foo: def baz_f(self: Any, *args: int, **kwargs: int) -> None: print("Baz", args, kwargs) -# Make an "interpreted" subtype of Foo -type2: Any = type -Bar = type2('Bar', (Foo,), {}) -Baz = type2('Baz', (Foo,), {'f': baz_f}) +def test_override() -> None: + # Make an "interpreted" subtype of Foo + type2: Any = type + Bar = type2('Bar', (Foo,), {}) + Baz = type2('Baz', (Foo,), {'f': baz_f}) -y: Foo = Bar() -y.f(1, z=2) -y.f() + y: Foo = Bar() + y.f(1, z=2) + y.f() -z: Foo = Baz() -z.f(1, z=2) -z.f() + z: Foo = Baz() + z.f(1, z=2) + z.f() [out] Foo 1 2 @@ -1330,9 +1332,10 @@ class Bar(Foo): def f(self, x: Optional[int]=None) -> None: print(x) -z: Foo = Bar() -z.f(1) -z.f() +def test_override() -> None: + z: Foo = Bar() + z.f(1) + z.f() [out] 1 @@ -1349,10 +1352,11 @@ class Bar(Foo): def f(self, *args: int, **kwargs: int) -> None: print("Bar", args, kwargs) -z: Foo = Bar() -z.f(1, z=2) -z.f(1, 2, 3) -# z.f(x=5) # Not tested because we (knowingly) do the wrong thing and pass it as positional +def test_override() -> None: + z: Foo = Bar() + z.f(1, z=2) + z.f(1, 2, 3) + # z.f(x=5) # Not tested because we (knowingly) do the wrong thing and pass it as positional [out] Bar (1,) {'z': 2} @@ -1370,10 +1374,11 @@ class Bar(Foo): def f(self, x: int = 10, *args: int, **kwargs: int) -> None: print("Bar", x, args, kwargs) -z: Foo = Bar() -z.f(1, z=2) -z.f(1, 2, 3) -z.f() +def test_override() -> None: + z: Foo = Bar() + z.f(1, z=2) + z.f(1, 2, 3) + z.f() [out] Bar 1 () {'z': 2} @@ -1397,18 +1402,19 @@ class Foo: def baz_f(self, a: int=30, y: int=50) -> None: print("Baz", a, y) -# Make an "interpreted" subtype of Foo -type2: Any = type -Baz = type2('Baz', (Foo,), {'f': baz_f}) +def test_override() -> None: + # Make an "interpreted" subtype of Foo + type2: Any = type + Baz = type2('Baz', (Foo,), {'f': baz_f}) -z: Foo = Baz() -z.f() -z.f(y=1) -z.f(1, 2) -# Not tested because we don't (and probably won't) match cpython here -# from testutil import assertRaises -# with assertRaises(TypeError): -# z.f(x=7) + z: Foo = Baz() + z.f() + z.f(y=1) + z.f(1, 2) + # Not tested because we don't (and probably won't) match cpython here + # from testutil import assertRaises + # with assertRaises(TypeError): + # z.f(x=7) [out] Baz 30 50 @@ -2591,7 +2597,8 @@ class Base: class Derived(Base): pass -assert Derived()() == 1 +def test_inherited() -> None: + assert Derived()() == 1 [case testClassWithFinalAttribute] from typing import Final diff --git a/mypyc/test-data/run-dunders-special.test b/mypyc/test-data/run-dunders-special.test index 30c618374f887..2672434e10ef2 100644 --- a/mypyc/test-data/run-dunders-special.test +++ b/mypyc/test-data/run-dunders-special.test @@ -6,4 +6,5 @@ class UsesNotImplemented: def __eq__(self, b: object) -> bool: return NotImplemented -assert UsesNotImplemented() != object() +def test_not_implemented() -> None: + assert UsesNotImplemented() != object() diff --git a/mypyc/test-data/run-functions.test b/mypyc/test-data/run-functions.test index 46f343fa37982..3d7f1f3cc7475 100644 --- a/mypyc/test-data/run-functions.test +++ b/mypyc/test-data/run-functions.test @@ -1235,13 +1235,11 @@ from contextlib import contextmanager def f() -> Iterator[None]: yield -def g() -> None: +def test_special_case() -> None: a = [''] with f(): a.pop() -g() - [case testUnpackKwargsCompiled] from typing import TypedDict from typing_extensions import Unpack @@ -1253,8 +1251,9 @@ class Person(TypedDict): def foo(**kwargs: Unpack[Person]) -> None: print(kwargs["name"]) -# This is not really supported yet, just test that we behave reasonably. -foo(name='Jennifer', age=38) +def test_unpack() -> None: + # This is not really supported yet, just test that we behave reasonably. + foo(name='Jennifer', age=38) [typing fixtures/typing-full.pyi] [out] Jennifer @@ -1269,8 +1268,9 @@ def foo() -> None: print(inner.__dict__) # type: ignore[attr-defined] print(inner.x) # type: ignore[attr-defined] -if sys.version_info >= (3, 12): # type: ignore - foo() +def test_nested() -> None: + if sys.version_info >= (3, 12): # type: ignore + foo() [out] [out version>=3.12] {} @@ -1285,7 +1285,8 @@ def bar() -> None: functools.update_wrapper(inner, bar) # type: ignore print(inner.__dict__) # type: ignore -bar() +def test_update() -> None: + bar() [typing fixtures/typing-full.pyi] [out] {'__module__': 'native', '__name__': 'bar', '__qualname__': 'bar', '__doc__': None, '__wrapped__': } diff --git a/mypyc/test-data/run-generators.test b/mypyc/test-data/run-generators.test index a43aff27dd45e..3b4581f849e9f 100644 --- a/mypyc/test-data/run-generators.test +++ b/mypyc/test-data/run-generators.test @@ -617,16 +617,22 @@ else: from typing import Iterator class Foo: - flag: bool + flag = False class C: - foo: Foo + foo = Foo() def genf(self) -> Iterator[None]: self.foo.flag = True yield self.foo.flag = False +def test_near_yield() -> None: + c = C() + for x in c.genf(): + pass + assert c.foo.flag == False + [case testGeneratorEarlyReturnWithBorrows] from typing import Iterator class Bar: @@ -639,6 +645,12 @@ class Foo: return yield 0 +def test_early_return() -> None: + foo = Foo() + for x in foo.f(): + pass + assert foo.bar.bar == 1 + [case testBorrowingInGeneratorInTupleAssignment] from typing import Iterator diff --git a/mypyc/test-data/run-generics.test b/mypyc/test-data/run-generics.test index bc78a3b8ab864..55e5adbbb4f9a 100644 --- a/mypyc/test-data/run-generics.test +++ b/mypyc/test-data/run-generics.test @@ -27,22 +27,23 @@ def fn_typeddict(t: T) -> None: print([x for x in t.keys()]) print({k: v for k, v in t.items()}) -fn_mapping({}) -print("=====") -fn_mapping({"a": 1, "b": 2}) -print("=====") +def test_mapping() -> None: + fn_mapping({}) + print("=====") + fn_mapping({"a": 1, "b": 2}) + print("=====") -fn_union({"a": 1, "b": 2}) -print("=====") -fn_union({"a": "1", "b": "2"}) -print("=====") + fn_union({"a": 1, "b": 2}) + print("=====") + fn_union({"a": "1", "b": "2"}) + print("=====") -orig: Union[Dict[str, int], Dict[str, str]] = {"a": 1, "b": 2} -fn_union(orig) -print("=====") + orig: Union[Dict[str, int], Dict[str, str]] = {"a": 1, "b": 2} + fn_union(orig) + print("=====") -td: TD = {"foo": 1} -fn_typeddict(td) + td: TD = {"foo": 1} + fn_typeddict(td) [typing fixtures/typing-full.pyi] [out] \[] @@ -96,8 +97,9 @@ def deco(func: Callable[P, int]) -> Callable[P, int]: def f(x: int, y: str) -> int: return x -assert f(1, 'a') == 1 -assert f(2, y='b') == 2 +def test_usable() -> None: + assert f(1, 'a') == 1 + assert f(2, y='b') == 2 [out] \[1, 'a'] {} diff --git a/mypyc/test-data/run-imports.test b/mypyc/test-data/run-imports.test index c5839d57820e1..ce83a882e2ded 100644 --- a/mypyc/test-data/run-imports.test +++ b/mypyc/test-data/run-imports.test @@ -212,9 +212,10 @@ import shared def do_import() -> None: import a -assert shared.counter == 0 -do_import() -assert shared.counter == 1 +def test_lazy() -> None: + assert shared.counter == 0 + do_import() + assert shared.counter == 1 [file a.py] import shared @@ -224,9 +225,10 @@ shared.counter += 1 counter = 0 [case testDelayedImport] -import a -print("inbetween") -import b +def test_delayed() -> None: + import a + print("inbetween") + import b [file a.py] print("first") @@ -240,19 +242,21 @@ inbetween last [case testImportErrorLineNumber] -try: - import enum - import dataclasses, missing # type: ignore[import] -except ImportError as e: - line = e.__traceback__.tb_lineno # type: ignore[attr-defined] - assert line == 3, f"traceback's line number is {line}, expected 3" +def test_error() -> None: + try: + import enum + import dataclasses, missing # type: ignore[import] + except ImportError as e: + line = e.__traceback__.tb_lineno # type: ignore[attr-defined] + assert line == 4, f"traceback's line number is {line}, expected 4" [case testImportGroupIsolation] def func() -> None: import second -import first -func() +def test_isolation() -> None: + import first + func() [file first.py] print("first") diff --git a/mypyc/test-data/run-misc.test b/mypyc/test-data/run-misc.test index f6a1c744cade3..129946a4c3300 100644 --- a/mypyc/test-data/run-misc.test +++ b/mypyc/test-data/run-misc.test @@ -109,10 +109,11 @@ def gen(b: bool) -> Generator[Any, None, None]: y = None yield y -assert f(False) == ((1, None), (None, 1)) -assert f(True) == ((None, 1), (1, None)) -assert next(gen(False)) is None -assert next(gen(True)) == 1 +def test_inferred() -> None: + assert f(False) == ((1, None), (None, 1)) + assert f(True) == ((None, 1), (1, None)) + assert next(gen(False)) is None + assert next(gen(True)) == 1 [case testWith] from typing import Any @@ -829,23 +830,23 @@ assert call_any_nested([[1, 1, 1], [1, 1], []]) == 1 assert call_any_nested([[1, 1, 1], [0, 1], []]) == 0 [case testSum] -[typing fixtures/typing-full.pyi] -from typing import Any, List +from typing import List +empty: List[int] = [] def test_sum_of_numbers() -> None: assert sum(x for x in [1, 2, 3]) == 6 - assert sum(x for x in [0.0, 1.2, 2]) == 6.2 + assert sum(x for x in [0.0, 1.2, 2]) == 3.2 assert sum(x for x in [1, 1j]) == 1 + 1j def test_sum_callables() -> None: - assert sum((lambda x: x == 0)(x) for x in []) == 0 + assert sum((lambda x: x == 0)(x) for x in empty) == 0 assert sum((lambda x: x == 0)(x) for x in [0]) == 1 assert sum((lambda x: x == 0)(x) for x in [0, 0, 0]) == 3 assert sum((lambda x: x == 0)(x) for x in [0, 1, 0]) == 2 assert sum((lambda x: x % 2 == 0)(x) for x in range(2**10)) == 2**9 def test_sum_comparisons() -> None: - assert sum(x == 0 for x in []) == 0 + assert sum(x == 0 for x in empty) == 0 assert sum(x == 0 for x in [0]) == 1 assert sum(x == 0 for x in [0, 0, 0]) == 3 assert sum(x == 0 for x in [0, 1, 0]) == 2 @@ -865,13 +866,14 @@ def test_sum_misc() -> None: def test_sum_start_given() -> None: a = 1 assert sum((x == 0 for x in [0, 1]), a) == 2 - assert sum(((lambda x: x == 0)(x) for x in []), 1) == 1 + assert sum(((lambda x: x == 0)(x) for x in empty), 1) == 1 assert sum(((lambda x: x == 0)(x) for x in [0]), 1) == 2 assert sum(((lambda x: x == 0)(x) for x in [0, 0, 0]), 1) == 4 assert sum(((lambda x: x == 0)(x) for x in [0, 1, 0]), 1) == 3 assert sum(((lambda x: x % 2 == 0)(x) for x in range(2**10)), 1) == 2**9 + 1 assert sum((x for x in [1, 1j]), 2j) == 1 + 3j assert sum((c == 'd' for c in 'abcdd'), 1) == 3 +[typing fixtures/typing-full.pyi] [case testNoneStuff] from typing import Optional @@ -1090,19 +1092,20 @@ def test_complex() -> None: from typing import cast import sys -A = sys.platform == 'x' and foobar -B = sys.platform == 'x' and sys.foobar -C = sys.platform == 'x' and f(a, -b, 'y') > [c + e, g(y=2)] -C = sys.platform == 'x' and cast(a, b[c]) -C = sys.platform == 'x' and (lambda x: y + x) -C = sys.platform == 'x' and (x for y in z) -C = sys.platform == 'x' and [x for y in z] -C = sys.platform == 'x' and {x: x for y in z} -C = sys.platform == 'x' and {x for y in z} - -assert not A -assert not B -assert not C +def test_unreachable() -> None: + A = sys.platform == 'x' and foobar + B = sys.platform == 'x' and sys.foobar + C = sys.platform == 'x' and f(a, -b, 'y') > [c + e, g(y=2)] + C = sys.platform == 'x' and cast(a, b[c]) + C = sys.platform == 'x' and (lambda x: y + x) + C = sys.platform == 'x' and (x for y in z) + C = sys.platform == 'x' and [x for y in z] + C = sys.platform == 'x' and {x: x for y in z} + C = sys.platform == 'x' and {x for y in z} + + assert not A + assert not B + assert not C [case testDoesntSegfaultWhenTopLevelFails] # make the initial import fail @@ -1126,6 +1129,10 @@ class B(A): def _(arg): pass def _(arg): pass +def test_underscore() -> None: + A() + B() + [case testGlobalRedefinition_toplevel] # mypy: allow-redefinition i = 0 diff --git a/mypyc/test-data/run-python38.test b/mypyc/test-data/run-python38.test index 7de43907cb867..cf7c7d7dea52e 100644 --- a/mypyc/test-data/run-python38.test +++ b/mypyc/test-data/run-python38.test @@ -75,11 +75,12 @@ class Bar(Foo): def f(self, *args: int, **kwargs: int) -> None: print("stuff", args, kwargs) -z: Foo = Bar() -z.f(1, z=50) -z.f() -z.f(1) -z.f(z=50) +def test_pos_only() -> None: + z: Foo = Bar() + z.f(1, z=50) + z.f() + z.f(1) + z.f(z=50) [out] stuff (1,) {'z': 50} diff --git a/mypyc/test-data/run-singledispatch.test b/mypyc/test-data/run-singledispatch.test index 61e4897c96d66..a119c325984ae 100644 --- a/mypyc/test-data/run-singledispatch.test +++ b/mypyc/test-data/run-singledispatch.test @@ -152,12 +152,14 @@ from functools import singledispatch def fun(arg) -> bool: return False -try: - @fun.register - def fun_specialized(arg: None) -> bool: - return True -except TypeError: - pass +def test_argument() -> None: + try: + @fun.register + def fun_specialized(arg: None) -> bool: + return True + assert False, "expected to raise an exception" + except TypeError: + pass [case testRegisteringTheSameFunctionSeveralTimes] from functools import singledispatch @@ -598,9 +600,11 @@ assert f(1) == 'default' def _(arg: B) -> str: return 'b' -assert f(A()) == 'a' -assert f(B()) == 'b' -assert f(1) == 'default' +# TODO: Move whole testcase to a function when mypyc#1118 is fixed. +def test_final() -> None: + assert f(A()) == 'a' + assert f(B()) == 'b' + assert f(1) == 'default' [case testDynamicallyRegisteringFunctionFromInterpretedCode] diff --git a/mypyc/test-data/run-strings.test b/mypyc/test-data/run-strings.test index c726c4c70896c..6551d9c352dfe 100644 --- a/mypyc/test-data/run-strings.test +++ b/mypyc/test-data/run-strings.test @@ -363,7 +363,6 @@ def test_str_min_max() -> None: assert max(x, z) == 'aaa' [case testStringFormattingCStyle] -[typing fixtures/typing-full.pyi] from typing import Tuple var = 'mypyc' @@ -408,6 +407,7 @@ def test_basics() -> None: inf_num = float('inf') assert '%s, %s' % (nan_num, inf_num) == 'nan, inf' assert '%f, %f' % (nan_num, inf_num) == 'nan, inf' +[typing fixtures/typing-full.pyi] [case testFStrings] import decimal From 82e0eb6bc516f32aaf056d10b5d50e567b706996 Mon Sep 17 00:00:00 2001 From: Piotr Sawicki Date: Fri, 11 Jul 2025 11:36:06 +0200 Subject: [PATCH 075/424] [mypyc] Add back test for next with list iterator (#19419) Adding back a test for `next` with a list iterator as argument. Previously this test only checked if compilation succeeded so I have removed it in #19416, now it calls the test function and checks the result. --- mypyc/test-data/run-lists.test | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/mypyc/test-data/run-lists.test b/mypyc/test-data/run-lists.test index ee1bd27e6352e..03d5741b9ecac 100644 --- a/mypyc/test-data/run-lists.test +++ b/mypyc/test-data/run-lists.test @@ -486,6 +486,17 @@ def test_from_gen() -> None: f = list("str:" + x for x in source_str) assert f == ["str:a", "str:b", "str:c", "str:d"] +[case testNext] +from typing import List + +def get_next(x: List[int]) -> int: + return next((i for i in x), -1) + +def test_next() -> None: + assert get_next([]) == -1 + assert get_next([1]) == 1 + assert get_next([3,2,1]) == 3 + [case testListGetItemWithBorrow] from typing import List From 095df178f7417f43d87e952a58bc90290a226f8f Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Fri, 11 Jul 2025 13:27:26 +0200 Subject: [PATCH 076/424] Allow adjacent conditionally-defined overloads (#19042) Fixes #19015. Fixes #17521. --- mypy/fastparse.py | 15 ++++++++---- test-data/unit/check-overloading.test | 34 +++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index e2af2198cdfdb..bb71242182f19 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -631,7 +631,7 @@ def fix_function_overloads(self, stmts: list[Statement]) -> list[Statement]: ret: list[Statement] = [] current_overload: list[OverloadPart] = [] current_overload_name: str | None = None - seen_unconditional_func_def = False + last_unconditional_func_def: str | None = None last_if_stmt: IfStmt | None = None last_if_overload: Decorator | FuncDef | OverloadedFuncDef | None = None last_if_stmt_overload_name: str | None = None @@ -641,7 +641,7 @@ def fix_function_overloads(self, stmts: list[Statement]) -> list[Statement]: if_overload_name: str | None = None if_block_with_overload: Block | None = None if_unknown_truth_value: IfStmt | None = None - if isinstance(stmt, IfStmt) and seen_unconditional_func_def is False: + if isinstance(stmt, IfStmt): # Check IfStmt block to determine if function overloads can be merged if_overload_name = self._check_ifstmt_for_overloads(stmt, current_overload_name) if if_overload_name is not None: @@ -669,11 +669,18 @@ def fix_function_overloads(self, stmts: list[Statement]) -> list[Statement]: last_if_unknown_truth_value = None current_overload.append(stmt) if isinstance(stmt, FuncDef): - seen_unconditional_func_def = True + # This is, strictly speaking, wrong: there might be a decorated + # implementation. However, it only affects the error message we show: + # ideally it's "already defined", but "implementation must come last" + # is also reasonable. + # TODO: can we get rid of this completely and just always emit + # "implementation must come last" instead? + last_unconditional_func_def = stmt.name elif ( current_overload_name is not None and isinstance(stmt, IfStmt) and if_overload_name == current_overload_name + and last_unconditional_func_def != current_overload_name ): # IfStmt only contains stmts relevant to current_overload. # Check if stmts are reachable and add them to current_overload, @@ -729,7 +736,7 @@ def fix_function_overloads(self, stmts: list[Statement]) -> list[Statement]: # most of mypy/mypyc assumes that all the functions in an OverloadedFuncDef are # related, but multiple underscore functions next to each other aren't necessarily # related - seen_unconditional_func_def = False + last_unconditional_func_def = None if isinstance(stmt, Decorator) and not unnamed_function(stmt.name): current_overload = [stmt] current_overload_name = stmt.name diff --git a/test-data/unit/check-overloading.test b/test-data/unit/check-overloading.test index e427d5b21d40e..0f0fc8747223b 100644 --- a/test-data/unit/check-overloading.test +++ b/test-data/unit/check-overloading.test @@ -6310,6 +6310,40 @@ reveal_type(f12(A())) # N: Revealed type is "__main__.A" [typing fixtures/typing-medium.pyi] +[case testAdjacentConditionalOverloads] +# flags: --always-true true_alias +from typing import overload + +true_alias = True + +if true_alias: + @overload + def ham(v: str) -> list[str]: ... + + @overload + def ham(v: int) -> list[int]: ... + +def ham(v: "int | str") -> "list[str] | list[int]": + return [] + +if true_alias: + @overload + def spam(v: str) -> str: ... + + @overload + def spam(v: int) -> int: ... + +def spam(v: "int | str") -> "str | int": + return "" + +reveal_type(ham) # N: Revealed type is "Overload(def (v: builtins.str) -> builtins.list[builtins.str], def (v: builtins.int) -> builtins.list[builtins.int])" +reveal_type(spam) # N: Revealed type is "Overload(def (v: builtins.str) -> builtins.str, def (v: builtins.int) -> builtins.int)" + +reveal_type(ham("")) # N: Revealed type is "builtins.list[builtins.str]" +reveal_type(ham(0)) # N: Revealed type is "builtins.list[builtins.int]" +reveal_type(spam("")) # N: Revealed type is "builtins.str" +reveal_type(spam(0)) # N: Revealed type is "builtins.int" + [case testOverloadIfUnconditionalFuncDef] # flags: --always-true True --always-false False from typing import overload From d0962dfa50341bb2702880ca7f9530ab292503fe Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 11 Jul 2025 13:37:31 +0100 Subject: [PATCH 077/424] [mypyc] Add a few native int helper irbuilder methods (#19423) I will use these in a follow-up PR. These simplify IR construction a bit. Only the most common operations have helpers for now, but we can add more later if it seems useful. --- mypyc/irbuild/ll_builder.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index 40f1d40b478b2..c3ea0725cfd48 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -2104,6 +2104,33 @@ def float_mod(self, lhs: Value, rhs: Value, line: int) -> Value: def compare_floats(self, lhs: Value, rhs: Value, op: int, line: int) -> Value: return self.add(FloatComparisonOp(lhs, rhs, op, line)) + def int_add(self, lhs: Value, rhs: Value | int) -> Value: + """Helper to add two native integers. + + The result has the type of lhs. + """ + if isinstance(rhs, int): + rhs = Integer(rhs, lhs.type) + return self.int_op(lhs.type, lhs, rhs, IntOp.ADD, line=-1) + + def int_sub(self, lhs: Value, rhs: Value | int) -> Value: + """Helper to subtract a native integer from another one. + + The result has the type of lhs. + """ + if isinstance(rhs, int): + rhs = Integer(rhs, lhs.type) + return self.int_op(lhs.type, lhs, rhs, IntOp.SUB, line=-1) + + def int_mul(self, lhs: Value, rhs: Value | int) -> Value: + """Helper to multiply two native integers. + + The result has the type of lhs. + """ + if isinstance(rhs, int): + rhs = Integer(rhs, lhs.type) + return self.int_op(lhs.type, lhs, rhs, IntOp.MUL, line=-1) + def fixed_width_int_op( self, type: RPrimitive, lhs: Value, rhs: Value, op: int, line: int ) -> Value: From 2f7ba4e52685cb81abdac81dc71a2ffe13e4dae8 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 11 Jul 2025 13:39:36 +0100 Subject: [PATCH 078/424] [mypyc] Remove the unused CPyList_GetItemUnsafe primitive (#19424) We now generate low-level IR directly instead of calling a primitive function. --- mypyc/lib-rt/CPy.h | 1 - mypyc/lib-rt/list_ops.c | 8 -------- 2 files changed, 9 deletions(-) diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index a0f1b06cc0d5b..29370ab2d5d76 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -646,7 +646,6 @@ PyObject *CPyObject_GetSlice(PyObject *obj, CPyTagged start, CPyTagged end); PyObject *CPyList_Build(Py_ssize_t len, ...); PyObject *CPyList_GetItem(PyObject *list, CPyTagged index); -PyObject *CPyList_GetItemUnsafe(PyObject *list, CPyTagged index); PyObject *CPyList_GetItemShort(PyObject *list, CPyTagged index); PyObject *CPyList_GetItemBorrow(PyObject *list, CPyTagged index); PyObject *CPyList_GetItemShortBorrow(PyObject *list, CPyTagged index); diff --git a/mypyc/lib-rt/list_ops.c b/mypyc/lib-rt/list_ops.c index 03af8a769c0e5..e141b99c091ea 100644 --- a/mypyc/lib-rt/list_ops.c +++ b/mypyc/lib-rt/list_ops.c @@ -59,14 +59,6 @@ PyObject *CPyList_Copy(PyObject *list) { return PyObject_CallMethodNoArgs(list, name); } - -PyObject *CPyList_GetItemUnsafe(PyObject *list, CPyTagged index) { - Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); - PyObject *result = PyList_GET_ITEM(list, n); - Py_INCREF(result); - return result; -} - PyObject *CPyList_GetItemShort(PyObject *list, CPyTagged index) { Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); Py_ssize_t size = PyList_GET_SIZE(list); From db6788868b07fd13bcc8606a165f2ca90a68c76c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 11 Jul 2025 15:45:25 +0100 Subject: [PATCH 079/424] [mypyc] Use native integers for some sequence indexing operations (#19426) For example, when iterating over a list, now we use a native integer for the index (which is not exposed to the user). Previously we used tagged integers, but in these use cases they provide no real benefit. This simplifies the IR and should slightly improve performance, as fewer tagged int to native int conversions are needed. Multiple ops have to be migrated in one go, as these interact with each other, and by only changing a subset of them would actually generate more verbose IR, as a bunch of extra coercions would be needed. List of impacted statements: * For loop over sequence * Assignment like `x, y = a` for tuple/list rvalue * Dict iteration * List comprehension For example, consider this example: ``` def foo(a: list[int]) -> None: for x in a: pass ``` Old generated IR was like this: ``` def foo(a): a :: list r0 :: short_int r1 :: ptr r2 :: native_int r3 :: short_int r4 :: bit r5 :: native_int r6, r7 :: ptr r8 :: native_int r9 :: ptr r10 :: object r11 :: int r12 :: short_int r13 :: None L0: r0 = 0 L1: r1 = get_element_ptr a ob_size :: PyVarObject r2 = load_mem r1 :: native_int* r3 = r2 << 1 r4 = r0 < r3 :: signed if r4 goto L2 else goto L5 :: bool L2: r5 = r0 >> 1 r6 = get_element_ptr a ob_item :: PyListObject r7 = load_mem r6 :: ptr* r8 = r5 * 8 r9 = r7 + r8 r10 = load_mem r9 :: builtins.object* inc_ref r10 r11 = unbox(int, r10) dec_ref r10 if is_error(r11) goto L6 (error at foo:2) else goto L3 L3: dec_ref r11 :: int L4: r12 = r0 + 2 r0 = r12 goto L1 L5: return 1 L6: r13 = :: None return r13 ``` Now the generated IR is simpler: ``` def foo(a): a :: list r0 :: native_int r1 :: ptr r2 :: native_int r3 :: bit r4, r5 :: ptr r6 :: native_int r7 :: ptr r8 :: object r9 :: int r10 :: native_int r11 :: None L0: r0 = 0 L1: r1 = get_element_ptr a ob_size :: PyVarObject r2 = load_mem r1 :: native_int* r3 = r0 < r2 :: signed if r3 goto L2 else goto L5 :: bool L2: r4 = get_element_ptr a ob_item :: PyListObject r5 = load_mem r4 :: ptr* r6 = r0 * 8 r7 = r5 + r6 r8 = load_mem r7 :: builtins.object* inc_ref r8 r9 = unbox(int, r8) dec_ref r8 if is_error(r9) goto L6 (error at foo:2) else goto L3 L3: dec_ref r9 :: int L4: r10 = r0 + 1 r0 = r10 goto L1 L5: return 1 L6: r11 = :: None return r11 ``` --- mypyc/irbuild/builder.py | 8 +- mypyc/irbuild/for_helpers.py | 22 +- mypyc/lib-rt/CPy.h | 10 +- mypyc/lib-rt/list_ops.c | 11 +- mypyc/lib-rt/tuple_ops.c | 19 +- mypyc/primitives/dict_ops.py | 2 +- mypyc/primitives/list_ops.py | 9 +- mypyc/primitives/tuple_ops.py | 19 +- mypyc/test-data/irbuild-basic.test | 210 +++---- mypyc/test-data/irbuild-dict.test | 306 +++++---- mypyc/test-data/irbuild-generics.test | 790 ++++++++++++------------ mypyc/test-data/irbuild-lists.test | 181 +++--- mypyc/test-data/irbuild-set.test | 204 +++--- mypyc/test-data/irbuild-statements.test | 294 +++++---- mypyc/test-data/irbuild-tuple.test | 148 +++-- mypyc/test-data/lowering-int.test | 54 +- mypyc/test-data/refcount.test | 58 +- 17 files changed, 1128 insertions(+), 1217 deletions(-) diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index 28ebcf2075fb2..7e63d482c786e 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -129,6 +129,7 @@ from mypyc.primitives.list_ops import list_get_item_unsafe_op, list_pop_last, to_list from mypyc.primitives.misc_ops import check_unpack_count_op, get_module_dict_op, import_op from mypyc.primitives.registry import CFunctionDescription, function_ops +from mypyc.primitives.tuple_ops import tuple_get_item_unsafe_op # These int binary operations can borrow their operands safely, since the # primitives take this into consideration. @@ -772,10 +773,15 @@ def process_sequence_assignment( values = [] for i in range(len(target.items)): item = target.items[i] - index = self.builder.load_int(i) + index: Value if is_list_rprimitive(rvalue.type): + index = Integer(i, c_pyssize_t_rprimitive) item_value = self.primitive_op(list_get_item_unsafe_op, [rvalue, index], line) + elif is_tuple_rprimitive(rvalue.type): + index = Integer(i, c_pyssize_t_rprimitive) + item_value = self.call_c(tuple_get_item_unsafe_op, [rvalue, index], line) else: + index = self.builder.load_int(i) item_value = self.builder.gen_method_call( rvalue, "__getitem__", [index], item.type, line ) diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index ab90a8c86b281..358f7cb76ba8a 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -44,6 +44,7 @@ RTuple, RType, bool_rprimitive, + c_pyssize_t_rprimitive, int_rprimitive, is_dict_rprimitive, is_fixed_width_rtype, @@ -75,6 +76,7 @@ from mypyc.primitives.misc_ops import stop_async_iteration_op from mypyc.primitives.registry import CFunctionDescription from mypyc.primitives.set_ops import set_add_op +from mypyc.primitives.tuple_ops import tuple_get_item_unsafe_op GenFunc = Callable[[], None] @@ -586,7 +588,9 @@ def gen_cleanup(self) -> None: def load_len(self, expr: Value | AssignmentTarget) -> Value: """A helper to get collection length, used by several subclasses.""" - return self.builder.builder.builtin_len(self.builder.read(expr, self.line), self.line) + return self.builder.builder.builtin_len( + self.builder.read(expr, self.line), self.line, use_pyssize_t=True + ) class ForIterable(ForGenerator): @@ -766,6 +770,8 @@ def unsafe_index(builder: IRBuilder, target: Value, index: Value, line: int) -> # so we just check manually. if is_list_rprimitive(target.type): return builder.primitive_op(list_get_item_unsafe_op, [target, index], line) + elif is_tuple_rprimitive(target.type): + return builder.call_c(tuple_get_item_unsafe_op, [target, index], line) else: return builder.gen_method_call(target, "__getitem__", [index], None, line) @@ -784,11 +790,9 @@ def init(self, expr_reg: Value, target_type: RType, reverse: bool) -> None: # environment class. self.expr_target = builder.maybe_spill(expr_reg) if not reverse: - index_reg: Value = Integer(0) + index_reg: Value = Integer(0, c_pyssize_t_rprimitive) else: - index_reg = builder.binary_op( - self.load_len(self.expr_target), Integer(1), "-", self.line - ) + index_reg = builder.builder.int_sub(self.load_len(self.expr_target), 1) self.index_target = builder.maybe_spill_assignable(index_reg) self.target_type = target_type @@ -838,13 +842,7 @@ def gen_step(self) -> None: builder = self.builder line = self.line step = 1 if not self.reverse else -1 - add = builder.int_op( - short_int_rprimitive, - builder.read(self.index_target, line), - Integer(step), - IntOp.ADD, - line, - ) + add = builder.builder.int_add(builder.read(self.index_target, line), step) builder.assign(self.index_target, add, line) diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index 29370ab2d5d76..dba84d44f3630 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -652,7 +652,7 @@ PyObject *CPyList_GetItemShortBorrow(PyObject *list, CPyTagged index); PyObject *CPyList_GetItemInt64(PyObject *list, int64_t index); PyObject *CPyList_GetItemInt64Borrow(PyObject *list, int64_t index); bool CPyList_SetItem(PyObject *list, CPyTagged index, PyObject *value); -bool CPyList_SetItemUnsafe(PyObject *list, CPyTagged index, PyObject *value); +void CPyList_SetItemUnsafe(PyObject *list, Py_ssize_t index, PyObject *value); bool CPyList_SetItemInt64(PyObject *list, int64_t index, PyObject *value); PyObject *CPyList_PopLast(PyObject *obj); PyObject *CPyList_Pop(PyObject *obj, CPyTagged index); @@ -703,14 +703,13 @@ tuple_T4CIOO CPyDict_NextItem(PyObject *dict_or_iter, CPyTagged offset); int CPyMapping_Check(PyObject *obj); // Check that dictionary didn't change size during iteration. -static inline char CPyDict_CheckSize(PyObject *dict, CPyTagged size) { +static inline char CPyDict_CheckSize(PyObject *dict, Py_ssize_t size) { if (!PyDict_CheckExact(dict)) { // Dict subclasses will be checked by Python runtime. return 1; } - Py_ssize_t py_size = CPyTagged_AsSsize_t(size); Py_ssize_t dict_size = PyDict_Size(dict); - if (py_size != dict_size) { + if (size != dict_size) { PyErr_SetString(PyExc_RuntimeError, "dictionary changed size during iteration"); return 0; } @@ -783,7 +782,8 @@ bool CPySet_Remove(PyObject *set, PyObject *key); PyObject *CPySequenceTuple_GetItem(PyObject *tuple, CPyTagged index); PyObject *CPySequenceTuple_GetSlice(PyObject *obj, CPyTagged start, CPyTagged end); -bool CPySequenceTuple_SetItemUnsafe(PyObject *tuple, CPyTagged index, PyObject *value); +PyObject *CPySequenceTuple_GetItemUnsafe(PyObject *tuple, Py_ssize_t index); +void CPySequenceTuple_SetItemUnsafe(PyObject *tuple, Py_ssize_t index, PyObject *value); // Exception operations diff --git a/mypyc/lib-rt/list_ops.c b/mypyc/lib-rt/list_ops.c index e141b99c091ea..31a0d5cec7d5a 100644 --- a/mypyc/lib-rt/list_ops.c +++ b/mypyc/lib-rt/list_ops.c @@ -231,15 +231,8 @@ bool CPyList_SetItemInt64(PyObject *list, int64_t index, PyObject *value) { } // This function should only be used to fill in brand new lists. -bool CPyList_SetItemUnsafe(PyObject *list, CPyTagged index, PyObject *value) { - if (CPyTagged_CheckShort(index)) { - Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); - PyList_SET_ITEM(list, n, value); - return true; - } else { - PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG); - return false; - } +void CPyList_SetItemUnsafe(PyObject *list, Py_ssize_t index, PyObject *value) { + PyList_SET_ITEM(list, index, value); } PyObject *CPyList_PopLast(PyObject *obj) diff --git a/mypyc/lib-rt/tuple_ops.c b/mypyc/lib-rt/tuple_ops.c index 64418974666fe..1df73f1907e2f 100644 --- a/mypyc/lib-rt/tuple_ops.c +++ b/mypyc/lib-rt/tuple_ops.c @@ -46,16 +46,17 @@ PyObject *CPySequenceTuple_GetSlice(PyObject *obj, CPyTagged start, CPyTagged en return CPyObject_GetSlice(obj, start, end); } +// No error checking +PyObject *CPySequenceTuple_GetItemUnsafe(PyObject *tuple, Py_ssize_t index) +{ + PyObject *result = PyTuple_GET_ITEM(tuple, index); + Py_INCREF(result); + return result; +} + // PyTuple_SET_ITEM does no error checking, // and should only be used to fill in brand new tuples. -bool CPySequenceTuple_SetItemUnsafe(PyObject *tuple, CPyTagged index, PyObject *value) +void CPySequenceTuple_SetItemUnsafe(PyObject *tuple, Py_ssize_t index, PyObject *value) { - if (CPyTagged_CheckShort(index)) { - Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); - PyTuple_SET_ITEM(tuple, n, value); - return true; - } else { - PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG); - return false; - } + PyTuple_SET_ITEM(tuple, index, value); } diff --git a/mypyc/primitives/dict_ops.py b/mypyc/primitives/dict_ops.py index ce7b9bb8d70e1..3f289c3c6f08d 100644 --- a/mypyc/primitives/dict_ops.py +++ b/mypyc/primitives/dict_ops.py @@ -289,7 +289,7 @@ # check that len(dict) == const during iteration dict_check_size_op = custom_op( - arg_types=[dict_rprimitive, int_rprimitive], + arg_types=[dict_rprimitive, c_pyssize_t_rprimitive], return_type=bit_rprimitive, c_function_name="CPyDict_CheckSize", error_kind=ERR_FALSE, diff --git a/mypyc/primitives/list_ops.py b/mypyc/primitives/list_ops.py index 7442e31c91182..57cb541fdbb83 100644 --- a/mypyc/primitives/list_ops.py +++ b/mypyc/primitives/list_ops.py @@ -13,6 +13,7 @@ object_rprimitive, pointer_rprimitive, short_int_rprimitive, + void_rtype, ) from mypyc.primitives.registry import ( ERR_NEG_INT, @@ -154,7 +155,7 @@ # that is in-bounds for the list. list_get_item_unsafe_op = custom_primitive_op( name="list_get_item_unsafe", - arg_types=[list_rprimitive, short_int_rprimitive], + arg_types=[list_rprimitive, c_pyssize_t_rprimitive], return_type=object_rprimitive, error_kind=ERR_NEVER, ) @@ -183,10 +184,10 @@ # PyList_SET_ITEM does no error checking, # and should only be used to fill in brand new lists. new_list_set_item_op = custom_op( - arg_types=[list_rprimitive, int_rprimitive, object_rprimitive], - return_type=bit_rprimitive, + arg_types=[list_rprimitive, c_pyssize_t_rprimitive, object_rprimitive], + return_type=void_rtype, c_function_name="CPyList_SetItemUnsafe", - error_kind=ERR_FALSE, + error_kind=ERR_NEVER, steals=[False, False, True], ) diff --git a/mypyc/primitives/tuple_ops.py b/mypyc/primitives/tuple_ops.py index a9bbaa80fb5cc..e680b6943d845 100644 --- a/mypyc/primitives/tuple_ops.py +++ b/mypyc/primitives/tuple_ops.py @@ -6,14 +6,14 @@ from __future__ import annotations -from mypyc.ir.ops import ERR_FALSE, ERR_MAGIC +from mypyc.ir.ops import ERR_MAGIC, ERR_NEVER from mypyc.ir.rtypes import ( - bit_rprimitive, c_pyssize_t_rprimitive, int_rprimitive, list_rprimitive, object_rprimitive, tuple_rprimitive, + void_rtype, ) from mypyc.primitives.registry import binary_op, custom_op, function_op, load_address_op, method_op @@ -29,6 +29,15 @@ error_kind=ERR_MAGIC, ) +# This is unsafe because it assumes that the index is a non-negative integer +# that is in-bounds for the tuple. +tuple_get_item_unsafe_op = custom_op( + arg_types=[tuple_rprimitive, c_pyssize_t_rprimitive], + return_type=object_rprimitive, + c_function_name="CPySequenceTuple_GetItemUnsafe", + error_kind=ERR_NEVER, +) + # Construct a boxed tuple from items: (item1, item2, ...) new_tuple_op = custom_op( arg_types=[c_pyssize_t_rprimitive], @@ -48,10 +57,10 @@ # PyTuple_SET_ITEM does no error checking, # and should only be used to fill in brand new tuples. new_tuple_set_item_op = custom_op( - arg_types=[tuple_rprimitive, int_rprimitive, object_rprimitive], - return_type=bit_rprimitive, + arg_types=[tuple_rprimitive, c_pyssize_t_rprimitive, object_rprimitive], + return_type=void_rtype, c_function_name="CPySequenceTuple_SetItemUnsafe", - error_kind=ERR_FALSE, + error_kind=ERR_NEVER, steals=[False, False, True], ) diff --git a/mypyc/test-data/irbuild-basic.test b/mypyc/test-data/irbuild-basic.test index d652cb9c9a149..ea1b3d06869ae 100644 --- a/mypyc/test-data/irbuild-basic.test +++ b/mypyc/test-data/irbuild-basic.test @@ -1895,18 +1895,16 @@ def f(): r0, r1 :: list r2, r3, r4 :: object r5 :: ptr - r6 :: short_int - r7 :: native_int - r8 :: short_int - r9 :: bit - r10 :: object - r11, x :: int - r12, r13 :: bit - r14 :: int - r15 :: object - r16 :: i32 - r17 :: bit - r18 :: short_int + r6, r7 :: native_int + r8 :: bit + r9 :: object + r10, x :: int + r11, r12 :: bit + r13 :: int + r14 :: object + r15 :: i32 + r16 :: bit + r17 :: native_int L0: r0 = PyList_New(0) r1 = PyList_New(3) @@ -1921,30 +1919,29 @@ L0: r6 = 0 L1: r7 = var_object_size r1 - r8 = r7 << 1 - r9 = int_lt r6, r8 - if r9 goto L2 else goto L8 :: bool + r8 = r6 < r7 :: signed + if r8 goto L2 else goto L8 :: bool L2: - r10 = list_get_item_unsafe r1, r6 - r11 = unbox(int, r10) - x = r11 - r12 = int_ne x, 4 - if r12 goto L4 else goto L3 :: bool + r9 = list_get_item_unsafe r1, r6 + r10 = unbox(int, r9) + x = r10 + r11 = int_ne x, 4 + if r11 goto L4 else goto L3 :: bool L3: goto L7 L4: - r13 = int_ne x, 6 - if r13 goto L6 else goto L5 :: bool + r12 = int_ne x, 6 + if r12 goto L6 else goto L5 :: bool L5: goto L7 L6: - r14 = CPyTagged_Multiply(x, x) - r15 = box(int, r14) - r16 = PyList_Append(r0, r15) - r17 = r16 >= 0 :: signed + r13 = CPyTagged_Multiply(x, x) + r14 = box(int, r13) + r15 = PyList_Append(r0, r14) + r16 = r15 >= 0 :: signed L7: - r18 = r6 + 2 - r6 = r18 + r17 = r6 + 1 + r6 = r17 goto L1 L8: return r0 @@ -1959,18 +1956,16 @@ def f(): r1 :: list r2, r3, r4 :: object r5 :: ptr - r6 :: short_int - r7 :: native_int - r8 :: short_int - r9 :: bit - r10 :: object - r11, x :: int - r12, r13 :: bit - r14 :: int - r15, r16 :: object - r17 :: i32 - r18 :: bit - r19 :: short_int + r6, r7 :: native_int + r8 :: bit + r9 :: object + r10, x :: int + r11, r12 :: bit + r13 :: int + r14, r15 :: object + r16 :: i32 + r17 :: bit + r18 :: native_int L0: r0 = PyDict_New() r1 = PyList_New(3) @@ -1985,31 +1980,30 @@ L0: r6 = 0 L1: r7 = var_object_size r1 - r8 = r7 << 1 - r9 = int_lt r6, r8 - if r9 goto L2 else goto L8 :: bool + r8 = r6 < r7 :: signed + if r8 goto L2 else goto L8 :: bool L2: - r10 = list_get_item_unsafe r1, r6 - r11 = unbox(int, r10) - x = r11 - r12 = int_ne x, 4 - if r12 goto L4 else goto L3 :: bool + r9 = list_get_item_unsafe r1, r6 + r10 = unbox(int, r9) + x = r10 + r11 = int_ne x, 4 + if r11 goto L4 else goto L3 :: bool L3: goto L7 L4: - r13 = int_ne x, 6 - if r13 goto L6 else goto L5 :: bool + r12 = int_ne x, 6 + if r12 goto L6 else goto L5 :: bool L5: goto L7 L6: - r14 = CPyTagged_Multiply(x, x) - r15 = box(int, x) - r16 = box(int, r14) - r17 = CPyDict_SetItem(r0, r15, r16) - r18 = r17 >= 0 :: signed + r13 = CPyTagged_Multiply(x, x) + r14 = box(int, x) + r15 = box(int, r13) + r16 = CPyDict_SetItem(r0, r14, r15) + r17 = r16 >= 0 :: signed L7: - r19 = r6 + 2 - r6 = r19 + r18 = r6 + 1 + r6 = r18 goto L1 L8: return r0 @@ -2023,74 +2017,66 @@ def f(l: List[Tuple[int, int, int]]) -> List[int]: [out] def f(l): l :: list - r0 :: short_int - r1 :: native_int - r2 :: short_int - r3 :: bit - r4 :: object - r5 :: tuple[int, int, int] - r6, x, r7, y, r8, z :: int - r9 :: short_int - r10 :: native_int - r11 :: list - r12 :: short_int - r13 :: native_int - r14 :: short_int - r15 :: bit - r16 :: object - r17 :: tuple[int, int, int] - r18, x_2, r19, y_2, r20, z_2, r21, r22 :: int - r23 :: object - r24 :: bit - r25 :: short_int + r0, r1 :: native_int + r2 :: bit + r3 :: object + r4 :: tuple[int, int, int] + r5, x, r6, y, r7, z :: int + r8, r9 :: native_int + r10 :: list + r11, r12 :: native_int + r13 :: bit + r14 :: object + r15 :: tuple[int, int, int] + r16, x_2, r17, y_2, r18, z_2, r19, r20 :: int + r21 :: object + r22 :: native_int L0: r0 = 0 L1: r1 = var_object_size l - r2 = r1 << 1 - r3 = int_lt r0, r2 - if r3 goto L2 else goto L4 :: bool + r2 = r0 < r1 :: signed + if r2 goto L2 else goto L4 :: bool L2: - r4 = list_get_item_unsafe l, r0 - r5 = unbox(tuple[int, int, int], r4) - r6 = r5[0] - x = r6 - r7 = r5[1] - y = r7 - r8 = r5[2] - z = r8 + r3 = list_get_item_unsafe l, r0 + r4 = unbox(tuple[int, int, int], r3) + r5 = r4[0] + x = r5 + r6 = r4[1] + y = r6 + r7 = r4[2] + z = r7 L3: - r9 = r0 + 2 - r0 = r9 + r8 = r0 + 1 + r0 = r8 goto L1 L4: - r10 = var_object_size l - r11 = PyList_New(r10) - r12 = 0 + r9 = var_object_size l + r10 = PyList_New(r9) + r11 = 0 L5: - r13 = var_object_size l - r14 = r13 << 1 - r15 = int_lt r12, r14 - if r15 goto L6 else goto L8 :: bool + r12 = var_object_size l + r13 = r11 < r12 :: signed + if r13 goto L6 else goto L8 :: bool L6: - r16 = list_get_item_unsafe l, r12 - r17 = unbox(tuple[int, int, int], r16) - r18 = r17[0] - x_2 = r18 - r19 = r17[1] - y_2 = r19 - r20 = r17[2] - z_2 = r20 - r21 = CPyTagged_Add(x_2, y_2) - r22 = CPyTagged_Add(r21, z_2) - r23 = box(int, r22) - r24 = CPyList_SetItemUnsafe(r11, r12, r23) + r14 = list_get_item_unsafe l, r11 + r15 = unbox(tuple[int, int, int], r14) + r16 = r15[0] + x_2 = r16 + r17 = r15[1] + y_2 = r17 + r18 = r15[2] + z_2 = r18 + r19 = CPyTagged_Add(x_2, y_2) + r20 = CPyTagged_Add(r19, z_2) + r21 = box(int, r20) + CPyList_SetItemUnsafe(r10, r11, r21) L7: - r25 = r12 + 2 - r12 = r25 + r22 = r11 + 1 + r11 = r22 goto L5 L8: - return r11 + return r10 [case testProperty] class PropertyHolder: diff --git a/mypyc/test-data/irbuild-dict.test b/mypyc/test-data/irbuild-dict.test index cacb14dae2736..e0c014f078138 100644 --- a/mypyc/test-data/irbuild-dict.test +++ b/mypyc/test-data/irbuild-dict.test @@ -152,41 +152,39 @@ def increment(d): d :: dict r0 :: short_int r1 :: native_int - r2 :: short_int - r3 :: object - r4 :: tuple[bool, short_int, object] - r5 :: short_int - r6 :: bool - r7 :: object - r8, k :: str - r9, r10, r11 :: object - r12 :: i32 - r13, r14, r15 :: bit + r2 :: object + r3 :: tuple[bool, short_int, object] + r4 :: short_int + r5 :: bool + r6 :: object + r7, k :: str + r8, r9, r10 :: object + r11 :: i32 + r12, r13, r14 :: bit L0: r0 = 0 r1 = PyDict_Size(d) - r2 = r1 << 1 - r3 = CPyDict_GetKeysIter(d) + r2 = CPyDict_GetKeysIter(d) L1: - r4 = CPyDict_NextKey(r3, r0) - r5 = r4[1] - r0 = r5 - r6 = r4[0] - if r6 goto L2 else goto L4 :: bool + r3 = CPyDict_NextKey(r2, r0) + r4 = r3[1] + r0 = r4 + r5 = r3[0] + if r5 goto L2 else goto L4 :: bool L2: - r7 = r4[2] - r8 = cast(str, r7) - k = r8 - r9 = CPyDict_GetItem(d, k) - r10 = object 1 - r11 = PyNumber_InPlaceAdd(r9, r10) - r12 = CPyDict_SetItem(d, k, r11) - r13 = r12 >= 0 :: signed + r6 = r3[2] + r7 = cast(str, r6) + k = r7 + r8 = CPyDict_GetItem(d, k) + r9 = object 1 + r10 = PyNumber_InPlaceAdd(r8, r9) + r11 = CPyDict_SetItem(d, k, r10) + r12 = r11 >= 0 :: signed L3: - r14 = CPyDict_CheckSize(d, r2) + r13 = CPyDict_CheckSize(d, r1) goto L1 L4: - r15 = CPy_NoErrOccurred() + r14 = CPy_NoErrOccurred() L5: return d @@ -244,192 +242,184 @@ def print_dict_methods(d1, d2): d1, d2 :: dict r0 :: short_int r1 :: native_int - r2 :: short_int - r3 :: object - r4 :: tuple[bool, short_int, object] - r5 :: short_int - r6 :: bool - r7 :: object - r8, v :: int - r9 :: object - r10 :: i32 - r11 :: bit - r12 :: bool - r13, r14 :: bit - r15 :: short_int - r16 :: native_int - r17 :: short_int - r18 :: object - r19 :: tuple[bool, short_int, object, object] - r20 :: short_int - r21 :: bool - r22, r23 :: object - r24, r25, k :: int - r26, r27, r28, r29, r30 :: object - r31 :: i32 - r32, r33, r34 :: bit + r2 :: object + r3 :: tuple[bool, short_int, object] + r4 :: short_int + r5 :: bool + r6 :: object + r7, v :: int + r8 :: object + r9 :: i32 + r10 :: bit + r11 :: bool + r12, r13 :: bit + r14 :: short_int + r15 :: native_int + r16 :: object + r17 :: tuple[bool, short_int, object, object] + r18 :: short_int + r19 :: bool + r20, r21 :: object + r22, r23, k :: int + r24, r25, r26, r27, r28 :: object + r29 :: i32 + r30, r31, r32 :: bit L0: r0 = 0 r1 = PyDict_Size(d1) - r2 = r1 << 1 - r3 = CPyDict_GetValuesIter(d1) + r2 = CPyDict_GetValuesIter(d1) L1: - r4 = CPyDict_NextValue(r3, r0) - r5 = r4[1] - r0 = r5 - r6 = r4[0] - if r6 goto L2 else goto L6 :: bool + r3 = CPyDict_NextValue(r2, r0) + r4 = r3[1] + r0 = r4 + r5 = r3[0] + if r5 goto L2 else goto L6 :: bool L2: - r7 = r4[2] - r8 = unbox(int, r7) - v = r8 - r9 = box(int, v) - r10 = PyDict_Contains(d2, r9) - r11 = r10 >= 0 :: signed - r12 = truncate r10: i32 to builtins.bool - if r12 goto L3 else goto L4 :: bool + r6 = r3[2] + r7 = unbox(int, r6) + v = r7 + r8 = box(int, v) + r9 = PyDict_Contains(d2, r8) + r10 = r9 >= 0 :: signed + r11 = truncate r9: i32 to builtins.bool + if r11 goto L3 else goto L4 :: bool L3: return 1 L4: L5: - r13 = CPyDict_CheckSize(d1, r2) + r12 = CPyDict_CheckSize(d1, r1) goto L1 L6: - r14 = CPy_NoErrOccurred() + r13 = CPy_NoErrOccurred() L7: - r15 = 0 - r16 = PyDict_Size(d2) - r17 = r16 << 1 - r18 = CPyDict_GetItemsIter(d2) + r14 = 0 + r15 = PyDict_Size(d2) + r16 = CPyDict_GetItemsIter(d2) L8: - r19 = CPyDict_NextItem(r18, r15) - r20 = r19[1] - r15 = r20 - r21 = r19[0] - if r21 goto L9 else goto L11 :: bool + r17 = CPyDict_NextItem(r16, r14) + r18 = r17[1] + r14 = r18 + r19 = r17[0] + if r19 goto L9 else goto L11 :: bool L9: - r22 = r19[2] - r23 = r19[3] - r24 = unbox(int, r22) - r25 = unbox(int, r23) - k = r24 - v = r25 - r26 = box(int, k) - r27 = CPyDict_GetItem(d2, r26) - r28 = box(int, v) - r29 = PyNumber_InPlaceAdd(r27, r28) - r30 = box(int, k) - r31 = CPyDict_SetItem(d2, r30, r29) - r32 = r31 >= 0 :: signed + r20 = r17[2] + r21 = r17[3] + r22 = unbox(int, r20) + r23 = unbox(int, r21) + k = r22 + v = r23 + r24 = box(int, k) + r25 = CPyDict_GetItem(d2, r24) + r26 = box(int, v) + r27 = PyNumber_InPlaceAdd(r25, r26) + r28 = box(int, k) + r29 = CPyDict_SetItem(d2, r28, r27) + r30 = r29 >= 0 :: signed L10: - r33 = CPyDict_CheckSize(d2, r17) + r31 = CPyDict_CheckSize(d2, r15) goto L8 L11: - r34 = CPy_NoErrOccurred() + r32 = CPy_NoErrOccurred() L12: return 1 def union_of_dicts(d): d, r0, new :: dict r1 :: short_int r2 :: native_int - r3 :: short_int - r4 :: object - r5 :: tuple[bool, short_int, object, object] - r6 :: short_int - r7 :: bool - r8, r9 :: object - r10 :: str - r11 :: union[int, str] + r3 :: object + r4 :: tuple[bool, short_int, object, object] + r5 :: short_int + r6 :: bool + r7, r8 :: object + r9 :: str + r10 :: union[int, str] k :: str v :: union[int, str] - r12 :: object - r13 :: object[1] - r14 :: object_ptr - r15 :: object - r16 :: int - r17 :: object - r18 :: i32 - r19, r20, r21 :: bit + r11 :: object + r12 :: object[1] + r13 :: object_ptr + r14 :: object + r15 :: int + r16 :: object + r17 :: i32 + r18, r19, r20 :: bit L0: r0 = PyDict_New() new = r0 r1 = 0 r2 = PyDict_Size(d) - r3 = r2 << 1 - r4 = CPyDict_GetItemsIter(d) + r3 = CPyDict_GetItemsIter(d) L1: - r5 = CPyDict_NextItem(r4, r1) - r6 = r5[1] - r1 = r6 - r7 = r5[0] - if r7 goto L2 else goto L4 :: bool + r4 = CPyDict_NextItem(r3, r1) + r5 = r4[1] + r1 = r5 + r6 = r4[0] + if r6 goto L2 else goto L4 :: bool L2: - r8 = r5[2] - r9 = r5[3] - r10 = cast(str, r8) - r11 = cast(union[int, str], r9) - k = r10 - v = r11 - r12 = load_address PyLong_Type - r13 = [v] - r14 = load_address r13 - r15 = PyObject_Vectorcall(r12, r14, 1, 0) + r7 = r4[2] + r8 = r4[3] + r9 = cast(str, r7) + r10 = cast(union[int, str], r8) + k = r9 + v = r10 + r11 = load_address PyLong_Type + r12 = [v] + r13 = load_address r12 + r14 = PyObject_Vectorcall(r11, r13, 1, 0) keep_alive v - r16 = unbox(int, r15) - r17 = box(int, r16) - r18 = CPyDict_SetItem(new, k, r17) - r19 = r18 >= 0 :: signed + r15 = unbox(int, r14) + r16 = box(int, r15) + r17 = CPyDict_SetItem(new, k, r16) + r18 = r17 >= 0 :: signed L3: - r20 = CPyDict_CheckSize(d, r3) + r19 = CPyDict_CheckSize(d, r2) goto L1 L4: - r21 = CPy_NoErrOccurred() + r20 = CPy_NoErrOccurred() L5: return 1 def typeddict(d): d :: dict r0 :: short_int r1 :: native_int - r2 :: short_int - r3 :: object - r4 :: tuple[bool, short_int, object, object] - r5 :: short_int - r6 :: bool - r7, r8 :: object - r9, k :: str + r2 :: object + r3 :: tuple[bool, short_int, object, object] + r4 :: short_int + r5 :: bool + r6, r7 :: object + r8, k :: str v :: object - r10 :: str - r11 :: bool + r9 :: str + r10 :: bool name :: object - r12, r13 :: bit + r11, r12 :: bit L0: r0 = 0 r1 = PyDict_Size(d) - r2 = r1 << 1 - r3 = CPyDict_GetItemsIter(d) + r2 = CPyDict_GetItemsIter(d) L1: - r4 = CPyDict_NextItem(r3, r0) - r5 = r4[1] - r0 = r5 - r6 = r4[0] - if r6 goto L2 else goto L6 :: bool + r3 = CPyDict_NextItem(r2, r0) + r4 = r3[1] + r0 = r4 + r5 = r3[0] + if r5 goto L2 else goto L6 :: bool L2: - r7 = r4[2] - r8 = r4[3] - r9 = cast(str, r7) - k = r9 - v = r8 - r10 = 'name' - r11 = CPyStr_Equal(k, r10) - if r11 goto L3 else goto L4 :: bool + r6 = r3[2] + r7 = r3[3] + r8 = cast(str, r6) + k = r8 + v = r7 + r9 = 'name' + r10 = CPyStr_Equal(k, r9) + if r10 goto L3 else goto L4 :: bool L3: name = v L4: L5: - r12 = CPyDict_CheckSize(d, r2) + r11 = CPyDict_CheckSize(d, r1) goto L1 L6: - r13 = CPy_NoErrOccurred() + r12 = CPy_NoErrOccurred() L7: return 1 diff --git a/mypyc/test-data/irbuild-generics.test b/mypyc/test-data/irbuild-generics.test index feb7b9db20fb8..d39d47e397a1f 100644 --- a/mypyc/test-data/irbuild-generics.test +++ b/mypyc/test-data/irbuild-generics.test @@ -226,153 +226,145 @@ def fn_mapping(m): r0 :: list r1 :: short_int r2 :: native_int - r3 :: short_int - r4 :: object - r5 :: tuple[bool, short_int, object] - r6 :: short_int - r7 :: bool - r8 :: object - r9, x :: str - r10 :: i32 - r11, r12, r13 :: bit - r14 :: list - r15 :: short_int - r16 :: native_int - r17 :: short_int - r18 :: object - r19 :: tuple[bool, short_int, object] - r20 :: short_int - r21 :: bool + r3 :: object + r4 :: tuple[bool, short_int, object] + r5 :: short_int + r6 :: bool + r7 :: object + r8, x :: str + r9 :: i32 + r10, r11, r12 :: bit + r13 :: list + r14 :: short_int + r15 :: native_int + r16 :: object + r17 :: tuple[bool, short_int, object] + r18 :: short_int + r19 :: bool + r20 :: object + r21, x_2 :: int r22 :: object - r23, x_2 :: int - r24 :: object - r25 :: i32 - r26, r27, r28 :: bit - r29 :: set - r30 :: short_int - r31 :: native_int + r23 :: i32 + r24, r25, r26 :: bit + r27 :: set + r28 :: short_int + r29 :: native_int + r30 :: object + r31 :: tuple[bool, short_int, object] r32 :: short_int - r33 :: object - r34 :: tuple[bool, short_int, object] - r35 :: short_int - r36 :: bool - r37 :: object - r38, x_3 :: str - r39 :: i32 - r40, r41, r42 :: bit - r43 :: dict - r44 :: short_int - r45 :: native_int - r46 :: short_int - r47 :: object - r48 :: tuple[bool, short_int, object, object] - r49 :: short_int - r50 :: bool - r51, r52 :: object - r53 :: str - r54 :: int + r33 :: bool + r34 :: object + r35, x_3 :: str + r36 :: i32 + r37, r38, r39 :: bit + r40 :: dict + r41 :: short_int + r42 :: native_int + r43 :: object + r44 :: tuple[bool, short_int, object, object] + r45 :: short_int + r46 :: bool + r47, r48 :: object + r49 :: str + r50 :: int k :: str v :: int - r55 :: object - r56 :: i32 - r57, r58, r59 :: bit + r51 :: object + r52 :: i32 + r53, r54, r55 :: bit L0: r0 = PyList_New(0) r1 = 0 r2 = PyDict_Size(m) - r3 = r2 << 1 - r4 = CPyDict_GetKeysIter(m) + r3 = CPyDict_GetKeysIter(m) L1: - r5 = CPyDict_NextKey(r4, r1) - r6 = r5[1] - r1 = r6 - r7 = r5[0] - if r7 goto L2 else goto L4 :: bool + r4 = CPyDict_NextKey(r3, r1) + r5 = r4[1] + r1 = r5 + r6 = r4[0] + if r6 goto L2 else goto L4 :: bool L2: - r8 = r5[2] - r9 = cast(str, r8) - x = r9 - r10 = PyList_Append(r0, x) - r11 = r10 >= 0 :: signed + r7 = r4[2] + r8 = cast(str, r7) + x = r8 + r9 = PyList_Append(r0, x) + r10 = r9 >= 0 :: signed L3: - r12 = CPyDict_CheckSize(m, r3) + r11 = CPyDict_CheckSize(m, r2) goto L1 L4: - r13 = CPy_NoErrOccurred() + r12 = CPy_NoErrOccurred() L5: - r14 = PyList_New(0) - r15 = 0 - r16 = PyDict_Size(m) - r17 = r16 << 1 - r18 = CPyDict_GetValuesIter(m) + r13 = PyList_New(0) + r14 = 0 + r15 = PyDict_Size(m) + r16 = CPyDict_GetValuesIter(m) L6: - r19 = CPyDict_NextValue(r18, r15) - r20 = r19[1] - r15 = r20 - r21 = r19[0] - if r21 goto L7 else goto L9 :: bool + r17 = CPyDict_NextValue(r16, r14) + r18 = r17[1] + r14 = r18 + r19 = r17[0] + if r19 goto L7 else goto L9 :: bool L7: - r22 = r19[2] - r23 = unbox(int, r22) - x_2 = r23 - r24 = box(int, x_2) - r25 = PyList_Append(r14, r24) - r26 = r25 >= 0 :: signed + r20 = r17[2] + r21 = unbox(int, r20) + x_2 = r21 + r22 = box(int, x_2) + r23 = PyList_Append(r13, r22) + r24 = r23 >= 0 :: signed L8: - r27 = CPyDict_CheckSize(m, r17) + r25 = CPyDict_CheckSize(m, r15) goto L6 L9: - r28 = CPy_NoErrOccurred() + r26 = CPy_NoErrOccurred() L10: - r29 = PySet_New(0) - r30 = 0 - r31 = PyDict_Size(m) - r32 = r31 << 1 - r33 = CPyDict_GetKeysIter(m) + r27 = PySet_New(0) + r28 = 0 + r29 = PyDict_Size(m) + r30 = CPyDict_GetKeysIter(m) L11: - r34 = CPyDict_NextKey(r33, r30) - r35 = r34[1] - r30 = r35 - r36 = r34[0] - if r36 goto L12 else goto L14 :: bool + r31 = CPyDict_NextKey(r30, r28) + r32 = r31[1] + r28 = r32 + r33 = r31[0] + if r33 goto L12 else goto L14 :: bool L12: - r37 = r34[2] - r38 = cast(str, r37) - x_3 = r38 - r39 = PySet_Add(r29, x_3) - r40 = r39 >= 0 :: signed + r34 = r31[2] + r35 = cast(str, r34) + x_3 = r35 + r36 = PySet_Add(r27, x_3) + r37 = r36 >= 0 :: signed L13: - r41 = CPyDict_CheckSize(m, r32) + r38 = CPyDict_CheckSize(m, r29) goto L11 L14: - r42 = CPy_NoErrOccurred() + r39 = CPy_NoErrOccurred() L15: - r43 = PyDict_New() - r44 = 0 - r45 = PyDict_Size(m) - r46 = r45 << 1 - r47 = CPyDict_GetItemsIter(m) + r40 = PyDict_New() + r41 = 0 + r42 = PyDict_Size(m) + r43 = CPyDict_GetItemsIter(m) L16: - r48 = CPyDict_NextItem(r47, r44) - r49 = r48[1] - r44 = r49 - r50 = r48[0] - if r50 goto L17 else goto L19 :: bool + r44 = CPyDict_NextItem(r43, r41) + r45 = r44[1] + r41 = r45 + r46 = r44[0] + if r46 goto L17 else goto L19 :: bool L17: - r51 = r48[2] - r52 = r48[3] - r53 = cast(str, r51) - r54 = unbox(int, r52) - k = r53 - v = r54 - r55 = box(int, v) - r56 = CPyDict_SetItem(r43, k, r55) - r57 = r56 >= 0 :: signed + r47 = r44[2] + r48 = r44[3] + r49 = cast(str, r47) + r50 = unbox(int, r48) + k = r49 + v = r50 + r51 = box(int, v) + r52 = CPyDict_SetItem(r40, k, r51) + r53 = r52 >= 0 :: signed L18: - r58 = CPyDict_CheckSize(m, r46) + r54 = CPyDict_CheckSize(m, r42) goto L16 L19: - r59 = CPy_NoErrOccurred() + r55 = CPy_NoErrOccurred() L20: return 1 def fn_union(m): @@ -380,149 +372,141 @@ def fn_union(m): r0 :: list r1 :: short_int r2 :: native_int - r3 :: short_int - r4 :: object - r5 :: tuple[bool, short_int, object] - r6 :: short_int - r7 :: bool - r8 :: object - r9, x :: str - r10 :: i32 - r11, r12, r13 :: bit - r14 :: list - r15 :: short_int - r16 :: native_int - r17 :: short_int - r18 :: object - r19 :: tuple[bool, short_int, object] - r20 :: short_int - r21 :: bool - r22 :: object - r23, x_2 :: union[int, str] - r24 :: i32 - r25, r26, r27 :: bit - r28 :: set - r29 :: short_int - r30 :: native_int + r3 :: object + r4 :: tuple[bool, short_int, object] + r5 :: short_int + r6 :: bool + r7 :: object + r8, x :: str + r9 :: i32 + r10, r11, r12 :: bit + r13 :: list + r14 :: short_int + r15 :: native_int + r16 :: object + r17 :: tuple[bool, short_int, object] + r18 :: short_int + r19 :: bool + r20 :: object + r21, x_2 :: union[int, str] + r22 :: i32 + r23, r24, r25 :: bit + r26 :: set + r27 :: short_int + r28 :: native_int + r29 :: object + r30 :: tuple[bool, short_int, object] r31 :: short_int - r32 :: object - r33 :: tuple[bool, short_int, object] - r34 :: short_int - r35 :: bool - r36 :: object - r37, x_3 :: str - r38 :: i32 - r39, r40, r41 :: bit - r42 :: dict - r43 :: short_int - r44 :: native_int - r45 :: short_int - r46 :: object - r47 :: tuple[bool, short_int, object, object] - r48 :: short_int - r49 :: bool - r50, r51 :: object - r52 :: str - r53 :: union[int, str] + r32 :: bool + r33 :: object + r34, x_3 :: str + r35 :: i32 + r36, r37, r38 :: bit + r39 :: dict + r40 :: short_int + r41 :: native_int + r42 :: object + r43 :: tuple[bool, short_int, object, object] + r44 :: short_int + r45 :: bool + r46, r47 :: object + r48 :: str + r49 :: union[int, str] k :: str v :: union[int, str] - r54 :: i32 - r55, r56, r57 :: bit + r50 :: i32 + r51, r52, r53 :: bit L0: r0 = PyList_New(0) r1 = 0 r2 = PyDict_Size(m) - r3 = r2 << 1 - r4 = CPyDict_GetKeysIter(m) + r3 = CPyDict_GetKeysIter(m) L1: - r5 = CPyDict_NextKey(r4, r1) - r6 = r5[1] - r1 = r6 - r7 = r5[0] - if r7 goto L2 else goto L4 :: bool + r4 = CPyDict_NextKey(r3, r1) + r5 = r4[1] + r1 = r5 + r6 = r4[0] + if r6 goto L2 else goto L4 :: bool L2: - r8 = r5[2] - r9 = cast(str, r8) - x = r9 - r10 = PyList_Append(r0, x) - r11 = r10 >= 0 :: signed + r7 = r4[2] + r8 = cast(str, r7) + x = r8 + r9 = PyList_Append(r0, x) + r10 = r9 >= 0 :: signed L3: - r12 = CPyDict_CheckSize(m, r3) + r11 = CPyDict_CheckSize(m, r2) goto L1 L4: - r13 = CPy_NoErrOccurred() + r12 = CPy_NoErrOccurred() L5: - r14 = PyList_New(0) - r15 = 0 - r16 = PyDict_Size(m) - r17 = r16 << 1 - r18 = CPyDict_GetValuesIter(m) + r13 = PyList_New(0) + r14 = 0 + r15 = PyDict_Size(m) + r16 = CPyDict_GetValuesIter(m) L6: - r19 = CPyDict_NextValue(r18, r15) - r20 = r19[1] - r15 = r20 - r21 = r19[0] - if r21 goto L7 else goto L9 :: bool + r17 = CPyDict_NextValue(r16, r14) + r18 = r17[1] + r14 = r18 + r19 = r17[0] + if r19 goto L7 else goto L9 :: bool L7: - r22 = r19[2] - r23 = cast(union[int, str], r22) - x_2 = r23 - r24 = PyList_Append(r14, x_2) - r25 = r24 >= 0 :: signed + r20 = r17[2] + r21 = cast(union[int, str], r20) + x_2 = r21 + r22 = PyList_Append(r13, x_2) + r23 = r22 >= 0 :: signed L8: - r26 = CPyDict_CheckSize(m, r17) + r24 = CPyDict_CheckSize(m, r15) goto L6 L9: - r27 = CPy_NoErrOccurred() + r25 = CPy_NoErrOccurred() L10: - r28 = PySet_New(0) - r29 = 0 - r30 = PyDict_Size(m) - r31 = r30 << 1 - r32 = CPyDict_GetKeysIter(m) + r26 = PySet_New(0) + r27 = 0 + r28 = PyDict_Size(m) + r29 = CPyDict_GetKeysIter(m) L11: - r33 = CPyDict_NextKey(r32, r29) - r34 = r33[1] - r29 = r34 - r35 = r33[0] - if r35 goto L12 else goto L14 :: bool + r30 = CPyDict_NextKey(r29, r27) + r31 = r30[1] + r27 = r31 + r32 = r30[0] + if r32 goto L12 else goto L14 :: bool L12: - r36 = r33[2] - r37 = cast(str, r36) - x_3 = r37 - r38 = PySet_Add(r28, x_3) - r39 = r38 >= 0 :: signed + r33 = r30[2] + r34 = cast(str, r33) + x_3 = r34 + r35 = PySet_Add(r26, x_3) + r36 = r35 >= 0 :: signed L13: - r40 = CPyDict_CheckSize(m, r31) + r37 = CPyDict_CheckSize(m, r28) goto L11 L14: - r41 = CPy_NoErrOccurred() + r38 = CPy_NoErrOccurred() L15: - r42 = PyDict_New() - r43 = 0 - r44 = PyDict_Size(m) - r45 = r44 << 1 - r46 = CPyDict_GetItemsIter(m) + r39 = PyDict_New() + r40 = 0 + r41 = PyDict_Size(m) + r42 = CPyDict_GetItemsIter(m) L16: - r47 = CPyDict_NextItem(r46, r43) - r48 = r47[1] - r43 = r48 - r49 = r47[0] - if r49 goto L17 else goto L19 :: bool + r43 = CPyDict_NextItem(r42, r40) + r44 = r43[1] + r40 = r44 + r45 = r43[0] + if r45 goto L17 else goto L19 :: bool L17: - r50 = r47[2] - r51 = r47[3] - r52 = cast(str, r50) - r53 = cast(union[int, str], r51) - k = r52 - v = r53 - r54 = CPyDict_SetItem(r42, k, v) - r55 = r54 >= 0 :: signed + r46 = r43[2] + r47 = r43[3] + r48 = cast(str, r46) + r49 = cast(union[int, str], r47) + k = r48 + v = r49 + r50 = CPyDict_SetItem(r39, k, v) + r51 = r50 >= 0 :: signed L18: - r56 = CPyDict_CheckSize(m, r45) + r52 = CPyDict_CheckSize(m, r41) goto L16 L19: - r57 = CPy_NoErrOccurred() + r53 = CPy_NoErrOccurred() L20: return 1 def fn_typeddict(t): @@ -530,144 +514,136 @@ def fn_typeddict(t): r0 :: list r1 :: short_int r2 :: native_int - r3 :: short_int - r4 :: object - r5 :: tuple[bool, short_int, object] - r6 :: short_int - r7 :: bool - r8 :: object - r9, x :: str - r10 :: i32 - r11, r12, r13 :: bit - r14 :: list - r15 :: short_int - r16 :: native_int - r17 :: short_int - r18 :: object - r19 :: tuple[bool, short_int, object] - r20 :: short_int - r21 :: bool - r22, x_2 :: object - r23 :: i32 - r24, r25, r26 :: bit - r27 :: set - r28 :: short_int - r29 :: native_int + r3 :: object + r4 :: tuple[bool, short_int, object] + r5 :: short_int + r6 :: bool + r7 :: object + r8, x :: str + r9 :: i32 + r10, r11, r12 :: bit + r13 :: list + r14 :: short_int + r15 :: native_int + r16 :: object + r17 :: tuple[bool, short_int, object] + r18 :: short_int + r19 :: bool + r20, x_2 :: object + r21 :: i32 + r22, r23, r24 :: bit + r25 :: set + r26 :: short_int + r27 :: native_int + r28 :: object + r29 :: tuple[bool, short_int, object] r30 :: short_int - r31 :: object - r32 :: tuple[bool, short_int, object] - r33 :: short_int - r34 :: bool - r35 :: object - r36, x_3 :: str - r37 :: i32 - r38, r39, r40 :: bit - r41 :: dict - r42 :: short_int - r43 :: native_int - r44 :: short_int - r45 :: object - r46 :: tuple[bool, short_int, object, object] - r47 :: short_int - r48 :: bool - r49, r50 :: object - r51, k :: str + r31 :: bool + r32 :: object + r33, x_3 :: str + r34 :: i32 + r35, r36, r37 :: bit + r38 :: dict + r39 :: short_int + r40 :: native_int + r41 :: object + r42 :: tuple[bool, short_int, object, object] + r43 :: short_int + r44 :: bool + r45, r46 :: object + r47, k :: str v :: object - r52 :: i32 - r53, r54, r55 :: bit + r48 :: i32 + r49, r50, r51 :: bit L0: r0 = PyList_New(0) r1 = 0 r2 = PyDict_Size(t) - r3 = r2 << 1 - r4 = CPyDict_GetKeysIter(t) + r3 = CPyDict_GetKeysIter(t) L1: - r5 = CPyDict_NextKey(r4, r1) - r6 = r5[1] - r1 = r6 - r7 = r5[0] - if r7 goto L2 else goto L4 :: bool + r4 = CPyDict_NextKey(r3, r1) + r5 = r4[1] + r1 = r5 + r6 = r4[0] + if r6 goto L2 else goto L4 :: bool L2: - r8 = r5[2] - r9 = cast(str, r8) - x = r9 - r10 = PyList_Append(r0, x) - r11 = r10 >= 0 :: signed + r7 = r4[2] + r8 = cast(str, r7) + x = r8 + r9 = PyList_Append(r0, x) + r10 = r9 >= 0 :: signed L3: - r12 = CPyDict_CheckSize(t, r3) + r11 = CPyDict_CheckSize(t, r2) goto L1 L4: - r13 = CPy_NoErrOccurred() + r12 = CPy_NoErrOccurred() L5: - r14 = PyList_New(0) - r15 = 0 - r16 = PyDict_Size(t) - r17 = r16 << 1 - r18 = CPyDict_GetValuesIter(t) + r13 = PyList_New(0) + r14 = 0 + r15 = PyDict_Size(t) + r16 = CPyDict_GetValuesIter(t) L6: - r19 = CPyDict_NextValue(r18, r15) - r20 = r19[1] - r15 = r20 - r21 = r19[0] - if r21 goto L7 else goto L9 :: bool + r17 = CPyDict_NextValue(r16, r14) + r18 = r17[1] + r14 = r18 + r19 = r17[0] + if r19 goto L7 else goto L9 :: bool L7: - r22 = r19[2] - x_2 = r22 - r23 = PyList_Append(r14, x_2) - r24 = r23 >= 0 :: signed + r20 = r17[2] + x_2 = r20 + r21 = PyList_Append(r13, x_2) + r22 = r21 >= 0 :: signed L8: - r25 = CPyDict_CheckSize(t, r17) + r23 = CPyDict_CheckSize(t, r15) goto L6 L9: - r26 = CPy_NoErrOccurred() + r24 = CPy_NoErrOccurred() L10: - r27 = PySet_New(0) - r28 = 0 - r29 = PyDict_Size(t) - r30 = r29 << 1 - r31 = CPyDict_GetKeysIter(t) + r25 = PySet_New(0) + r26 = 0 + r27 = PyDict_Size(t) + r28 = CPyDict_GetKeysIter(t) L11: - r32 = CPyDict_NextKey(r31, r28) - r33 = r32[1] - r28 = r33 - r34 = r32[0] - if r34 goto L12 else goto L14 :: bool + r29 = CPyDict_NextKey(r28, r26) + r30 = r29[1] + r26 = r30 + r31 = r29[0] + if r31 goto L12 else goto L14 :: bool L12: - r35 = r32[2] - r36 = cast(str, r35) - x_3 = r36 - r37 = PySet_Add(r27, x_3) - r38 = r37 >= 0 :: signed + r32 = r29[2] + r33 = cast(str, r32) + x_3 = r33 + r34 = PySet_Add(r25, x_3) + r35 = r34 >= 0 :: signed L13: - r39 = CPyDict_CheckSize(t, r30) + r36 = CPyDict_CheckSize(t, r27) goto L11 L14: - r40 = CPy_NoErrOccurred() + r37 = CPy_NoErrOccurred() L15: - r41 = PyDict_New() - r42 = 0 - r43 = PyDict_Size(t) - r44 = r43 << 1 - r45 = CPyDict_GetItemsIter(t) + r38 = PyDict_New() + r39 = 0 + r40 = PyDict_Size(t) + r41 = CPyDict_GetItemsIter(t) L16: - r46 = CPyDict_NextItem(r45, r42) - r47 = r46[1] - r42 = r47 - r48 = r46[0] - if r48 goto L17 else goto L19 :: bool + r42 = CPyDict_NextItem(r41, r39) + r43 = r42[1] + r39 = r43 + r44 = r42[0] + if r44 goto L17 else goto L19 :: bool L17: - r49 = r46[2] - r50 = r46[3] - r51 = cast(str, r49) - k = r51 - v = r50 - r52 = CPyDict_SetItem(r41, k, v) - r53 = r52 >= 0 :: signed + r45 = r42[2] + r46 = r42[3] + r47 = cast(str, r45) + k = r47 + v = r46 + r48 = CPyDict_SetItem(r38, k, v) + r49 = r48 >= 0 :: signed L18: - r54 = CPyDict_CheckSize(t, r44) + r50 = CPyDict_CheckSize(t, r40) goto L16 L19: - r55 = CPy_NoErrOccurred() + r51 = CPy_NoErrOccurred() L20: return 1 @@ -713,38 +689,34 @@ def inner_deco_obj.__call__(__mypyc_self__, args, kwargs): r0 :: __main__.deco_env r1 :: native_int r2 :: list - r3 :: short_int - r4 :: native_int - r5 :: short_int - r6 :: bit - r7, x :: object - r8 :: bit - r9 :: short_int + r3, r4 :: native_int + r5 :: bit + r6, x :: object + r7 :: native_int can_listcomp :: list - r10 :: dict - r11 :: short_int - r12 :: native_int + r8 :: dict + r9 :: short_int + r10 :: native_int + r11 :: object + r12 :: tuple[bool, short_int, object, object] r13 :: short_int - r14 :: object - r15 :: tuple[bool, short_int, object, object] - r16 :: short_int - r17 :: bool - r18, r19 :: object - r20, k :: str + r14 :: bool + r15, r16 :: object + r17, k :: str v :: object - r21 :: i32 - r22, r23, r24 :: bit + r18 :: i32 + r19, r20, r21 :: bit can_dictcomp :: dict - r25, can_iter, r26, can_use_keys, r27, can_use_values :: list - r28 :: object - r29 :: list - r30 :: object - r31 :: dict - r32 :: i32 - r33 :: bit - r34 :: tuple - r35 :: object - r36 :: int + r22, can_iter, r23, can_use_keys, r24, can_use_values :: list + r25 :: object + r26 :: list + r27 :: object + r28 :: dict + r29 :: i32 + r30 :: bit + r31 :: tuple + r32 :: object + r33 :: int L0: r0 = __mypyc_self__.__mypyc_env__ r1 = var_object_size args @@ -752,61 +724,59 @@ L0: r3 = 0 L1: r4 = var_object_size args - r5 = r4 << 1 - r6 = int_lt r3, r5 - if r6 goto L2 else goto L4 :: bool + r5 = r3 < r4 :: signed + if r5 goto L2 else goto L4 :: bool L2: - r7 = CPySequenceTuple_GetItem(args, r3) - x = r7 - r8 = CPyList_SetItemUnsafe(r2, r3, x) + r6 = CPySequenceTuple_GetItemUnsafe(args, r3) + x = r6 + CPyList_SetItemUnsafe(r2, r3, x) L3: - r9 = r3 + 2 - r3 = r9 + r7 = r3 + 1 + r3 = r7 goto L1 L4: can_listcomp = r2 - r10 = PyDict_New() - r11 = 0 - r12 = PyDict_Size(kwargs) - r13 = r12 << 1 - r14 = CPyDict_GetItemsIter(kwargs) + r8 = PyDict_New() + r9 = 0 + r10 = PyDict_Size(kwargs) + r11 = CPyDict_GetItemsIter(kwargs) L5: - r15 = CPyDict_NextItem(r14, r11) - r16 = r15[1] - r11 = r16 - r17 = r15[0] - if r17 goto L6 else goto L8 :: bool + r12 = CPyDict_NextItem(r11, r9) + r13 = r12[1] + r9 = r13 + r14 = r12[0] + if r14 goto L6 else goto L8 :: bool L6: - r18 = r15[2] - r19 = r15[3] - r20 = cast(str, r18) - k = r20 - v = r19 - r21 = CPyDict_SetItem(r10, k, v) - r22 = r21 >= 0 :: signed + r15 = r12[2] + r16 = r12[3] + r17 = cast(str, r15) + k = r17 + v = r16 + r18 = CPyDict_SetItem(r8, k, v) + r19 = r18 >= 0 :: signed L7: - r23 = CPyDict_CheckSize(kwargs, r13) + r20 = CPyDict_CheckSize(kwargs, r10) goto L5 L8: - r24 = CPy_NoErrOccurred() + r21 = CPy_NoErrOccurred() L9: - can_dictcomp = r10 - r25 = PySequence_List(kwargs) - can_iter = r25 - r26 = CPyDict_Keys(kwargs) - can_use_keys = r26 - r27 = CPyDict_Values(kwargs) - can_use_values = r27 - r28 = r0.func - r29 = PyList_New(0) - r30 = CPyList_Extend(r29, args) - r31 = PyDict_New() - r32 = CPyDict_UpdateInDisplay(r31, kwargs) - r33 = r32 >= 0 :: signed - r34 = PyList_AsTuple(r29) - r35 = PyObject_Call(r28, r34, r31) - r36 = unbox(int, r35) - return r36 + can_dictcomp = r8 + r22 = PySequence_List(kwargs) + can_iter = r22 + r23 = CPyDict_Keys(kwargs) + can_use_keys = r23 + r24 = CPyDict_Values(kwargs) + can_use_values = r24 + r25 = r0.func + r26 = PyList_New(0) + r27 = CPyList_Extend(r26, args) + r28 = PyDict_New() + r29 = CPyDict_UpdateInDisplay(r28, kwargs) + r30 = r29 >= 0 :: signed + r31 = PyList_AsTuple(r26) + r32 = PyObject_Call(r25, r31, r28) + r33 = unbox(int, r32) + return r33 def deco(func): func :: object r0 :: __main__.deco_env diff --git a/mypyc/test-data/irbuild-lists.test b/mypyc/test-data/irbuild-lists.test index efd38870974d9..06120e077af95 100644 --- a/mypyc/test-data/irbuild-lists.test +++ b/mypyc/test-data/irbuild-lists.test @@ -371,27 +371,21 @@ def f(source): source :: list r0 :: native_int r1 :: list - r2 :: short_int - r3 :: native_int - r4 :: short_int - r5 :: bit - r6 :: object - r7, x, r8 :: int - r9 :: object - r10 :: bit - r11 :: short_int + r2, r3 :: native_int + r4 :: bit + r5 :: object + r6, x, r7 :: int + r8 :: object + r9 :: native_int a :: list - r12 :: native_int - r13 :: list - r14 :: short_int - r15 :: native_int - r16 :: short_int - r17 :: bit + r10 :: native_int + r11 :: list + r12, r13 :: native_int + r14 :: bit + r15 :: object + r16, x_2, r17 :: int r18 :: object - r19, x_2, r20 :: int - r21 :: object - r22 :: bit - r23 :: short_int + r19 :: native_int b :: list L0: r0 = var_object_size source @@ -399,43 +393,41 @@ L0: r2 = 0 L1: r3 = var_object_size source - r4 = r3 << 1 - r5 = int_lt r2, r4 - if r5 goto L2 else goto L4 :: bool + r4 = r2 < r3 :: signed + if r4 goto L2 else goto L4 :: bool L2: - r6 = list_get_item_unsafe source, r2 - r7 = unbox(int, r6) - x = r7 - r8 = CPyTagged_Add(x, 2) - r9 = box(int, r8) - r10 = CPyList_SetItemUnsafe(r1, r2, r9) + r5 = list_get_item_unsafe source, r2 + r6 = unbox(int, r5) + x = r6 + r7 = CPyTagged_Add(x, 2) + r8 = box(int, r7) + CPyList_SetItemUnsafe(r1, r2, r8) L3: - r11 = r2 + 2 - r2 = r11 + r9 = r2 + 1 + r2 = r9 goto L1 L4: a = r1 - r12 = var_object_size source - r13 = PyList_New(r12) - r14 = 0 + r10 = var_object_size source + r11 = PyList_New(r10) + r12 = 0 L5: - r15 = var_object_size source - r16 = r15 << 1 - r17 = int_lt r14, r16 - if r17 goto L6 else goto L8 :: bool + r13 = var_object_size source + r14 = r12 < r13 :: signed + if r14 goto L6 else goto L8 :: bool L6: - r18 = list_get_item_unsafe source, r14 - r19 = unbox(int, r18) - x_2 = r19 - r20 = CPyTagged_Add(x_2, 2) - r21 = box(int, r20) - r22 = CPyList_SetItemUnsafe(r13, r14, r21) + r15 = list_get_item_unsafe source, r12 + r16 = unbox(int, r15) + x_2 = r16 + r17 = CPyTagged_Add(x_2, 2) + r18 = box(int, r17) + CPyList_SetItemUnsafe(r11, r12, r18) L7: - r23 = r14 + 2 - r14 = r23 + r19 = r12 + 1 + r12 = r19 goto L5 L8: - b = r13 + b = r11 return 1 [case testGeneratorNext] @@ -446,40 +438,37 @@ def test(x: List[int]) -> None: [out] def test(x): x :: list - r0 :: short_int - r1 :: native_int - r2 :: short_int - r3 :: bit - r4 :: object - r5, i :: int - r6 :: object - r7 :: union[int, None] - r8 :: short_int - r9 :: object + r0, r1 :: native_int + r2 :: bit + r3 :: object + r4, i :: int + r5 :: object + r6 :: union[int, None] + r7 :: native_int + r8 :: object res :: union[int, None] L0: r0 = 0 L1: r1 = var_object_size x - r2 = r1 << 1 - r3 = int_lt r0, r2 - if r3 goto L2 else goto L4 :: bool + r2 = r0 < r1 :: signed + if r2 goto L2 else goto L4 :: bool L2: - r4 = list_get_item_unsafe x, r0 - r5 = unbox(int, r4) - i = r5 - r6 = box(int, i) - r7 = r6 + r3 = list_get_item_unsafe x, r0 + r4 = unbox(int, r3) + i = r4 + r5 = box(int, i) + r6 = r5 goto L5 L3: - r8 = r0 + 2 - r0 = r8 + r7 = r0 + 1 + r0 = r7 goto L1 L4: - r9 = box(None, 1) - r7 = r9 + r8 = box(None, 1) + r6 = r8 L5: - res = r7 + res = r6 return 1 [case testSimplifyListUnion] @@ -517,53 +506,47 @@ L2: return r4 def loop(a): a :: list - r0 :: short_int - r1 :: native_int - r2 :: short_int - r3 :: bit - r4 :: object - r5, x :: union[str, bytes] - r6 :: short_int + r0, r1 :: native_int + r2 :: bit + r3 :: object + r4, x :: union[str, bytes] + r5 :: native_int L0: r0 = 0 L1: r1 = var_object_size a - r2 = r1 << 1 - r3 = int_lt r0, r2 - if r3 goto L2 else goto L4 :: bool + r2 = r0 < r1 :: signed + if r2 goto L2 else goto L4 :: bool L2: - r4 = list_get_item_unsafe a, r0 - r5 = cast(union[str, bytes], r4) - x = r5 + r3 = list_get_item_unsafe a, r0 + r4 = cast(union[str, bytes], r3) + x = r4 L3: - r6 = r0 + 2 - r0 = r6 + r5 = r0 + 1 + r0 = r5 goto L1 L4: return 1 def nested_union(a): a :: list - r0 :: short_int - r1 :: native_int - r2 :: short_int - r3 :: bit - r4 :: object - r5, x :: union[str, None] - r6 :: short_int + r0, r1 :: native_int + r2 :: bit + r3 :: object + r4, x :: union[str, None] + r5 :: native_int L0: r0 = 0 L1: r1 = var_object_size a - r2 = r1 << 1 - r3 = int_lt r0, r2 - if r3 goto L2 else goto L4 :: bool + r2 = r0 < r1 :: signed + if r2 goto L2 else goto L4 :: bool L2: - r4 = list_get_item_unsafe a, r0 - r5 = cast(union[str, None], r4) - x = r5 + r3 = list_get_item_unsafe a, r0 + r4 = cast(union[str, None], r3) + x = r4 L3: - r6 = r0 + 2 - r0 = r6 + r5 = r0 + 1 + r0 = r5 goto L1 L4: return 1 diff --git a/mypyc/test-data/irbuild-set.test b/mypyc/test-data/irbuild-set.test index c42a1fa74a757..5586a2bf4cfb4 100644 --- a/mypyc/test-data/irbuild-set.test +++ b/mypyc/test-data/irbuild-set.test @@ -85,16 +85,14 @@ def test1(): r4 :: ptr tmp_list :: list r5 :: set - r6 :: short_int - r7 :: native_int - r8 :: short_int - r9 :: bit - r10 :: object - r11, x, r12 :: int - r13 :: object - r14 :: i32 - r15 :: bit - r16 :: short_int + r6, r7 :: native_int + r8 :: bit + r9 :: object + r10, x, r11 :: int + r12 :: object + r13 :: i32 + r14 :: bit + r15 :: native_int a :: set L0: r0 = PyList_New(3) @@ -111,20 +109,19 @@ L0: r6 = 0 L1: r7 = var_object_size tmp_list - r8 = r7 << 1 - r9 = int_lt r6, r8 - if r9 goto L2 else goto L4 :: bool + r8 = r6 < r7 :: signed + if r8 goto L2 else goto L4 :: bool L2: - r10 = list_get_item_unsafe tmp_list, r6 - r11 = unbox(int, r10) - x = r11 - r12 = f(x) - r13 = box(int, r12) - r14 = PySet_Add(r5, r13) - r15 = r14 >= 0 :: signed + r9 = list_get_item_unsafe tmp_list, r6 + r10 = unbox(int, r9) + x = r10 + r11 = f(x) + r12 = box(int, r11) + r13 = PySet_Add(r5, r12) + r14 = r13 >= 0 :: signed L3: - r16 = r6 + 2 - r6 = r16 + r15 = r6 + 1 + r6 = r15 goto L1 L4: a = r5 @@ -168,16 +165,15 @@ def test3(): r7 :: set r8 :: short_int r9 :: native_int - r10 :: short_int - r11 :: object - r12 :: tuple[bool, short_int, object] - r13 :: short_int - r14 :: bool - r15 :: object - r16, x, r17 :: int - r18 :: object - r19 :: i32 - r20, r21, r22 :: bit + r10 :: object + r11 :: tuple[bool, short_int, object] + r12 :: short_int + r13 :: bool + r14 :: object + r15, x, r16 :: int + r17 :: object + r18 :: i32 + r19, r20, r21 :: bit c :: set L0: r0 = '1' @@ -191,27 +187,26 @@ L0: r7 = PySet_New(0) r8 = 0 r9 = PyDict_Size(tmp_dict) - r10 = r9 << 1 - r11 = CPyDict_GetKeysIter(tmp_dict) + r10 = CPyDict_GetKeysIter(tmp_dict) L1: - r12 = CPyDict_NextKey(r11, r8) - r13 = r12[1] - r8 = r13 - r14 = r12[0] - if r14 goto L2 else goto L4 :: bool + r11 = CPyDict_NextKey(r10, r8) + r12 = r11[1] + r8 = r12 + r13 = r11[0] + if r13 goto L2 else goto L4 :: bool L2: - r15 = r12[2] - r16 = unbox(int, r15) - x = r16 - r17 = f(x) - r18 = box(int, r17) - r19 = PySet_Add(r7, r18) - r20 = r19 >= 0 :: signed + r14 = r11[2] + r15 = unbox(int, r14) + x = r15 + r16 = f(x) + r17 = box(int, r16) + r18 = PySet_Add(r7, r17) + r19 = r18 >= 0 :: signed L3: - r21 = CPyDict_CheckSize(tmp_dict, r10) + r20 = CPyDict_CheckSize(tmp_dict, r9) goto L1 L4: - r22 = CPy_NoErrOccurred() + r21 = CPy_NoErrOccurred() L5: c = r7 return 1 @@ -313,28 +308,26 @@ def test(): tmp_list :: list r7 :: set r8, r9 :: list - r10 :: short_int - r11 :: native_int - r12 :: short_int - r13 :: bit - r14 :: object - r15, z :: int - r16 :: bit - r17 :: int - r18 :: object - r19 :: i32 - r20 :: bit - r21 :: short_int - r22, r23, r24 :: object - r25, y, r26 :: int - r27 :: object - r28 :: i32 - r29, r30 :: bit - r31, r32, r33 :: object - r34, x, r35 :: int - r36 :: object - r37 :: i32 - r38, r39 :: bit + r10, r11 :: native_int + r12 :: bit + r13 :: object + r14, z :: int + r15 :: bit + r16 :: int + r17 :: object + r18 :: i32 + r19 :: bit + r20 :: native_int + r21, r22, r23 :: object + r24, y, r25 :: int + r26 :: object + r27 :: i32 + r28, r29 :: bit + r30, r31, r32 :: object + r33, x, r34 :: int + r35 :: object + r36 :: i32 + r37, r38 :: bit a :: set L0: r0 = PyList_New(5) @@ -357,60 +350,59 @@ L0: r10 = 0 L1: r11 = var_object_size tmp_list - r12 = r11 << 1 - r13 = int_lt r10, r12 - if r13 goto L2 else goto L6 :: bool + r12 = r10 < r11 :: signed + if r12 goto L2 else goto L6 :: bool L2: - r14 = list_get_item_unsafe tmp_list, r10 - r15 = unbox(int, r14) - z = r15 - r16 = int_lt z, 8 - if r16 goto L4 else goto L3 :: bool + r13 = list_get_item_unsafe tmp_list, r10 + r14 = unbox(int, r13) + z = r14 + r15 = int_lt z, 8 + if r15 goto L4 else goto L3 :: bool L3: goto L5 L4: - r17 = f1(z) - r18 = box(int, r17) - r19 = PyList_Append(r9, r18) - r20 = r19 >= 0 :: signed + r16 = f1(z) + r17 = box(int, r16) + r18 = PyList_Append(r9, r17) + r19 = r18 >= 0 :: signed L5: - r21 = r10 + 2 - r10 = r21 + r20 = r10 + 1 + r10 = r20 goto L1 L6: - r22 = PyObject_GetIter(r9) - r23 = PyObject_GetIter(r22) + r21 = PyObject_GetIter(r9) + r22 = PyObject_GetIter(r21) L7: - r24 = PyIter_Next(r23) - if is_error(r24) goto L10 else goto L8 + r23 = PyIter_Next(r22) + if is_error(r23) goto L10 else goto L8 L8: - r25 = unbox(int, r24) - y = r25 - r26 = f2(y) - r27 = box(int, r26) - r28 = PyList_Append(r8, r27) - r29 = r28 >= 0 :: signed + r24 = unbox(int, r23) + y = r24 + r25 = f2(y) + r26 = box(int, r25) + r27 = PyList_Append(r8, r26) + r28 = r27 >= 0 :: signed L9: goto L7 L10: - r30 = CPy_NoErrOccurred() + r29 = CPy_NoErrOccurred() L11: - r31 = PyObject_GetIter(r8) - r32 = PyObject_GetIter(r31) + r30 = PyObject_GetIter(r8) + r31 = PyObject_GetIter(r30) L12: - r33 = PyIter_Next(r32) - if is_error(r33) goto L15 else goto L13 + r32 = PyIter_Next(r31) + if is_error(r32) goto L15 else goto L13 L13: - r34 = unbox(int, r33) - x = r34 - r35 = f3(x) - r36 = box(int, r35) - r37 = PySet_Add(r7, r36) - r38 = r37 >= 0 :: signed + r33 = unbox(int, r32) + x = r33 + r34 = f3(x) + r35 = box(int, r34) + r36 = PySet_Add(r7, r35) + r37 = r36 >= 0 :: signed L14: goto L12 L15: - r39 = CPy_NoErrOccurred() + r38 = CPy_NoErrOccurred() L16: a = r7 return 1 diff --git a/mypyc/test-data/irbuild-statements.test b/mypyc/test-data/irbuild-statements.test index 1f9336d321409..48b8e0e318b8b 100644 --- a/mypyc/test-data/irbuild-statements.test +++ b/mypyc/test-data/irbuild-statements.test @@ -230,30 +230,27 @@ def f(ls: List[int]) -> int: def f(ls): ls :: list y :: int - r0 :: short_int - r1 :: native_int - r2 :: short_int - r3 :: bit - r4 :: object - r5, x, r6 :: int - r7 :: short_int + r0, r1 :: native_int + r2 :: bit + r3 :: object + r4, x, r5 :: int + r6 :: native_int L0: y = 0 r0 = 0 L1: r1 = var_object_size ls - r2 = r1 << 1 - r3 = int_lt r0, r2 - if r3 goto L2 else goto L4 :: bool + r2 = r0 < r1 :: signed + if r2 goto L2 else goto L4 :: bool L2: - r4 = list_get_item_unsafe ls, r0 - r5 = unbox(int, r4) - x = r5 - r6 = CPyTagged_Add(y, x) - y = r6 + r3 = list_get_item_unsafe ls, r0 + r4 = unbox(int, r3) + x = r4 + r5 = CPyTagged_Add(y, x) + y = r5 L3: - r7 = r0 + 2 - r0 = r7 + r6 = r0 + 1 + r0 = r6 goto L1 L4: return y @@ -269,39 +266,37 @@ def f(d): d :: dict r0 :: short_int r1 :: native_int - r2 :: short_int - r3 :: object - r4 :: tuple[bool, short_int, object] - r5 :: short_int - r6 :: bool - r7 :: object - r8, key :: int - r9, r10 :: object - r11 :: int - r12, r13 :: bit + r2 :: object + r3 :: tuple[bool, short_int, object] + r4 :: short_int + r5 :: bool + r6 :: object + r7, key :: int + r8, r9 :: object + r10 :: int + r11, r12 :: bit L0: r0 = 0 r1 = PyDict_Size(d) - r2 = r1 << 1 - r3 = CPyDict_GetKeysIter(d) + r2 = CPyDict_GetKeysIter(d) L1: - r4 = CPyDict_NextKey(r3, r0) - r5 = r4[1] - r0 = r5 - r6 = r4[0] - if r6 goto L2 else goto L4 :: bool + r3 = CPyDict_NextKey(r2, r0) + r4 = r3[1] + r0 = r4 + r5 = r3[0] + if r5 goto L2 else goto L4 :: bool L2: - r7 = r4[2] - r8 = unbox(int, r7) - key = r8 - r9 = box(int, key) - r10 = CPyDict_GetItem(d, r9) - r11 = unbox(int, r10) + r6 = r3[2] + r7 = unbox(int, r6) + key = r7 + r8 = box(int, key) + r9 = CPyDict_GetItem(d, r8) + r10 = unbox(int, r9) L3: - r12 = CPyDict_CheckSize(d, r2) + r11 = CPyDict_CheckSize(d, r1) goto L1 L4: - r13 = CPy_NoErrOccurred() + r12 = CPy_NoErrOccurred() L5: return 1 @@ -321,54 +316,52 @@ def sum_over_even_values(d): s :: int r0 :: short_int r1 :: native_int - r2 :: short_int - r3 :: object - r4 :: tuple[bool, short_int, object] - r5 :: short_int - r6 :: bool - r7 :: object - r8, key :: int - r9, r10 :: object - r11, r12 :: int - r13 :: bit - r14, r15 :: object - r16, r17 :: int - r18, r19 :: bit + r2 :: object + r3 :: tuple[bool, short_int, object] + r4 :: short_int + r5 :: bool + r6 :: object + r7, key :: int + r8, r9 :: object + r10, r11 :: int + r12 :: bit + r13, r14 :: object + r15, r16 :: int + r17, r18 :: bit L0: s = 0 r0 = 0 r1 = PyDict_Size(d) - r2 = r1 << 1 - r3 = CPyDict_GetKeysIter(d) + r2 = CPyDict_GetKeysIter(d) L1: - r4 = CPyDict_NextKey(r3, r0) - r5 = r4[1] - r0 = r5 - r6 = r4[0] - if r6 goto L2 else goto L6 :: bool + r3 = CPyDict_NextKey(r2, r0) + r4 = r3[1] + r0 = r4 + r5 = r3[0] + if r5 goto L2 else goto L6 :: bool L2: - r7 = r4[2] - r8 = unbox(int, r7) - key = r8 - r9 = box(int, key) - r10 = CPyDict_GetItem(d, r9) - r11 = unbox(int, r10) - r12 = CPyTagged_Remainder(r11, 4) - r13 = r12 != 0 - if r13 goto L3 else goto L4 :: bool + r6 = r3[2] + r7 = unbox(int, r6) + key = r7 + r8 = box(int, key) + r9 = CPyDict_GetItem(d, r8) + r10 = unbox(int, r9) + r11 = CPyTagged_Remainder(r10, 4) + r12 = r11 != 0 + if r12 goto L3 else goto L4 :: bool L3: goto L5 L4: - r14 = box(int, key) - r15 = CPyDict_GetItem(d, r14) - r16 = unbox(int, r15) - r17 = CPyTagged_Add(s, r16) - s = r17 + r13 = box(int, key) + r14 = CPyDict_GetItem(d, r13) + r15 = unbox(int, r14) + r16 = CPyTagged_Add(s, r15) + s = r16 L5: - r18 = CPyDict_CheckSize(d, r2) + r17 = CPyDict_CheckSize(d, r1) goto L1 L6: - r19 = CPy_NoErrOccurred() + r18 = CPy_NoErrOccurred() L7: return s @@ -597,16 +590,16 @@ L0: r0 = CPySequence_CheckUnpackCount(l, 2) r1 = r0 >= 0 :: signed r2 = list_get_item_unsafe l, 0 - r3 = list_get_item_unsafe l, 2 + r3 = list_get_item_unsafe l, 1 x = r2 r4 = unbox(int, r3) y = r4 r5 = CPySequence_CheckUnpackCount(t, 2) r6 = r5 >= 0 :: signed - r7 = CPySequenceTuple_GetItem(t, 0) - r8 = CPySequenceTuple_GetItem(t, 2) - r9 = unbox(int, r8) + r7 = CPySequenceTuple_GetItemUnsafe(t, 0) + r8 = CPySequenceTuple_GetItemUnsafe(t, 1) x = r7 + r9 = unbox(int, r8) y = r9 return 1 @@ -872,33 +865,32 @@ def g(x: Iterable[int]) -> None: [out] def f(a): a :: list - r0, r1 :: short_int - r2 :: native_int - r3 :: short_int - r4 :: bit + r0 :: short_int + r1, r2 :: native_int + r3 :: bit i :: int - r5 :: object - r6, x, r7 :: int - r8, r9 :: short_int + r4 :: object + r5, x, r6 :: int + r7 :: short_int + r8 :: native_int L0: r0 = 0 r1 = 0 L1: r2 = var_object_size a - r3 = r2 << 1 - r4 = int_lt r1, r3 - if r4 goto L2 else goto L4 :: bool + r3 = r1 < r2 :: signed + if r3 goto L2 else goto L4 :: bool L2: i = r0 - r5 = list_get_item_unsafe a, r1 - r6 = unbox(int, r5) - x = r6 - r7 = CPyTagged_Add(i, x) + r4 = list_get_item_unsafe a, r1 + r5 = unbox(int, r4) + x = r5 + r6 = CPyTagged_Add(i, x) L3: - r8 = r0 + 2 - r0 = r8 - r9 = r1 + 2 - r1 = r9 + r7 = r0 + 2 + r0 = r7 + r8 = r1 + 1 + r1 = r8 goto L1 L4: L5: @@ -944,66 +936,65 @@ def g(a: Iterable[bool], b: List[int]) -> None: def f(a, b): a :: list b :: object - r0 :: short_int + r0 :: native_int r1 :: object r2 :: native_int - r3 :: short_int - r4 :: bit - r5, r6 :: object - r7, x :: int - r8, y :: bool - r9 :: i32 - r10 :: bit - r11 :: bool - r12 :: short_int - r13 :: bit + r3 :: bit + r4, r5 :: object + r6, x :: int + r7, y :: bool + r8 :: i32 + r9 :: bit + r10 :: bool + r11 :: native_int + r12 :: bit L0: r0 = 0 r1 = PyObject_GetIter(b) L1: r2 = var_object_size a - r3 = r2 << 1 - r4 = int_lt r0, r3 - if r4 goto L2 else goto L7 :: bool + r3 = r0 < r2 :: signed + if r3 goto L2 else goto L7 :: bool L2: - r5 = PyIter_Next(r1) - if is_error(r5) goto L7 else goto L3 + r4 = PyIter_Next(r1) + if is_error(r4) goto L7 else goto L3 L3: - r6 = list_get_item_unsafe a, r0 - r7 = unbox(int, r6) - x = r7 - r8 = unbox(bool, r5) - y = r8 - r9 = PyObject_IsTrue(b) - r10 = r9 >= 0 :: signed - r11 = truncate r9: i32 to builtins.bool - if r11 goto L4 else goto L5 :: bool + r5 = list_get_item_unsafe a, r0 + r6 = unbox(int, r5) + x = r6 + r7 = unbox(bool, r4) + y = r7 + r8 = PyObject_IsTrue(b) + r9 = r8 >= 0 :: signed + r10 = truncate r8: i32 to builtins.bool + if r10 goto L4 else goto L5 :: bool L4: x = 2 L5: L6: - r12 = r0 + 2 - r0 = r12 + r11 = r0 + 1 + r0 = r11 goto L1 L7: - r13 = CPy_NoErrOccurred() + r12 = CPy_NoErrOccurred() L8: return 1 def g(a, b): a :: object b :: list r0 :: object - r1, r2 :: short_int + r1 :: native_int + r2 :: short_int z :: int r3 :: object r4 :: native_int - r5 :: short_int - r6, r7 :: bit - r8, x :: bool - r9 :: object - r10, y :: int - r11, r12 :: short_int - r13 :: bit + r5, r6 :: bit + r7, x :: bool + r8 :: object + r9, y :: int + r10 :: native_int + r11 :: short_int + r12 :: bit L0: r0 = PyObject_GetIter(a) r1 = 0 @@ -1014,28 +1005,27 @@ L1: if is_error(r3) goto L6 else goto L2 L2: r4 = var_object_size b - r5 = r4 << 1 - r6 = int_lt r1, r5 - if r6 goto L3 else goto L6 :: bool + r5 = r1 < r4 :: signed + if r5 goto L3 else goto L6 :: bool L3: - r7 = int_lt r2, 10 - if r7 goto L4 else goto L6 :: bool + r6 = int_lt r2, 10 + if r6 goto L4 else goto L6 :: bool L4: - r8 = unbox(bool, r3) - x = r8 - r9 = list_get_item_unsafe b, r1 - r10 = unbox(int, r9) - y = r10 + r7 = unbox(bool, r3) + x = r7 + r8 = list_get_item_unsafe b, r1 + r9 = unbox(int, r8) + y = r9 x = 0 L5: - r11 = r1 + 2 - r1 = r11 - r12 = r2 + 2 - r2 = r12 - z = r12 + r10 = r1 + 1 + r1 = r10 + r11 = r2 + 2 + r2 = r11 + z = r11 goto L1 L6: - r13 = CPy_NoErrOccurred() + r12 = CPy_NoErrOccurred() L7: return 1 diff --git a/mypyc/test-data/irbuild-tuple.test b/mypyc/test-data/irbuild-tuple.test index 2220217510805..c39968fc139e9 100644 --- a/mypyc/test-data/irbuild-tuple.test +++ b/mypyc/test-data/irbuild-tuple.test @@ -127,27 +127,24 @@ def f(xs: Tuple[str, ...]) -> None: [out] def f(xs): xs :: tuple - r0 :: short_int - r1 :: native_int - r2 :: short_int - r3 :: bit - r4 :: object - r5, x :: str - r6 :: short_int + r0, r1 :: native_int + r2 :: bit + r3 :: object + r4, x :: str + r5 :: native_int L0: r0 = 0 L1: r1 = var_object_size xs - r2 = r1 << 1 - r3 = int_lt r0, r2 - if r3 goto L2 else goto L4 :: bool + r2 = r0 < r1 :: signed + if r2 goto L2 else goto L4 :: bool L2: - r4 = CPySequenceTuple_GetItem(xs, r0) - r5 = cast(str, r4) - x = r5 + r3 = CPySequenceTuple_GetItemUnsafe(xs, r0) + r4 = cast(str, r3) + x = r4 L3: - r6 = r0 + 2 - r0 = r6 + r5 = r0 + 1 + r0 = r5 goto L1 L4: return 1 @@ -234,16 +231,13 @@ def test(): source :: list r5 :: native_int r6 :: tuple - r7 :: short_int - r8 :: native_int - r9 :: short_int - r10 :: bit - r11 :: object - r12, x :: int - r13 :: bool - r14 :: object - r15 :: bit - r16 :: short_int + r7, r8 :: native_int + r9 :: bit + r10 :: object + r11, x :: int + r12 :: bool + r13 :: object + r14 :: native_int a :: tuple L0: r0 = PyList_New(3) @@ -261,25 +255,24 @@ L0: r7 = 0 L1: r8 = var_object_size source - r9 = r8 << 1 - r10 = int_lt r7, r9 - if r10 goto L2 else goto L4 :: bool + r9 = r7 < r8 :: signed + if r9 goto L2 else goto L4 :: bool L2: - r11 = list_get_item_unsafe source, r7 - r12 = unbox(int, r11) - x = r12 - r13 = f(x) - r14 = box(bool, r13) - r15 = CPySequenceTuple_SetItemUnsafe(r6, r7, r14) + r10 = list_get_item_unsafe source, r7 + r11 = unbox(int, r10) + x = r11 + r12 = f(x) + r13 = box(bool, r12) + CPySequenceTuple_SetItemUnsafe(r6, r7, r13) L3: - r16 = r7 + 2 - r7 = r16 + r14 = r7 + 1 + r7 = r14 goto L1 L4: a = r6 return 1 -[case testTupleBuiltFromStr] +[case testTupleBuiltFromStr_64bit] def f2(val: str) -> str: return val + "f2" @@ -298,14 +291,11 @@ def test(): r1 :: native_int r2 :: bit r3 :: tuple - r4 :: short_int - r5 :: native_int - r6 :: bit - r7 :: short_int - r8 :: bit - r9, x, r10 :: str - r11 :: bit - r12 :: short_int + r4, r5 :: native_int + r6, r7, r8, r9 :: bit + r10, r11, r12 :: int + r13, x, r14 :: str + r15 :: native_int a :: tuple L0: r0 = 'abc' @@ -317,19 +307,31 @@ L0: L1: r5 = CPyStr_Size_size_t(source) r6 = r5 >= 0 :: signed - r7 = r5 << 1 - r8 = int_lt r4, r7 - if r8 goto L2 else goto L4 :: bool + r7 = r4 < r5 :: signed + if r7 goto L2 else goto L8 :: bool L2: - r9 = CPyStr_GetItem(source, r4) - x = r9 - r10 = f2(x) - r11 = CPySequenceTuple_SetItemUnsafe(r3, r4, r10) + r8 = r4 <= 4611686018427387903 :: signed + if r8 goto L3 else goto L4 :: bool L3: - r12 = r4 + 2 - r4 = r12 - goto L1 + r9 = r4 >= -4611686018427387904 :: signed + if r9 goto L5 else goto L4 :: bool L4: + r10 = CPyTagged_FromInt64(r4) + r11 = r10 + goto L6 +L5: + r12 = r4 << 1 + r11 = r12 +L6: + r13 = CPyStr_GetItem(source, r11) + x = r13 + r14 = f2(x) + CPySequenceTuple_SetItemUnsafe(r3, r4, r14) +L7: + r15 = r4 + 1 + r4 = r15 + goto L1 +L8: a = r3 return 1 @@ -351,15 +353,12 @@ def test(source): source :: tuple r0 :: native_int r1 :: tuple - r2 :: short_int - r3 :: native_int - r4 :: short_int - r5 :: bit - r6 :: object - r7, x, r8 :: bool - r9 :: object - r10 :: bit - r11 :: short_int + r2, r3 :: native_int + r4 :: bit + r5 :: object + r6, x, r7 :: bool + r8 :: object + r9 :: native_int a :: tuple L0: r0 = var_object_size source @@ -367,19 +366,18 @@ L0: r2 = 0 L1: r3 = var_object_size source - r4 = r3 << 1 - r5 = int_lt r2, r4 - if r5 goto L2 else goto L4 :: bool + r4 = r2 < r3 :: signed + if r4 goto L2 else goto L4 :: bool L2: - r6 = CPySequenceTuple_GetItem(source, r2) - r7 = unbox(bool, r6) - x = r7 - r8 = f(x) - r9 = box(bool, r8) - r10 = CPySequenceTuple_SetItemUnsafe(r1, r2, r9) + r5 = CPySequenceTuple_GetItemUnsafe(source, r2) + r6 = unbox(bool, r5) + x = r6 + r7 = f(x) + r8 = box(bool, r7) + CPySequenceTuple_SetItemUnsafe(r1, r2, r8) L3: - r11 = r2 + 2 - r2 = r11 + r9 = r2 + 1 + r2 = r9 goto L1 L4: a = r1 diff --git a/mypyc/test-data/lowering-int.test b/mypyc/test-data/lowering-int.test index ad561c5618722..b4fe14db59c41 100644 --- a/mypyc/test-data/lowering-int.test +++ b/mypyc/test-data/lowering-int.test @@ -341,47 +341,43 @@ def f(l: list[int]) -> None: [out] def f(l): l :: list - r0 :: short_int + r0 :: native_int r1 :: ptr r2 :: native_int - r3 :: short_int - r4 :: bit - r5 :: native_int - r6, r7 :: ptr - r8 :: native_int - r9 :: ptr - r10 :: object - r11, x :: int - r12 :: short_int - r13 :: None + r3 :: bit + r4, r5 :: ptr + r6 :: native_int + r7 :: ptr + r8 :: object + r9, x :: int + r10 :: native_int + r11 :: None L0: r0 = 0 L1: r1 = get_element_ptr l ob_size :: PyVarObject r2 = load_mem r1 :: native_int* - r3 = r2 << 1 - r4 = r0 < r3 :: signed - if r4 goto L2 else goto L5 :: bool + r3 = r0 < r2 :: signed + if r3 goto L2 else goto L5 :: bool L2: - r5 = r0 >> 1 - r6 = get_element_ptr l ob_item :: PyListObject - r7 = load_mem r6 :: ptr* - r8 = r5 * 8 - r9 = r7 + r8 - r10 = load_mem r9 :: builtins.object* - inc_ref r10 - r11 = unbox(int, r10) - dec_ref r10 - if is_error(r11) goto L6 (error at f:4) else goto L3 + r4 = get_element_ptr l ob_item :: PyListObject + r5 = load_mem r4 :: ptr* + r6 = r0 * 8 + r7 = r5 + r6 + r8 = load_mem r7 :: builtins.object* + inc_ref r8 + r9 = unbox(int, r8) + dec_ref r8 + if is_error(r9) goto L6 (error at f:4) else goto L3 L3: - x = r11 + x = r9 dec_ref x :: int L4: - r12 = r0 + 2 - r0 = r12 + r10 = r0 + 1 + r0 = r10 goto L1 L5: return 1 L6: - r13 = :: None - return r13 + r11 = :: None + return r11 diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index 22153cff5a918..a831d9baf86ea 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -730,49 +730,47 @@ def f(d): d :: dict r0 :: short_int r1 :: native_int - r2 :: short_int - r3 :: object - r4 :: tuple[bool, short_int, object] - r5 :: short_int - r6 :: bool - r7 :: object - r8, key :: int - r9, r10 :: object - r11 :: int - r12, r13 :: bit + r2 :: object + r3 :: tuple[bool, short_int, object] + r4 :: short_int + r5 :: bool + r6 :: object + r7, key :: int + r8, r9 :: object + r10 :: int + r11, r12 :: bit L0: r0 = 0 r1 = PyDict_Size(d) - r2 = r1 << 1 - r3 = CPyDict_GetKeysIter(d) + r2 = CPyDict_GetKeysIter(d) L1: - r4 = CPyDict_NextKey(r3, r0) - r5 = r4[1] - r0 = r5 - r6 = r4[0] - if r6 goto L2 else goto L6 :: bool + r3 = CPyDict_NextKey(r2, r0) + r4 = r3[1] + r0 = r4 + r5 = r3[0] + if r5 goto L2 else goto L6 :: bool L2: - r7 = r4[2] - dec_ref r4 - r8 = unbox(int, r7) - dec_ref r7 - key = r8 - r9 = box(int, key) - r10 = CPyDict_GetItem(d, r9) + r6 = r3[2] + dec_ref r3 + r7 = unbox(int, r6) + dec_ref r6 + key = r7 + r8 = box(int, key) + r9 = CPyDict_GetItem(d, r8) + dec_ref r8 + r10 = unbox(int, r9) dec_ref r9 - r11 = unbox(int, r10) - dec_ref r10 - dec_ref r11 :: int + dec_ref r10 :: int L3: - r12 = CPyDict_CheckSize(d, r2) + r11 = CPyDict_CheckSize(d, r1) goto L1 L4: - r13 = CPy_NoErrOccurred() + r12 = CPy_NoErrOccurred() L5: return 1 L6: + dec_ref r2 dec_ref r3 - dec_ref r4 goto L4 [case testBorrowRefs] From f49a88f55fb84eb02b0b0b1db369b9ee0f138e3b Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 14 Jul 2025 13:24:58 +0100 Subject: [PATCH 080/424] [mypyc] Simplify IR generated for "for" loops over strings (#19434) Add unsafe list get item primitive. The new primitive just calls the primary get item primitive, but we could later provide an optimized primitive if this turns out to be a performance bottleneck. --- mypyc/irbuild/for_helpers.py | 3 +++ mypyc/lib-rt/CPy.h | 1 + mypyc/lib-rt/str_ops.c | 5 ++++ mypyc/primitives/str_ops.py | 9 +++++++ mypyc/test-data/irbuild-tuple.test | 38 ++++++++++-------------------- 5 files changed, 30 insertions(+), 26 deletions(-) diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index 358f7cb76ba8a..a7ed97ac8eab6 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -76,6 +76,7 @@ from mypyc.primitives.misc_ops import stop_async_iteration_op from mypyc.primitives.registry import CFunctionDescription from mypyc.primitives.set_ops import set_add_op +from mypyc.primitives.str_ops import str_get_item_unsafe_op from mypyc.primitives.tuple_ops import tuple_get_item_unsafe_op GenFunc = Callable[[], None] @@ -772,6 +773,8 @@ def unsafe_index(builder: IRBuilder, target: Value, index: Value, line: int) -> return builder.primitive_op(list_get_item_unsafe_op, [target, index], line) elif is_tuple_rprimitive(target.type): return builder.call_c(tuple_get_item_unsafe_op, [target, index], line) + elif is_str_rprimitive(target.type): + return builder.call_c(str_get_item_unsafe_op, [target, index], line) else: return builder.gen_method_call(target, "__getitem__", [index], None, line) diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index dba84d44f3630..698e65155da46 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -727,6 +727,7 @@ static inline char CPyDict_CheckSize(PyObject *dict, Py_ssize_t size) { char CPyStr_Equal(PyObject *str1, PyObject *str2); PyObject *CPyStr_Build(Py_ssize_t len, ...); PyObject *CPyStr_GetItem(PyObject *str, CPyTagged index); +PyObject *CPyStr_GetItemUnsafe(PyObject *str, Py_ssize_t index); CPyTagged CPyStr_Find(PyObject *str, PyObject *substr, CPyTagged start, int direction); CPyTagged CPyStr_FindWithEnd(PyObject *str, PyObject *substr, CPyTagged start, CPyTagged end, int direction); PyObject *CPyStr_Split(PyObject *str, PyObject *sep, CPyTagged max_split); diff --git a/mypyc/lib-rt/str_ops.c b/mypyc/lib-rt/str_ops.c index 5fd376f21cfae..a2d10aacea46e 100644 --- a/mypyc/lib-rt/str_ops.c +++ b/mypyc/lib-rt/str_ops.c @@ -117,6 +117,11 @@ PyObject *CPyStr_GetItem(PyObject *str, CPyTagged index) { } } +PyObject *CPyStr_GetItemUnsafe(PyObject *str, Py_ssize_t index) { + // This is unsafe since we don't check for overflow when doing <<. + return CPyStr_GetItem(str, index << 1); +} + // A simplification of _PyUnicode_JoinArray() from CPython 3.9.6 PyObject *CPyStr_Build(Py_ssize_t len, ...) { Py_ssize_t i; diff --git a/mypyc/primitives/str_ops.py b/mypyc/primitives/str_ops.py index 37dbdf21bb5d1..e3f0b9dbbc2a8 100644 --- a/mypyc/primitives/str_ops.py +++ b/mypyc/primitives/str_ops.py @@ -95,6 +95,15 @@ error_kind=ERR_MAGIC, ) +# This is unsafe since it assumes that the index is within reasonable bounds. +# In the future this might do no bounds checking at all. +str_get_item_unsafe_op = custom_op( + arg_types=[str_rprimitive, c_pyssize_t_rprimitive], + return_type=str_rprimitive, + c_function_name="CPyStr_GetItemUnsafe", + error_kind=ERR_MAGIC, +) + # str[begin:end] str_slice_op = custom_op( arg_types=[str_rprimitive, int_rprimitive, int_rprimitive], diff --git a/mypyc/test-data/irbuild-tuple.test b/mypyc/test-data/irbuild-tuple.test index c39968fc139e9..5c5ec27b18829 100644 --- a/mypyc/test-data/irbuild-tuple.test +++ b/mypyc/test-data/irbuild-tuple.test @@ -272,7 +272,7 @@ L4: a = r6 return 1 -[case testTupleBuiltFromStr_64bit] +[case testTupleBuiltFromStr] def f2(val: str) -> str: return val + "f2" @@ -292,10 +292,9 @@ def test(): r2 :: bit r3 :: tuple r4, r5 :: native_int - r6, r7, r8, r9 :: bit - r10, r11, r12 :: int - r13, x, r14 :: str - r15 :: native_int + r6, r7 :: bit + r8, x, r9 :: str + r10 :: native_int a :: tuple L0: r0 = 'abc' @@ -308,30 +307,17 @@ L1: r5 = CPyStr_Size_size_t(source) r6 = r5 >= 0 :: signed r7 = r4 < r5 :: signed - if r7 goto L2 else goto L8 :: bool + if r7 goto L2 else goto L4 :: bool L2: - r8 = r4 <= 4611686018427387903 :: signed - if r8 goto L3 else goto L4 :: bool + r8 = CPyStr_GetItemUnsafe(source, r4) + x = r8 + r9 = f2(x) + CPySequenceTuple_SetItemUnsafe(r3, r4, r9) L3: - r9 = r4 >= -4611686018427387904 :: signed - if r9 goto L5 else goto L4 :: bool -L4: - r10 = CPyTagged_FromInt64(r4) - r11 = r10 - goto L6 -L5: - r12 = r4 << 1 - r11 = r12 -L6: - r13 = CPyStr_GetItem(source, r11) - x = r13 - r14 = f2(x) - CPySequenceTuple_SetItemUnsafe(r3, r4, r14) -L7: - r15 = r4 + 1 - r4 = r15 + r10 = r4 + 1 + r4 = r10 goto L1 -L8: +L4: a = r3 return 1 From 9bc098505d8c363f4ff03b165517414872ca7869 Mon Sep 17 00:00:00 2001 From: esarp <11684270+esarp@users.noreply.github.com> Date: Mon, 14 Jul 2025 10:34:40 -0500 Subject: [PATCH 081/424] Initial changelog for 1.17 release (#19427) --- CHANGELOG.md | 123 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a1470b7d50c3e..e4f148fe63820 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ ## Next Release +## Mypy 1.17 (Unreleased) + +We’ve just uploaded mypy 1.17 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). +Mypy is a static type checker for Python. This release includes new features and bug fixes. +You can install it as follows: + + python3 -m pip install -U mypy + +You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). + ### Remove Support for targeting Python 3.8 Mypy now requires `--python-version 3.9` or greater. Support for only Python 3.8 is @@ -29,6 +39,119 @@ Mypy only supports Python 3.9+. The \--force-uppercase-builtins flag is now depr Contributed by Marc Mueller (PR [19176](https://github.com/python/mypy/pull/19176)) +### Mypyc Fixes and Improvements + +* Fix exception swallowing in async try/finally blocks with await (Chainfire, PR [19353](https://github.com/python/mypy/pull/19353)) +* Fix AttributeError in async try/finally with mixed return paths (Chainfire, PR [19361](https://github.com/python/mypy/pull/19361)) +* Derive .c file name from full module name if using multi_file (Jukka Lehtosalo, PR [19278](https://github.com/python/mypy/pull/19278)) +* Support overriding the group name used in output files (Jukka Lehtosalo, PR [19272](https://github.com/python/mypy/pull/19272)) +* Make generated generator helper method internal (Jukka Lehtosalo, PR [19268](https://github.com/python/mypy/pull/19268)) +* Add note about using non-native class to subclass built-in types (Jukka Lehtosalo, PR [19236](https://github.com/python/mypy/pull/19236)) +* Make some generated classes implicitly final (Jukka Lehtosalo, PR [19235](https://github.com/python/mypy/pull/19235)) +* Free coroutine after await encounters StopIteration (Jukka Lehtosalo, PR [19231](https://github.com/python/mypy/pull/19231)) +* Use non-tagged integer for generator label (Jukka Lehtosalo, PR [19218](https://github.com/python/mypy/pull/19218)) +* Merge generator and environment classes in simple cases (Jukka Lehtosalo, PR [19207](https://github.com/python/mypy/pull/19207)) +* Don't simplify module prefixes if using separate compilation (Jukka Lehtosalo, PR [19206](https://github.com/python/mypy/pull/19206)) +* Test function nesting with async functions (Jukka Lehtosalo, PR [19203](https://github.com/python/mypy/pull/19203)) +* Enable partial, unsafe support for free-threading (Jukka Lehtosalo, PR [19167](https://github.com/python/mypy/pull/19167)) +* Add comment about incref/decref and free-threaded builds (Jukka Lehtosalo, PR [19155](https://github.com/python/mypy/pull/19155)) +* Refactor extension module C generation and generated C (Jukka Lehtosalo, PR [19126](https://github.com/python/mypy/pull/19126)) +* Fix incref/decref on free-threaded builds (Jukka Lehtosalo, PR [19127](https://github.com/python/mypy/pull/19127)) +* Remove last unreachable block from mypyc code (Stanislav Terliakov, PR [19086](https://github.com/python/mypy/pull/19086)) + +### Stubgen Improvements + +* stubgen: add test case for handling `Incomplete` return types (Alexey Makridenko, PR [19253](https://github.com/python/mypy/pull/19253)) +* stubgen: add import for `types` in `__exit__` method signature (Alexey Makridenko, PR [19120](https://github.com/python/mypy/pull/19120)) +* stubgenc: add support for including class and property docstrings (Chad Dombrova, PR [17964](https://github.com/python/mypy/pull/17964)) +* stubgen: Don't generate `Incomplete | None = None` argument annotation (Sebastian Rittau, PR [19097](https://github.com/python/mypy/pull/19097)) +* Support several more constructs in stubgen's AliasPrinter (Stanislav Terliakov, PR [18888](https://github.com/python/mypy/pull/18888)) + +### Stubtest Improvements + +* Syntax error messages capitalization (Charulata, PR [19114](https://github.com/python/mypy/pull/19114)) + +### Miscellaneous Fixes and Improvements + +* Combine the revealed types of multiple iteration steps in a more robust manner (Christoph Tyralla, PR [19324](https://github.com/python/mypy/pull/19324)) +* Improve the handling of "iteration dependent" errors and notes in finally clauses (Christoph Tyralla, PR [19270](https://github.com/python/mypy/pull/19270)) +* Lessen dmypy suggest path limitations for Windows machines (CoolCat467, PR [19337](https://github.com/python/mypy/pull/19337)) +* Type ignore comments erroneously marked as unused by dmypy (Charlie Denton, PR [15043](https://github.com/python/mypy/pull/15043)) +* Handle corner case: protocol vs classvar vs descriptor (Ivan Levkivskyi, PR [19277](https://github.com/python/mypy/pull/19277)) +* Fix `exhaustive-match` error code in title (johnthagen, PR [19276](https://github.com/python/mypy/pull/19276)) +* Fix couple inconsistencies in protocols vs TypeType (Ivan Levkivskyi, PR [19267](https://github.com/python/mypy/pull/19267)) +* Fix missing error context for unpacking assignment involving star expression (Brian Schubert, PR [19258](https://github.com/python/mypy/pull/19258)) +* Fix and simplify error de-duplication (Ivan Levkivskyi, PR [19247](https://github.com/python/mypy/pull/19247)) +* Add regression test for narrowing union of mixins (Shantanu, PR [19266](https://github.com/python/mypy/pull/19266)) +* Disallow `ClassVar` in type aliases (Brian Schubert, PR [19263](https://github.com/python/mypy/pull/19263)) +* Refactor/unify access to static attributes (Ivan Levkivskyi, PR [19254](https://github.com/python/mypy/pull/19254)) +* Clean-up and move operator access to checkmember.py (Ivan Levkivskyi, PR [19250](https://github.com/python/mypy/pull/19250)) +* Add script that prints compiled files when self compiling (Jukka Lehtosalo, PR [19260](https://github.com/python/mypy/pull/19260)) +* Fix help message url for "None and Optional handling" section (Guy Wilson, PR [19252](https://github.com/python/mypy/pull/19252)) +* Display FQN for imported base classes in errors about incompatible overrides (Mikhail Golubev, PR [19115](https://github.com/python/mypy/pull/19115)) +* Fix a minor merge conflict caused by #19118 (Christoph Tyralla, PR [19246](https://github.com/python/mypy/pull/19246)) +* Avoid false `unreachable`, `redundant-expr`, and `redundant-casts` warnings in loops more robustly and efficiently, and avoid multiple `revealed type` notes for the same line (Christoph Tyralla, PR [19118](https://github.com/python/mypy/pull/19118)) +* Fix type extraction from `isinstance` checks (Stanislav Terliakov, PR [19223](https://github.com/python/mypy/pull/19223)) +* Erase stray typevars in functools.partial generic (Stanislav Terliakov, PR [18954](https://github.com/python/mypy/pull/18954)) +* Make infer_condition_value recognize the whole truth table (Stanislav Terliakov, PR [18944](https://github.com/python/mypy/pull/18944)) +* Support type aliases, `NamedTuple` and `TypedDict` in constrained TypeVar defaults (Stanislav Terliakov, PR [18884](https://github.com/python/mypy/pull/18884)) +* Move dataclass kw_only fields to the end of the signature (Stanislav Terliakov, PR [19018](https://github.com/python/mypy/pull/19018)) +* Deprecated --force-uppercase-builtins flag (Marc Mueller, PR [19176](https://github.com/python/mypy/pull/19176)) +* Provide a better fallback value for the python_version option (Marc Mueller, PR [19162](https://github.com/python/mypy/pull/19162)) +* Avoid spurious non-overlapping eq error with metaclass with `__eq__` (Michael J. Sullivan, PR [19220](https://github.com/python/mypy/pull/19220)) +* Remove --show-speed-regression in primer (Shantanu, PR [19226](https://github.com/python/mypy/pull/19226)) +* Add flag to raise error if match statement does not match exaustively (Donal Burns, PR [19144](https://github.com/python/mypy/pull/19144)) +* Narrow type variable bounds in binder (Ivan Levkivskyi, PR [19183](https://github.com/python/mypy/pull/19183)) +* Add regression test for dataclass typeguard (Shantanu, PR [19214](https://github.com/python/mypy/pull/19214)) +* Add classifier for Python 3.14 (Marc Mueller, PR [19199](https://github.com/python/mypy/pull/19199)) +* Further cleanup after dropping Python 3.8 (Marc Mueller, PR [19197](https://github.com/python/mypy/pull/19197)) +* Fix nondeterministic type checking by making join with explicit Protocol and type promotion commute (Shantanu, PR [18402](https://github.com/python/mypy/pull/18402)) +* Infer constraints eagerly if actual is Any (Ivan Levkivskyi, PR [19190](https://github.com/python/mypy/pull/19190)) +* Include walrus assignments in conditional inference (Stanislav Terliakov, PR [19038](https://github.com/python/mypy/pull/19038)) +* Use PEP 604 syntax for TypeStrVisitor (Marc Mueller, PR [19179](https://github.com/python/mypy/pull/19179)) +* Use checkmember.py to check protocol subtyping (Ivan Levkivskyi, PR [18943](https://github.com/python/mypy/pull/18943)) +* Update test requirements (Marc Mueller, PR [19163](https://github.com/python/mypy/pull/19163)) +* Use more lower case builtins in error messages (Marc Mueller, PR [19177](https://github.com/python/mypy/pull/19177)) +* Remove force_uppercase_builtins default from test helpers (Marc Mueller, PR [19173](https://github.com/python/mypy/pull/19173)) +* Start testing Python 3.14 (Marc Mueller, PR [19164](https://github.com/python/mypy/pull/19164)) +* Fix example to use correct method of Stack (Łukasz Kwieciński, PR [19123](https://github.com/python/mypy/pull/19123)) +* Fix nondeterministic type checking caused by nonassociative of None joins (Shantanu, PR [19158](https://github.com/python/mypy/pull/19158)) +* Drop support for --python-version 3.8 (Marc Mueller, PR [19157](https://github.com/python/mypy/pull/19157)) +* Fix nondeterministic type checking caused by nonassociativity of joins (Shantanu, PR [19147](https://github.com/python/mypy/pull/19147)) +* Fix nondeterministic type checking by making join between TypeType and TypeVar commute (Shantanu, PR [19149](https://github.com/python/mypy/pull/19149)) +* Forbid `.pop` of `Readonly` `NotRequired` TypedDict items (Stanislav Terliakov, PR [19133](https://github.com/python/mypy/pull/19133)) +* Emit a friendlier warning on invalid exclude regex, instead of a stacktrace (wyattscarpenter, PR [19102](https://github.com/python/mypy/pull/19102)) +* Update dmypy/client.py: Enable ANSI color codes for windows cmd (wyattscarpenter, PR [19088](https://github.com/python/mypy/pull/19088)) +* Extend special case for context-based typevar inference to typevar unions in return position (Stanislav Terliakov, PR [18976](https://github.com/python/mypy/pull/18976)) + +### Acknowledgements + +Thanks to all mypy contributors who contributed to this release: + +* Alexey Makridenko +* Brian Schubert +* Chad Dombrova +* Chainfire +* Charlie Denton +* Charulata +* Christoph Tyralla +* CoolCat467 +* Donal Burns +* Guy Wilson +* Ivan Levkivskyi +* johnthagen +* Jukka Lehtosalo +* Łukasz Kwieciński +* Marc Mueller +* Michael J. Sullivan +* Mikhail Golubev +* Sebastian Rittau +* Shantanu +* Stanislav Terliakov +* wyattscarpenter + +I’d also like to thank my employer, Dropbox, for supporting mypy development. + ## Mypy 1.16 We’ve just uploaded mypy 1.16 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). From c66417d11a33eb4c9d1e34dcb75dd5b826cb7420 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 14 Jul 2025 16:51:20 +0100 Subject: [PATCH 082/424] Updates to 1.17 changelog (#19436) Add a few sections and do some editing. --- CHANGELOG.md | 168 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 107 insertions(+), 61 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4f148fe63820..a74fb46aba6b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Next Release -## Mypy 1.17 (Unreleased) +## Mypy 1.17 We’ve just uploaded mypy 1.17 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). Mypy is a static type checker for Python. This release includes new features and bug fixes. @@ -12,11 +12,60 @@ You can install it as follows: You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). -### Remove Support for targeting Python 3.8 +### Optionally Check That Match Is Exhaustive -Mypy now requires `--python-version 3.9` or greater. Support for only Python 3.8 is -fully removed now. Given an unsupported version, mypy will default to the oldest -supported one, currently 3.9. +Mypy can now optionally generate an error if a match statement does not +match exhaustively, without having to use `assert_never(...)`. Enable +this by using `--enable-error-code exhaustive-match`. + +Example: + +```python +# mypy: enable-error-code=exhaustive-match + +import enum + +class Color(enum.Enum): + RED = 1 + BLUE = 2 + +def show_color(val: Color) -> None: + # error: Unhandled case for values of type "Literal[Color.BLUE]" + match val: + case Color.RED: + print("red") +``` + +This feature was contributed by Donal Burns (PR [19144](https://github.com/python/mypy/pull/19144)). + +### Further Improvements to Attribute Resolution + +This release includes additional improvements to how attribute types +and kinds are resolved. These fix many bugs and overall improve consistency. + +* Handle corner case: protocol/class variable/descriptor (Ivan Levkivskyi, PR [19277](https://github.com/python/mypy/pull/19277)) +* Fix a few inconsistencies in protocol/type object interactions (Ivan Levkivskyi, PR [19267](https://github.com/python/mypy/pull/19267)) +* Refactor/unify access to static attributes (Ivan Levkivskyi, PR [19254](https://github.com/python/mypy/pull/19254)) +* Remove inconsistencies in operator handling (Ivan Levkivskyi, PR [19250](https://github.com/python/mypy/pull/19250)) +* Make protocol subtyping more consistent (Ivan Levkivskyi, PR [18943](https://github.com/python/mypy/pull/18943)) + +### Fixes to Nondeterministic Type Checking + +Previous mypy versions could infer different types for certain expressions +across different runs (typically depending on which order certain types +were processed, and this order was nondeterministic). This release includes +fixes to several such issues. + +* Fix nondeterministic type checking by making join with explicit Protocol and type promotion commute (Shantanu, PR [18402](https://github.com/python/mypy/pull/18402)) +* Fix nondeterministic type checking caused by nonassociative of None joins (Shantanu, PR [19158](https://github.com/python/mypy/pull/19158)) +* Fix nondeterministic type checking caused by nonassociativity of joins (Shantanu, PR [19147](https://github.com/python/mypy/pull/19147)) +* Fix nondeterministic type checking by making join between `type` and TypeVar commute (Shantanu, PR [19149](https://github.com/python/mypy/pull/19149)) + +### Remove Support for Targeting Python 3.8 + +Mypy now requires `--python-version 3.9` or greater. Support for targeting Python 3.8 is +fully removed now. Since 3.8 is an unsupported version, mypy will default to the oldest +supported version (currently 3.9) if you still try to target 3.8. This change is necessary because typeshed stopped supporting Python 3.8 after it reached its End of Life in October 2024. @@ -27,102 +76,99 @@ Contributed by Marc Mueller ### Initial Support for Python 3.14 Mypy is now tested on 3.14 and mypyc works with 3.14.0b3 and later. -Mypyc compiled wheels of mypy itself will be available for new versions after 3.14.0rc1 is released. +Binary wheels compiled with mypyc for mypy itself will be available for 3.14 +some time after 3.14.0rc1 has been released. -Note that not all new features might be supported just yet. +Note that not all features are supported just yet. Contributed by Marc Mueller (PR [19164](https://github.com/python/mypy/pull/19164)) -### Deprecated Flag: \--force-uppercase-builtins +### Deprecated Flag: `--force-uppercase-builtins` -Mypy only supports Python 3.9+. The \--force-uppercase-builtins flag is now deprecated and a no-op. It will be removed in a future version. +Mypy only supports Python 3.9+. The `--force-uppercase-builtins` flag is now +deprecated as unnecessary, and a no-op. It will be removed in a future version. Contributed by Marc Mueller (PR [19176](https://github.com/python/mypy/pull/19176)) -### Mypyc Fixes and Improvements +### Mypyc: Improvements to Generators and Async Functions + +This release includes both performance improvements and bug fixes related +to generators and async functions (these share many implementation details). * Fix exception swallowing in async try/finally blocks with await (Chainfire, PR [19353](https://github.com/python/mypy/pull/19353)) * Fix AttributeError in async try/finally with mixed return paths (Chainfire, PR [19361](https://github.com/python/mypy/pull/19361)) -* Derive .c file name from full module name if using multi_file (Jukka Lehtosalo, PR [19278](https://github.com/python/mypy/pull/19278)) -* Support overriding the group name used in output files (Jukka Lehtosalo, PR [19272](https://github.com/python/mypy/pull/19272)) * Make generated generator helper method internal (Jukka Lehtosalo, PR [19268](https://github.com/python/mypy/pull/19268)) -* Add note about using non-native class to subclass built-in types (Jukka Lehtosalo, PR [19236](https://github.com/python/mypy/pull/19236)) -* Make some generated classes implicitly final (Jukka Lehtosalo, PR [19235](https://github.com/python/mypy/pull/19235)) * Free coroutine after await encounters StopIteration (Jukka Lehtosalo, PR [19231](https://github.com/python/mypy/pull/19231)) * Use non-tagged integer for generator label (Jukka Lehtosalo, PR [19218](https://github.com/python/mypy/pull/19218)) * Merge generator and environment classes in simple cases (Jukka Lehtosalo, PR [19207](https://github.com/python/mypy/pull/19207)) -* Don't simplify module prefixes if using separate compilation (Jukka Lehtosalo, PR [19206](https://github.com/python/mypy/pull/19206)) -* Test function nesting with async functions (Jukka Lehtosalo, PR [19203](https://github.com/python/mypy/pull/19203)) + +### Mypyc: Partial, Unsafe Support for Free Threading + +Mypyc has minimal, quite memory-unsafe support for the free threaded +builds of 3.14. It is also only lightly tested. Bug reports and experience +reports are welcome! + +Here are some of the major limitations: +* Free threading only works when compiling a single module at a time. +* If there is concurrent access to an object while another thread is mutating the same + object, it's possible to encounter segfaults and memory corruption. +* There are no efficient native primitives for thread synthronization, though the + regular `threading` module can be used. +* Some workloads don't scale well to multiple threads for no clear reason. + +Related PRs: + * Enable partial, unsafe support for free-threading (Jukka Lehtosalo, PR [19167](https://github.com/python/mypy/pull/19167)) -* Add comment about incref/decref and free-threaded builds (Jukka Lehtosalo, PR [19155](https://github.com/python/mypy/pull/19155)) -* Refactor extension module C generation and generated C (Jukka Lehtosalo, PR [19126](https://github.com/python/mypy/pull/19126)) * Fix incref/decref on free-threaded builds (Jukka Lehtosalo, PR [19127](https://github.com/python/mypy/pull/19127)) -* Remove last unreachable block from mypyc code (Stanislav Terliakov, PR [19086](https://github.com/python/mypy/pull/19086)) -### Stubgen Improvements +### Other Mypyc Fixes and Improvements -* stubgen: add test case for handling `Incomplete` return types (Alexey Makridenko, PR [19253](https://github.com/python/mypy/pull/19253)) -* stubgen: add import for `types` in `__exit__` method signature (Alexey Makridenko, PR [19120](https://github.com/python/mypy/pull/19120)) -* stubgenc: add support for including class and property docstrings (Chad Dombrova, PR [17964](https://github.com/python/mypy/pull/17964)) -* stubgen: Don't generate `Incomplete | None = None` argument annotation (Sebastian Rittau, PR [19097](https://github.com/python/mypy/pull/19097)) -* Support several more constructs in stubgen's AliasPrinter (Stanislav Terliakov, PR [18888](https://github.com/python/mypy/pull/18888)) +* Derive .c file name from full module name if using multi_file (Jukka Lehtosalo, PR [19278](https://github.com/python/mypy/pull/19278)) +* Support overriding the group name used in output files (Jukka Lehtosalo, PR [19272](https://github.com/python/mypy/pull/19272)) +* Add note about using non-native class to subclass built-in types (Jukka Lehtosalo, PR [19236](https://github.com/python/mypy/pull/19236)) +* Make some generated classes implicitly final (Jukka Lehtosalo, PR [19235](https://github.com/python/mypy/pull/19235)) +* Don't simplify module prefixes if using separate compilation (Jukka Lehtosalo, PR [19206](https://github.com/python/mypy/pull/19206)) -### Stubtest Improvements +### Stubgen Improvements -* Syntax error messages capitalization (Charulata, PR [19114](https://github.com/python/mypy/pull/19114)) +* Add import for `types` in `__exit__` method signature (Alexey Makridenko, PR [19120](https://github.com/python/mypy/pull/19120)) +* Add support for including class and property docstrings (Chad Dombrova, PR [17964](https://github.com/python/mypy/pull/17964)) +* Don't generate `Incomplete | None = None` argument annotation (Sebastian Rittau, PR [19097](https://github.com/python/mypy/pull/19097)) +* Support several more constructs in stubgen's alias printer (Stanislav Terliakov, PR [18888](https://github.com/python/mypy/pull/18888)) ### Miscellaneous Fixes and Improvements * Combine the revealed types of multiple iteration steps in a more robust manner (Christoph Tyralla, PR [19324](https://github.com/python/mypy/pull/19324)) * Improve the handling of "iteration dependent" errors and notes in finally clauses (Christoph Tyralla, PR [19270](https://github.com/python/mypy/pull/19270)) * Lessen dmypy suggest path limitations for Windows machines (CoolCat467, PR [19337](https://github.com/python/mypy/pull/19337)) -* Type ignore comments erroneously marked as unused by dmypy (Charlie Denton, PR [15043](https://github.com/python/mypy/pull/15043)) -* Handle corner case: protocol vs classvar vs descriptor (Ivan Levkivskyi, PR [19277](https://github.com/python/mypy/pull/19277)) -* Fix `exhaustive-match` error code in title (johnthagen, PR [19276](https://github.com/python/mypy/pull/19276)) -* Fix couple inconsistencies in protocols vs TypeType (Ivan Levkivskyi, PR [19267](https://github.com/python/mypy/pull/19267)) +* Fix type ignore comments erroneously marked as unused by dmypy (Charlie Denton, PR [15043](https://github.com/python/mypy/pull/15043)) +* Fix misspelled `exhaustive-match` error code (johnthagen, PR [19276](https://github.com/python/mypy/pull/19276)) * Fix missing error context for unpacking assignment involving star expression (Brian Schubert, PR [19258](https://github.com/python/mypy/pull/19258)) * Fix and simplify error de-duplication (Ivan Levkivskyi, PR [19247](https://github.com/python/mypy/pull/19247)) -* Add regression test for narrowing union of mixins (Shantanu, PR [19266](https://github.com/python/mypy/pull/19266)) * Disallow `ClassVar` in type aliases (Brian Schubert, PR [19263](https://github.com/python/mypy/pull/19263)) -* Refactor/unify access to static attributes (Ivan Levkivskyi, PR [19254](https://github.com/python/mypy/pull/19254)) -* Clean-up and move operator access to checkmember.py (Ivan Levkivskyi, PR [19250](https://github.com/python/mypy/pull/19250)) -* Add script that prints compiled files when self compiling (Jukka Lehtosalo, PR [19260](https://github.com/python/mypy/pull/19260)) +* Add script that prints list of compiled files when compiling mypy (Jukka Lehtosalo, PR [19260](https://github.com/python/mypy/pull/19260)) * Fix help message url for "None and Optional handling" section (Guy Wilson, PR [19252](https://github.com/python/mypy/pull/19252)) -* Display FQN for imported base classes in errors about incompatible overrides (Mikhail Golubev, PR [19115](https://github.com/python/mypy/pull/19115)) -* Fix a minor merge conflict caused by #19118 (Christoph Tyralla, PR [19246](https://github.com/python/mypy/pull/19246)) +* Display fully qualified name of imported base classes in errors about incompatible overrides (Mikhail Golubev, PR [19115](https://github.com/python/mypy/pull/19115)) * Avoid false `unreachable`, `redundant-expr`, and `redundant-casts` warnings in loops more robustly and efficiently, and avoid multiple `revealed type` notes for the same line (Christoph Tyralla, PR [19118](https://github.com/python/mypy/pull/19118)) * Fix type extraction from `isinstance` checks (Stanislav Terliakov, PR [19223](https://github.com/python/mypy/pull/19223)) -* Erase stray typevars in functools.partial generic (Stanislav Terliakov, PR [18954](https://github.com/python/mypy/pull/18954)) -* Make infer_condition_value recognize the whole truth table (Stanislav Terliakov, PR [18944](https://github.com/python/mypy/pull/18944)) +* Erase stray type variables in `functools.partial` (Stanislav Terliakov, PR [18954](https://github.com/python/mypy/pull/18954)) +* Make inferring condition value recognize the whole truth table (Stanislav Terliakov, PR [18944](https://github.com/python/mypy/pull/18944)) * Support type aliases, `NamedTuple` and `TypedDict` in constrained TypeVar defaults (Stanislav Terliakov, PR [18884](https://github.com/python/mypy/pull/18884)) -* Move dataclass kw_only fields to the end of the signature (Stanislav Terliakov, PR [19018](https://github.com/python/mypy/pull/19018)) -* Deprecated --force-uppercase-builtins flag (Marc Mueller, PR [19176](https://github.com/python/mypy/pull/19176)) -* Provide a better fallback value for the python_version option (Marc Mueller, PR [19162](https://github.com/python/mypy/pull/19162)) -* Avoid spurious non-overlapping eq error with metaclass with `__eq__` (Michael J. Sullivan, PR [19220](https://github.com/python/mypy/pull/19220)) -* Remove --show-speed-regression in primer (Shantanu, PR [19226](https://github.com/python/mypy/pull/19226)) -* Add flag to raise error if match statement does not match exaustively (Donal Burns, PR [19144](https://github.com/python/mypy/pull/19144)) -* Narrow type variable bounds in binder (Ivan Levkivskyi, PR [19183](https://github.com/python/mypy/pull/19183)) -* Add regression test for dataclass typeguard (Shantanu, PR [19214](https://github.com/python/mypy/pull/19214)) +* Move dataclass `kw_only` fields to the end of the signature (Stanislav Terliakov, PR [19018](https://github.com/python/mypy/pull/19018)) +* Provide a better fallback value for the `python_version` option (Marc Mueller, PR [19162](https://github.com/python/mypy/pull/19162)) +* Avoid spurious non-overlapping equality error with metaclass with `__eq__` (Michael J. Sullivan, PR [19220](https://github.com/python/mypy/pull/19220)) +* Narrow type variable bounds (Ivan Levkivskyi, PR [19183](https://github.com/python/mypy/pull/19183)) * Add classifier for Python 3.14 (Marc Mueller, PR [19199](https://github.com/python/mypy/pull/19199)) -* Further cleanup after dropping Python 3.8 (Marc Mueller, PR [19197](https://github.com/python/mypy/pull/19197)) -* Fix nondeterministic type checking by making join with explicit Protocol and type promotion commute (Shantanu, PR [18402](https://github.com/python/mypy/pull/18402)) +* Capitalize syntax error messages (Charulata, PR [19114](https://github.com/python/mypy/pull/19114)) * Infer constraints eagerly if actual is Any (Ivan Levkivskyi, PR [19190](https://github.com/python/mypy/pull/19190)) * Include walrus assignments in conditional inference (Stanislav Terliakov, PR [19038](https://github.com/python/mypy/pull/19038)) -* Use PEP 604 syntax for TypeStrVisitor (Marc Mueller, PR [19179](https://github.com/python/mypy/pull/19179)) -* Use checkmember.py to check protocol subtyping (Ivan Levkivskyi, PR [18943](https://github.com/python/mypy/pull/18943)) -* Update test requirements (Marc Mueller, PR [19163](https://github.com/python/mypy/pull/19163)) -* Use more lower case builtins in error messages (Marc Mueller, PR [19177](https://github.com/python/mypy/pull/19177)) -* Remove force_uppercase_builtins default from test helpers (Marc Mueller, PR [19173](https://github.com/python/mypy/pull/19173)) -* Start testing Python 3.14 (Marc Mueller, PR [19164](https://github.com/python/mypy/pull/19164)) +* Use PEP 604 syntax when converting types to strings (Marc Mueller, PR [19179](https://github.com/python/mypy/pull/19179)) +* Use more lower-case builtin types in error messages (Marc Mueller, PR [19177](https://github.com/python/mypy/pull/19177)) * Fix example to use correct method of Stack (Łukasz Kwieciński, PR [19123](https://github.com/python/mypy/pull/19123)) -* Fix nondeterministic type checking caused by nonassociative of None joins (Shantanu, PR [19158](https://github.com/python/mypy/pull/19158)) -* Drop support for --python-version 3.8 (Marc Mueller, PR [19157](https://github.com/python/mypy/pull/19157)) -* Fix nondeterministic type checking caused by nonassociativity of joins (Shantanu, PR [19147](https://github.com/python/mypy/pull/19147)) -* Fix nondeterministic type checking by making join between TypeType and TypeVar commute (Shantanu, PR [19149](https://github.com/python/mypy/pull/19149)) * Forbid `.pop` of `Readonly` `NotRequired` TypedDict items (Stanislav Terliakov, PR [19133](https://github.com/python/mypy/pull/19133)) * Emit a friendlier warning on invalid exclude regex, instead of a stacktrace (wyattscarpenter, PR [19102](https://github.com/python/mypy/pull/19102)) -* Update dmypy/client.py: Enable ANSI color codes for windows cmd (wyattscarpenter, PR [19088](https://github.com/python/mypy/pull/19088)) -* Extend special case for context-based typevar inference to typevar unions in return position (Stanislav Terliakov, PR [18976](https://github.com/python/mypy/pull/18976)) +* Enable ANSI color codes for dmypy client in Windows (wyattscarpenter, PR [19088](https://github.com/python/mypy/pull/19088)) +* Extend special case for context-based type variable inference to unions in return position (Stanislav Terliakov, PR [18976](https://github.com/python/mypy/pull/18976)) ### Acknowledgements From 32f57e4b09d60b3d492e7c4e9e3682272fe9c566 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 15 Jul 2025 12:30:51 +0100 Subject: [PATCH 083/424] [mypyc] Add SetElement op for initializing struct values (#19437) Also add Undef value type that can currently only used as the operand for SetElement to signify that we are creating a new value instead of modifying an existing value. A new struct value can be created by starting with Undef and setting each element sequentially. Each operation produces a new struct value, but the temporaries will be optimized away in the later passes (currently by the C compiler, but we could do something more clever here in the future). This is needed to support packed arrays, which are represented as structs. I extracted this from my packed array branch, and it's currently unused outside tests. --- mypyc/analysis/dataflow.py | 7 +++- mypyc/analysis/ircheck.py | 8 ++++- mypyc/analysis/selfleaks.py | 4 +++ mypyc/codegen/emitfunc.py | 27 +++++++++++++++ mypyc/ir/ops.py | 58 +++++++++++++++++++++++++++++++++ mypyc/ir/pprint.py | 9 ++++- mypyc/test/test_emitfunc.py | 18 ++++++++++ mypyc/transform/ir_transform.py | 7 ++++ mypyc/transform/refcount.py | 3 +- 9 files changed, 137 insertions(+), 4 deletions(-) diff --git a/mypyc/analysis/dataflow.py b/mypyc/analysis/dataflow.py index db62ef1700fa0..827c70a0eb4db 100644 --- a/mypyc/analysis/dataflow.py +++ b/mypyc/analysis/dataflow.py @@ -45,12 +45,14 @@ RegisterOp, Return, SetAttr, + SetElement, SetMem, Truncate, TupleGet, TupleSet, Unborrow, Unbox, + Undef, Unreachable, Value, ) @@ -272,6 +274,9 @@ def visit_load_mem(self, op: LoadMem) -> GenAndKill[T]: def visit_get_element_ptr(self, op: GetElementPtr) -> GenAndKill[T]: return self.visit_register_op(op) + def visit_set_element(self, op: SetElement) -> GenAndKill[T]: + return self.visit_register_op(op) + def visit_load_address(self, op: LoadAddress) -> GenAndKill[T]: return self.visit_register_op(op) @@ -444,7 +449,7 @@ def visit_set_mem(self, op: SetMem) -> GenAndKill[Value]: def non_trivial_sources(op: Op) -> set[Value]: result = set() for source in op.sources(): - if not isinstance(source, (Integer, Float)): + if not isinstance(source, (Integer, Float, Undef)): result.add(source) return result diff --git a/mypyc/analysis/ircheck.py b/mypyc/analysis/ircheck.py index 88737ac208de2..4ad2a52c1036b 100644 --- a/mypyc/analysis/ircheck.py +++ b/mypyc/analysis/ircheck.py @@ -17,6 +17,7 @@ ControlOp, DecRef, Extend, + Float, FloatComparisonOp, FloatNeg, FloatOp, @@ -42,12 +43,14 @@ Register, Return, SetAttr, + SetElement, SetMem, Truncate, TupleGet, TupleSet, Unborrow, Unbox, + Undef, Unreachable, Value, ) @@ -148,7 +151,7 @@ def check_op_sources_valid(fn: FuncIR) -> list[FnError]: for block in fn.blocks: for op in block.ops: for source in op.sources(): - if isinstance(source, Integer): + if isinstance(source, (Integer, Float, Undef)): pass elif isinstance(source, Op): if source not in valid_ops: @@ -423,6 +426,9 @@ def visit_set_mem(self, op: SetMem) -> None: def visit_get_element_ptr(self, op: GetElementPtr) -> None: pass + def visit_set_element(self, op: SetElement) -> None: + pass + def visit_load_address(self, op: LoadAddress) -> None: pass diff --git a/mypyc/analysis/selfleaks.py b/mypyc/analysis/selfleaks.py index 9f7e00db78d27..8f46cbe3312bc 100644 --- a/mypyc/analysis/selfleaks.py +++ b/mypyc/analysis/selfleaks.py @@ -35,6 +35,7 @@ RegisterOp, Return, SetAttr, + SetElement, SetMem, Truncate, TupleGet, @@ -181,6 +182,9 @@ def visit_load_mem(self, op: LoadMem) -> GenAndKill: def visit_get_element_ptr(self, op: GetElementPtr) -> GenAndKill: return CLEAN + def visit_set_element(self, op: SetElement) -> GenAndKill: + return CLEAN + def visit_load_address(self, op: LoadAddress) -> GenAndKill: return CLEAN diff --git a/mypyc/codegen/emitfunc.py b/mypyc/codegen/emitfunc.py index 3fdd08037d1af..9012f072f96b8 100644 --- a/mypyc/codegen/emitfunc.py +++ b/mypyc/codegen/emitfunc.py @@ -70,12 +70,14 @@ Register, Return, SetAttr, + SetElement, SetMem, Truncate, TupleGet, TupleSet, Unborrow, Unbox, + Undef, Unreachable, Value, ) @@ -813,6 +815,31 @@ def visit_get_element_ptr(self, op: GetElementPtr) -> None: ) ) + def visit_set_element(self, op: SetElement) -> None: + dest = self.reg(op) + item = self.reg(op.item) + field = op.field + if isinstance(op.src, Undef): + # First assignment to an undefined struct is trivial. + self.emit_line(f"{dest}.{field} = {item};") + else: + # In the general case create a copy of the struct with a single + # item modified. + # + # TODO: Can we do better if only a subset of fields are initialized? + # TODO: Make this less verbose in the common case + # TODO: Support tuples (or use RStruct for tuples)? + src = self.reg(op.src) + src_type = op.src.type + assert isinstance(src_type, RStruct), src_type + init_items = [] + for n in src_type.names: + if n != field: + init_items.append(f"{src}.{n}") + else: + init_items.append(item) + self.emit_line(f"{dest} = ({self.ctype(src_type)}) {{ {', '.join(init_items)} }};") + def visit_load_address(self, op: LoadAddress) -> None: typ = op.type dest = self.reg(op) diff --git a/mypyc/ir/ops.py b/mypyc/ir/ops.py index f362b0cca1979..4829dd6a903dc 100644 --- a/mypyc/ir/ops.py +++ b/mypyc/ir/ops.py @@ -34,6 +34,7 @@ class to enable the new behavior. Sometimes adding a new abstract from mypyc.ir.rtypes import ( RArray, RInstance, + RStruct, RTuple, RType, RVoid, @@ -244,6 +245,26 @@ def __init__(self, value: bytes, line: int = -1) -> None: self.line = line +@final +class Undef(Value): + """An undefined value. + + Use Undef() as the initial value followed by one or more SetElement + ops to initialize a struct. Pseudocode example: + + r0 = set_element undef MyStruct, "field1", f1 + r1 = set_element r0, "field2", f2 + # r1 now has new struct value with two fields set + + Warning: Always initialize undefined values before using them, + as otherwise the values are garbage. You shouldn't expect that + undefined values are zeroed, in particular. + """ + + def __init__(self, rtype: RType) -> None: + self.type = rtype + + class Op(Value): """Abstract base class for all IR operations. @@ -1636,6 +1657,39 @@ def accept(self, visitor: OpVisitor[T]) -> T: return visitor.visit_get_element_ptr(self) +@final +class SetElement(RegisterOp): + """Set the value of a struct element. + + This evaluates to a new struct with the changed value. + + Use together with Undef to initialize a fresh struct value + (see Undef for more details). + """ + + error_kind = ERR_NEVER + + def __init__(self, src: Value, field: str, item: Value, line: int = -1) -> None: + super().__init__(line) + assert isinstance(src.type, RStruct), src.type + self.type = src.type + self.src = src + self.item = item + self.field = field + + def sources(self) -> list[Value]: + return [self.src] + + def set_sources(self, new: list[Value]) -> None: + (self.src,) = new + + def stolen(self) -> list[Value]: + return [self.src] + + def accept(self, visitor: OpVisitor[T]) -> T: + return visitor.visit_set_element(self) + + @final class LoadAddress(RegisterOp): """Get the address of a value: result = (type)&src @@ -1908,6 +1962,10 @@ def visit_set_mem(self, op: SetMem) -> T: def visit_get_element_ptr(self, op: GetElementPtr) -> T: raise NotImplementedError + @abstractmethod + def visit_set_element(self, op: SetElement) -> T: + raise NotImplementedError + @abstractmethod def visit_load_address(self, op: LoadAddress) -> T: raise NotImplementedError diff --git a/mypyc/ir/pprint.py b/mypyc/ir/pprint.py index 5bb11cc231ccc..2a239a0b4d9dc 100644 --- a/mypyc/ir/pprint.py +++ b/mypyc/ir/pprint.py @@ -50,12 +50,14 @@ Register, Return, SetAttr, + SetElement, SetMem, Truncate, TupleGet, TupleSet, Unborrow, Unbox, + Undef, Unreachable, Value, ) @@ -273,6 +275,9 @@ def visit_set_mem(self, op: SetMem) -> str: def visit_get_element_ptr(self, op: GetElementPtr) -> str: return self.format("%r = get_element_ptr %r %s :: %t", op, op.src, op.field, op.src_type) + def visit_set_element(self, op: SetElement) -> str: + return self.format("%r = set_element %r, %s, %r", op, op.src, op.field, op.item) + def visit_load_address(self, op: LoadAddress) -> str: if isinstance(op.src, Register): return self.format("%r = load_address %r", op, op.src) @@ -330,6 +335,8 @@ def format(self, fmt: str, *args: Any) -> str: result.append(repr(arg.value)) elif isinstance(arg, CString): result.append(f"CString({arg.value!r})") + elif isinstance(arg, Undef): + result.append(f"undef {arg.type.name}") else: result.append(self.names[arg]) elif typespec == "d": @@ -486,7 +493,7 @@ def generate_names_for_ir(args: list[Register], blocks: list[BasicBlock]) -> dic continue if isinstance(value, Register) and value.name: name = value.name - elif isinstance(value, (Integer, Float)): + elif isinstance(value, (Integer, Float, Undef)): continue else: name = "r%d" % temp_index diff --git a/mypyc/test/test_emitfunc.py b/mypyc/test/test_emitfunc.py index 6be4875dafa12..6382271cfe94c 100644 --- a/mypyc/test/test_emitfunc.py +++ b/mypyc/test/test_emitfunc.py @@ -35,9 +35,11 @@ Register, Return, SetAttr, + SetElement, SetMem, TupleGet, Unbox, + Undef, Unreachable, Value, ) @@ -121,6 +123,11 @@ def add_local(name: str, rtype: RType) -> Register: self.r = add_local("r", RInstance(ir)) self.none = add_local("none", none_rprimitive) + self.struct_type = RStruct( + "Foo", ["b", "x", "y"], [bool_rprimitive, int32_rprimitive, int64_rprimitive] + ) + self.st = add_local("st", self.struct_type) + self.context = EmitterContext(NameGenerator([["mod"]])) def test_goto(self) -> None: @@ -674,6 +681,17 @@ def test_get_element_ptr(self) -> None: GetElementPtr(self.o, r, "i64"), """cpy_r_r0 = (CPyPtr)&((Foo *)cpy_r_o)->i64;""" ) + def test_set_element(self) -> None: + # Use compact syntax when setting the initial element of an undefined value + self.assert_emit( + SetElement(Undef(self.struct_type), "b", self.b), """cpy_r_r0.b = cpy_r_b;""" + ) + # We propagate the unchanged values in subsequent assignments + self.assert_emit( + SetElement(self.st, "x", self.i32), + """cpy_r_r0 = (Foo) { cpy_r_st.b, cpy_r_i32, cpy_r_st.y };""", + ) + def test_load_address(self) -> None: self.assert_emit( LoadAddress(object_rprimitive, "PyDict_Type"), diff --git a/mypyc/transform/ir_transform.py b/mypyc/transform/ir_transform.py index 7834fed394657..bcb6db9b0daf5 100644 --- a/mypyc/transform/ir_transform.py +++ b/mypyc/transform/ir_transform.py @@ -39,6 +39,7 @@ RaiseStandardError, Return, SetAttr, + SetElement, SetMem, Truncate, TupleGet, @@ -214,6 +215,9 @@ def visit_set_mem(self, op: SetMem) -> Value | None: def visit_get_element_ptr(self, op: GetElementPtr) -> Value | None: return self.add(op) + def visit_set_element(self, op: SetElement) -> Value | None: + return self.add(op) + def visit_load_address(self, op: LoadAddress) -> Value | None: return self.add(op) @@ -354,6 +358,9 @@ def visit_set_mem(self, op: SetMem) -> None: def visit_get_element_ptr(self, op: GetElementPtr) -> None: op.src = self.fix_op(op.src) + def visit_set_element(self, op: SetElement) -> None: + op.src = self.fix_op(op.src) + def visit_load_address(self, op: LoadAddress) -> None: if isinstance(op.src, LoadStatic): new = self.fix_op(op.src) diff --git a/mypyc/transform/refcount.py b/mypyc/transform/refcount.py index c589918986f07..60daebc415fd6 100644 --- a/mypyc/transform/refcount.py +++ b/mypyc/transform/refcount.py @@ -43,6 +43,7 @@ Op, Register, RegisterOp, + Undef, Value, ) @@ -94,7 +95,7 @@ def is_maybe_undefined(post_must_defined: set[Value], src: Value) -> bool: def maybe_append_dec_ref( ops: list[Op], dest: Value, defined: AnalysisDict[Value], key: tuple[BasicBlock, int] ) -> None: - if dest.type.is_refcounted and not isinstance(dest, Integer): + if dest.type.is_refcounted and not isinstance(dest, (Integer, Undef)): ops.append(DecRef(dest, is_xdec=is_maybe_undefined(defined[key], dest))) From 38cdacfd142291eae3f30c1a4a83f4042a159975 Mon Sep 17 00:00:00 2001 From: Piotr Sawicki Date: Tue, 15 Jul 2025 14:08:01 +0200 Subject: [PATCH 084/424] [mypyc] Add primitives for isinstance of built-in types (#19435) Follow-up to #19416 adding primitives for `isinstance(obj, type)` where type is built-in. --- mypyc/irbuild/specialize.py | 23 +++- mypyc/primitives/bytes_ops.py | 21 ++- mypyc/primitives/dict_ops.py | 9 ++ mypyc/primitives/float_ops.py | 10 ++ mypyc/primitives/int_ops.py | 9 ++ mypyc/primitives/list_ops.py | 2 +- mypyc/primitives/misc_ops.py | 9 ++ mypyc/primitives/set_ops.py | 20 ++- mypyc/primitives/str_ops.py | 9 ++ mypyc/primitives/tuple_ops.py | 10 ++ mypyc/test-data/irbuild-basic.test | 37 ++---- mypyc/test-data/irbuild-i64.test | 28 ++-- mypyc/test-data/irbuild-isinstance.test | 162 ++++++++++++++++++------ mypyc/test-data/irbuild-optional.test | 30 ++--- mypyc/test-data/run-bools.test | 26 ++++ mypyc/test-data/run-bytes.test | 51 ++++++++ mypyc/test-data/run-dicts.test | 32 +++++ mypyc/test-data/run-floats.test | 31 +++++ mypyc/test-data/run-integers.test | 34 +++++ mypyc/test-data/run-sets.test | 52 ++++++++ mypyc/test-data/run-strings.test | 35 +++++ mypyc/test-data/run-tuples.test | 32 +++++ 22 files changed, 568 insertions(+), 104 deletions(-) diff --git a/mypyc/irbuild/specialize.py b/mypyc/irbuild/specialize.py index b490c2a52e574..3015640fb3fd5 100644 --- a/mypyc/irbuild/specialize.py +++ b/mypyc/irbuild/specialize.py @@ -83,19 +83,26 @@ join_formatted_strings, tokenizer_format_call, ) +from mypyc.primitives.bytes_ops import isinstance_bytearray, isinstance_bytes from mypyc.primitives.dict_ops import ( dict_items_op, dict_keys_op, dict_setdefault_spec_init_op, dict_values_op, + isinstance_dict, ) +from mypyc.primitives.float_ops import isinstance_float +from mypyc.primitives.int_ops import isinstance_int from mypyc.primitives.list_ops import isinstance_list, new_list_set_item_op +from mypyc.primitives.misc_ops import isinstance_bool +from mypyc.primitives.set_ops import isinstance_frozenset, isinstance_set from mypyc.primitives.str_ops import ( + isinstance_str, str_encode_ascii_strict, str_encode_latin1_strict, str_encode_utf8_strict, ) -from mypyc.primitives.tuple_ops import new_tuple_set_item_op +from mypyc.primitives.tuple_ops import isinstance_tuple, new_tuple_set_item_op # Specializers are attempted before compiling the arguments to the # function. Specializers can return None to indicate that they failed @@ -546,7 +553,19 @@ def gen_inner_stmts() -> None: return retval -isinstance_primitives: Final = {"builtins.list": isinstance_list} +isinstance_primitives: Final = { + "builtins.bool": isinstance_bool, + "builtins.bytearray": isinstance_bytearray, + "builtins.bytes": isinstance_bytes, + "builtins.dict": isinstance_dict, + "builtins.float": isinstance_float, + "builtins.frozenset": isinstance_frozenset, + "builtins.int": isinstance_int, + "builtins.list": isinstance_list, + "builtins.set": isinstance_set, + "builtins.str": isinstance_str, + "builtins.tuple": isinstance_tuple, +} @specialize_function("builtins.isinstance") diff --git a/mypyc/primitives/bytes_ops.py b/mypyc/primitives/bytes_ops.py index 1afd196cff846..c88e89d1a2bad 100644 --- a/mypyc/primitives/bytes_ops.py +++ b/mypyc/primitives/bytes_ops.py @@ -2,9 +2,10 @@ from __future__ import annotations -from mypyc.ir.ops import ERR_MAGIC +from mypyc.ir.ops import ERR_MAGIC, ERR_NEVER from mypyc.ir.rtypes import ( RUnion, + bit_rprimitive, bytes_rprimitive, c_int_rprimitive, c_pyssize_t_rprimitive, @@ -35,6 +36,15 @@ error_kind=ERR_MAGIC, ) +# translate isinstance(obj, bytes) +isinstance_bytes = function_op( + name="builtins.isinstance", + arg_types=[object_rprimitive], + return_type=bit_rprimitive, + c_function_name="PyBytes_Check", + error_kind=ERR_NEVER, +) + # bytearray(obj) function_op( name="builtins.bytearray", @@ -44,6 +54,15 @@ error_kind=ERR_MAGIC, ) +# translate isinstance(obj, bytearray) +isinstance_bytearray = function_op( + name="builtins.isinstance", + arg_types=[object_rprimitive], + return_type=bit_rprimitive, + c_function_name="PyByteArray_Check", + error_kind=ERR_NEVER, +) + # bytes ==/!= (return -1/0/1) bytes_compare = custom_op( arg_types=[bytes_rprimitive, bytes_rprimitive], diff --git a/mypyc/primitives/dict_ops.py b/mypyc/primitives/dict_ops.py index 3f289c3c6f08d..ac928bb0eb504 100644 --- a/mypyc/primitives/dict_ops.py +++ b/mypyc/primitives/dict_ops.py @@ -71,6 +71,15 @@ error_kind=ERR_MAGIC, ) +# translate isinstance(obj, dict) +isinstance_dict = function_op( + name="builtins.isinstance", + arg_types=[object_rprimitive], + return_type=bit_rprimitive, + c_function_name="PyDict_Check", + error_kind=ERR_NEVER, +) + # dict[key] dict_get_item_op = method_op( name="__getitem__", diff --git a/mypyc/primitives/float_ops.py b/mypyc/primitives/float_ops.py index 14e8d4caf09cf..542192add5420 100644 --- a/mypyc/primitives/float_ops.py +++ b/mypyc/primitives/float_ops.py @@ -4,6 +4,7 @@ from mypyc.ir.ops import ERR_MAGIC, ERR_MAGIC_OVERLAPPING, ERR_NEVER from mypyc.ir.rtypes import ( + bit_rprimitive, bool_rprimitive, float_rprimitive, int_rprimitive, @@ -166,3 +167,12 @@ c_function_name="CPyFloat_IsNaN", error_kind=ERR_NEVER, ) + +# translate isinstance(obj, float) +isinstance_float = function_op( + name="builtins.isinstance", + arg_types=[object_rprimitive], + return_type=bit_rprimitive, + c_function_name="PyFloat_Check", + error_kind=ERR_NEVER, +) diff --git a/mypyc/primitives/int_ops.py b/mypyc/primitives/int_ops.py index 9b8b48da602d1..d723c9b63a86c 100644 --- a/mypyc/primitives/int_ops.py +++ b/mypyc/primitives/int_ops.py @@ -296,3 +296,12 @@ def int_unary_op(name: str, c_function_name: str) -> PrimitiveDescription: c_function_name="CPyUInt8_Overflow", error_kind=ERR_ALWAYS, ) + +# translate isinstance(obj, int) +isinstance_int = function_op( + name="builtints.isinstance", + arg_types=[object_rprimitive], + return_type=bit_rprimitive, + c_function_name="PyLong_Check", + error_kind=ERR_NEVER, +) diff --git a/mypyc/primitives/list_ops.py b/mypyc/primitives/list_ops.py index 57cb541fdbb83..516d9e1a4e026 100644 --- a/mypyc/primitives/list_ops.py +++ b/mypyc/primitives/list_ops.py @@ -56,7 +56,7 @@ extra_int_constants=[(0, int_rprimitive)], ) -# isinstance(obj, list) +# translate isinstance(obj, list) isinstance_list = function_op( name="builtins.isinstance", arg_types=[object_rprimitive], diff --git a/mypyc/primitives/misc_ops.py b/mypyc/primitives/misc_ops.py index 7494b46790cec..114a5f0a9823e 100644 --- a/mypyc/primitives/misc_ops.py +++ b/mypyc/primitives/misc_ops.py @@ -191,6 +191,15 @@ truncated_type=bool_rprimitive, ) +# isinstance(obj, bool) +isinstance_bool = function_op( + name="builtins.isinstance", + arg_types=[object_rprimitive], + return_type=bit_rprimitive, + c_function_name="PyBool_Check", + error_kind=ERR_NEVER, +) + # slice(start, stop, step) new_slice_op = function_op( name="builtins.slice", diff --git a/mypyc/primitives/set_ops.py b/mypyc/primitives/set_ops.py index eb7c9b46609d5..786de008746dd 100644 --- a/mypyc/primitives/set_ops.py +++ b/mypyc/primitives/set_ops.py @@ -2,7 +2,7 @@ from __future__ import annotations -from mypyc.ir.ops import ERR_FALSE, ERR_MAGIC +from mypyc.ir.ops import ERR_FALSE, ERR_MAGIC, ERR_NEVER from mypyc.ir.rtypes import ( bit_rprimitive, bool_rprimitive, @@ -64,6 +64,24 @@ error_kind=ERR_MAGIC, ) +# translate isinstance(obj, set) +isinstance_set = function_op( + name="builtins.isinstance", + arg_types=[object_rprimitive], + return_type=bit_rprimitive, + c_function_name="PySet_Check", + error_kind=ERR_NEVER, +) + +# translate isinstance(obj, frozenset) +isinstance_frozenset = function_op( + name="builtins.isinstance", + arg_types=[object_rprimitive], + return_type=bit_rprimitive, + c_function_name="PyFrozenSet_Check", + error_kind=ERR_NEVER, +) + # item in set set_in_op = binary_op( name="in", diff --git a/mypyc/primitives/str_ops.py b/mypyc/primitives/str_ops.py index e3f0b9dbbc2a8..f07081c6aaa50 100644 --- a/mypyc/primitives/str_ops.py +++ b/mypyc/primitives/str_ops.py @@ -48,6 +48,15 @@ error_kind=ERR_MAGIC, ) +# translate isinstance(obj, str) +isinstance_str = function_op( + name="builtins.isinstance", + arg_types=[object_rprimitive], + return_type=bit_rprimitive, + c_function_name="PyUnicode_Check", + error_kind=ERR_NEVER, +) + # str1 + str2 binary_op( name="+", diff --git a/mypyc/primitives/tuple_ops.py b/mypyc/primitives/tuple_ops.py index e680b6943d845..d95161acf853e 100644 --- a/mypyc/primitives/tuple_ops.py +++ b/mypyc/primitives/tuple_ops.py @@ -8,6 +8,7 @@ from mypyc.ir.ops import ERR_MAGIC, ERR_NEVER from mypyc.ir.rtypes import ( + bit_rprimitive, c_pyssize_t_rprimitive, int_rprimitive, list_rprimitive, @@ -83,6 +84,15 @@ error_kind=ERR_MAGIC, ) +# translate isinstance(obj, tuple) +isinstance_tuple = function_op( + name="builtins.isinstance", + arg_types=[object_rprimitive], + return_type=bit_rprimitive, + c_function_name="PyTuple_Check", + error_kind=ERR_NEVER, +) + # tuple + tuple binary_op( name="+", diff --git a/mypyc/test-data/irbuild-basic.test b/mypyc/test-data/irbuild-basic.test index ea1b3d06869ae..4a7d315ec8367 100644 --- a/mypyc/test-data/irbuild-basic.test +++ b/mypyc/test-data/irbuild-basic.test @@ -1581,24 +1581,18 @@ def main() -> None: [out] def foo(x): x :: union[int, str] - r0 :: object - r1 :: i32 - r2 :: bit - r3 :: bool - r4 :: __main__.B - r5 :: __main__.A + r0 :: bit + r1 :: __main__.B + r2 :: __main__.A L0: - r0 = load_address PyLong_Type - r1 = PyObject_IsInstance(x, r0) - r2 = r1 >= 0 :: signed - r3 = truncate r1: i32 to builtins.bool - if r3 goto L1 else goto L2 :: bool + r0 = PyLong_Check(x) + if r0 goto L1 else goto L2 :: bool L1: - r4 = B() - return r4 + r1 = B() + return r1 L2: - r5 = A() - return r5 + r2 = A() + return r2 def main(): r0 :: object r1 :: __main__.A @@ -3389,16 +3383,11 @@ def f(x: object) -> bool: return isinstance(x, bool) [out] def f(x): - x, r0 :: object - r1 :: i32 - r2 :: bit - r3 :: bool + x :: object + r0 :: bit L0: - r0 = load_address PyBool_Type - r1 = PyObject_IsInstance(x, r0) - r2 = r1 >= 0 :: signed - r3 = truncate r1: i32 to builtins.bool - return r3 + r0 = PyBool_Check(x) + return r0 [case testRangeObject] def range_object() -> None: diff --git a/mypyc/test-data/irbuild-i64.test b/mypyc/test-data/irbuild-i64.test index c59e306b09df1..e55c3bfe2acc2 100644 --- a/mypyc/test-data/irbuild-i64.test +++ b/mypyc/test-data/irbuild-i64.test @@ -2046,27 +2046,21 @@ L2: return r6 def narrow2(x): x :: union[__main__.C, i64] - r0 :: object - r1 :: i32 - r2 :: bit - r3 :: bool - r4 :: i64 - r5 :: __main__.C - r6 :: i64 + r0 :: bit + r1 :: i64 + r2 :: __main__.C + r3 :: i64 L0: - r0 = load_address PyLong_Type - r1 = PyObject_IsInstance(x, r0) - r2 = r1 >= 0 :: signed - r3 = truncate r1: i32 to builtins.bool - if r3 goto L1 else goto L2 :: bool + r0 = PyLong_Check(x) + if r0 goto L1 else goto L2 :: bool L1: - r4 = unbox(i64, x) - return r4 + r1 = unbox(i64, x) + return r1 L2: - r5 = borrow cast(__main__.C, x) - r6 = r5.a + r2 = borrow cast(__main__.C, x) + r3 = r2.a keep_alive x - return r6 + return r3 [case testI64ConvertBetweenTuples_64bit] from __future__ import annotations diff --git a/mypyc/test-data/irbuild-isinstance.test b/mypyc/test-data/irbuild-isinstance.test index 78da2e9c1e196..30adfe61e3840 100644 --- a/mypyc/test-data/irbuild-isinstance.test +++ b/mypyc/test-data/irbuild-isinstance.test @@ -4,16 +4,11 @@ def is_int(value: object) -> bool: [out] def is_int(value): - value, r0 :: object - r1 :: i32 - r2 :: bit - r3 :: bool + value :: object + r0 :: bit L0: - r0 = load_address PyLong_Type - r1 = PyObject_IsInstance(value, r0) - r2 = r1 >= 0 :: signed - r3 = truncate r1: i32 to builtins.bool - return r3 + r0 = PyLong_Check(value) + return r0 [case testIsinstanceNotBool1] def is_not_bool(value: object) -> bool: @@ -21,17 +16,12 @@ def is_not_bool(value: object) -> bool: [out] def is_not_bool(value): - value, r0 :: object - r1 :: i32 - r2 :: bit - r3, r4 :: bool + value :: object + r0, r1 :: bit L0: - r0 = load_address PyBool_Type - r1 = PyObject_IsInstance(value, r0) - r2 = r1 >= 0 :: signed - r3 = truncate r1: i32 to builtins.bool - r4 = r3 ^ 1 - return r4 + r0 = PyBool_Check(value) + r1 = r0 ^ 1 + return r1 [case testIsinstanceIntAndNotBool] # This test is to ensure that 'value' doesn't get coerced to int when we are @@ -41,32 +31,22 @@ def is_not_bool_and_is_int(value: object) -> bool: [out] def is_not_bool_and_is_int(value): - value, r0 :: object - r1 :: i32 - r2 :: bit - r3, r4 :: bool - r5 :: object - r6 :: i32 - r7 :: bit - r8, r9 :: bool + value :: object + r0 :: bit + r1 :: bool + r2, r3 :: bit L0: - r0 = load_address PyLong_Type - r1 = PyObject_IsInstance(value, r0) - r2 = r1 >= 0 :: signed - r3 = truncate r1: i32 to builtins.bool - if r3 goto L2 else goto L1 :: bool + r0 = PyLong_Check(value) + if r0 goto L2 else goto L1 :: bool L1: - r4 = r3 + r1 = r0 goto L3 L2: - r5 = load_address PyBool_Type - r6 = PyObject_IsInstance(value, r5) - r7 = r6 >= 0 :: signed - r8 = truncate r6: i32 to builtins.bool - r9 = r8 ^ 1 - r4 = r9 + r2 = PyBool_Check(value) + r3 = r2 ^ 1 + r1 = r3 L3: - return r4 + return r1 [case testBorrowSpecialCaseWithIsinstance] class C: @@ -107,3 +87,105 @@ L1: keep_alive x L2: return 1 + +[case testBytes] +from typing import Any + +def is_bytes(x: Any) -> bool: + return isinstance(x, bytes) + +def is_bytearray(x: Any) -> bool: + return isinstance(x, bytearray) + +[out] +def is_bytes(x): + x :: object + r0 :: bit +L0: + r0 = PyBytes_Check(x) + return r0 +def is_bytearray(x): + x :: object + r0 :: bit +L0: + r0 = PyByteArray_Check(x) + return r0 + +[case testDict] +from typing import Any + +def is_dict(x: Any) -> bool: + return isinstance(x, dict) + +[out] +def is_dict(x): + x :: object + r0 :: bit +L0: + r0 = PyDict_Check(x) + return r0 + +[case testFloat] +from typing import Any + +def is_float(x: Any) -> bool: + return isinstance(x, float) + +[out] +def is_float(x): + x :: object + r0 :: bit +L0: + r0 = PyFloat_Check(x) + return r0 + +[case testSet] +from typing import Any + +def is_set(x: Any) -> bool: + return isinstance(x, set) + +def is_frozenset(x: Any) -> bool: + return isinstance(x, frozenset) + +[out] +def is_set(x): + x :: object + r0 :: bit +L0: + r0 = PySet_Check(x) + return r0 +def is_frozenset(x): + x :: object + r0 :: bit +L0: + r0 = PyFrozenSet_Check(x) + return r0 + +[case testStr] +from typing import Any + +def is_str(x: Any) -> bool: + return isinstance(x, str) + +[out] +def is_str(x): + x :: object + r0 :: bit +L0: + r0 = PyUnicode_Check(x) + return r0 + +[case testTuple] +from typing import Any + +def is_tuple(x: Any) -> bool: + return isinstance(x, tuple) + +[out] +def is_tuple(x): + x :: object + r0 :: bit +L0: + r0 = PyTuple_Check(x) + return r0 diff --git a/mypyc/test-data/irbuild-optional.test b/mypyc/test-data/irbuild-optional.test index 75c0085869995..b81465d362bab 100644 --- a/mypyc/test-data/irbuild-optional.test +++ b/mypyc/test-data/irbuild-optional.test @@ -251,28 +251,22 @@ def f(x: Union[int, A]) -> int: [out] def f(x): x :: union[int, __main__.A] - r0 :: object - r1 :: i32 - r2 :: bit - r3 :: bool - r4, r5 :: int - r6 :: __main__.A - r7 :: int + r0 :: bit + r1, r2 :: int + r3 :: __main__.A + r4 :: int L0: - r0 = load_address PyLong_Type - r1 = PyObject_IsInstance(x, r0) - r2 = r1 >= 0 :: signed - r3 = truncate r1: i32 to builtins.bool - if r3 goto L1 else goto L2 :: bool + r0 = PyLong_Check(x) + if r0 goto L1 else goto L2 :: bool L1: - r4 = unbox(int, x) - r5 = CPyTagged_Add(r4, 2) - return r5 + r1 = unbox(int, x) + r2 = CPyTagged_Add(r1, 2) + return r2 L2: - r6 = borrow cast(__main__.A, x) - r7 = r6.a + r3 = borrow cast(__main__.A, x) + r4 = r3.a keep_alive x - return r7 + return r4 L3: unreachable diff --git a/mypyc/test-data/run-bools.test b/mypyc/test-data/run-bools.test index 3409665bfb377..b34fedebaa9fc 100644 --- a/mypyc/test-data/run-bools.test +++ b/mypyc/test-data/run-bools.test @@ -228,3 +228,29 @@ def test_mix() -> None: print((y or 0) and True) [out] 0 + +[case testIsInstance] +from typing import Any +def test_built_in() -> None: + true: Any = True + false: Any = False + assert isinstance(true, bool) + assert isinstance(false, bool) + + assert not isinstance(set(), bool) + assert not isinstance((), bool) + assert not isinstance((True, False), bool) + assert not isinstance({False, True}, bool) + assert not isinstance(int() + 1, bool) + assert not isinstance(str() + 'False', bool) + +def test_user_defined() -> None: + from userdefinedbool import bool + + b: Any = True + assert isinstance(bool(), bool) + assert not isinstance(b, bool) + +[file userdefinedbool.py] +class bool: + pass diff --git a/mypyc/test-data/run-bytes.test b/mypyc/test-data/run-bytes.test index bee6b6fe9f76a..5a285320c849e 100644 --- a/mypyc/test-data/run-bytes.test +++ b/mypyc/test-data/run-bytes.test @@ -323,3 +323,54 @@ class A: def test_bytes_dunder() -> None: assert b'%b' % A() == b'aaa' assert b'%s' % A() == b'aaa' + +[case testIsInstance] +from copysubclass import subbytes, subbytearray +from typing import Any +def test_bytes() -> None: + b: Any = b'' + assert isinstance(b, bytes) + assert isinstance(b + b'123', bytes) + assert isinstance(b + b'\xff', bytes) + assert isinstance(subbytes(), bytes) + assert isinstance(subbytes(b + b'123'), bytes) + assert isinstance(subbytes(b + b'\xff'), bytes) + + assert not isinstance(set(), bytes) + assert not isinstance((), bytes) + assert not isinstance((b'1',b'2',b'3'), bytes) + assert not isinstance({b'a',b'b'}, bytes) + assert not isinstance(int() + 1, bytes) + assert not isinstance(str() + 'a', bytes) + +def test_user_defined_bytes() -> None: + from userdefinedbytes import bytes + + assert isinstance(bytes(), bytes) + assert not isinstance(b'\x7f', bytes) + +def test_bytearray() -> None: + assert isinstance(bytearray(), bytearray) + assert isinstance(bytearray(b'123'), bytearray) + assert isinstance(bytearray(b'\xff'), bytearray) + assert isinstance(subbytearray(), bytearray) + assert isinstance(subbytearray(bytearray(b'123')), bytearray) + assert isinstance(subbytearray(bytearray(b'\xff')), bytearray) + + assert not isinstance(set(), bytearray) + assert not isinstance((), bytearray) + assert not isinstance((bytearray(b'1'),bytearray(b'2'),bytearray(b'3')), bytearray) + assert not isinstance([bytearray(b'a'),bytearray(b'b')], bytearray) + assert not isinstance(int() + 1, bytearray) + assert not isinstance(str() + 'a', bytearray) + +[file copysubclass.py] +class subbytes(bytes): + pass + +class subbytearray(bytearray): + pass + +[file userdefinedbytes.py] +class bytes: + pass diff --git a/mypyc/test-data/run-dicts.test b/mypyc/test-data/run-dicts.test index 2a3be188ad00f..2b75b32c906e0 100644 --- a/mypyc/test-data/run-dicts.test +++ b/mypyc/test-data/run-dicts.test @@ -336,3 +336,35 @@ def test_dict_to_bool() -> None: for x in tmp_list: assert is_true(x) assert not is_false(x) + +[case testIsInstance] +from copysubclass import subc +def test_built_in() -> None: + assert isinstance({}, dict) + assert isinstance({'one': 1, 'two': 2}, dict) + assert isinstance({1: 1, 'two': 2}, dict) + assert isinstance(subc(), dict) + assert isinstance(subc({'a': 1, 'b': 2}), dict) + assert isinstance(subc({1: 'a', 2: 'b'}), dict) + + assert not isinstance(set(), dict) + assert not isinstance((), dict) + assert not isinstance((1,2,3), dict) + assert not isinstance({'a','b'}, dict) + assert not isinstance(int() + 1, dict) + assert not isinstance(str() + 'a', dict) + +def test_user_defined() -> None: + from userdefineddict import dict + + assert isinstance(dict(), dict) + assert not isinstance({1: dict()}, dict) + +[file copysubclass.py] +from typing import Any +class subc(dict[Any, Any]): + pass + +[file userdefineddict.py] +class dict: + pass diff --git a/mypyc/test-data/run-floats.test b/mypyc/test-data/run-floats.test index 49620f6448c79..424d52cdb0d55 100644 --- a/mypyc/test-data/run-floats.test +++ b/mypyc/test-data/run-floats.test @@ -512,3 +512,34 @@ def test_implement_trait_attribute() -> None: a.y = 8.0 assert a.x == 7 assert a.y == 8 + +[case testIsInstance] +from copysubclass import subc +from testutil import float_vals +from typing import Any +def test_built_in() -> None: + for f in float_vals: + assert isinstance(float(0) + f, float) + assert isinstance(subc(f), float) + + assert not isinstance(set(), float) + assert not isinstance((), float) + assert not isinstance((1.0, 2.0), float) + assert not isinstance({3.14}, float) + assert not isinstance(int() + 1, float) + assert not isinstance(str() + '4.2', float) + +def test_user_defined() -> None: + from userdefinedfloat import float + + f: Any = 3.14 + assert isinstance(float(), float) + assert not isinstance(f, float) + +[file copysubclass.py] +class subc(float): + pass + +[file userdefinedfloat.py] +class float: + pass diff --git a/mypyc/test-data/run-integers.test b/mypyc/test-data/run-integers.test index d575e141b5671..1163c9d942f78 100644 --- a/mypyc/test-data/run-integers.test +++ b/mypyc/test-data/run-integers.test @@ -538,3 +538,37 @@ def test_int_bool_min_max() -> None: assert min(u, z) == -10 assert max(u, y) == False assert max(u, z) == True + +[case testIsInstance] +from copysubclass import subc +from typing import Any +def test_built_in() -> None: + i: Any = 0 + assert isinstance(i + 0, int) + assert isinstance(i + 9223372036854775808, int) + assert isinstance(i + -9223372036854775808, int) + assert isinstance(subc(), int) + assert isinstance(subc(9223372036854775808), int) + assert isinstance(subc(-9223372036854775808), int) + + assert not isinstance(set(), int) + assert not isinstance((), int) + assert not isinstance((1,2,3), int) + assert not isinstance({1,2}, int) + assert not isinstance(float(0) + 1.0, int) + assert not isinstance(str() + '1', int) + +def test_user_defined() -> None: + from userdefinedint import int + + i: Any = 42 + assert isinstance(int(), int) + assert not isinstance(i, int) + +[file copysubclass.py] +class subc(int): + pass + +[file userdefinedint.py] +class int: + pass diff --git a/mypyc/test-data/run-sets.test b/mypyc/test-data/run-sets.test index 68edd1e6b77d4..2668d63bcdaca 100644 --- a/mypyc/test-data/run-sets.test +++ b/mypyc/test-data/run-sets.test @@ -265,3 +265,55 @@ def test_in_set() -> None: def test_for_set() -> None: assert not s ^ {None, False, 1, 2.0, "3", b"4", 5j, (6,), CONST}, s + +[case testIsInstance] +from copysubclass import subset, subfrozenset +def test_built_in_set() -> None: + assert isinstance(set(), set) + assert isinstance({'one', 'two'}, set) + assert isinstance({'a', 1}, set) + assert isinstance(subset(), set) + assert isinstance(subset({'one', 'two'}), set) + assert isinstance(subset({'a', 1}), set) + + assert not isinstance(frozenset(), set) + assert not isinstance({}, set) + assert not isinstance([], set) + assert not isinstance((1,2,3), set) + assert not isinstance({1:'a', 2:'b'}, set) + assert not isinstance(int() + 1, set) + assert not isinstance(str() + 'a', set) + +def test_user_defined_set() -> None: + from userdefinedset import set + + assert isinstance(set(), set) + assert not isinstance({set()}, set) + +def test_built_in_frozenset() -> None: + assert isinstance(frozenset(), frozenset) + assert isinstance(frozenset({'one', 'two'}), frozenset) + assert isinstance(frozenset({'a', 1}), frozenset) + assert isinstance(subfrozenset(), frozenset) + assert isinstance(subfrozenset({'one', 'two'}), frozenset) + assert isinstance(subfrozenset({'a', 1}), frozenset) + + assert not isinstance(set(), frozenset) + assert not isinstance({}, frozenset) + assert not isinstance([], frozenset) + assert not isinstance((1,2,3), frozenset) + assert not isinstance({1:'a', 2:'b'}, frozenset) + assert not isinstance(int() + 1, frozenset) + assert not isinstance(str() + 'a', frozenset) + +[file copysubclass.py] +from typing import Any +class subset(set[Any]): + pass + +class subfrozenset(frozenset[Any]): + pass + +[file userdefinedset.py] +class set: + pass diff --git a/mypyc/test-data/run-strings.test b/mypyc/test-data/run-strings.test index 6551d9c352dfe..8a914c08bfb21 100644 --- a/mypyc/test-data/run-strings.test +++ b/mypyc/test-data/run-strings.test @@ -964,3 +964,38 @@ def test_count_multi_start_end_emoji() -> None: assert string.count("😴😴😴", 0, 12) == 1, string.count("😴😴😴", 0, 12) assert string.count("🚀🚀🚀", 0, 12) == 2, string.count("🚀🚀🚀", 0, 12) assert string.count("ñññ", 0, 12) == 1, string.count("ñññ", 0, 12) + +[case testIsInstance] +from copysubclass import subc +from typing import Any +def test_built_in() -> None: + s: Any = str() + assert isinstance(s, str) + assert isinstance(s + "test", str) + assert isinstance(s + "ñññ", str) + assert isinstance(subc(), str) + assert isinstance(subc("test"), str) + assert isinstance(subc("ñññ"), str) + + assert not isinstance(set(), str) + assert not isinstance((), str) + assert not isinstance(('a','b'), str) + assert not isinstance({'a','b'}, str) + assert not isinstance(int() + 1, str) + assert not isinstance(['a','b'], str) + +def test_user_defined() -> None: + from userdefinedstr import str + + s: Any = "str" + assert isinstance(str(), str) + assert not isinstance(s, str) + +[file copysubclass.py] +from typing import Any +class subc(str): + pass + +[file userdefinedstr.py] +class str: + pass diff --git a/mypyc/test-data/run-tuples.test b/mypyc/test-data/run-tuples.test index fe9a8dff08c63..ea0a1cb8d8529 100644 --- a/mypyc/test-data/run-tuples.test +++ b/mypyc/test-data/run-tuples.test @@ -294,3 +294,35 @@ def test_multiply() -> None: assert (1,) * 3 == res assert 3 * (1,) == res assert multiply((1,), 3) == res + +[case testIsInstance] +from copysubclass import subc +def test_built_in() -> None: + assert isinstance((), tuple) + assert isinstance((1, 2), tuple) + assert isinstance(('a', 'b', 'c'), tuple) + assert isinstance(subc(()), tuple) + assert isinstance(subc((1, 2)), tuple) + assert isinstance(subc(('a', 'b', 'c')), tuple) + + assert not isinstance(set(), tuple) + assert not isinstance({}, tuple) + assert not isinstance([1,2,3], tuple) + assert not isinstance({'a','b'}, tuple) + assert not isinstance(int() + 1, tuple) + assert not isinstance(str() + 'a', tuple) + +def test_user_defined() -> None: + from userdefinedtuple import tuple + + assert isinstance(tuple(), tuple) + assert not isinstance((1, tuple()), tuple) + +[file copysubclass.py] +from typing import Any +class subc(tuple[Any]): + pass + +[file userdefinedtuple.py] +class tuple: + pass From 70d78812b7190ccc6b272d745cc442412ba0bbb3 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 15 Jul 2025 13:31:35 +0100 Subject: [PATCH 085/424] [mypyc] Refactor: make LoadMem not borrow by default (#19445) Borrowing is a dangerous default. It can only be used in very specific circumstances, so it shouldn't be the default. This is also arguably more consistent with other read ops which don't borrow by default. --- mypyc/codegen/emitfunc.py | 2 ++ mypyc/ir/ops.py | 7 +++---- mypyc/ir/pprint.py | 4 +++- mypyc/irbuild/for_helpers.py | 2 +- mypyc/irbuild/ll_builder.py | 7 +++++-- mypyc/lower/list_ops.py | 8 +++----- mypyc/test-data/irbuild-classes.test | 14 +++++++------- mypyc/test-data/irbuild-isinstance.test | 2 +- mypyc/test-data/irbuild-optional.test | 10 +++++----- mypyc/test-data/irbuild-singledispatch.test | 8 ++++---- mypyc/test-data/lowering-int.test | 1 - mypyc/test-data/refcount.test | 2 +- 12 files changed, 35 insertions(+), 32 deletions(-) diff --git a/mypyc/codegen/emitfunc.py b/mypyc/codegen/emitfunc.py index 9012f072f96b8..086be293d5b36 100644 --- a/mypyc/codegen/emitfunc.py +++ b/mypyc/codegen/emitfunc.py @@ -793,6 +793,8 @@ def visit_load_mem(self, op: LoadMem) -> None: # TODO: we shouldn't dereference to type that are pointer type so far type = self.ctype(op.type) self.emit_line(f"{dest} = *({type} *){src};") + if not op.is_borrowed: + self.emit_inc_ref(dest, op.type) def visit_set_mem(self, op: SetMem) -> None: dest = self.reg(op.dest) diff --git a/mypyc/ir/ops.py b/mypyc/ir/ops.py index 4829dd6a903dc..62ac9b8d48e4c 100644 --- a/mypyc/ir/ops.py +++ b/mypyc/ir/ops.py @@ -1579,14 +1579,13 @@ class LoadMem(RegisterOp): error_kind = ERR_NEVER - def __init__(self, type: RType, src: Value, line: int = -1) -> None: + def __init__(self, type: RType, src: Value, line: int = -1, *, borrow: bool = False) -> None: super().__init__(line) self.type = type - # TODO: for now we enforce that the src memory address should be Py_ssize_t - # later we should also support same width unsigned int + # TODO: Support other native integer types assert is_pointer_rprimitive(src.type) self.src = src - self.is_borrowed = True + self.is_borrowed = borrow and type.is_refcounted def sources(self) -> list[Value]: return [self.src] diff --git a/mypyc/ir/pprint.py b/mypyc/ir/pprint.py index 2a239a0b4d9dc..b0de041e1eaef 100644 --- a/mypyc/ir/pprint.py +++ b/mypyc/ir/pprint.py @@ -267,7 +267,9 @@ def visit_float_comparison_op(self, op: FloatComparisonOp) -> str: return self.format("%r = %r %s %r", op, op.lhs, op.op_str[op.op], op.rhs) def visit_load_mem(self, op: LoadMem) -> str: - return self.format("%r = load_mem %r :: %t*", op, op.src, op.type) + return self.format( + "%r = %sload_mem %r :: %t*", op, self.borrow_prefix(op), op.src, op.type + ) def visit_set_mem(self, op: SetMem) -> str: return self.format("set_mem %r, %r :: %t*", op.dest, op.src, op.dest_type) diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index a7ed97ac8eab6..5cf89f579ec48 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -733,7 +733,7 @@ def gen_condition(self) -> None: def except_match() -> Value: addr = builder.add(LoadAddress(pointer_rprimitive, stop_async_iteration_op.src, line)) - return builder.add(LoadMem(stop_async_iteration_op.type, addr)) + return builder.add(LoadMem(stop_async_iteration_op.type, addr, borrow=True)) def try_body() -> None: awaitable = builder.call_c(anext_op, [builder.read(self.iter_target)], line) diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index c3ea0725cfd48..79ad4cc62822e 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -286,6 +286,9 @@ def goto_and_activate(self, block: BasicBlock) -> None: def keep_alive(self, values: list[Value], *, steal: bool = False) -> None: self.add(KeepAlive(values, steal=steal)) + def load_mem(self, ptr: Value, value_type: RType, *, borrow: bool = False) -> Value: + return self.add(LoadMem(value_type, ptr, borrow=borrow)) + def push_error_handler(self, handler: BasicBlock | None) -> None: self.error_handlers.append(handler) @@ -660,7 +663,7 @@ def other() -> Value: def get_type_of_obj(self, obj: Value, line: int) -> Value: ob_type_address = self.add(GetElementPtr(obj, PyObject, "ob_type", line)) - ob_type = self.add(LoadMem(object_rprimitive, ob_type_address)) + ob_type = self.load_mem(ob_type_address, object_rprimitive, borrow=True) self.add(KeepAlive([obj])) return ob_type @@ -2261,7 +2264,7 @@ def builtin_len(self, val: Value, line: int, use_pyssize_t: bool = False) -> Val size_value = self.primitive_op(var_object_size, [val], line) elif is_set_rprimitive(typ) or is_frozenset_rprimitive(typ): elem_address = self.add(GetElementPtr(val, PySetObject, "used")) - size_value = self.add(LoadMem(c_pyssize_t_rprimitive, elem_address)) + size_value = self.load_mem(elem_address, c_pyssize_t_rprimitive) self.add(KeepAlive([val])) elif is_dict_rprimitive(typ): size_value = self.call_c(dict_ssize_t_size_op, [val], line) diff --git a/mypyc/lower/list_ops.py b/mypyc/lower/list_ops.py index 63a1ecca8d114..631008db5db66 100644 --- a/mypyc/lower/list_ops.py +++ b/mypyc/lower/list_ops.py @@ -1,7 +1,7 @@ from __future__ import annotations from mypyc.common import PLATFORM_SIZE -from mypyc.ir.ops import GetElementPtr, IncRef, Integer, IntOp, LoadMem, SetMem, Value +from mypyc.ir.ops import GetElementPtr, Integer, IntOp, SetMem, Value from mypyc.ir.rtypes import ( PyListObject, c_pyssize_t_rprimitive, @@ -42,7 +42,7 @@ def buf_init_item(builder: LowLevelIRBuilder, args: list[Value], line: int) -> V @lower_primitive_op("list_items") def list_items(builder: LowLevelIRBuilder, args: list[Value], line: int) -> Value: ob_item_ptr = builder.add(GetElementPtr(args[0], PyListObject, "ob_item", line)) - return builder.add(LoadMem(pointer_rprimitive, ob_item_ptr, line)) + return builder.load_mem(ob_item_ptr, pointer_rprimitive) def list_item_ptr(builder: LowLevelIRBuilder, obj: Value, index: Value, line: int) -> Value: @@ -68,6 +68,4 @@ def list_item_ptr(builder: LowLevelIRBuilder, obj: Value, index: Value, line: in def list_get_item_unsafe(builder: LowLevelIRBuilder, args: list[Value], line: int) -> Value: index = builder.coerce(args[1], c_pyssize_t_rprimitive, line) item_ptr = list_item_ptr(builder, args[0], index, line) - value = builder.add(LoadMem(object_rprimitive, item_ptr, line)) - builder.add(IncRef(value)) - return value + return builder.load_mem(item_ptr, object_rprimitive) diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index 1543568fccad4..1a2c237cc3c9b 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -363,7 +363,7 @@ def f(x): L0: r0 = __main__.B :: type r1 = get_element_ptr x ob_type :: PyObject - r2 = load_mem r1 :: builtins.object* + r2 = borrow load_mem r1 :: builtins.object* keep_alive x r3 = r2 == r0 if r3 goto L1 else goto L2 :: bool @@ -402,7 +402,7 @@ def f(x): L0: r0 = __main__.A :: type r1 = get_element_ptr x ob_type :: PyObject - r2 = load_mem r1 :: builtins.object* + r2 = borrow load_mem r1 :: builtins.object* keep_alive x r3 = r2 == r0 if r3 goto L1 else goto L2 :: bool @@ -412,7 +412,7 @@ L1: L2: r5 = __main__.B :: type r6 = get_element_ptr x ob_type :: PyObject - r7 = load_mem r6 :: builtins.object* + r7 = borrow load_mem r6 :: builtins.object* keep_alive x r8 = r7 == r5 r4 = r8 @@ -449,7 +449,7 @@ def f(x): L0: r0 = __main__.A :: type r1 = get_element_ptr x ob_type :: PyObject - r2 = load_mem r1 :: builtins.object* + r2 = borrow load_mem r1 :: builtins.object* keep_alive x r3 = r2 == r0 if r3 goto L1 else goto L2 :: bool @@ -459,7 +459,7 @@ L1: L2: r5 = __main__.R :: type r6 = get_element_ptr x ob_type :: PyObject - r7 = load_mem r6 :: builtins.object* + r7 = borrow load_mem r6 :: builtins.object* keep_alive x r8 = r7 == r5 r4 = r8 @@ -500,7 +500,7 @@ def f(x): L0: r0 = __main__.A :: type r1 = get_element_ptr x ob_type :: PyObject - r2 = load_mem r1 :: builtins.object* + r2 = borrow load_mem r1 :: builtins.object* keep_alive x r3 = r2 == r0 if r3 goto L1 else goto L2 :: bool @@ -510,7 +510,7 @@ L1: L2: r5 = __main__.C :: type r6 = get_element_ptr x ob_type :: PyObject - r7 = load_mem r6 :: builtins.object* + r7 = borrow load_mem r6 :: builtins.object* keep_alive x r8 = r7 == r5 r4 = r8 diff --git a/mypyc/test-data/irbuild-isinstance.test b/mypyc/test-data/irbuild-isinstance.test index 30adfe61e3840..0df9448b819f2 100644 --- a/mypyc/test-data/irbuild-isinstance.test +++ b/mypyc/test-data/irbuild-isinstance.test @@ -77,7 +77,7 @@ L0: x = r0 r1 = __main__.C :: type r2 = get_element_ptr x ob_type :: PyObject - r3 = load_mem r2 :: builtins.object* + r3 = borrow load_mem r2 :: builtins.object* keep_alive x r4 = r3 == r1 if r4 goto L1 else goto L2 :: bool diff --git a/mypyc/test-data/irbuild-optional.test b/mypyc/test-data/irbuild-optional.test index b81465d362bab..fbf7cb148b089 100644 --- a/mypyc/test-data/irbuild-optional.test +++ b/mypyc/test-data/irbuild-optional.test @@ -311,7 +311,7 @@ def get(o): L0: r0 = __main__.A :: type r1 = get_element_ptr o ob_type :: PyObject - r2 = load_mem r1 :: builtins.object* + r2 = borrow load_mem r1 :: builtins.object* keep_alive o r3 = r2 == r0 if r3 goto L1 else goto L2 :: bool @@ -390,7 +390,7 @@ def g(o): L0: r0 = __main__.A :: type r1 = get_element_ptr o ob_type :: PyObject - r2 = load_mem r1 :: builtins.object* + r2 = borrow load_mem r1 :: builtins.object* keep_alive o r3 = r2 == r0 if r3 goto L1 else goto L2 :: bool @@ -403,7 +403,7 @@ L1: L2: r8 = __main__.B :: type r9 = get_element_ptr o ob_type :: PyObject - r10 = load_mem r9 :: builtins.object* + r10 = borrow load_mem r9 :: builtins.object* keep_alive o r11 = r10 == r8 if r11 goto L3 else goto L4 :: bool @@ -456,7 +456,7 @@ def f(o): L0: r0 = __main__.A :: type r1 = get_element_ptr o ob_type :: PyObject - r2 = load_mem r1 :: builtins.object* + r2 = borrow load_mem r1 :: builtins.object* keep_alive o r3 = r2 == r0 if r3 goto L1 else goto L2 :: bool @@ -488,7 +488,7 @@ def g(o): L0: r0 = __main__.A :: type r1 = get_element_ptr o ob_type :: PyObject - r2 = load_mem r1 :: builtins.object* + r2 = borrow load_mem r1 :: builtins.object* keep_alive o r3 = r2 == r0 if r3 goto L1 else goto L2 :: bool diff --git a/mypyc/test-data/irbuild-singledispatch.test b/mypyc/test-data/irbuild-singledispatch.test index c95e832cc5dfa..981208cb52ee5 100644 --- a/mypyc/test-data/irbuild-singledispatch.test +++ b/mypyc/test-data/irbuild-singledispatch.test @@ -57,7 +57,7 @@ def f_obj.__call__(__mypyc_self__, arg): r27 :: bool L0: r0 = get_element_ptr arg ob_type :: PyObject - r1 = load_mem r0 :: builtins.object* + r1 = borrow load_mem r0 :: builtins.object* keep_alive arg r2 = __mypyc_self__.dispatch_cache r3 = CPyDict_GetWithNone(r2, r1) @@ -82,7 +82,7 @@ L2: L3: r16 = load_address PyLong_Type r17 = get_element_ptr r6 ob_type :: PyObject - r18 = load_mem r17 :: builtins.object* + r18 = borrow load_mem r17 :: builtins.object* keep_alive r6 r19 = r18 == r16 if r19 goto L4 else goto L7 :: bool @@ -195,7 +195,7 @@ def f_obj.__call__(__mypyc_self__, x): r24 :: None L0: r0 = get_element_ptr x ob_type :: PyObject - r1 = load_mem r0 :: builtins.object* + r1 = borrow load_mem r0 :: builtins.object* keep_alive x r2 = __mypyc_self__.dispatch_cache r3 = CPyDict_GetWithNone(r2, r1) @@ -220,7 +220,7 @@ L2: L3: r16 = load_address PyLong_Type r17 = get_element_ptr r6 ob_type :: PyObject - r18 = load_mem r17 :: builtins.object* + r18 = borrow load_mem r17 :: builtins.object* keep_alive r6 r19 = r18 == r16 if r19 goto L4 else goto L5 :: bool diff --git a/mypyc/test-data/lowering-int.test b/mypyc/test-data/lowering-int.test index b4fe14db59c41..c2bcba54e444d 100644 --- a/mypyc/test-data/lowering-int.test +++ b/mypyc/test-data/lowering-int.test @@ -365,7 +365,6 @@ L2: r6 = r0 * 8 r7 = r5 + r6 r8 = load_mem r7 :: builtins.object* - inc_ref r8 r9 = unbox(int, r8) dec_ref r8 if is_error(r9) goto L6 (error at f:4) else goto L3 diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index a831d9baf86ea..a71c53041cf7e 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -1117,7 +1117,7 @@ L0: r0 = borrow x.a r1 = __main__.D :: type r2 = get_element_ptr r0 ob_type :: PyObject - r3 = load_mem r2 :: builtins.object* + r3 = borrow load_mem r2 :: builtins.object* r4 = r3 == r1 if r4 goto L1 else goto L2 :: bool L1: From 640375293f140f573bda1c7dfc5e1bde851198fd Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Wed, 16 Jul 2025 03:10:02 +0200 Subject: [PATCH 086/424] Prevent a crash when InitVar is redefined with a method in a subclass (#19453) Fixes #19443. This case is too niche (and should be trivially avoidable), so just not crashing should be good enough. The value is indeed redefined, and trying to massage the plugin to move the `X-redefinition` back to `X` in names is not worth the effort IMO. --- mypy/checker.py | 9 ++++++++- test-data/unit/check-dataclasses.test | 16 ++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/mypy/checker.py b/mypy/checker.py index 159569849061c..7579c36a97d05 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -2188,7 +2188,14 @@ def check_method_override_for_base_with_name( else: override_class_or_static = defn.func.is_class or defn.func.is_static typ, _ = self.node_type_from_base(defn.name, defn.info, defn) - assert typ is not None + if typ is None: + # This may only happen if we're checking `x-redefinition` member + # and `x` itself is for some reason gone. Normally the node should + # be reachable from the containing class by its name. + # The redefinition is never removed, use this as a sanity check to verify + # the reasoning above. + assert f"{defn.name}-redefinition" in defn.info.names + return False original_node = base_attr.node # `original_type` can be partial if (e.g.) it is originally an diff --git a/test-data/unit/check-dataclasses.test b/test-data/unit/check-dataclasses.test index 2ead202bd6af1..a6ac30e20c36c 100644 --- a/test-data/unit/check-dataclasses.test +++ b/test-data/unit/check-dataclasses.test @@ -2666,3 +2666,19 @@ class PersonBad(TypedDict): class JobBad: person: PersonBad = field(default_factory=PersonBad) # E: Argument "default_factory" to "field" has incompatible type "type[PersonBad]"; expected "Callable[[], PersonBad]" [builtins fixtures/dict.pyi] + +[case testDataclassInitVarRedefinitionNoCrash] +# https://github.com/python/mypy/issues/19443 +from dataclasses import InitVar, dataclass + +class ClassA: + def value(self) -> int: + return 0 + +@dataclass +class ClassB(ClassA): + value: InitVar[int] + + def value(self) -> int: # E: Name "value" already defined on line 10 + return 0 +[builtins fixtures/dict.pyi] From b546953a9beb145b582f691308ed1878e3599ef5 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Wed, 16 Jul 2025 10:50:29 +0200 Subject: [PATCH 087/424] Sync typeshed (#19446) Source commit: https://github.com/python/typeshed/commit/84e41f2853d7af3d651d620f093031cba849bd1d --- mypy/typeshed/stdlib/_interpreters.pyi | 21 ++- mypy/typeshed/stdlib/_json.pyi | 4 +- mypy/typeshed/stdlib/_markupbase.pyi | 4 +- mypy/typeshed/stdlib/ast.pyi | 38 +++--- mypy/typeshed/stdlib/asyncio/__init__.pyi | 2 +- mypy/typeshed/stdlib/builtins.pyi | 2 +- mypy/typeshed/stdlib/collections/__init__.pyi | 4 + .../compression/{bz2/__init__.pyi => bz2.pyi} | 0 .../{gzip/__init__.pyi => gzip.pyi} | 0 .../{lzma/__init__.pyi => lzma.pyi} | 0 .../{zlib/__init__.pyi => zlib.pyi} | 0 .../stdlib/compression/zstd/_zstdfile.pyi | 6 +- mypy/typeshed/stdlib/ctypes/__init__.pyi | 2 +- mypy/typeshed/stdlib/html/parser.pyi | 2 +- .../stdlib/importlib/metadata/__init__.pyi | 2 + mypy/typeshed/stdlib/string/templatelib.pyi | 2 +- mypy/typeshed/stdlib/sys/__init__.pyi | 4 + mypy/typeshed/stdlib/tarfile.pyi | 121 +++++++++++++++++- mypy/typeshed/stdlib/types.pyi | 4 +- mypy/typeshed/stdlib/typing.pyi | 5 +- mypy/typeshed/stdlib/urllib/error.pyi | 5 + mypy/typeshed/stdlib/weakref.pyi | 4 + test-data/unit/pythoneval.test | 3 +- 23 files changed, 193 insertions(+), 42 deletions(-) rename mypy/typeshed/stdlib/compression/{bz2/__init__.pyi => bz2.pyi} (100%) rename mypy/typeshed/stdlib/compression/{gzip/__init__.pyi => gzip.pyi} (100%) rename mypy/typeshed/stdlib/compression/{lzma/__init__.pyi => lzma.pyi} (100%) rename mypy/typeshed/stdlib/compression/{zlib/__init__.pyi => zlib.pyi} (100%) diff --git a/mypy/typeshed/stdlib/_interpreters.pyi b/mypy/typeshed/stdlib/_interpreters.pyi index caa1115e9d3d6..ad8eccbe33284 100644 --- a/mypy/typeshed/stdlib/_interpreters.pyi +++ b/mypy/typeshed/stdlib/_interpreters.pyi @@ -1,9 +1,10 @@ import types -from collections.abc import Callable, Mapping -from typing import Final, Literal, SupportsIndex +from collections.abc import Callable +from typing import Any, Final, Literal, SupportsIndex from typing_extensions import TypeAlias _Configs: TypeAlias = Literal["default", "isolated", "legacy", "empty", ""] +_SharedDict: TypeAlias = dict[str, Any] # many objects can be shared class InterpreterError(Exception): ... class InterpreterNotFoundError(InterpreterError): ... @@ -22,7 +23,11 @@ def is_running(id: SupportsIndex, *, restrict: bool = False) -> bool: ... def get_config(id: SupportsIndex, *, restrict: bool = False) -> types.SimpleNamespace: ... def whence(id: SupportsIndex) -> int: ... def exec( - id: SupportsIndex, code: str | types.CodeType | Callable[[], object], shared: bool | None = None, *, restrict: bool = False + id: SupportsIndex, + code: str | types.CodeType | Callable[[], object], + shared: _SharedDict | None = None, + *, + restrict: bool = False, ) -> None | types.SimpleNamespace: ... def call( id: SupportsIndex, @@ -33,12 +38,16 @@ def call( restrict: bool = False, ) -> object: ... def run_string( - id: SupportsIndex, script: str | types.CodeType | Callable[[], object], shared: bool | None = None, *, restrict: bool = False + id: SupportsIndex, + script: str | types.CodeType | Callable[[], object], + shared: _SharedDict | None = None, + *, + restrict: bool = False, ) -> None: ... def run_func( - id: SupportsIndex, func: types.CodeType | Callable[[], object], shared: bool | None = None, *, restrict: bool = False + id: SupportsIndex, func: types.CodeType | Callable[[], object], shared: _SharedDict | None = None, *, restrict: bool = False ) -> None: ... -def set___main___attrs(id: SupportsIndex, updates: Mapping[str, object], *, restrict: bool = False) -> None: ... +def set___main___attrs(id: SupportsIndex, updates: _SharedDict, *, restrict: bool = False) -> None: ... def incref(id: SupportsIndex, *, implieslink: bool = False, restrict: bool = False) -> None: ... def decref(id: SupportsIndex, *, restrict: bool = False) -> None: ... def is_shareable(obj: object) -> bool: ... diff --git a/mypy/typeshed/stdlib/_json.pyi b/mypy/typeshed/stdlib/_json.pyi index 5296b8e62a028..cc59146ed982b 100644 --- a/mypy/typeshed/stdlib/_json.pyi +++ b/mypy/typeshed/stdlib/_json.pyi @@ -11,7 +11,7 @@ class make_encoder: @property def key_separator(self) -> str: ... @property - def indent(self) -> int | None: ... + def indent(self) -> str | None: ... @property def markers(self) -> dict[int, Any] | None: ... @property @@ -25,7 +25,7 @@ class make_encoder: markers: dict[int, Any] | None, default: Callable[[Any], Any], encoder: Callable[[str], str], - indent: int | None, + indent: str | None, key_separator: str, item_separator: str, sort_keys: bool, diff --git a/mypy/typeshed/stdlib/_markupbase.pyi b/mypy/typeshed/stdlib/_markupbase.pyi index 62bad25e5cccc..597bd09b700b0 100644 --- a/mypy/typeshed/stdlib/_markupbase.pyi +++ b/mypy/typeshed/stdlib/_markupbase.pyi @@ -5,9 +5,9 @@ class ParserBase: def reset(self) -> None: ... def getpos(self) -> tuple[int, int]: ... def unknown_decl(self, data: str) -> None: ... - def parse_comment(self, i: int, report: int = 1) -> int: ... # undocumented + def parse_comment(self, i: int, report: bool = True) -> int: ... # undocumented def parse_declaration(self, i: int) -> int: ... # undocumented - def parse_marked_section(self, i: int, report: int = 1) -> int: ... # undocumented + def parse_marked_section(self, i: int, report: bool = True) -> int: ... # undocumented def updatepos(self, i: int, j: int) -> int: ... # undocumented if sys.version_info < (3, 10): # Removed from ParserBase: https://bugs.python.org/issue31844 diff --git a/mypy/typeshed/stdlib/ast.pyi b/mypy/typeshed/stdlib/ast.pyi index 613940f5da6ab..fcd6e8b01e743 100644 --- a/mypy/typeshed/stdlib/ast.pyi +++ b/mypy/typeshed/stdlib/ast.pyi @@ -1096,8 +1096,16 @@ class Constant(expr): kind: str | None if sys.version_info < (3, 14): # Aliases for value, for backwards compatibility - s: _ConstantValue - n: _ConstantValue + @deprecated("Will be removed in Python 3.14; use value instead") + @property + def n(self) -> _ConstantValue: ... + @n.setter + def n(self, value: _ConstantValue) -> None: ... + @deprecated("Will be removed in Python 3.14; use value instead") + @property + def s(self) -> _ConstantValue: ... + @s.setter + def s(self, value: _ConstantValue) -> None: ... def __init__(self, value: _ConstantValue, kind: str | None = None, **kwargs: Unpack[_Attributes]) -> None: ... @@ -1495,11 +1503,11 @@ if sys.version_info >= (3, 10): class MatchSingleton(pattern): __match_args__ = ("value",) - value: Literal[True, False] | None - def __init__(self, value: Literal[True, False] | None, **kwargs: Unpack[_Attributes[int]]) -> None: ... + value: bool | None + def __init__(self, value: bool | None, **kwargs: Unpack[_Attributes[int]]) -> None: ... if sys.version_info >= (3, 14): - def __replace__(self, *, value: Literal[True, False] | None = ..., **kwargs: Unpack[_Attributes[int]]) -> Self: ... + def __replace__(self, *, value: bool | None = ..., **kwargs: Unpack[_Attributes[int]]) -> Self: ... class MatchSequence(pattern): __match_args__ = ("patterns",) @@ -1696,25 +1704,23 @@ class _ABC(type): if sys.version_info < (3, 14): @deprecated("Replaced by ast.Constant; removed in Python 3.14") class Num(Constant, metaclass=_ABC): - value: int | float | complex + def __new__(cls, n: complex, **kwargs: Unpack[_Attributes]) -> Constant: ... # type: ignore[misc] # pyright: ignore[reportInconsistentConstructor] @deprecated("Replaced by ast.Constant; removed in Python 3.14") class Str(Constant, metaclass=_ABC): - value: str - # Aliases for value, for backwards compatibility - s: str + def __new__(cls, s: str, **kwargs: Unpack[_Attributes]) -> Constant: ... # type: ignore[misc] # pyright: ignore[reportInconsistentConstructor] @deprecated("Replaced by ast.Constant; removed in Python 3.14") class Bytes(Constant, metaclass=_ABC): - value: bytes - # Aliases for value, for backwards compatibility - s: bytes + def __new__(cls, s: bytes, **kwargs: Unpack[_Attributes]) -> Constant: ... # type: ignore[misc] # pyright: ignore[reportInconsistentConstructor] @deprecated("Replaced by ast.Constant; removed in Python 3.14") - class NameConstant(Constant, metaclass=_ABC): ... + class NameConstant(Constant, metaclass=_ABC): + def __new__(cls, value: _ConstantValue, kind: str | None, **kwargs: Unpack[_Attributes]) -> Constant: ... # type: ignore[misc] # pyright: ignore[reportInconsistentConstructor] @deprecated("Replaced by ast.Constant; removed in Python 3.14") - class Ellipsis(Constant, metaclass=_ABC): ... + class Ellipsis(Constant, metaclass=_ABC): + def __new__(cls, **kwargs: Unpack[_Attributes]) -> Constant: ... # type: ignore[misc] # pyright: ignore[reportInconsistentConstructor] # everything below here is defined in ast.py @@ -1797,7 +1803,7 @@ if sys.version_info >= (3, 13): type_comments: bool = False, feature_version: None | int | tuple[int, int] = None, optimize: Literal[-1, 0, 1, 2] = -1, - ) -> AST: ... + ) -> mod: ... else: @overload @@ -1868,7 +1874,7 @@ else: *, type_comments: bool = False, feature_version: None | int | tuple[int, int] = None, - ) -> AST: ... + ) -> mod: ... def literal_eval(node_or_string: str | AST) -> Any: ... diff --git a/mypy/typeshed/stdlib/asyncio/__init__.pyi b/mypy/typeshed/stdlib/asyncio/__init__.pyi index 68e44a88face6..58739816a67eb 100644 --- a/mypy/typeshed/stdlib/asyncio/__init__.pyi +++ b/mypy/typeshed/stdlib/asyncio/__init__.pyi @@ -1,4 +1,4 @@ -# ruff: noqa: PLR5501 # This condition is so big, it's clearer to keep to platform condition in two blocks +# This condition is so big, it's clearer to keep to platform condition in two blocks # Can't NOQA on a specific line: https://github.com/plinss/flake8-noqa/issues/22 import sys from collections.abc import Awaitable, Coroutine, Generator diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi index 6e983ef9ef29f..b853330b18fba 100644 --- a/mypy/typeshed/stdlib/builtins.pyi +++ b/mypy/typeshed/stdlib/builtins.pyi @@ -609,7 +609,7 @@ class bytes(Sequence[int]): def strip(self, bytes: ReadableBuffer | None = None, /) -> bytes: ... def swapcase(self) -> bytes: ... def title(self) -> bytes: ... - def translate(self, table: ReadableBuffer | None, /, delete: bytes = b"") -> bytes: ... + def translate(self, table: ReadableBuffer | None, /, delete: ReadableBuffer = b"") -> bytes: ... def upper(self) -> bytes: ... def zfill(self, width: SupportsIndex, /) -> bytes: ... @classmethod diff --git a/mypy/typeshed/stdlib/collections/__init__.pyi b/mypy/typeshed/stdlib/collections/__init__.pyi index b9e4f84ec0b63..bc33d91caa1d0 100644 --- a/mypy/typeshed/stdlib/collections/__init__.pyi +++ b/mypy/typeshed/stdlib/collections/__init__.pyi @@ -108,6 +108,8 @@ class UserDict(MutableMapping[_KT, _VT]): @overload def get(self, key: _KT, default: None = None) -> _VT | None: ... @overload + def get(self, key: _KT, default: _VT) -> _VT: ... + @overload def get(self, key: _KT, default: _T) -> _VT | _T: ... class UserList(MutableSequence[_T]): @@ -452,6 +454,8 @@ class ChainMap(MutableMapping[_KT, _VT]): @overload def get(self, key: _KT, default: None = None) -> _VT | None: ... @overload + def get(self, key: _KT, default: _VT) -> _VT: ... + @overload def get(self, key: _KT, default: _T) -> _VT | _T: ... def __missing__(self, key: _KT) -> _VT: ... # undocumented def __bool__(self) -> bool: ... diff --git a/mypy/typeshed/stdlib/compression/bz2/__init__.pyi b/mypy/typeshed/stdlib/compression/bz2.pyi similarity index 100% rename from mypy/typeshed/stdlib/compression/bz2/__init__.pyi rename to mypy/typeshed/stdlib/compression/bz2.pyi diff --git a/mypy/typeshed/stdlib/compression/gzip/__init__.pyi b/mypy/typeshed/stdlib/compression/gzip.pyi similarity index 100% rename from mypy/typeshed/stdlib/compression/gzip/__init__.pyi rename to mypy/typeshed/stdlib/compression/gzip.pyi diff --git a/mypy/typeshed/stdlib/compression/lzma/__init__.pyi b/mypy/typeshed/stdlib/compression/lzma.pyi similarity index 100% rename from mypy/typeshed/stdlib/compression/lzma/__init__.pyi rename to mypy/typeshed/stdlib/compression/lzma.pyi diff --git a/mypy/typeshed/stdlib/compression/zlib/__init__.pyi b/mypy/typeshed/stdlib/compression/zlib.pyi similarity index 100% rename from mypy/typeshed/stdlib/compression/zlib/__init__.pyi rename to mypy/typeshed/stdlib/compression/zlib.pyi diff --git a/mypy/typeshed/stdlib/compression/zstd/_zstdfile.pyi b/mypy/typeshed/stdlib/compression/zstd/_zstdfile.pyi index 045b2d35acfe0..e67b3d992f2f9 100644 --- a/mypy/typeshed/stdlib/compression/zstd/_zstdfile.pyi +++ b/mypy/typeshed/stdlib/compression/zstd/_zstdfile.pyi @@ -3,7 +3,7 @@ from collections.abc import Mapping from compression._common import _streams from compression.zstd import ZstdDict from io import TextIOWrapper, _WrappedBuffer -from typing import Literal, overload, type_check_only +from typing import Literal, Protocol, overload, type_check_only from typing_extensions import TypeAlias from _zstd import ZstdCompressor, _ZstdCompressorFlushBlock, _ZstdCompressorFlushFrame @@ -16,11 +16,11 @@ _ReadTextMode: TypeAlias = Literal["rt"] _WriteTextMode: TypeAlias = Literal["wt", "xt", "at"] @type_check_only -class _FileBinaryRead(_streams._Reader): +class _FileBinaryRead(_streams._Reader, Protocol): def close(self) -> None: ... @type_check_only -class _FileBinaryWrite(SupportsWrite[bytes]): +class _FileBinaryWrite(SupportsWrite[bytes], Protocol): def close(self) -> None: ... class ZstdFile(_streams.BaseStream): diff --git a/mypy/typeshed/stdlib/ctypes/__init__.pyi b/mypy/typeshed/stdlib/ctypes/__init__.pyi index 0b14bd856784c..52288d011e984 100644 --- a/mypy/typeshed/stdlib/ctypes/__init__.pyi +++ b/mypy/typeshed/stdlib/ctypes/__init__.pyi @@ -32,7 +32,7 @@ if sys.platform == "win32": from _ctypes import FormatError as FormatError, get_last_error as get_last_error, set_last_error as set_last_error if sys.version_info >= (3, 14): - from _ctypes import COMError as COMError + from _ctypes import COMError as COMError, CopyComPointer as CopyComPointer if sys.version_info >= (3, 11): from ctypes._endian import BigEndianUnion as BigEndianUnion, LittleEndianUnion as LittleEndianUnion diff --git a/mypy/typeshed/stdlib/html/parser.pyi b/mypy/typeshed/stdlib/html/parser.pyi index d322ade965d94..5d38c9c0d800c 100644 --- a/mypy/typeshed/stdlib/html/parser.pyi +++ b/mypy/typeshed/stdlib/html/parser.pyi @@ -21,7 +21,7 @@ class HTMLParser(ParserBase): def check_for_whole_start_tag(self, i: int) -> int: ... # undocumented def clear_cdata_mode(self) -> None: ... # undocumented def goahead(self, end: bool) -> None: ... # undocumented - def parse_bogus_comment(self, i: int, report: bool = ...) -> int: ... # undocumented + def parse_bogus_comment(self, i: int, report: bool = True) -> int: ... # undocumented def parse_endtag(self, i: int) -> int: ... # undocumented def parse_html_declaration(self, i: int) -> int: ... # undocumented def parse_pi(self, i: int) -> int: ... # undocumented diff --git a/mypy/typeshed/stdlib/importlib/metadata/__init__.pyi b/mypy/typeshed/stdlib/importlib/metadata/__init__.pyi index 15d8b50b09d21..789878382ceb8 100644 --- a/mypy/typeshed/stdlib/importlib/metadata/__init__.pyi +++ b/mypy/typeshed/stdlib/importlib/metadata/__init__.pyi @@ -140,6 +140,8 @@ if sys.version_info >= (3, 10) and sys.version_info < (3, 12): @overload def get(self, name: _KT, default: None = None) -> _VT | None: ... @overload + def get(self, name: _KT, default: _VT) -> _VT: ... + @overload def get(self, name: _KT, default: _T) -> _VT | _T: ... def __iter__(self) -> Iterator[_KT]: ... def __contains__(self, *args: object) -> bool: ... diff --git a/mypy/typeshed/stdlib/string/templatelib.pyi b/mypy/typeshed/stdlib/string/templatelib.pyi index 324447f5f34ce..3f460006a7965 100644 --- a/mypy/typeshed/stdlib/string/templatelib.pyi +++ b/mypy/typeshed/stdlib/string/templatelib.pyi @@ -26,6 +26,6 @@ class Interpolation: __match_args__ = ("value", "expression", "conversion", "format_spec") def __new__( - cls, value: Any, expression: str, conversion: Literal["a", "r", "s"] | None = None, format_spec: str = "" + cls, value: Any, expression: str = "", conversion: Literal["a", "r", "s"] | None = None, format_spec: str = "" ) -> Interpolation: ... def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... diff --git a/mypy/typeshed/stdlib/sys/__init__.pyi b/mypy/typeshed/stdlib/sys/__init__.pyi index 054fe91b17c6f..0ca30396a8785 100644 --- a/mypy/typeshed/stdlib/sys/__init__.pyi +++ b/mypy/typeshed/stdlib/sys/__init__.pyi @@ -345,6 +345,10 @@ else: def _current_frames() -> dict[int, FrameType]: ... def _getframe(depth: int = 0, /) -> FrameType: ... + +if sys.version_info >= (3, 12): + def _getframemodulename(depth: int = 0) -> str | None: ... + def _debugmallocstats() -> None: ... def __displayhook__(object: object, /) -> None: ... def __excepthook__(exctype: type[BaseException], value: BaseException, traceback: TracebackType | None, /) -> None: ... diff --git a/mypy/typeshed/stdlib/tarfile.pyi b/mypy/typeshed/stdlib/tarfile.pyi index a18ef0b823f9b..dba250f2d3533 100644 --- a/mypy/typeshed/stdlib/tarfile.pyi +++ b/mypy/typeshed/stdlib/tarfile.pyi @@ -9,6 +9,9 @@ from types import TracebackType from typing import IO, ClassVar, Literal, Protocol, overload from typing_extensions import Self, TypeAlias, deprecated +if sys.version_info >= (3, 14): + from compression.zstd import ZstdDict + __all__ = [ "TarFile", "TarInfo", @@ -186,6 +189,30 @@ class TarFile: debug: int | None = ..., errorlevel: int | None = ..., ) -> Self: ... + if sys.version_info >= (3, 14): + @overload + @classmethod + def open( + cls, + name: StrOrBytesPath | None, + mode: Literal["r:zst"], + fileobj: _Fileobj | None = None, + bufsize: int = 10240, + *, + format: int | None = ..., + tarinfo: type[TarInfo] | None = ..., + dereference: bool | None = ..., + ignore_zeros: bool | None = ..., + encoding: str | None = ..., + errors: str = ..., + pax_headers: Mapping[str, str] | None = ..., + debug: int | None = ..., + errorlevel: int | None = ..., + level: None = None, + options: Mapping[int, int] | None = None, + zstd_dict: ZstdDict | None = None, + ) -> Self: ... + @overload @classmethod def open( @@ -304,12 +331,56 @@ class TarFile: errorlevel: int | None = ..., preset: Literal[0, 1, 2, 3, 4, 5, 6, 7, 8, 9] | None = ..., ) -> Self: ... + if sys.version_info >= (3, 14): + @overload + @classmethod + def open( + cls, + name: StrOrBytesPath | None, + mode: Literal["x:zst", "w:zst"], + fileobj: _Fileobj | None = None, + bufsize: int = 10240, + *, + format: int | None = ..., + tarinfo: type[TarInfo] | None = ..., + dereference: bool | None = ..., + ignore_zeros: bool | None = ..., + encoding: str | None = ..., + errors: str = ..., + pax_headers: Mapping[str, str] | None = ..., + debug: int | None = ..., + errorlevel: int | None = ..., + options: Mapping[int, int] | None = None, + zstd_dict: ZstdDict | None = None, + ) -> Self: ... + @overload + @classmethod + def open( + cls, + name: StrOrBytesPath | None = None, + *, + mode: Literal["x:zst", "w:zst"], + fileobj: _Fileobj | None = None, + bufsize: int = 10240, + format: int | None = ..., + tarinfo: type[TarInfo] | None = ..., + dereference: bool | None = ..., + ignore_zeros: bool | None = ..., + encoding: str | None = ..., + errors: str = ..., + pax_headers: Mapping[str, str] | None = ..., + debug: int | None = ..., + errorlevel: int | None = ..., + options: Mapping[int, int] | None = None, + zstd_dict: ZstdDict | None = None, + ) -> Self: ... + @overload @classmethod def open( cls, name: StrOrBytesPath | ReadableBuffer | None, - mode: Literal["r|*", "r|", "r|gz", "r|bz2", "r|xz"], + mode: Literal["r|*", "r|", "r|gz", "r|bz2", "r|xz", "r|zst"], fileobj: _Fileobj | None = None, bufsize: int = 10240, *, @@ -329,7 +400,7 @@ class TarFile: cls, name: StrOrBytesPath | ReadableBuffer | None = None, *, - mode: Literal["r|*", "r|", "r|gz", "r|bz2", "r|xz"], + mode: Literal["r|*", "r|", "r|gz", "r|bz2", "r|xz", "r|zst"], fileobj: _Fileobj | None = None, bufsize: int = 10240, format: int | None = ..., @@ -347,7 +418,7 @@ class TarFile: def open( cls, name: StrOrBytesPath | WriteableBuffer | None, - mode: Literal["w|", "w|xz"], + mode: Literal["w|", "w|xz", "w|zst"], fileobj: _Fileobj | None = None, bufsize: int = 10240, *, @@ -367,7 +438,7 @@ class TarFile: cls, name: StrOrBytesPath | WriteableBuffer | None = None, *, - mode: Literal["w|", "w|xz"], + mode: Literal["w|", "w|xz", "w|zst"], fileobj: _Fileobj | None = None, bufsize: int = 10240, format: int | None = ..., @@ -526,6 +597,48 @@ class TarFile: debug: int | None = ..., errorlevel: int | None = ..., ) -> Self: ... + if sys.version_info >= (3, 14): + @overload + @classmethod + def zstopen( + cls, + name: StrOrBytesPath | None, + mode: Literal["r"] = "r", + fileobj: IO[bytes] | None = None, + level: None = None, + options: Mapping[int, int] | None = None, + zstd_dict: ZstdDict | None = None, + *, + format: int | None = ..., + tarinfo: type[TarInfo] | None = ..., + dereference: bool | None = ..., + ignore_zeros: bool | None = ..., + encoding: str | None = ..., + pax_headers: Mapping[str, str] | None = ..., + debug: int | None = ..., + errorlevel: int | None = ..., + ) -> Self: ... + @overload + @classmethod + def zstopen( + cls, + name: StrOrBytesPath | None, + mode: Literal["w", "x"], + fileobj: IO[bytes] | None = None, + level: int | None = None, + options: Mapping[int, int] | None = None, + zstd_dict: ZstdDict | None = None, + *, + format: int | None = ..., + tarinfo: type[TarInfo] | None = ..., + dereference: bool | None = ..., + ignore_zeros: bool | None = ..., + encoding: str | None = ..., + pax_headers: Mapping[str, str] | None = ..., + debug: int | None = ..., + errorlevel: int | None = ..., + ) -> Self: ... + def getmember(self, name: str) -> TarInfo: ... def getmembers(self) -> _list[TarInfo]: ... def getnames(self) -> _list[str]: ... diff --git a/mypy/typeshed/stdlib/types.pyi b/mypy/typeshed/stdlib/types.pyi index 582cb653422f5..44bd3eeb3f533 100644 --- a/mypy/typeshed/stdlib/types.pyi +++ b/mypy/typeshed/stdlib/types.pyi @@ -323,7 +323,9 @@ class MappingProxyType(Mapping[_KT, _VT_co]): @overload def get(self, key: _KT, /) -> _VT_co | None: ... @overload - def get(self, key: _KT, default: _VT_co | _T2, /) -> _VT_co | _T2: ... + def get(self, key: _KT, default: _VT_co, /) -> _VT_co: ... # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] # Covariant type as parameter + @overload + def get(self, key: _KT, default: _T2, /) -> _VT_co | _T2: ... def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... def __reversed__(self) -> Iterator[_KT]: ... def __or__(self, value: Mapping[_T1, _T2], /) -> dict[_KT | _T1, _VT_co | _T2]: ... diff --git a/mypy/typeshed/stdlib/typing.pyi b/mypy/typeshed/stdlib/typing.pyi index 79ab9eee924f2..d296c8d921498 100644 --- a/mypy/typeshed/stdlib/typing.pyi +++ b/mypy/typeshed/stdlib/typing.pyi @@ -1,5 +1,4 @@ # Since this module defines "overload" it is not recognized by Ruff as typing.overload -# ruff: noqa: F811 # TODO: The collections import is required, otherwise mypy crashes. # https://github.com/python/mypy/issues/16744 import collections # noqa: F401 # pyright: ignore[reportUnusedImport] @@ -746,7 +745,9 @@ class Mapping(Collection[_KT], Generic[_KT, _VT_co]): @overload def get(self, key: _KT, /) -> _VT_co | None: ... @overload - def get(self, key: _KT, /, default: _VT_co | _T) -> _VT_co | _T: ... + def get(self, key: _KT, /, default: _VT_co) -> _VT_co: ... # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] # Covariant type as parameter + @overload + def get(self, key: _KT, /, default: _T) -> _VT_co | _T: ... def items(self) -> ItemsView[_KT, _VT_co]: ... def keys(self) -> KeysView[_KT]: ... def values(self) -> ValuesView[_VT_co]: ... diff --git a/mypy/typeshed/stdlib/urllib/error.pyi b/mypy/typeshed/stdlib/urllib/error.pyi index 89cec9bf289c4..2173d7e6efaa5 100644 --- a/mypy/typeshed/stdlib/urllib/error.pyi +++ b/mypy/typeshed/stdlib/urllib/error.pyi @@ -6,6 +6,8 @@ __all__ = ["URLError", "HTTPError", "ContentTooShortError"] class URLError(OSError): reason: str | BaseException + # The `filename` attribute only exists if it was provided to `__init__` and wasn't `None`. + filename: str def __init__(self, reason: str | BaseException, filename: str | None = None) -> None: ... class HTTPError(URLError, addinfourl): @@ -16,6 +18,9 @@ class HTTPError(URLError, addinfourl): @property def reason(self) -> str: ... # type: ignore[override] code: int + msg: str + hdrs: Message + fp: IO[bytes] def __init__(self, url: str, code: int, msg: str, hdrs: Message, fp: IO[bytes] | None) -> None: ... class ContentTooShortError(URLError): diff --git a/mypy/typeshed/stdlib/weakref.pyi b/mypy/typeshed/stdlib/weakref.pyi index 593eb4615c8f4..334fab7e7468c 100644 --- a/mypy/typeshed/stdlib/weakref.pyi +++ b/mypy/typeshed/stdlib/weakref.pyi @@ -99,6 +99,8 @@ class WeakValueDictionary(MutableMapping[_KT, _VT]): @overload def get(self, key: _KT, default: None = None) -> _VT | None: ... @overload + def get(self, key: _KT, default: _VT) -> _VT: ... + @overload def get(self, key: _KT, default: _T) -> _VT | _T: ... # These are incompatible with Mapping def keys(self) -> Iterator[_KT]: ... # type: ignore[override] @@ -149,6 +151,8 @@ class WeakKeyDictionary(MutableMapping[_KT, _VT]): @overload def get(self, key: _KT, default: None = None) -> _VT | None: ... @overload + def get(self, key: _KT, default: _VT) -> _VT: ... + @overload def get(self, key: _KT, default: _T) -> _VT | _T: ... # These are incompatible with Mapping def keys(self) -> Iterator[_KT]: ... # type: ignore[override] diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index 3cd509d442904..72c00a3b9b1c6 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -1050,7 +1050,8 @@ _testTypedDictGet.py:8: note: Revealed type is "builtins.object" _testTypedDictGet.py:9: error: All overload variants of "get" of "Mapping" require at least one argument _testTypedDictGet.py:9: note: Possible overload variants: _testTypedDictGet.py:9: note: def get(self, str, /) -> object -_testTypedDictGet.py:9: note: def [_T] get(self, str, /, default: object) -> object +_testTypedDictGet.py:9: note: def get(self, str, /, default: object) -> object +_testTypedDictGet.py:9: note: def [_T] get(self, str, /, default: _T) -> object _testTypedDictGet.py:11: note: Revealed type is "builtins.object" [case testTypedDictMappingMethods] From eb07c060cd334ea2e6d9049c214153c5236d741e Mon Sep 17 00:00:00 2001 From: Piotr Sawicki Date: Wed, 16 Jul 2025 11:14:49 +0200 Subject: [PATCH 088/424] [mypyc] Report error when registering a nested function (#19450) Fixes https://github.com/mypyc/mypyc/issues/1118 Added a check for nested `@singledispatch` functions and nested functions registered to `@singledispatch` functions to report an error in mypyc. Currently either of those cases causes mypyc to crash because of the different handling of nested and top-level functions. Changed to abort the compilation early when these errors are found so that in the transform code we can assume that the singledispatch functions are valid. This means that mypyc might not report as many errors until it quits as before, so I have split the error output test in `commandline.test` into two. --- mypyc/irbuild/main.py | 2 + mypyc/irbuild/prepare.py | 16 ++++ mypyc/test-data/commandline.test | 99 +++++++++++++-------- mypyc/test-data/irbuild-singledispatch.test | 55 ++++++++++++ 4 files changed, 133 insertions(+), 39 deletions(-) diff --git a/mypyc/irbuild/main.py b/mypyc/irbuild/main.py index 894e8f277723d..d2c8924a7298e 100644 --- a/mypyc/irbuild/main.py +++ b/mypyc/irbuild/main.py @@ -71,6 +71,8 @@ def build_ir( singledispatch_info = find_singledispatch_register_impls(modules, errors) result: ModuleIRs = {} + if errors.num_errors > 0: + return result # Generate IR for all modules. class_irs = [] diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index 4eff90f90b7d9..1d6117ab7b1ed 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -616,6 +616,12 @@ def __init__(self, errors: Errors) -> None: self.decorators_to_remove: dict[FuncDef, list[int]] = {} self.errors: Errors = errors + self.func_stack_depth = 0 + + def visit_func_def(self, o: FuncDef) -> None: + self.func_stack_depth += 1 + super().visit_func_def(o) + self.func_stack_depth -= 1 def visit_decorator(self, dec: Decorator) -> None: if dec.decorators: @@ -627,6 +633,10 @@ def visit_decorator(self, dec: Decorator) -> None: for i, d in enumerate(decorators_to_store): impl = get_singledispatch_register_call_info(d, dec.func) if impl is not None: + if self.func_stack_depth > 0: + self.errors.error( + "Registering nested functions not supported", self.current_path, d.line + ) self.singledispatch_impls[impl.singledispatch_func].append( (impl.dispatch_type, dec.func) ) @@ -643,6 +653,12 @@ def visit_decorator(self, dec: Decorator) -> None: ) else: if refers_to_fullname(d, "functools.singledispatch"): + if self.func_stack_depth > 0: + self.errors.error( + "Nested singledispatch functions not supported", + self.current_path, + d.line, + ) decorators_to_remove.append(i) # make sure that we still treat the function as a singledispatch function # even if we don't find any registered implementations (which might happen diff --git a/mypyc/test-data/commandline.test b/mypyc/test-data/commandline.test index 77c2e08bcf344..392ad3620790b 100644 --- a/mypyc/test-data/commandline.test +++ b/mypyc/test-data/commandline.test @@ -101,13 +101,71 @@ assert a.f(10) == 100 def f(x: int) -> int: return x*x -[case testErrorOutput] +[case testErrorOutput1] +# cmd: test.py + +[file test.py] +from functools import singledispatch +from mypy_extensions import trait +from typing import Any + +def decorator(x: Any) -> Any: + return x + +class NeverMetaclass(type): # E: Inheriting from most builtin types is unimplemented \ + # N: Potential workaround: @mypy_extensions.mypyc_attr(native_class=False) \ + # N: https://mypyc.readthedocs.io/en/stable/native_classes.html#defining-non-native-classes + pass + +class Concrete1: + pass + +@trait +class Trait1: + pass + +class Concrete2: + pass + +@decorator +class NonExt(Concrete1): # E: Non-extension classes may not inherit from extension classes + pass + +class NopeMultipleInheritanceAndBadOrder3(Trait1, Concrete1, Concrete2): # E: Non-trait base must appear first in parent list + pass + +class NopeBadOrder(Trait1, Concrete2): # E: Non-trait base must appear first in parent list + pass + +class Foo: + pass + +@singledispatch +def a(arg) -> None: + pass + +@decorator # E: Calling decorator after registering function not supported +@a.register +def g(arg: int) -> None: + pass + +@a.register +@decorator +def h(arg: str) -> None: + pass + +@decorator +@decorator # E: Calling decorator after registering function not supported +@a.register +def i(arg: Foo) -> None: + pass + +[case testErrorOutput2] # cmd: test.py [file test.py] from typing import Final, List, Any, AsyncIterable from mypy_extensions import trait, mypyc_attr -from functools import singledispatch def busted(b: bool) -> None: for i in range(1, 10, 0): # E: range() step can't be zero @@ -138,11 +196,6 @@ Foo.lol = 50 # E: Only class variables defined as ClassVar can be assigned to def decorator(x: Any) -> Any: return x -class NeverMetaclass(type): # E: Inheriting from most builtin types is unimplemented \ - # N: Potential workaround: @mypy_extensions.mypyc_attr(native_class=False) \ - # N: https://mypyc.readthedocs.io/en/stable/native_classes.html#defining-non-native-classes - pass - class Concrete1: pass @@ -161,11 +214,6 @@ class Concrete2: class Trait2(Concrete2): pass -@decorator -class NonExt(Concrete1): # E: Non-extension classes may not inherit from extension classes - pass - - class NopeMultipleInheritance(Concrete1, Concrete2): # E: Multiple inheritance is not supported (except for traits) pass @@ -175,13 +223,6 @@ class NopeMultipleInheritanceAndBadOrder(Concrete1, Trait1, Concrete2): # E: Mu class NopeMultipleInheritanceAndBadOrder2(Concrete1, Concrete2, Trait1): # E: Multiple inheritance is not supported (except for traits) pass -class NopeMultipleInheritanceAndBadOrder3(Trait1, Concrete1, Concrete2): # E: Non-trait base must appear first in parent list # E: Multiple inheritance is not supported (except for traits) - pass - -class NopeBadOrder(Trait1, Concrete2): # E: Non-trait base must appear first in parent list - pass - - @decorator class NonExt2: @property # E: Property setters not supported in non-extension classes @@ -219,26 +260,6 @@ class AllowInterp2(PureTrait): # E: Base class "test.PureTrait" does not allow async def async_generators() -> AsyncIterable[int]: yield 1 # E: async generators are unimplemented -@singledispatch -def a(arg) -> None: - pass - -@decorator # E: Calling decorator after registering function not supported -@a.register -def g(arg: int) -> None: - pass - -@a.register -@decorator -def h(arg: str) -> None: - pass - -@decorator -@decorator # E: Calling decorator after registering function not supported -@a.register -def i(arg: Foo) -> None: - pass - [case testOnlyWarningOutput] # cmd: test.py diff --git a/mypyc/test-data/irbuild-singledispatch.test b/mypyc/test-data/irbuild-singledispatch.test index 981208cb52ee5..ef11ae04dc648 100644 --- a/mypyc/test-data/irbuild-singledispatch.test +++ b/mypyc/test-data/irbuild-singledispatch.test @@ -274,3 +274,58 @@ L0: r1 = f(r0) r2 = box(None, 1) return r2 + +[case registerNestedFunctionError] +from functools import singledispatch +from typing import Any, overload + +def dec(x: Any) -> Any: + return x + +def f() -> None: + @singledispatch # E: Nested singledispatch functions not supported + def singledispatch_in_func(x: Any) -> None: + pass + +@dec +def g() -> None: + @singledispatch # E: Nested singledispatch functions not supported + def singledispatch_in_decorated(x: Any) -> None: + pass + +@overload +def h(x: int) -> None: + pass +@overload +def h(x: str) -> None: + pass +def h(x: Any) -> None: + @singledispatch # E: Nested singledispatch functions not supported + def singledispatch_in_overload(x: Any) -> None: + pass + +@singledispatch +def outside(x: Any) -> None: + pass + +def i() -> None: + @outside.register # E: Registering nested functions not supported + def register_in_func(x: int) -> None: + pass + +@dec +def j() -> None: + @outside.register # E: Registering nested functions not supported + def register_in_decorated(x: int) -> None: + pass + +@overload +def k(x: int) -> None: + pass +@overload +def k(x: str) -> None: + pass +def k(x: Any) -> None: + @outside.register # E: Registering nested functions not supported + def register_in_overload(x: int) -> None: + pass From 02a472a0dbb7d21810ef2c56626eba2113e5a049 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 16 Jul 2025 13:43:40 +0100 Subject: [PATCH 089/424] [mypyc] Optionally log a sampled operation trace to a file (#19457) Logging executed ops is useful for performance analysis. For example, we can look for functions which perform many slow operations and try to optimize them. I've already used this successfully to implement several optimizations. A typical optimization that this helps with is replacing a generic Python function call operation with a native call. This has also helped me identify inefficient code generated by mypyc. Compile using `MYPYC_LOG_TRACE=1 mypyc ...` to enable trace logging. The log will be written to `mypyc_trace.txt`. Roughly 1/1000 of ops of certain kinds (e.g. primitive calls) are logged. This can also be enabled by passing `log_trace=True` to `mypycify`. Compared to profiling, this logging data is frequency-oriented instead of CPU time oriented, and it's mostly helpful for micro-optimizations. It also needs some understanding of mypyc internals to be useful. It's not generally possible to reconstruct call stacks from the event data (but we could improve this). However, there is very little noise in the data and even small improvements can be visible. Logging isn't impacted by C compiler optimizations, so for a faster iteration loop, optimizations can be disabled. In the future this could possibly be used for profile-guided optimizations, but we'd probably need to adjust the data collection a bit for this use case. This is currently not documented and mostly intended for mypy or mypyc maintainers for now. Also no tests yet, since this is not a user-evel feature and it's disabled by default. Random example of log entries from mypy self check: ``` mypy.typeops.TypeVarExtractor._merge:1146:call_c:CPyList_Extend mypy.semanal.SemanticAnalyzer.lookup::primitive_op:list_get_item_unsafe mypy.expandtype.ExpandTypeVisitor.visit_type_var__TypeVisitor_glue:239:call:mypy.expandtype.ExpandTypeVisitor.visit_type_var mypy.applytype.apply_generic_arguments:111:call_c:CPy_NoErrOccurred mypy.indirection.TypeIndirectionVisitor.visit_callable_type__TypeVisitor_glue:118:call:mypy.indirection.TypeIndirectionVisitor.visit_callable_type mypy.fastparse.ASTConverter.visit_Call::primitive_op:buf_init_item mypy.semanal.SemanticAnalyzer.is_func_scope::primitive_op:int_eq mypy.meet.is_overlapping_types:397:call:mypy.meet._is_subtype_is_overlapping_types_obj mypy.types.CallableType.serialize:2287:call_c:CPyList_SetItemUnsafe mypy.checkexpr.ExpressionChecker.check_argument_types:2576:call_c:CPyList_SetItemUnsafe ``` For example, let's look at this line: ``` mypy.typeops.TypeVarExtractor._merge:1146:call_c:CPyList_Extend ``` In method `TypeVarExtractor._merge`, on line 1146 of `mypy/typeops.py`, the C primitive CPyList_Extend was called (corresponds to `list.extend`). I'll later add some documentation to the wiki or other developer docs and give examples of using and analyzing the data. --- mypyc/__main__.py | 20 +++++++-- mypyc/build.py | 10 +++++ mypyc/codegen/emitmodule.py | 4 ++ mypyc/lib-rt/CPy.h | 1 + mypyc/lib-rt/misc_ops.c | 28 ++++++++++++ mypyc/options.py | 5 +++ mypyc/primitives/misc_ops.py | 11 +++++ mypyc/transform/log_trace.py | 83 ++++++++++++++++++++++++++++++++++++ 8 files changed, 159 insertions(+), 3 deletions(-) create mode 100644 mypyc/transform/log_trace.py diff --git a/mypyc/__main__.py b/mypyc/__main__.py index 653199e0fb558..9b3973710efac 100644 --- a/mypyc/__main__.py +++ b/mypyc/__main__.py @@ -23,8 +23,15 @@ from setuptools import setup from mypyc.build import mypycify -setup(name='mypyc_output', - ext_modules=mypycify({}, opt_level="{}", debug_level="{}", strict_dunder_typing={}), +setup( + name='mypyc_output', + ext_modules=mypycify( + {}, + opt_level="{}", + debug_level="{}", + strict_dunder_typing={}, + log_trace={}, + ), ) """ @@ -39,10 +46,17 @@ def main() -> None: opt_level = os.getenv("MYPYC_OPT_LEVEL", "3") debug_level = os.getenv("MYPYC_DEBUG_LEVEL", "1") strict_dunder_typing = bool(int(os.getenv("MYPYC_STRICT_DUNDER_TYPING", "0"))) + # If enabled, compiled code writes a sampled log of executed ops (or events) to + # mypyc_trace.txt. + log_trace = bool(int(os.getenv("MYPYC_LOG_TRACE", "0"))) setup_file = os.path.join(build_dir, "setup.py") with open(setup_file, "w") as f: - f.write(setup_format.format(sys.argv[1:], opt_level, debug_level, strict_dunder_typing)) + f.write( + setup_format.format( + sys.argv[1:], opt_level, debug_level, strict_dunder_typing, log_trace + ) + ) # We don't use run_setup (like we do in the test suite) because it throws # away the error code from distutils, and we don't care about the slight diff --git a/mypyc/build.py b/mypyc/build.py index ab7ba5393614b..8ddbf4d22a278 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -485,6 +485,7 @@ def mypycify( include_runtime_files: bool | None = None, strict_dunder_typing: bool = False, group_name: str | None = None, + log_trace: bool = False, ) -> list[Extension]: """Main entry point to building using mypyc. @@ -531,6 +532,10 @@ def mypycify( the hash of module names. This is used for the names of the output C files and the shared library. This is only supported if there is a single group. [Experimental] + log_trace: If True, compiled code writes a trace log of events in + mypyc_trace.txt (derived from executed operations). This is + useful for performance analysis, such as analyzing which + primitive ops are used the most and on which lines. """ # Figure out our configuration @@ -543,6 +548,7 @@ def mypycify( include_runtime_files=include_runtime_files, strict_dunder_typing=strict_dunder_typing, group_name=group_name, + log_trace=log_trace, ) # Generate all the actual important C code @@ -583,6 +589,8 @@ def mypycify( # See https://github.com/mypyc/mypyc/issues/956 "-Wno-cpp", ] + if log_trace: + cflags.append("-DMYPYC_LOG_TRACE") elif compiler.compiler_type == "msvc": # msvc doesn't have levels, '/O2' is full and '/Od' is disable if opt_level == "0": @@ -607,6 +615,8 @@ def mypycify( # that we actually get the compilation speed and memory # use wins that multi-file mode is intended for. cflags += ["/GL-", "/wd9025"] # warning about overriding /GL + if log_trace: + cflags.append("/DMYPYC_LOG_TRACE") # If configured to (defaults to yes in multi-file mode), copy the # runtime library in. Otherwise it just gets #included to save on diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 2a6f17cea5e2e..7037409ff40bf 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -65,6 +65,7 @@ from mypyc.transform.copy_propagation import do_copy_propagation from mypyc.transform.exceptions import insert_exception_handling from mypyc.transform.flag_elimination import do_flag_elimination +from mypyc.transform.log_trace import insert_event_trace_logging from mypyc.transform.lower import lower_ir from mypyc.transform.refcount import insert_ref_count_opcodes from mypyc.transform.spill import insert_spills @@ -253,6 +254,9 @@ def compile_scc_to_ir( if fn in env_user_functions: insert_spills(fn, env_user_functions[fn]) + if compiler_options.log_trace: + insert_event_trace_logging(fn, compiler_options) + # Switch to lower abstraction level IR. lower_ir(fn, compiler_options) # Perform optimizations. diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index 698e65155da46..e7a7f9a076265 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -929,6 +929,7 @@ PyObject *CPySingledispatch_RegisterFunction(PyObject *singledispatch_func, PyOb PyObject *CPy_GetAIter(PyObject *obj); PyObject *CPy_GetANext(PyObject *aiter); void CPy_SetTypeAliasTypeComputeFunction(PyObject *alias, PyObject *compute_value); +void CPyTrace_LogEvent(const char *location, const char *line, const char *op, const char *details); #ifdef __cplusplus } diff --git a/mypyc/lib-rt/misc_ops.c b/mypyc/lib-rt/misc_ops.c index d234138b2ff7b..8aa25cc11e023 100644 --- a/mypyc/lib-rt/misc_ops.c +++ b/mypyc/lib-rt/misc_ops.c @@ -1030,6 +1030,34 @@ PyObject *CPy_GetANext(PyObject *aiter) return NULL; } +#ifdef MYPYC_LOG_TRACE + +// This is only compiled in if trace logging is enabled by user + +static int TraceCounter = 0; +static const int TRACE_EVERY_NTH = 1009; // Should be a prime number +#define TRACE_LOG_FILE_NAME "mypyc_trace.txt" +static FILE *TraceLogFile = NULL; + +// Log a tracing event on every Nth call +void CPyTrace_LogEvent(const char *location, const char *line, const char *op, const char *details) { + if (TraceLogFile == NULL) { + if ((TraceLogFile = fopen(TRACE_LOG_FILE_NAME, "w")) == NULL) { + fprintf(stderr, "error: Could not open trace file %s\n", TRACE_LOG_FILE_NAME); + abort(); + } + } + if (TraceCounter == 0) { + fprintf(TraceLogFile, "%s:%s:%s:%s\n", location, line, op, details); + } + TraceCounter++; + if (TraceCounter == TRACE_EVERY_NTH) { + TraceCounter = 0; + } +} + +#endif + #ifdef CPY_3_12_FEATURES // Copied from Python 3.12.3, since this struct is internal to CPython. It defines diff --git a/mypyc/options.py b/mypyc/options.py index 51114926f6b23..50c76d3c0656e 100644 --- a/mypyc/options.py +++ b/mypyc/options.py @@ -16,6 +16,7 @@ def __init__( python_version: tuple[int, int] | None = None, strict_dunder_typing: bool = False, group_name: str | None = None, + log_trace: bool = False, ) -> None: self.strip_asserts = strip_asserts self.multi_file = multi_file @@ -45,3 +46,7 @@ def __init__( # library is generated (with shims). This can be used to make the output # file names more predictable. self.group_name = group_name + # If enabled, write a trace log of events based on executed operations to + # mypyc_trace.txt when compiled module is executed. This is useful for + # performance analysis. + self.log_trace = log_trace diff --git a/mypyc/primitives/misc_ops.py b/mypyc/primitives/misc_ops.py index 114a5f0a9823e..e2a1aea1a8d6a 100644 --- a/mypyc/primitives/misc_ops.py +++ b/mypyc/primitives/misc_ops.py @@ -9,6 +9,7 @@ c_int_rprimitive, c_pointer_rprimitive, c_pyssize_t_rprimitive, + cstring_rprimitive, dict_rprimitive, int_rprimitive, object_pointer_rprimitive, @@ -300,3 +301,13 @@ return_type=void_rtype, error_kind=ERR_NEVER, ) + +# Log an event to a trace log, which is written to a file during execution. +log_trace_event = custom_primitive_op( + name="log_trace_event", + c_function_name="CPyTrace_LogEvent", + # (fullname of function/location, line number, operation name, operation details) + arg_types=[cstring_rprimitive, cstring_rprimitive, cstring_rprimitive, cstring_rprimitive], + return_type=void_rtype, + error_kind=ERR_NEVER, +) diff --git a/mypyc/transform/log_trace.py b/mypyc/transform/log_trace.py new file mode 100644 index 0000000000000..5b20940c66bb6 --- /dev/null +++ b/mypyc/transform/log_trace.py @@ -0,0 +1,83 @@ +"""This optional pass adds logging of various executed operations. + +Some subset of the executed operations are logged to the mypyc_trace.txt file. + +This is useful for performance analysis. For example, it's possible +to identify how frequently various primitive functions are called, +and in which code locations they are called. +""" + +from __future__ import annotations + +from mypyc.ir.func_ir import FuncIR +from mypyc.ir.ops import Call, CallC, CString, LoadLiteral, LoadStatic, Op, PrimitiveOp, Value +from mypyc.irbuild.ll_builder import LowLevelIRBuilder +from mypyc.options import CompilerOptions +from mypyc.primitives.misc_ops import log_trace_event +from mypyc.transform.ir_transform import IRTransform + + +def insert_event_trace_logging(fn: FuncIR, options: CompilerOptions) -> None: + builder = LowLevelIRBuilder(None, options) + transform = LogTraceEventTransform(builder, fn.decl.fullname) + transform.transform_blocks(fn.blocks) + fn.blocks = builder.blocks + + +def get_load_global_name(op: CallC) -> str | None: + name = op.function_name + if name == "CPyDict_GetItem": + arg = op.args[0] + if ( + isinstance(arg, LoadStatic) + and arg.namespace == "static" + and arg.identifier == "globals" + and isinstance(op.args[1], LoadLiteral) + ): + return str(op.args[1].value) + return None + + +class LogTraceEventTransform(IRTransform): + def __init__(self, builder: LowLevelIRBuilder, fullname: str) -> None: + super().__init__(builder) + self.fullname = fullname.encode("utf-8") + + def visit_call(self, op: Call) -> Value: + # TODO: Use different op name when constructing an instance + return self.log(op, "call", op.fn.fullname) + + def visit_primitive_op(self, op: PrimitiveOp) -> Value: + return self.log(op, "primitive_op", op.desc.name) + + def visit_call_c(self, op: CallC) -> Value: + if global_name := get_load_global_name(op): + return self.log(op, "globals_dict_get_item", global_name) + + func_name = op.function_name + if func_name == "PyObject_Vectorcall" and isinstance(op.args[0], CallC): + if global_name := get_load_global_name(op.args[0]): + return self.log(op, "python_call_global", global_name) + elif func_name == "CPyObject_GetAttr" and isinstance(op.args[1], LoadLiteral): + return self.log(op, "python_get_attr", str(op.args[1].value)) + elif func_name == "PyObject_VectorcallMethod" and isinstance(op.args[0], LoadLiteral): + return self.log(op, "python_call_method", str(op.args[0].value)) + + return self.log(op, "call_c", func_name) + + def log(self, op: Op, name: str, details: str) -> Value: + if op.line >= 0: + line_str = str(op.line) + else: + line_str = "" + self.builder.primitive_op( + log_trace_event, + [ + CString(self.fullname), + CString(line_str.encode("ascii")), + CString(name.encode("utf-8")), + CString(details.encode("utf-8")), + ], + op.line, + ) + return self.add(op) From edd491f30aa65d1d8798d774a88bc2f6741360f3 Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Wed, 16 Jul 2025 16:19:38 +0200 Subject: [PATCH 090/424] perf: return from `is_subtype` early (#19400) Our equality implementation is conservative: if two types compare equal, we know for sure they do indeed refer to the same type, and any type is subtype of itself. This saved 0.9% in my local benchmark run. --- misc/perf_compare.py | 2 ++ mypy/subtypes.py | 4 ++++ mypy/types.py | 2 ++ test-data/unit/check-generics.test | 4 ++-- test-data/unit/check-parameter-specification.test | 8 ++++---- 5 files changed, 14 insertions(+), 6 deletions(-) mode change 100644 => 100755 misc/perf_compare.py diff --git a/misc/perf_compare.py b/misc/perf_compare.py old mode 100644 new mode 100755 index 39dd22b313391..7d22f43d1c459 --- a/misc/perf_compare.py +++ b/misc/perf_compare.py @@ -1,3 +1,5 @@ +#! /usr/bin/env python + """Compare performance of mypyc-compiled mypy between one or more commits/branches. Simple usage: diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 428e6dec6749e..1aa8543505ec9 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -145,6 +145,8 @@ def is_subtype( between the type arguments (e.g., A and B), taking the variance of the type var into account. """ + if left == right: + return True if subtype_context is None: subtype_context = SubtypeContext( ignore_type_params=ignore_type_params, @@ -206,6 +208,8 @@ def is_proper_subtype( (this is useful for runtime isinstance() checks). If keep_erased_types is True, do not consider ErasedType a subtype of all types (used by type inference against unions). """ + if left == right: + return True if subtype_context is None: subtype_context = SubtypeContext( ignore_promotions=ignore_promotions, diff --git a/mypy/types.py b/mypy/types.py index 05b02acc68c00..e9d299dbc8fc3 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -2272,6 +2272,8 @@ def __eq__(self, other: object) -> bool: and self.name == other.name and self.is_type_obj() == other.is_type_obj() and self.is_ellipsis_args == other.is_ellipsis_args + and self.type_guard == other.type_guard + and self.type_is == other.type_is and self.fallback == other.fallback ) else: diff --git a/test-data/unit/check-generics.test b/test-data/unit/check-generics.test index 0be9d918c69f2..78680684f69bf 100644 --- a/test-data/unit/check-generics.test +++ b/test-data/unit/check-generics.test @@ -2929,8 +2929,8 @@ def mix(fs: List[Callable[[S], T]]) -> Callable[[S], List[T]]: def id(__x: U) -> U: ... fs = [id, id, id] -reveal_type(mix(fs)) # N: Revealed type is "def [S] (S`11) -> builtins.list[S`11]" -reveal_type(mix([id, id, id])) # N: Revealed type is "def [S] (S`13) -> builtins.list[S`13]" +reveal_type(mix(fs)) # N: Revealed type is "def [S] (S`7) -> builtins.list[S`7]" +reveal_type(mix([id, id, id])) # N: Revealed type is "def [S] (S`9) -> builtins.list[S`9]" [builtins fixtures/list.pyi] [case testInferenceAgainstGenericCurry] diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index e53c45b5b512b..0835ba7ac57d1 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -921,8 +921,8 @@ class A: def func(self, action: Callable[_P, _R], *args: _P.args, **kwargs: _P.kwargs) -> _R: ... -reveal_type(A.func) # N: Revealed type is "def [_P, _R] (self: __main__.A, action: def (*_P.args, **_P.kwargs) -> _R`6, *_P.args, **_P.kwargs) -> _R`6" -reveal_type(A().func) # N: Revealed type is "def [_P, _R] (action: def (*_P.args, **_P.kwargs) -> _R`10, *_P.args, **_P.kwargs) -> _R`10" +reveal_type(A.func) # N: Revealed type is "def [_P, _R] (self: __main__.A, action: def (*_P.args, **_P.kwargs) -> _R`4, *_P.args, **_P.kwargs) -> _R`4" +reveal_type(A().func) # N: Revealed type is "def [_P, _R] (action: def (*_P.args, **_P.kwargs) -> _R`8, *_P.args, **_P.kwargs) -> _R`8" def f(x: int) -> int: ... @@ -953,8 +953,8 @@ class A: def func(self, action: Job[_P, None]) -> Job[_P, None]: ... -reveal_type(A.func) # N: Revealed type is "def [_P] (self: __main__.A, action: __main__.Job[_P`4, None]) -> __main__.Job[_P`4, None]" -reveal_type(A().func) # N: Revealed type is "def [_P] (action: __main__.Job[_P`6, None]) -> __main__.Job[_P`6, None]" +reveal_type(A.func) # N: Revealed type is "def [_P] (self: __main__.A, action: __main__.Job[_P`3, None]) -> __main__.Job[_P`3, None]" +reveal_type(A().func) # N: Revealed type is "def [_P] (action: __main__.Job[_P`5, None]) -> __main__.Job[_P`5, None]" reveal_type(A().func(Job(lambda x: x))) # N: Revealed type is "__main__.Job[[x: Any], None]" def f(x: int, y: int) -> None: ... From 6f7d716c108dc8a8385c3bd7f7015b04c0c374b8 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 16 Jul 2025 18:17:47 +0100 Subject: [PATCH 091/424] Speed up the default plugin (#19462) Use precalculated set objects in more places. This is similar to #19385. Some cases were still unoptimized. I used trace logging (#19457) to identify functions where we were creating many set objects, and I noticed that these were unnecessary. This is a part of a set of micro-optimizations that improve self check performance by ~5.5%. --- mypy/plugins/default.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mypy/plugins/default.py b/mypy/plugins/default.py index 3d27ca99302ff..e492b8dd73351 100644 --- a/mypy/plugins/default.py +++ b/mypy/plugins/default.py @@ -82,6 +82,7 @@ TD_SETDEFAULT_NAMES: Final = {n + ".setdefault" for n in TPDICT_FB_NAMES} TD_POP_NAMES: Final = {n + ".pop" for n in TPDICT_FB_NAMES} +TD_DELITEM_NAMES: Final = {n + ".__delitem__" for n in TPDICT_FB_NAMES} TD_UPDATE_METHOD_NAMES: Final = ( {n + ".update" for n in TPDICT_FB_NAMES} @@ -144,11 +145,11 @@ def get_method_hook(self, fullname: str) -> Callable[[MethodContext], Type] | No return int_pos_callback elif fullname in ("builtins.tuple.__mul__", "builtins.tuple.__rmul__"): return tuple_mul_callback - elif fullname in {n + ".setdefault" for n in TPDICT_FB_NAMES}: + elif fullname in TD_SETDEFAULT_NAMES: return typed_dict_setdefault_callback - elif fullname in {n + ".pop" for n in TPDICT_FB_NAMES}: + elif fullname in TD_POP_NAMES: return typed_dict_pop_callback - elif fullname in {n + ".__delitem__" for n in TPDICT_FB_NAMES}: + elif fullname in TD_DELITEM_NAMES: return typed_dict_delitem_callback elif fullname == "_ctypes.Array.__getitem__": return array_getitem_callback From ca738e5d43be9d8fe6cdafcfe08a3cdf0fd435ea Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 16 Jul 2025 18:18:23 +0100 Subject: [PATCH 092/424] Micro-optimization: Avoid temporary set creation in is_proper_subtype (#19463) This was suggested by @sterliakov in https://github.com/python/mypy/pull/19384#issuecomment-3044364827 This is a part of a set of micro-optimizations that improve self check performance by ~5.5%. --- mypy/subtypes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 1aa8543505ec9..7da258a827f33 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -217,8 +217,8 @@ def is_proper_subtype( keep_erased_types=keep_erased_types, ) else: - assert not any( - {ignore_promotions, erase_instances, keep_erased_types} + assert ( + not ignore_promotions and not erase_instances and not keep_erased_types ), "Don't pass both context and individual flags" if type_state.is_assumed_proper_subtype(left, right): return True From 5f5871dc7646d9a7791c7b0e5a872f63eb24dd41 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 17 Jul 2025 10:10:36 +0100 Subject: [PATCH 093/424] Micro-optimize ExpandTypeVisitor (#19461) Specialize a hot for loop for the concrete `tuple` and `list` types. Also add a fast path for empty type arguments. The approach is similar to what I used in #19459. This is a part of a set of micro-optimizations that improve self check performance by ~5.5%. --- mypy/expandtype.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/mypy/expandtype.py b/mypy/expandtype.py index d27105f48ed3b..f704df3b010e3 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -1,6 +1,6 @@ from __future__ import annotations -from collections.abc import Iterable, Mapping, Sequence +from collections.abc import Iterable, Mapping from typing import Final, TypeVar, cast, overload from mypy.nodes import ARG_STAR, FakeInfo, Var @@ -209,7 +209,11 @@ def visit_erased_type(self, t: ErasedType) -> Type: return t def visit_instance(self, t: Instance) -> Type: - args = self.expand_types_with_unpack(list(t.args)) + if len(t.args) == 0: + # TODO: Why do we need to create a copy here? + return t.copy_modified() + + args = self.expand_type_tuple_with_unpack(t.args) if isinstance(t.type, FakeInfo): # The type checker expands function definitions and bodies @@ -431,7 +435,7 @@ def visit_overloaded(self, t: Overloaded) -> Type: items.append(new_item) return Overloaded(items) - def expand_types_with_unpack(self, typs: Sequence[Type]) -> list[Type]: + def expand_type_list_with_unpack(self, typs: list[Type]) -> list[Type]: """Expands a list of types that has an unpack.""" items: list[Type] = [] for item in typs: @@ -441,8 +445,19 @@ def expand_types_with_unpack(self, typs: Sequence[Type]) -> list[Type]: items.append(item.accept(self)) return items + def expand_type_tuple_with_unpack(self, typs: tuple[Type, ...]) -> list[Type]: + """Expands a tuple of types that has an unpack.""" + # Micro-optimization: Specialized variant of expand_type_list_with_unpack + items: list[Type] = [] + for item in typs: + if isinstance(item, UnpackType) and isinstance(item.type, TypeVarTupleType): + items.extend(self.expand_unpack(item)) + else: + items.append(item.accept(self)) + return items + def visit_tuple_type(self, t: TupleType) -> Type: - items = self.expand_types_with_unpack(t.items) + items = self.expand_type_list_with_unpack(t.items) if len(items) == 1: # Normalize Tuple[*Tuple[X, ...]] -> Tuple[X, ...] item = items[0] @@ -510,7 +525,7 @@ def visit_type_type(self, t: TypeType) -> Type: def visit_type_alias_type(self, t: TypeAliasType) -> Type: # Target of the type alias cannot contain type variables (not bound by the type # alias itself), so we just expand the arguments. - args = self.expand_types_with_unpack(t.args) + args = self.expand_type_list_with_unpack(t.args) # TODO: normalize if target is Tuple, and args are [*tuple[X, ...]]? return t.copy_modified(args=args) From a82948b87609eb422492db6733f18c3bbddc2920 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 17 Jul 2025 10:11:59 +0100 Subject: [PATCH 094/424] Micro-optimize chained plugin (#19464) Avoid using lambdas in the most expensive hooks, since they are slower than direct method calls. Also use `if hook is None` checks instead of `if hook`, since the prior is more efficient when compiled. I used trace logging to look for generic/unoptimized function calls, and it was clear that ChainedPlugin was doing many unoptimized calls that were easy to avoid. This duplicates some code, but I think it's fine since this code is updated very rarely but the code paths are very hot. This is a part of a set of micro-optimizations that improve self check performance by ~5.5%. --- mypy/plugin.py | 37 +++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/mypy/plugin.py b/mypy/plugin.py index 831721eb193c4..9019e3c2256f1 100644 --- a/mypy/plugin.py +++ b/mypy/plugin.py @@ -846,12 +846,22 @@ def get_additional_deps(self, file: MypyFile) -> list[tuple[int, str, int]]: return deps def get_type_analyze_hook(self, fullname: str) -> Callable[[AnalyzeTypeContext], Type] | None: - return self._find_hook(lambda plugin: plugin.get_type_analyze_hook(fullname)) + # Micro-optimization: Inline iteration over plugins + for plugin in self._plugins: + hook = plugin.get_type_analyze_hook(fullname) + if hook is not None: + return hook + return None def get_function_signature_hook( self, fullname: str ) -> Callable[[FunctionSigContext], FunctionLike] | None: - return self._find_hook(lambda plugin: plugin.get_function_signature_hook(fullname)) + # Micro-optimization: Inline iteration over plugins + for plugin in self._plugins: + hook = plugin.get_function_signature_hook(fullname) + if hook is not None: + return hook + return None def get_function_hook(self, fullname: str) -> Callable[[FunctionContext], Type] | None: return self._find_hook(lambda plugin: plugin.get_function_hook(fullname)) @@ -859,13 +869,28 @@ def get_function_hook(self, fullname: str) -> Callable[[FunctionContext], Type] def get_method_signature_hook( self, fullname: str ) -> Callable[[MethodSigContext], FunctionLike] | None: - return self._find_hook(lambda plugin: plugin.get_method_signature_hook(fullname)) + # Micro-optimization: Inline iteration over plugins + for plugin in self._plugins: + hook = plugin.get_method_signature_hook(fullname) + if hook is not None: + return hook + return None def get_method_hook(self, fullname: str) -> Callable[[MethodContext], Type] | None: - return self._find_hook(lambda plugin: plugin.get_method_hook(fullname)) + # Micro-optimization: Inline iteration over plugins + for plugin in self._plugins: + hook = plugin.get_method_hook(fullname) + if hook is not None: + return hook + return None def get_attribute_hook(self, fullname: str) -> Callable[[AttributeContext], Type] | None: - return self._find_hook(lambda plugin: plugin.get_attribute_hook(fullname)) + # Micro-optimization: Inline iteration over plugins + for plugin in self._plugins: + hook = plugin.get_attribute_hook(fullname) + if hook is not None: + return hook + return None def get_class_attribute_hook(self, fullname: str) -> Callable[[AttributeContext], Type] | None: return self._find_hook(lambda plugin: plugin.get_class_attribute_hook(fullname)) @@ -897,6 +922,6 @@ def get_dynamic_class_hook( def _find_hook(self, lookup: Callable[[Plugin], T]) -> T | None: for plugin in self._plugins: hook = lookup(plugin) - if hook: + if hook is not None: return hook return None From 8bd159b755529e067cf67e489b7f345a0b442468 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 17 Jul 2025 10:14:48 +0100 Subject: [PATCH 095/424] Micro-optimize TypeTraverserVisitor (#19459) Specialize for concrete sequence types (`list` and `tuple`) for faster iteration, since the traversal code is very hot. I used trace logging (#19457) to identify functions where `PyObject_GetIter` was called frequently. This is a part of a set of micro-optimizations that improve self check performance by ~5.5%. --- mypy/typetraverser.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/mypy/typetraverser.py b/mypy/typetraverser.py index cc6d4b637d2e2..047c5caf6daea 100644 --- a/mypy/typetraverser.py +++ b/mypy/typetraverser.py @@ -67,7 +67,7 @@ def visit_param_spec(self, t: ParamSpecType, /) -> None: t.default.accept(self) def visit_parameters(self, t: Parameters, /) -> None: - self.traverse_types(t.arg_types) + self.traverse_type_list(t.arg_types) def visit_type_var_tuple(self, t: TypeVarTupleType, /) -> None: t.default.accept(self) @@ -78,11 +78,11 @@ def visit_literal_type(self, t: LiteralType, /) -> None: # Composite types def visit_instance(self, t: Instance, /) -> None: - self.traverse_types(t.args) + self.traverse_type_tuple(t.args) def visit_callable_type(self, t: CallableType, /) -> None: # FIX generics - self.traverse_types(t.arg_types) + self.traverse_type_list(t.arg_types) t.ret_type.accept(self) t.fallback.accept(self) @@ -93,7 +93,7 @@ def visit_callable_type(self, t: CallableType, /) -> None: t.type_is.accept(self) def visit_tuple_type(self, t: TupleType, /) -> None: - self.traverse_types(t.items) + self.traverse_type_list(t.items) t.partial_fallback.accept(self) def visit_typeddict_type(self, t: TypedDictType, /) -> None: @@ -101,7 +101,7 @@ def visit_typeddict_type(self, t: TypedDictType, /) -> None: t.fallback.accept(self) def visit_union_type(self, t: UnionType, /) -> None: - self.traverse_types(t.items) + self.traverse_type_list(t.items) def visit_overloaded(self, t: Overloaded, /) -> None: self.traverse_types(t.items) @@ -115,16 +115,16 @@ def visit_callable_argument(self, t: CallableArgument, /) -> None: t.typ.accept(self) def visit_unbound_type(self, t: UnboundType, /) -> None: - self.traverse_types(t.args) + self.traverse_type_tuple(t.args) def visit_type_list(self, t: TypeList, /) -> None: - self.traverse_types(t.items) + self.traverse_type_list(t.items) def visit_ellipsis_type(self, t: EllipsisType, /) -> None: pass def visit_placeholder_type(self, t: PlaceholderType, /) -> None: - self.traverse_types(t.args) + self.traverse_type_list(t.args) def visit_partial_type(self, t: PartialType, /) -> None: pass @@ -136,7 +136,7 @@ def visit_type_alias_type(self, t: TypeAliasType, /) -> None: # TODO: sometimes we want to traverse target as well # We need to find a way to indicate explicitly the intent, # maybe make this method abstract (like for TypeTranslator)? - self.traverse_types(t.args) + self.traverse_type_list(t.args) def visit_unpack_type(self, t: UnpackType, /) -> None: t.type.accept(self) @@ -146,3 +146,13 @@ def visit_unpack_type(self, t: UnpackType, /) -> None: def traverse_types(self, types: Iterable[Type], /) -> None: for typ in types: typ.accept(self) + + def traverse_type_list(self, types: list[Type], /) -> None: + # Micro-optimization: Specialized for lists + for typ in types: + typ.accept(self) + + def traverse_type_tuple(self, types: tuple[Type, ...], /) -> None: + # Micro-optimization: Specialized for tuples + for typ in types: + typ.accept(self) From 32752ebc58011bc3067dba239e50c91bc6304bd4 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 17 Jul 2025 17:23:53 +0100 Subject: [PATCH 096/424] [mypyc] Add script to compile mypy with trace logging and run mypy (#19475) Trace logging helps when analyzing low-level performance issues in mypy. --- misc/log_trace_check.py | 85 +++++++++++++++++++++++++++++++++++++++++ misc/perf_compare.py | 13 ++++++- setup.py | 2 + 3 files changed, 98 insertions(+), 2 deletions(-) create mode 100644 misc/log_trace_check.py diff --git a/misc/log_trace_check.py b/misc/log_trace_check.py new file mode 100644 index 0000000000000..677c164fe9925 --- /dev/null +++ b/misc/log_trace_check.py @@ -0,0 +1,85 @@ +"""Compile mypy using mypyc with trace logging enabled, and collect a trace. + +The trace log can be used to analyze low-level performance bottlenecks. + +By default does a self check as the workload. + +This works on all supported platforms, unlike some of our other performance tools. +""" + +from __future__ import annotations + +import argparse +import glob +import os +import shutil +import subprocess +import sys +import time + +from perf_compare import build_mypy, clone + +# Generated files, including binaries, go under this directory to avoid overwriting user state. +TARGET_DIR = "mypy.log_trace.tmpdir" + + +def perform_type_check(target_dir: str, code: str | None) -> None: + cache_dir = os.path.join(target_dir, ".mypy_cache") + if os.path.exists(cache_dir): + shutil.rmtree(cache_dir) + args = [] + if code is None: + args.extend(["--config-file", "mypy_self_check.ini"]) + for pat in "mypy/*.py", "mypy/*/*.py", "mypyc/*.py", "mypyc/test/*.py": + args.extend(glob.glob(pat)) + else: + args.extend(["-c", code]) + check_cmd = ["python", "-m", "mypy"] + args + t0 = time.time() + subprocess.run(check_cmd, cwd=target_dir, check=True) + elapsed = time.time() - t0 + print(f"{elapsed:.2f}s elapsed") + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Compile mypy and collect a trace log while type checking (by default, self check)." + ) + parser.add_argument( + "--multi-file", + action="store_true", + help="compile mypy into one C file per module (to reduce RAM use during compilation)", + ) + parser.add_argument( + "--skip-compile", action="store_true", help="use compiled mypy from previous run" + ) + parser.add_argument( + "-c", + metavar="CODE", + default=None, + type=str, + help="type check Python code fragment instead of mypy self-check", + ) + args = parser.parse_args() + multi_file: bool = args.multi_file + skip_compile: bool = args.skip_compile + code: str | None = args.c + + target_dir = TARGET_DIR + + if not skip_compile: + clone(target_dir, "HEAD") + + print(f"Building mypy in {target_dir} with trace logging enabled...") + build_mypy(target_dir, multi_file, log_trace=True, opt_level="0") + elif not os.path.isdir(target_dir): + sys.exit("error: Can't find compile mypy from previous run -- can't use --skip-compile") + + perform_type_check(target_dir, code) + + trace_fnam = os.path.join(target_dir, "mypyc_trace.txt") + print(f"Generated event trace log in {trace_fnam}") + + +if __name__ == "__main__": + main() diff --git a/misc/perf_compare.py b/misc/perf_compare.py index 7d22f43d1c459..aa05270a8c00f 100755 --- a/misc/perf_compare.py +++ b/misc/perf_compare.py @@ -37,13 +37,22 @@ def heading(s: str) -> None: print() -def build_mypy(target_dir: str, multi_file: bool, *, cflags: str | None = None) -> None: +def build_mypy( + target_dir: str, + multi_file: bool, + *, + cflags: str | None = None, + log_trace: bool = False, + opt_level: str = "2", +) -> None: env = os.environ.copy() env["CC"] = "clang" - env["MYPYC_OPT_LEVEL"] = "2" + env["MYPYC_OPT_LEVEL"] = opt_level env["PYTHONHASHSEED"] = "1" if multi_file: env["MYPYC_MULTI_FILE"] = "1" + if log_trace: + env["MYPYC_LOG_TRACE"] = "1" if cflags is not None: env["CFLAGS"] = cflags cmd = [sys.executable, "setup.py", "--use-mypyc", "build_ext", "--inplace"] diff --git a/setup.py b/setup.py index 12cc1aad4d724..e085b0be38464 100644 --- a/setup.py +++ b/setup.py @@ -145,6 +145,7 @@ def run(self) -> None: opt_level = os.getenv("MYPYC_OPT_LEVEL", "3") debug_level = os.getenv("MYPYC_DEBUG_LEVEL", "1") force_multifile = os.getenv("MYPYC_MULTI_FILE", "") == "1" + log_trace = bool(int(os.getenv("MYPYC_LOG_TRACE", "0"))) ext_modules = mypycify( mypyc_targets + ["--config-file=mypy_bootstrap.ini"], opt_level=opt_level, @@ -152,6 +153,7 @@ def run(self) -> None: # Use multi-file compilation mode on windows because without it # our Appveyor builds run out of memory sometimes. multi_file=sys.platform == "win32" or force_multifile, + log_trace=log_trace, ) else: From 23e6072cb969aab81b6b07b096f7e6d07d522a46 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 18 Jul 2025 09:28:05 +0100 Subject: [PATCH 097/424] Micro-optimize type indirection visitor (#19460) Specialize iteration for the concrete types `list` and `tuple`, since this is faster than iterating over an abstract iterable in compiled code. Also avoid constructing many temporary lists. Duplicate some very hot code in the hopes further improving performance slightly, through inlining and possibly also better branch prediction. The approach is similar to what I used in #19459. This is a part of a set of micro-optimizations that improve self check performance by ~5.5%. --- mypy/indirection.py | 42 ++++++++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/mypy/indirection.py b/mypy/indirection.py index 4f455d2c1dc99..06a158818fbed 100644 --- a/mypy/indirection.py +++ b/mypy/indirection.py @@ -32,11 +32,29 @@ def find_modules(self, typs: Iterable[types.Type]) -> set[str]: self.modules = set() self.seen_fullnames = set() self.seen_aliases = set() - self._visit(typs) + for typ in typs: + self._visit(typ) return self.modules - def _visit(self, typ_or_typs: types.Type | Iterable[types.Type]) -> None: - typs = [typ_or_typs] if isinstance(typ_or_typs, types.Type) else typ_or_typs + def _visit(self, typ: types.Type) -> None: + if isinstance(typ, types.TypeAliasType): + # Avoid infinite recursion for recursive type aliases. + if typ not in self.seen_aliases: + self.seen_aliases.add(typ) + typ.accept(self) + + def _visit_type_tuple(self, typs: tuple[types.Type, ...]) -> None: + # Micro-optimization: Specialized version of _visit for lists + for typ in typs: + if isinstance(typ, types.TypeAliasType): + # Avoid infinite recursion for recursive type aliases. + if typ in self.seen_aliases: + continue + self.seen_aliases.add(typ) + typ.accept(self) + + def _visit_type_list(self, typs: list[types.Type]) -> None: + # Micro-optimization: Specialized version of _visit for tuples for typ in typs: if isinstance(typ, types.TypeAliasType): # Avoid infinite recursion for recursive type aliases. @@ -50,7 +68,7 @@ def _visit_module_name(self, module_name: str) -> None: self.modules.update(split_module_names(module_name)) def visit_unbound_type(self, t: types.UnboundType) -> None: - self._visit(t.args) + self._visit_type_tuple(t.args) def visit_any(self, t: types.AnyType) -> None: pass @@ -68,7 +86,7 @@ def visit_deleted_type(self, t: types.DeletedType) -> None: pass def visit_type_var(self, t: types.TypeVarType) -> None: - self._visit(t.values) + self._visit_type_list(t.values) self._visit(t.upper_bound) self._visit(t.default) @@ -84,10 +102,10 @@ def visit_unpack_type(self, t: types.UnpackType) -> None: t.type.accept(self) def visit_parameters(self, t: types.Parameters) -> None: - self._visit(t.arg_types) + self._visit_type_list(t.arg_types) def visit_instance(self, t: types.Instance) -> None: - self._visit(t.args) + self._visit_type_tuple(t.args) if t.type: # Uses of a class depend on everything in the MRO, # as changes to classes in the MRO can add types to methods, @@ -98,7 +116,7 @@ def visit_instance(self, t: types.Instance) -> None: self._visit_module_name(t.type.metaclass_type.type.module_name) def visit_callable_type(self, t: types.CallableType) -> None: - self._visit(t.arg_types) + self._visit_type_list(t.arg_types) self._visit(t.ret_type) if t.definition is not None: fullname = t.definition.fullname @@ -107,22 +125,22 @@ def visit_callable_type(self, t: types.CallableType) -> None: self.seen_fullnames.add(fullname) def visit_overloaded(self, t: types.Overloaded) -> None: - self._visit(t.items) + self._visit_type_list(list(t.items)) self._visit(t.fallback) def visit_tuple_type(self, t: types.TupleType) -> None: - self._visit(t.items) + self._visit_type_list(t.items) self._visit(t.partial_fallback) def visit_typeddict_type(self, t: types.TypedDictType) -> None: - self._visit(t.items.values()) + self._visit_type_list(list(t.items.values())) self._visit(t.fallback) def visit_literal_type(self, t: types.LiteralType) -> None: self._visit(t.fallback) def visit_union_type(self, t: types.UnionType) -> None: - self._visit(t.items) + self._visit_type_list(t.items) def visit_partial_type(self, t: types.PartialType) -> None: pass From abb1cc787455501e43f344c8af2de7d0d262e5ef Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 18 Jul 2025 15:34:19 +0100 Subject: [PATCH 098/424] Uninhabited should have all attributes (#19300) Although this can hide some mypy bugs, TBH this always bothered me as something conceptually wrong. The work on `checkmember` stuff inspired me to actually try it. --- mypy/checkexpr.py | 4 ++++ mypy/checkmember.py | 5 +++++ test-data/unit/check-unreachable-code.test | 16 ++++++++++++++++ 3 files changed, 25 insertions(+) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 8223ccfe4ca08..24f0c8c85d61b 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1671,6 +1671,10 @@ def check_call( object_type, original_type=callee, ) + elif isinstance(callee, UninhabitedType): + ret = UninhabitedType() + ret.ambiguous = callee.ambiguous + return callee, ret else: return self.msg.not_callable(callee, context), AnyType(TypeOfAny.from_error) diff --git a/mypy/checkmember.py b/mypy/checkmember.py index ef38cc3a0dcf0..7ce7e69e21d85 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -69,6 +69,7 @@ TypeVarLikeType, TypeVarTupleType, TypeVarType, + UninhabitedType, UnionType, get_proper_type, ) @@ -268,6 +269,10 @@ def _analyze_member_access( if not mx.suppress_errors: mx.msg.deleted_as_rvalue(typ, mx.context) return AnyType(TypeOfAny.from_error) + elif isinstance(typ, UninhabitedType): + attr_type = UninhabitedType() + attr_type.ambiguous = typ.ambiguous + return attr_type return report_missing_attribute(mx.original_type, typ, name, mx) diff --git a/test-data/unit/check-unreachable-code.test b/test-data/unit/check-unreachable-code.test index 368431127b760..f425410a97748 100644 --- a/test-data/unit/check-unreachable-code.test +++ b/test-data/unit/check-unreachable-code.test @@ -1587,3 +1587,19 @@ x = 0 # not unreachable f2: Callable[[], NoReturn] = lambda: foo() x = 0 # not unreachable + +[case testAttributeNoReturn] +# flags: --warn-unreachable +from typing import Optional, NoReturn, TypeVar + +def foo() -> NoReturn: + raise + +T = TypeVar("T") +def bar(x: Optional[list[T]] = None) -> T: + ... + +reveal_type(bar().attr) # N: Revealed type is "Never" +1 # not unreachable +reveal_type(foo().attr) # N: Revealed type is "Never" +1 # E: Statement is unreachable From a0665e1645ec7ff646d08e78f39113f2a5e4387d Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Fri, 18 Jul 2025 16:41:41 +0200 Subject: [PATCH 099/424] Fix "ignored exception in `hasattr`" in dmypy (#19428) Fixes #19425. That property has no setter so it should safe to exclude. It can raise in inconsistent state that arises when it is not copied last? --- mypy/server/astmerge.py | 15 ++++++++++----- pyproject.toml | 8 ++++++++ 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/mypy/server/astmerge.py b/mypy/server/astmerge.py index 8cd574628bb88..33e2d2b799cbe 100644 --- a/mypy/server/astmerge.py +++ b/mypy/server/astmerge.py @@ -345,13 +345,11 @@ def visit_type_alias(self, node: TypeAlias) -> None: def fixup(self, node: SN) -> SN: if node in self.replacements: new = self.replacements[node] - skip_slots: tuple[str, ...] = () if isinstance(node, TypeInfo) and isinstance(new, TypeInfo): # Special case: special_alias is not exposed in symbol tables, but may appear # in external types (e.g. named tuples), so we need to update it manually. - skip_slots = ("special_alias",) replace_object_state(new.special_alias, node.special_alias) - replace_object_state(new, node, skip_slots=skip_slots) + replace_object_state(new, node, skip_slots=_get_ignored_slots(new)) return cast(SN, new) return node @@ -556,9 +554,16 @@ def replace_nodes_in_symbol_table( if node.node in replacements: new = replacements[node.node] old = node.node - # Needed for TypeInfo, see comment in fixup() above. - replace_object_state(new, old, skip_slots=("special_alias",)) + replace_object_state(new, old, skip_slots=_get_ignored_slots(new)) node.node = new if isinstance(node.node, (Var, TypeAlias)): # Handle them here just in case these aren't exposed through the AST. node.node.accept(NodeReplaceVisitor(replacements)) + + +def _get_ignored_slots(node: SymbolNode) -> tuple[str, ...]: + if isinstance(node, OverloadedFuncDef): + return ("setter",) + if isinstance(node, TypeInfo): + return ("special_alias",) + return () diff --git a/pyproject.toml b/pyproject.toml index 1870e0931407c..032bfcb609e78 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -222,6 +222,14 @@ addopts = "-nauto --strict-markers --strict-config" # treat xpasses as test failures so they get converted to regular tests as soon as possible xfail_strict = true +# Force warnings as errors +filterwarnings = [ + "error", + # Some testcases may contain code that emits SyntaxWarnings, and they are not yet + # handled consistently in 3.14 (PEP 765) + "default::SyntaxWarning", +] + [tool.coverage.run] branch = true source = ["mypy"] From f42c93685521cb276fd59acdeaedc1d1b1148204 Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Mon, 21 Jul 2025 20:30:59 +0200 Subject: [PATCH 100/424] Retain `None` as constraints bottom if no bottoms were provided (#19485) Current version replaces `None` (which indicates "no items") with an empty union (=`Uninhabited`). This breaks inference in some cases. Fixes #19483. --- mypy/solve.py | 6 ++++-- test-data/unit/check-recursive-types.test | 4 ++-- test-data/unit/check-typeddict.test | 18 ++++++++++++++++++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/mypy/solve.py b/mypy/solve.py index 098d926bc7893..fbbcac2520ad0 100644 --- a/mypy/solve.py +++ b/mypy/solve.py @@ -270,6 +270,7 @@ def solve_one(lowers: Iterable[Type], uppers: Iterable[Type]) -> Type | None: uppers = new_uppers # ...unless this is the only information we have, then we just pass it on. + lowers = list(lowers) if not uppers and not lowers: candidate = UninhabitedType() candidate.ambiguous = True @@ -281,10 +282,11 @@ def solve_one(lowers: Iterable[Type], uppers: Iterable[Type]) -> Type | None: # Process each bound separately, and calculate the lower and upper # bounds based on constraints. Note that we assume that the constraint # targets do not have constraint references. - if type_state.infer_unions: + if type_state.infer_unions and lowers: # This deviates from the general mypy semantics because # recursive types are union-heavy in 95% of cases. - bottom = UnionType.make_union(list(lowers)) + # Retain `None` when no bottoms were provided to avoid bogus `Never` inference. + bottom = UnionType.make_union(lowers) else: # The order of lowers is non-deterministic. # We attempt to sort lowers because joins are non-associative. For instance: diff --git a/test-data/unit/check-recursive-types.test b/test-data/unit/check-recursive-types.test index 7ed5ea53c27e0..86e9f02b52636 100644 --- a/test-data/unit/check-recursive-types.test +++ b/test-data/unit/check-recursive-types.test @@ -54,7 +54,7 @@ reveal_type(flatten([1, [2, [3]]])) # N: Revealed type is "builtins.list[builti class Bad: ... x: Nested[int] = [1, [2, [3]]] -x = [1, [Bad()]] # E: List item 1 has incompatible type "list[Bad]"; expected "Union[int, Nested[int]]" +x = [1, [Bad()]] # E: List item 0 has incompatible type "Bad"; expected "Union[int, Nested[int]]" [builtins fixtures/isinstancelist.pyi] [case testRecursiveAliasGenericInferenceNested] @@ -605,7 +605,7 @@ class NT(NamedTuple, Generic[T]): class A: ... class B(A): ... -nti: NT[int] = NT(key=0, value=NT(key=1, value=A())) # E: Argument "value" to "NT" has incompatible type "NT[A]"; expected "Union[int, NT[int]]" +nti: NT[int] = NT(key=0, value=NT(key=1, value=A())) # E: Argument "value" to "NT" has incompatible type "A"; expected "Union[int, NT[int]]" reveal_type(nti) # N: Revealed type is "tuple[builtins.int, Union[builtins.int, ...], fallback=__main__.NT[builtins.int]]" nta: NT[A] diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index a068a63274ca4..be5a6c655d8ca 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -4271,3 +4271,21 @@ reveal_type(dicts.TF) # N: Revealed type is "def (*, user_id: builtins.int =) - reveal_type(dicts.TotalFalse) # N: Revealed type is "def (*, user_id: builtins.int =) -> TypedDict('__main__.Dicts.TF', {'user_id'?: builtins.int})" [builtins fixtures/dict.pyi] [typing fixtures/typing-typeddict.pyi] + +[case testRecursiveNestedTypedDictInference] +from typing import TypedDict, Sequence +from typing_extensions import NotRequired + +class Component(TypedDict): + type: str + components: NotRequired[Sequence['Component']] + +inputs: Sequence[Component] = [{ + 'type': 'tuple', + 'components': [ + {'type': 'uint256'}, + {'type': 'address'}, + ] +}] +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] From d5287070fc60c9a30b305372bd12ce3c1985bf28 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 21 Jul 2025 19:31:50 +0100 Subject: [PATCH 101/424] Fix decorated methods with self-types in protocols (#19484) Fixes https://github.com/python/mypy/issues/19482 The regression is caused by overlap of three problems (directly or indirectly): * Using `mx.original_type` instead of `mx.self_type` for expanding (not otherwise bound) self-types. * Having a weird special case for `...` vs self-types in constraint inference. * Not refreshing type variable ids during protocol subtype checks (hack needed for technical reasons). I fix the first one, and limit the blast radius of the second one. --- mypy/checkmember.py | 2 +- mypy/constraints.py | 5 ++++- test-data/unit/check-protocols.test | 14 ++++++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/mypy/checkmember.py b/mypy/checkmember.py index 7ce7e69e21d85..7eedab2e399a9 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -956,7 +956,7 @@ def expand_and_bind_callable( ) -> Type: if not mx.preserve_type_var_ids: functype = freshen_all_functions_type_vars(functype) - typ = get_proper_type(expand_self_type(var, functype, mx.original_type)) + typ = get_proper_type(expand_self_type(var, functype, mx.self_type)) assert isinstance(typ, FunctionLike) if is_trivial_self: typ = bind_self_fast(typ, mx.self_type) diff --git a/mypy/constraints.py b/mypy/constraints.py index 9eeea3cb2c262..6416791fa74a8 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -1110,7 +1110,10 @@ def visit_callable_type(self, template: CallableType) -> list[Constraint]: # like U -> U, should be Callable[..., Any], but if U is a self-type, we can # allow it to leak, to be later bound to self. A bunch of existing code # depends on this old behaviour. - and not any(tv.id.is_self() for tv in cactual.variables) + and not ( + any(tv.id.is_self() for tv in cactual.variables) + and template.is_ellipsis_args + ) ): # If the actual callable is generic, infer constraints in the opposite # direction, and indicate to the solver there are extra type variables diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index 79207c9aad56d..0f19b404082ef 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -4646,3 +4646,17 @@ reveal_type(t.bar) # N: Revealed type is "def () -> builtins.int" tt: Type[P] = C reveal_type(tt.foo) # N: Revealed type is "def (builtins.object) -> builtins.int" reveal_type(tt.bar) # N: Revealed type is "def (builtins.object) -> builtins.int" + +[case testProtocolDecoratedSelfBound] +from abc import abstractmethod +from typing import Protocol, Self + +class Proto(Protocol): + @abstractmethod + def foo(self, x: Self) -> None: ... + +class Impl: + def foo(self, x: Self) -> None: + pass + +x: Proto = Impl() From 2e5d7eecd7673f9098f54505e68444762855d812 Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Wed, 23 Jul 2025 01:55:40 +0200 Subject: [PATCH 102/424] Use normalized tuples for fallback calculation (#19111) Fixes #19105. I haven't checked this with namedtuples magic, nor am I certain that this is the best way. --- mypy/semanal_shared.py | 3 ++- mypy/semanal_typeargs.py | 6 ++++++ test-data/unit/check-typevar-tuple.test | 27 +++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/mypy/semanal_shared.py b/mypy/semanal_shared.py index bdd01ef6a6f35..e94604b66381a 100644 --- a/mypy/semanal_shared.py +++ b/mypy/semanal_shared.py @@ -46,6 +46,7 @@ TypeVarLikeType, TypeVarTupleType, UnpackType, + flatten_nested_tuples, get_proper_type, ) @@ -290,7 +291,7 @@ def calculate_tuple_fallback(typ: TupleType) -> None: fallback = typ.partial_fallback assert fallback.type.fullname == "builtins.tuple" items = [] - for item in typ.items: + for item in flatten_nested_tuples(typ.items): # TODO: this duplicates some logic in typeops.tuple_fallback(). if isinstance(item, UnpackType): unpacked_type = get_proper_type(item.type) diff --git a/mypy/semanal_typeargs.py b/mypy/semanal_typeargs.py index 435abb78ca43b..be39a8259c2e4 100644 --- a/mypy/semanal_typeargs.py +++ b/mypy/semanal_typeargs.py @@ -102,6 +102,8 @@ def visit_type_alias_type(self, t: TypeAliasType) -> None: # If there was already an error for the alias itself, there is no point in checking # the expansion, most likely it will result in the same kind of error. get_proper_type(t).accept(self) + if t.alias is not None: + t.alias.accept(self) def visit_tuple_type(self, t: TupleType) -> None: t.items = flatten_nested_tuples(t.items) @@ -254,6 +256,10 @@ def visit_unpack_type(self, typ: UnpackType) -> None: def check_type_var_values( self, name: str, actuals: list[Type], arg_name: str, valids: list[Type], context: Context ) -> bool: + if self.in_type_alias_expr: + # See testValidTypeAliasValues - we do not enforce typevar compatibility + # at the definition site. We check instantiation validity later. + return False is_error = False for actual in get_proper_types(actuals): # We skip UnboundType here, since they may appear in defn.bases, diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index db0e26ba2b364..e931c0c01aeeb 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -2666,3 +2666,30 @@ def identity(smth: _FT) -> _FT: class S(tuple[Unpack[Ts]], Generic[T, Unpack[Ts]]): def f(self, x: T, /) -> T: ... [builtins fixtures/tuple.pyi] + +[case testNoCrashSubclassingTupleWithTrivialUnpack] +# https://github.com/python/mypy/issues/19105 +from typing import Unpack + +class A(tuple[Unpack[tuple[int]]]): ... +class B(tuple[Unpack[tuple[()]]]): ... + +a: A +reveal_type(tuple(a)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +(x,) = a + +b: B +(_,) = b # E: Need more than 0 values to unpack (1 expected) +[builtins fixtures/tuple.pyi] + +[case testNoCrashSubclassingTupleWithVariadicUnpack] +# https://github.com/python/mypy/issues/19105 +from typing import Unpack + +class A(tuple[Unpack[tuple[int, ...]]]): ... + +a: A +tuple(a) +(x,) = a +(_,) = a +[builtins fixtures/tuple.pyi] From a56adc82f7dd747f400bb9267ec32b73d22de64e Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 25 Jul 2025 09:33:18 +0100 Subject: [PATCH 103/424] Optimize generic inference passes (#19501) This gives around 5% perf boost for interpreted mypy (on self-check), but only 1.5-2% for compiled one. Note this is _not_ a no-op, this is a faster (and IMO actually more correct) inference passes logic. Btw, for more generics-rich code the savings may be bigger, e.g. `mypy -c "import colours"` becomes 30% faster (compiled). --- mypy/checkexpr.py | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 24f0c8c85d61b..86f8d94104762 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1755,14 +1755,6 @@ def check_callable_call( return AnyType(TypeOfAny.from_error), callee seen_unpack = True - formal_to_actual = map_actuals_to_formals( - arg_kinds, - arg_names, - callee.arg_kinds, - callee.arg_names, - lambda i: self.accept(args[i]), - ) - # This is tricky: return type may contain its own type variables, like in # def [S] (S) -> def [T] (T) -> tuple[S, T], so we need to update their ids # to avoid possible id clashes if this call itself appears in a generic @@ -1773,27 +1765,29 @@ def check_callable_call( freeze_all_type_vars(fresh_ret_type) callee = callee.copy_modified(ret_type=fresh_ret_type) + if callee.is_generic(): + callee = freshen_function_type_vars(callee) + callee = self.infer_function_type_arguments_using_context(callee, context) + + formal_to_actual = map_actuals_to_formals( + arg_kinds, + arg_names, + callee.arg_kinds, + callee.arg_names, + lambda i: self.accept(args[i]), + ) + if callee.is_generic(): need_refresh = any( isinstance(v, (ParamSpecType, TypeVarTupleType)) for v in callee.variables ) - callee = freshen_function_type_vars(callee) - callee = self.infer_function_type_arguments_using_context(callee, context) - if need_refresh: - # Argument kinds etc. may have changed due to - # ParamSpec or TypeVarTuple variables being replaced with an arbitrary - # number of arguments; recalculate actual-to-formal map - formal_to_actual = map_actuals_to_formals( - arg_kinds, - arg_names, - callee.arg_kinds, - callee.arg_names, - lambda i: self.accept(args[i]), - ) callee = self.infer_function_type_arguments( callee, args, arg_kinds, arg_names, formal_to_actual, need_refresh, context ) if need_refresh: + # Argument kinds etc. may have changed due to + # ParamSpec or TypeVarTuple variables being replaced with an arbitrary + # number of arguments; recalculate actual-to-formal map formal_to_actual = map_actuals_to_formals( arg_kinds, arg_names, @@ -2258,6 +2252,11 @@ def infer_function_type_arguments_pass2( if isinstance(arg, (NoneType, UninhabitedType)) or has_erased_component(arg): inferred_args[i] = None callee_type = self.apply_generic_arguments(callee_type, inferred_args, context) + + if not callee_type.is_generic(): + # Fast path, second pass can't give new information. + return callee_type, [] + if need_refresh: formal_to_actual = map_actuals_to_formals( arg_kinds, From afcf6b227d96cdc2b8d07f62108cacc20ca42ece Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Sat, 26 Jul 2025 03:21:13 +0200 Subject: [PATCH 104/424] docs: update cython and setuptools RTD URLs (#19512) Fixes docs build failure discovered in #19510. Updated setuptools URL too because previous CI runs say that > intersphinx inventory has moved: https://setuptools.readthedocs.io/en/latest/objects.inv -> https://setuptools.pypa.io/en/latest/objects.inv --- docs/source/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 79a5c06196155..02caa44dce117 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -278,9 +278,9 @@ intersphinx_mapping = { "python": ("https://docs.python.org/3", None), "attrs": ("https://www.attrs.org/en/stable/", None), - "cython": ("https://docs.cython.org/en/latest", None), + "cython": ("https://cython.readthedocs.io/en/stable", None), "monkeytype": ("https://monkeytype.readthedocs.io/en/latest", None), - "setuptools": ("https://setuptools.readthedocs.io/en/latest", None), + "setuptools": ("https://setuptools.pypa.io/en/latest", None), } From 31adfb416e42ec4de65e823cce2774595b1f70e9 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 27 Jul 2025 00:29:55 +0100 Subject: [PATCH 105/424] Avoid duplicate visit in check_boolean_op() (#19515) Surprisingly, this simple change gives 0.5-1% perf boost on self-check (for some reasons this is quite hot function). --- mypy/checkexpr.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 86f8d94104762..a75ccf05612bd 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -3468,7 +3468,7 @@ def visit_op_expr(self, e: OpExpr) -> Type: # It's actually a type expression X | Y. return self.accept(e.analyzed) if e.op == "and" or e.op == "or": - return self.check_boolean_op(e, e) + return self.check_boolean_op(e) if e.op == "*" and isinstance(e.left, ListExpr): # Expressions of form [...] * e get special type inference. return self.check_list_multiply(e) @@ -4255,20 +4255,18 @@ def check_op( context=context, ) - def check_boolean_op(self, e: OpExpr, context: Context) -> Type: + def check_boolean_op(self, e: OpExpr) -> Type: """Type check a boolean operation ('and' or 'or').""" # A boolean operation can evaluate to either of the operands. - # We use the current type context to guide the type inference of of + # We use the current type context to guide the type inference of # the left operand. We also use the left operand type to guide the type # inference of the right operand so that expressions such as # '[1] or []' are inferred correctly. ctx = self.type_context[-1] left_type = self.accept(e.left, ctx) - expanded_left_type = try_expanding_sum_type_to_union( - self.accept(e.left, ctx), "builtins.bool" - ) + expanded_left_type = try_expanding_sum_type_to_union(left_type, "builtins.bool") assert e.op in ("and", "or") # Checked by visit_op_expr From c6b40df63ce0fecab6bced800ce778d99587c9b8 Mon Sep 17 00:00:00 2001 From: Omer Hadari Date: Sun, 27 Jul 2025 07:15:42 +0300 Subject: [PATCH 106/424] Prevent final reassignment in match case (#19496) Fixes #19507 This PR adds a check to prevent reassignment of final variables in a match statement. Currently, the following passes without an error: ```Python from typing import Final FOO: Final = 8 a = 10 match a: case FOO: pass print(FOO) # FOO is reassigned, prints 10 ``` MyPy already checks that the type of FOO isn't changed if used like this. I added a check in the same place that makes sure it's not `Final` either. Since this tests the match syntax, I put the test where I did. If it's not the appropriate place for it I will move it as instructed :) --- mypy/checker.py | 2 ++ test-data/unit/check-python310.test | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/mypy/checker.py b/mypy/checker.py index 7579c36a97d05..e4ee0bf4d14c3 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -5577,6 +5577,8 @@ def infer_variable_types_from_type_maps( previous_type, _, _ = self.check_lvalue(expr) if previous_type is not None: already_exists = True + if isinstance(expr.node, Var) and expr.node.is_final: + self.msg.cant_assign_to_final(expr.name, False, expr) if self.check_subtype( typ, previous_type, diff --git a/test-data/unit/check-python310.test b/test-data/unit/check-python310.test index bb8f038eb1eb2..80fd64fa3569a 100644 --- a/test-data/unit/check-python310.test +++ b/test-data/unit/check-python310.test @@ -2839,3 +2839,13 @@ match value_type: case _: assert_never(value_type) [builtins fixtures/tuple.pyi] + +[case testAssignmentToFinalInMatchCaseNotAllowed] +from typing import Final + +FOO: Final[int] = 10 + +val: int = 8 +match val: + case FOO: # E: Cannot assign to final name "FOO" + pass From a8d2f1334b48dfbee5fe04d6bd1f0b145a13d9d9 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 27 Jul 2025 22:49:16 +0100 Subject: [PATCH 107/424] Cache type_object_type() (#19514) This gives almost 4% performance boost (Python 3.12, compiled). Note there is an old bug in `type_object_type()`, we treat not ready types as `Any` without deferring, I disable caching in this case. Unfortunately, using this in fine-grained mode is tricky, essentially I have three options: * Use some horrible hacks to invalidate cache when needed * Add (expensive) class target dependency from `__init__`/`__new__` * Only allow constructor caching during initial load, but disable it in fine-grained increments I decided to choose the last option. I think it has the best balance complexity/benefits. --- mypy/checker.py | 7 ++++++- mypy/checker_shared.py | 1 + mypy/nodes.py | 6 ++++++ mypy/semanal_infer.py | 3 +++ mypy/server/update.py | 6 ++++-- mypy/typeops.py | 40 +++++++++++++++++++++++++++++++++++----- 6 files changed, 55 insertions(+), 8 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index e4ee0bf4d14c3..24076fca67103 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -397,6 +397,7 @@ def __init__( self.is_stub = tree.is_stub self.is_typeshed_stub = tree.is_typeshed_file(options) self.inferred_attribute_types = None + self.allow_constructor_cache = True # If True, process function definitions. If False, don't. This is used # for processing module top levels in fine-grained incremental mode. @@ -500,12 +501,16 @@ def check_first_pass(self) -> None: ) def check_second_pass( - self, todo: Sequence[DeferredNode | FineGrainedDeferredNode] | None = None + self, + todo: Sequence[DeferredNode | FineGrainedDeferredNode] | None = None, + *, + allow_constructor_cache: bool = True, ) -> bool: """Run second or following pass of type checking. This goes through deferred nodes, returning True if there were any. """ + self.allow_constructor_cache = allow_constructor_cache self.recurse_into_functions = True with state.strict_optional_set(self.options.strict_optional), checker_state.set(self): if not todo and not self.deferred_nodes: diff --git a/mypy/checker_shared.py b/mypy/checker_shared.py index 65cec41d52023..7a5e9cb52c70f 100644 --- a/mypy/checker_shared.py +++ b/mypy/checker_shared.py @@ -137,6 +137,7 @@ class TypeCheckerSharedApi(CheckerPluginInterface): module_refs: set[str] scope: CheckerScope checking_missing_await: bool + allow_constructor_cache: bool @property @abstractmethod diff --git a/mypy/nodes.py b/mypy/nodes.py index fc2656ce2130e..921620866a06e 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -3022,6 +3022,7 @@ class is generic then it will be a type constructor of higher kind. "dataclass_transform_spec", "is_type_check_only", "deprecated", + "type_object_type", ) _fullname: str # Fully qualified name @@ -3178,6 +3179,10 @@ class is generic then it will be a type constructor of higher kind. # The type's deprecation message (in case it is deprecated) deprecated: str | None + # Cached value of class constructor type, i.e. the type of class object when it + # appears in runtime context. + type_object_type: mypy.types.FunctionLike | None + FLAGS: Final = [ "is_abstract", "is_enum", @@ -3236,6 +3241,7 @@ def __init__(self, names: SymbolTable, defn: ClassDef, module_name: str) -> None self.dataclass_transform_spec = None self.is_type_check_only = False self.deprecated = None + self.type_object_type = None def add_type_vars(self) -> None: self.has_type_var_tuple_type = False diff --git a/mypy/semanal_infer.py b/mypy/semanal_infer.py index a146b56dc2d36..89a073cdad473 100644 --- a/mypy/semanal_infer.py +++ b/mypy/semanal_infer.py @@ -31,6 +31,7 @@ def infer_decorator_signature_if_simple( """ if dec.var.is_property: # Decorators are expected to have a callable type (it's a little odd). + # TODO: this may result in wrong type if @property is applied to decorated method. if dec.func.type is None: dec.var.type = CallableType( [AnyType(TypeOfAny.special_form)], @@ -47,6 +48,8 @@ def infer_decorator_signature_if_simple( for expr in dec.decorators: preserve_type = False if isinstance(expr, RefExpr) and isinstance(expr.node, FuncDef): + if expr.fullname == "typing.no_type_check": + return if expr.node.type and is_identity_signature(expr.node.type): preserve_type = True if not preserve_type: diff --git a/mypy/server/update.py b/mypy/server/update.py index ea336154ae56e..839090ca45ac9 100644 --- a/mypy/server/update.py +++ b/mypy/server/update.py @@ -1025,10 +1025,12 @@ def key(node: FineGrainedDeferredNode) -> int: # We seem to need additional passes in fine-grained incremental mode. checker.pass_num = 0 checker.last_pass = 3 - more = checker.check_second_pass(nodes) + # It is tricky to reliably invalidate constructor cache in fine-grained increments. + # See PR 19514 description for details. + more = checker.check_second_pass(nodes, allow_constructor_cache=False) while more: more = False - if graph[module_id].type_checker().check_second_pass(): + if graph[module_id].type_checker().check_second_pass(allow_constructor_cache=False): more = True if manager.options.export_types: diff --git a/mypy/typeops.py b/mypy/typeops.py index 9aa08b40a991d..1c22a1711944a 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -11,6 +11,7 @@ from collections.abc import Iterable, Sequence from typing import Any, Callable, TypeVar, cast +from mypy.checker_state import checker_state from mypy.copytype import copy_type from mypy.expandtype import expand_type, expand_type_by_instance from mypy.maptype import map_instance_to_supertype @@ -145,6 +146,15 @@ def type_object_type(info: TypeInfo, named_type: Callable[[str], Instance]) -> P where ... are argument types for the __init__/__new__ method (without the self argument). Also, the fallback type will be 'type' instead of 'function'. """ + allow_cache = ( + checker_state.type_checker is not None + and checker_state.type_checker.allow_constructor_cache + ) + + if info.type_object_type is not None: + if allow_cache: + return info.type_object_type + info.type_object_type = None # We take the type from whichever of __init__ and __new__ is first # in the MRO, preferring __init__ if there is a tie. @@ -167,7 +177,15 @@ def type_object_type(info: TypeInfo, named_type: Callable[[str], Instance]) -> P init_index = info.mro.index(init_method.node.info) new_index = info.mro.index(new_method.node.info) - fallback = info.metaclass_type or named_type("builtins.type") + if info.metaclass_type is not None: + fallback = info.metaclass_type + elif checker_state.type_checker: + # Prefer direct call when it is available. It is faster, and, + # unfortunately, some callers provide bogus callback. + fallback = checker_state.type_checker.named_type("builtins.type") + else: + fallback = named_type("builtins.type") + if init_index < new_index: method: FuncBase | Decorator = init_method.node is_new = False @@ -189,7 +207,10 @@ def type_object_type(info: TypeInfo, named_type: Callable[[str], Instance]) -> P is_bound=True, fallback=named_type("builtins.function"), ) - return class_callable(sig, info, fallback, None, is_new=False) + result: FunctionLike = class_callable(sig, info, fallback, None, is_new=False) + if allow_cache: + info.type_object_type = result + return result # Otherwise prefer __init__ in a tie. It isn't clear that this # is the right thing, but __new__ caused problems with @@ -199,12 +220,19 @@ def type_object_type(info: TypeInfo, named_type: Callable[[str], Instance]) -> P # Construct callable type based on signature of __init__. Adjust # return type and insert type arguments. if isinstance(method, FuncBase): + if isinstance(method, OverloadedFuncDef) and not method.type: + # Do not cache if the type is not ready. Same logic for decorators is + # achieved in early return above because is_valid_constructor() is False. + allow_cache = False t = function_type(method, fallback) else: assert isinstance(method.type, ProperType) assert isinstance(method.type, FunctionLike) # is_valid_constructor() ensures this t = method.type - return type_object_type_from_function(t, info, method.info, fallback, is_new) + result = type_object_type_from_function(t, info, method.info, fallback, is_new) + if allow_cache: + info.type_object_type = result + return result def is_valid_constructor(n: SymbolNode | None) -> bool: @@ -865,8 +893,8 @@ def function_type(func: FuncBase, fallback: Instance) -> FunctionLike: if isinstance(func, FuncItem): return callable_type(func, fallback) else: - # Broken overloads can have self.type set to None. - # TODO: should we instead always set the type in semantic analyzer? + # Either a broken overload, or decorated overload type is not ready. + # TODO: make sure the caller defers if possible. assert isinstance(func, OverloadedFuncDef) any_type = AnyType(TypeOfAny.from_error) dummy = CallableType( @@ -1254,6 +1282,8 @@ def get_protocol_member( if member == "__call__" and class_obj: # Special case: class objects always have __call__ that is just the constructor. + # TODO: this is wrong, it creates callables that are not recognized as type objects. + # Long-term, we should probably get rid of this callback argument altogether. def named_type(fullname: str) -> Instance: return Instance(left.type.mro[-1], []) From 0c1f1044fc5e7c6b36f9c0d16169837a3e5a2dba Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 28 Jul 2025 10:24:12 +0100 Subject: [PATCH 108/424] Various minor docstring and comment updates (#19519) Mostly grammar improvements. --- mypy/build.py | 14 +++++++------- mypyc/build.py | 10 +++++----- mypyc/codegen/emitmodule.py | 29 ++++++++++++++--------------- 3 files changed, 26 insertions(+), 27 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 355ba861385e4..71575de9d8773 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -194,7 +194,7 @@ def default_flush_errors( result.errors = messages return result except CompileError as e: - # CompileErrors raised from an errors object carry all of the + # CompileErrors raised from an errors object carry all the # messages that have not been reported out by error streaming. # Patch it up to contain either none or all none of the messages, # depending on whether we are flushing errors. @@ -802,11 +802,11 @@ def correct_rel_imp(imp: ImportFrom | ImportAll) -> str: res.append((pri, sub_id, imp.line)) else: all_are_submodules = False - # Add cur_id as a dependency, even if all of the + # Add cur_id as a dependency, even if all the # imports are submodules. Processing import from will try # to look through cur_id, so we should depend on it. - # As a workaround for for some bugs in cycle handling (#4498), - # if all of the imports are submodules, do the import at a lower + # As a workaround for some bugs in cycle handling (#4498), + # if all the imports are submodules, do the import at a lower # priority. pri = import_priority(imp, PRI_HIGH if not all_are_submodules else PRI_LOW) res.append((pri, cur_id, imp.line)) @@ -929,7 +929,7 @@ def write_deps_cache( ) -> None: """Write cache files for fine-grained dependencies. - Serialize fine-grained dependencies map for fine grained mode. + Serialize fine-grained dependencies map for fine-grained mode. Dependencies on some module 'm' is stored in the dependency cache file m.deps.json. This entails some spooky action at a distance: @@ -943,7 +943,7 @@ def write_deps_cache( fine-grained dependencies in a global cache file: * We take a snapshot of current sources to later check consistency between the fine-grained dependency cache and module cache metadata - * We store the mtime of all of the dependency files to verify they + * We store the mtime of all the dependency files to verify they haven't changed """ metastore = manager.metastore @@ -1111,7 +1111,7 @@ def read_deps_cache(manager: BuildManager, graph: Graph) -> dict[str, FgDepMeta] if deps_meta is None: return None meta_snapshot = deps_meta["snapshot"] - # Take a snapshot of the source hashes from all of the metas we found. + # Take a snapshot of the source hashes from all the metas we found. # (Including the ones we rejected because they were out of date.) # We use this to verify that they match up with the proto_deps. current_meta_snapshot = { diff --git a/mypyc/build.py b/mypyc/build.py index 8ddbf4d22a278..b7d3e1b25366a 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -270,12 +270,12 @@ def build_using_shared_lib( ) -> list[Extension]: """Produce the list of extension modules when a shared library is needed. - This creates one shared library extension module that all of the - others import and then one shim extension module for each - module in the build, that simply calls an initialization function + This creates one shared library extension module that all the + others import, and one shim extension module for each + module in the build. Each shim simply calls an initialization function in the shared library. - The shared library (which lib_name is the name of) is a python + The shared library (which lib_name is the name of) is a Python extension module that exports the real initialization functions in Capsules stored in module attributes. """ @@ -511,7 +511,7 @@ def mypycify( separate: Should compiled modules be placed in separate extension modules. If False, all modules are placed in a single shared library. If True, every module is placed in its own library. - Otherwise separate should be a list of + Otherwise, separate should be a list of (file name list, optional shared library name) pairs specifying groups of files that should be placed in the same shared library (while all other modules will be placed in its own library). diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 7037409ff40bf..13a3727cd1883 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -1,7 +1,7 @@ """Generate C code for a Python C extension module from Python source code.""" # FIXME: Basically nothing in this file operates on the level of a -# single module and it should be renamed. +# single module and it should be renamed. from __future__ import annotations @@ -71,7 +71,7 @@ from mypyc.transform.spill import insert_spills from mypyc.transform.uninit import insert_uninit_checks -# All of the modules being compiled are divided into "groups". A group +# All the modules being compiled are divided into "groups". A group # is a set of modules that are placed into the same shared library. # Two common configurations are that every module is placed in a group # by itself (fully separate compilation) and that every module is @@ -164,7 +164,7 @@ def report_config_data(self, ctx: ReportConfigContext) -> tuple[str | None, list if hash_digest(meta_json) != ir_data["meta_hash"]: return None - # Check that all of the source files are present and as + # Check that all the source files are present and as # expected. The main situation where this would come up is the # user deleting the build directory without deleting # .mypy_cache, which we should handle gracefully. @@ -215,8 +215,8 @@ def compile_scc_to_ir( ) -> ModuleIRs: """Compile an SCC into ModuleIRs. - Any modules that this SCC depends on must have either compiled or - loaded from a cache into mapper. + Any modules that this SCC depends on must have either been compiled, + type checked, or loaded from a cache into mapper. Arguments: scc: The list of MypyFiles to compile @@ -244,11 +244,11 @@ def compile_scc_to_ir( for module in modules.values(): for fn in module.functions: - # Insert uninit checks. + # Insert checks for uninitialized values. insert_uninit_checks(fn) # Insert exception handling. insert_exception_handling(fn) - # Insert refcount handling. + # Insert reference count handling. insert_ref_count_opcodes(fn) if fn in env_user_functions: @@ -369,7 +369,7 @@ def write_cache( cache are in sync and refer to the same version of the code. This is particularly important if mypyc crashes/errors/is stopped after mypy has written its cache but before mypyc has. - * The hashes of all of the source file outputs for the group + * The hashes of all the source file outputs for the group the module is in. This is so that the module will be recompiled if the source outputs are missing. """ @@ -429,7 +429,7 @@ def compile_modules_to_c( Each shared library module provides, for each module in its group, a PyCapsule containing an initialization function. Additionally, it provides a capsule containing an export table of - pointers to all of the group's functions and static variables. + pointers to all the group's functions and static variables. Arguments: result: The BuildResult from the mypy front-end @@ -504,7 +504,7 @@ def __init__( The code for a compilation group contains an internal and an external .h file, and then one .c if not in multi_file mode or - one .c file per module if in multi_file mode.) + one .c file per module if in multi_file mode. Arguments: modules: (name, ir) pairs for each module in the group @@ -512,8 +512,7 @@ def __init__( group_name: The name of the group (or None if this is single-module compilation) group_map: A map of modules to their group names names: The name generator for the compilation - multi_file: Whether to put each module in its own source file regardless - of group structure. + compiler_options: Mypyc specific options, including multi_file mode """ self.modules = modules self.source_paths = source_paths @@ -642,7 +641,7 @@ def generate_c_for_modules(self) -> list[tuple[str, str]]: decls = ext_declarations if declaration.is_type else declarations if not declaration.is_type: decls.emit_lines(f"extern {declaration.decl[0]}", *declaration.decl[1:]) - # If there is a definition, emit it. Otherwise repeat the declaration + # If there is a definition, emit it. Otherwise, repeat the declaration # (without an extern). if declaration.defn: emitter.emit_lines(*declaration.defn) @@ -770,13 +769,13 @@ def generate_export_table(self, decl_emitter: Emitter, code_emitter: Emitter) -> def generate_shared_lib_init(self, emitter: Emitter) -> None: """Generate the init function for a shared library. - A shared library contains all of the actual code for a + A shared library contains all the actual code for a compilation group. The init function is responsible for creating Capsules that wrap pointers to the initialization function of all the real init functions for modules in this shared library as well as - the export table containing all of the exported functions and + the export table containing all the exported functions and values from all the modules. These capsules are stored in attributes of the shared library. From 3a2b7888a7eb0543d572761464ab5295c585d0a9 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 28 Jul 2025 14:18:14 +0100 Subject: [PATCH 109/424] [mypyc] Only generate an export table if using separate compilation (#19521) When not using separate compilation, the export table is not used. Also, there's actually no simple and safe way to use it without separate compilation, since we'd have to first ensure that the structure of the export table is compatible, as otherwise the order of fields or the types of the fields could be incompatible and cause segfaults and other fun stuff. --- mypyc/codegen/emitmodule.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 13a3727cd1883..047309ec71e38 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -651,7 +651,8 @@ def generate_c_for_modules(self) -> list[tuple[str, str]]: decls.emit_lines(*declaration.decl) if self.group_name: - self.generate_export_table(ext_declarations, emitter) + if self.compiler_options.separate: + self.generate_export_table(ext_declarations, emitter) self.generate_shared_lib_init(emitter) @@ -808,20 +809,21 @@ def generate_shared_lib_init(self, emitter: Emitter) -> None: "", ) - emitter.emit_lines( - 'capsule = PyCapsule_New(&exports, "{}.exports", NULL);'.format( - shared_lib_name(self.group_name) - ), - "if (!capsule) {", - "goto fail;", - "}", - 'res = PyObject_SetAttrString(module, "exports", capsule);', - "Py_DECREF(capsule);", - "if (res < 0) {", - "goto fail;", - "}", - "", - ) + if self.compiler_options.separate: + emitter.emit_lines( + 'capsule = PyCapsule_New(&exports, "{}.exports", NULL);'.format( + shared_lib_name(self.group_name) + ), + "if (!capsule) {", + "goto fail;", + "}", + 'res = PyObject_SetAttrString(module, "exports", capsule);', + "Py_DECREF(capsule);", + "if (res < 0) {", + "goto fail;", + "}", + "", + ) for mod in self.modules: name = exported_name(mod) From e40c36c014710d76a13c90b5e40a828b92296f39 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 28 Jul 2025 17:17:48 +0100 Subject: [PATCH 110/424] [mypyc] Fix list.pop primitive on free-threaded builds (#19522) The old implementation caused segfaults on free-threaded builds. Provide a slower but working implementation for free-threaded builds. Also improve test coverage, since it used to be spotty. Tested on a manually compiled Python 3.14.0b4 free-threaded build on macOS. --- mypyc/lib-rt/list_ops.c | 36 ++++++++++++++++++++++++++++++++-- mypyc/primitives/list_ops.py | 2 +- mypyc/test-data/run-lists.test | 29 +++++++++++++++++++++------ 3 files changed, 58 insertions(+), 9 deletions(-) diff --git a/mypyc/lib-rt/list_ops.c b/mypyc/lib-rt/list_ops.c index 31a0d5cec7d5a..c611907fb6016 100644 --- a/mypyc/lib-rt/list_ops.c +++ b/mypyc/lib-rt/list_ops.c @@ -235,19 +235,51 @@ void CPyList_SetItemUnsafe(PyObject *list, Py_ssize_t index, PyObject *value) { PyList_SET_ITEM(list, index, value); } -PyObject *CPyList_PopLast(PyObject *obj) +#ifdef Py_GIL_DISABLED +// The original optimized list.pop implementation doesn't work on free-threaded +// builds, so provide an alternative that is a bit slower but works. +// +// Note that this implementation isn't intended to be atomic. +static inline PyObject *list_pop_index(PyObject *list, Py_ssize_t index) { + PyObject *item = PyList_GetItemRef(list, index); + if (item == NULL) { + return NULL; + } + if (PySequence_DelItem(list, index) < 0) { + Py_DECREF(item); + return NULL; + } + return item; +} +#endif + +PyObject *CPyList_PopLast(PyObject *list) { +#ifdef Py_GIL_DISABLED + // The other implementation causes segfaults on a free-threaded Python 3.14b4 build. + Py_ssize_t index = PyList_GET_SIZE(list) - 1; + return list_pop_index(list, index); +#else // I tried a specalized version of pop_impl for just removing the // last element and it wasn't any faster in microbenchmarks than // the generic one so I ditched it. - return list_pop_impl((PyListObject *)obj, -1); + return list_pop_impl((PyListObject *)list, -1); +#endif } PyObject *CPyList_Pop(PyObject *obj, CPyTagged index) { if (CPyTagged_CheckShort(index)) { Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); +#ifdef Py_GIL_DISABLED + // We must use a slower implementation on free-threaded builds. + if (n < 0) { + n += PyList_GET_SIZE(obj); + } + return list_pop_index(obj, n); +#else return list_pop_impl((PyListObject *)obj, n); +#endif } else { PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG); return NULL; diff --git a/mypyc/primitives/list_ops.py b/mypyc/primitives/list_ops.py index 516d9e1a4e026..b9d20a25bea35 100644 --- a/mypyc/primitives/list_ops.py +++ b/mypyc/primitives/list_ops.py @@ -219,7 +219,7 @@ ) # list.pop(index) -list_pop = method_op( +method_op( name="pop", arg_types=[list_rprimitive, int_rprimitive], return_type=object_rprimitive, diff --git a/mypyc/test-data/run-lists.test b/mypyc/test-data/run-lists.test index 03d5741b9ecac..54bcc03846049 100644 --- a/mypyc/test-data/run-lists.test +++ b/mypyc/test-data/run-lists.test @@ -150,7 +150,9 @@ print(primes(13)) \[0, 0, 1, 1] \[0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1] -[case testListBuild] +[case testListPrimitives] +from testutil import assertRaises + def test_list_build() -> None: # Currently LIST_BUILDING_EXPANSION_THRESHOLD equals to 10 # long list built by list_build_op @@ -169,9 +171,6 @@ def test_list_build() -> None: l3.append('a') assert l3 == ['a'] -[case testListPrims] -from typing import List - def test_append() -> None: l = [1, 2] l.append(10) @@ -189,10 +188,28 @@ def test_pop_last() -> None: def test_pop_index() -> None: l = [1, 2, 10, 3] - l.pop(2) + assert l.pop(2) == 10 assert l == [1, 2, 3] - l.pop(-2) + assert l.pop(-2) == 2 assert l == [1, 3] + assert l.pop(-2) == 1 + assert l.pop(0) == 3 + assert l == [] + l = [int() + 1000, int() + 1001, int() + 1002] + assert l.pop(0) == 1000 + assert l.pop(-1) == 1002 + assert l == [1001] + +def test_pop_index_errors() -> None: + l = [int() + 1000] + with assertRaises(IndexError): + l.pop(1) + with assertRaises(IndexError): + l.pop(-2) + with assertRaises(OverflowError): + l.pop(1 << 100) + with assertRaises(OverflowError): + l.pop(-(1 << 100)) def test_count() -> None: l = [1, 3] From bd94bcbb0a548ec480b78bcf71a955a45dc9d77c Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 28 Jul 2025 18:17:58 +0100 Subject: [PATCH 111/424] Try simple-minded call expression cache (#19505) This gives a modest 1% improvement on self-check (compiled), but it gives almost 40% on `mypy -c "import colour"`. Some comments: * I only cache `CallExpr`, `ListExpr`, and `TupleExpr`, this is not very principled, I found this as a best balance between rare cases like `colour`, and more common cases like self-check. * Caching is fragile within lambdas, so I simply disable it, it rarely matters anyway. * I cache both messages and the type map, surprisingly the latter only affects couple test cases, but I still do this generally for peace of mind. * It looks like there are only three things that require cache invalidation: binder, partial types, and deferrals. In general, this is a bit scary (as this a major change), but also perf improvements for slow libraries are very tempting. --- mypy/binder.py | 6 +++ mypy/checker.py | 7 +++- mypy/checkexpr.py | 45 +++++++++++++++++++++- mypy/errors.py | 6 ++- test-data/unit/check-overloading.test | 23 +++++++++++ test-data/unit/fixtures/isinstancelist.pyi | 2 + 6 files changed, 84 insertions(+), 5 deletions(-) diff --git a/mypy/binder.py b/mypy/binder.py index d3482d1dad4f8..2ae58dad1fe07 100644 --- a/mypy/binder.py +++ b/mypy/binder.py @@ -138,6 +138,10 @@ def __init__(self, options: Options) -> None: # flexible inference of variable types (--allow-redefinition-new). self.bind_all = options.allow_redefinition_new + # This tracks any externally visible changes in binder to invalidate + # expression caches when needed. + self.version = 0 + def _get_id(self) -> int: self.next_id += 1 return self.next_id @@ -158,6 +162,7 @@ def push_frame(self, conditional_frame: bool = False) -> Frame: return f def _put(self, key: Key, type: Type, from_assignment: bool, index: int = -1) -> None: + self.version += 1 self.frames[index].types[key] = CurrentType(type, from_assignment) def _get(self, key: Key, index: int = -1) -> CurrentType | None: @@ -185,6 +190,7 @@ def put(self, expr: Expression, typ: Type, *, from_assignment: bool = True) -> N self._put(key, typ, from_assignment) def unreachable(self) -> None: + self.version += 1 self.frames[-1].unreachable = True def suppress_unreachable_warnings(self) -> None: diff --git a/mypy/checker.py b/mypy/checker.py index 24076fca67103..97c1e10259c98 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -449,7 +449,6 @@ def reset(self) -> None: self.binder = ConditionalTypeBinder(self.options) self._type_maps[1:] = [] self._type_maps[0].clear() - self.temp_type_map = None self.expr_checker.reset() self.deferred_nodes = [] self.partial_types = [] @@ -3024,6 +3023,8 @@ def visit_block(self, b: Block) -> None: break else: self.accept(s) + # Clear expression cache after each statement to avoid unlimited growth. + self.expr_checker.expr_cache.clear() def should_report_unreachable_issues(self) -> bool: return ( @@ -4005,7 +4006,7 @@ def check_multi_assignment_from_union( for t, lv in zip(transposed, self.flatten_lvalues(lvalues)): # We can access _type_maps directly since temporary type maps are # only created within expressions. - t.append(self._type_maps[0].pop(lv, AnyType(TypeOfAny.special_form))) + t.append(self._type_maps[-1].pop(lv, AnyType(TypeOfAny.special_form))) union_types = tuple(make_simplified_union(col) for col in transposed) for expr, items in assignments.items(): # Bind a union of types collected in 'assignments' to every expression. @@ -4664,6 +4665,8 @@ def replace_partial_type( ) -> None: """Replace the partial type of var with a non-partial type.""" var.type = new_type + # Updating a partial type should invalidate expression caches. + self.binder.version += 1 del partial_types[var] if self.options.allow_redefinition_new: # When using --allow-redefinition-new, binder tracks all types of diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index a75ccf05612bd..a8afebbd9923f 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -19,7 +19,7 @@ from mypy.checkmember import analyze_member_access, has_operator from mypy.checkstrformat import StringFormatterChecker from mypy.erasetype import erase_type, remove_instance_last_known_values, replace_meta_vars -from mypy.errors import ErrorWatcher, report_internal_error +from mypy.errors import ErrorInfo, ErrorWatcher, report_internal_error from mypy.expandtype import ( expand_type, expand_type_by_instance, @@ -355,9 +355,15 @@ def __init__( type_state.infer_polymorphic = not self.chk.options.old_type_inference self._arg_infer_context_cache = None + self.expr_cache: dict[ + tuple[Expression, Type | None], + tuple[int, Type, list[ErrorInfo], dict[Expression, Type]], + ] = {} + self.in_lambda_expr = False def reset(self) -> None: self.resolved_type = {} + self.expr_cache.clear() def visit_name_expr(self, e: NameExpr) -> Type: """Type check a name expression. @@ -5402,6 +5408,8 @@ def find_typeddict_context( def visit_lambda_expr(self, e: LambdaExpr) -> Type: """Type check lambda expression.""" + old_in_lambda = self.in_lambda_expr + self.in_lambda_expr = True self.chk.check_default_args(e, body_is_trivial=False) inferred_type, type_override = self.infer_lambda_type_using_context(e) if not inferred_type: @@ -5422,6 +5430,7 @@ def visit_lambda_expr(self, e: LambdaExpr) -> Type: ret_type = self.accept(e.expr(), allow_none_return=True) fallback = self.named_type("builtins.function") self.chk.return_types.pop() + self.in_lambda_expr = old_in_lambda return callable_type(e, fallback, ret_type) else: # Type context available. @@ -5434,6 +5443,7 @@ def visit_lambda_expr(self, e: LambdaExpr) -> Type: self.accept(e.expr(), allow_none_return=True) ret_type = self.chk.lookup_type(e.expr()) self.chk.return_types.pop() + self.in_lambda_expr = old_in_lambda return replace_callable_return_type(inferred_type, ret_type) def infer_lambda_type_using_context( @@ -5978,6 +5988,24 @@ def accept( typ = self.visit_conditional_expr(node, allow_none_return=True) elif allow_none_return and isinstance(node, AwaitExpr): typ = self.visit_await_expr(node, allow_none_return=True) + # Deeply nested generic calls can deteriorate performance dramatically. + # Although in most cases caching makes little difference, in worst case + # it avoids exponential complexity. + # We cannot use cache inside lambdas, because they skip immediate type + # context, and use enclosing one, see infer_lambda_type_using_context(). + # TODO: consider using cache for more expression kinds. + elif isinstance(node, (CallExpr, ListExpr, TupleExpr)) and not ( + self.in_lambda_expr or self.chk.current_node_deferred + ): + if (node, type_context) in self.expr_cache: + binder_version, typ, messages, type_map = self.expr_cache[(node, type_context)] + if binder_version == self.chk.binder.version: + self.chk.store_types(type_map) + self.msg.add_errors(messages) + else: + typ = self.accept_maybe_cache(node, type_context=type_context) + else: + typ = self.accept_maybe_cache(node, type_context=type_context) else: typ = node.accept(self) except Exception as err: @@ -6008,6 +6036,21 @@ def accept( self.in_expression = False return result + def accept_maybe_cache(self, node: Expression, type_context: Type | None = None) -> Type: + binder_version = self.chk.binder.version + # Micro-optimization: inline local_type_map() as it is somewhat slow in mypyc. + type_map: dict[Expression, Type] = {} + self.chk._type_maps.append(type_map) + with self.msg.filter_errors(filter_errors=True, save_filtered_errors=True) as msg: + typ = node.accept(self) + messages = msg.filtered_errors() + if binder_version == self.chk.binder.version and not self.chk.current_node_deferred: + self.expr_cache[(node, type_context)] = (binder_version, typ, messages, type_map) + self.chk._type_maps.pop() + self.chk.store_types(type_map) + self.msg.add_errors(messages) + return typ + def named_type(self, name: str) -> Instance: """Return an instance type with type given by the name and no type arguments. Alias for TypeChecker.named_type. diff --git a/mypy/errors.py b/mypy/errors.py index 5c135146bcb71..d75c1c62a1edb 100644 --- a/mypy/errors.py +++ b/mypy/errors.py @@ -390,7 +390,7 @@ class Errors: # in some cases to avoid reporting huge numbers of errors. seen_import_error = False - _watchers: list[ErrorWatcher] = [] + _watchers: list[ErrorWatcher] def __init__( self, @@ -421,6 +421,7 @@ def initialize(self) -> None: self.scope = None self.target_module = None self.seen_import_error = False + self._watchers = [] def reset(self) -> None: self.initialize() @@ -931,7 +932,8 @@ def prefer_simple_messages(self) -> bool: if self.file in self.ignored_files: # Errors ignored, so no point generating fancy messages return True - for _watcher in self._watchers: + if self._watchers: + _watcher = self._watchers[-1] if _watcher._filter is True and _watcher._filtered is None: # Errors are filtered return True diff --git a/test-data/unit/check-overloading.test b/test-data/unit/check-overloading.test index 0f0fc8747223b..22221416f151d 100644 --- a/test-data/unit/check-overloading.test +++ b/test-data/unit/check-overloading.test @@ -6801,3 +6801,26 @@ class D(Generic[T]): a: D[str] # E: Type argument "str" of "D" must be a subtype of "C" reveal_type(a.f(1)) # N: Revealed type is "builtins.int" reveal_type(a.f("x")) # N: Revealed type is "builtins.str" + +[case testMultiAssignFromUnionInOverloadCached] +from typing import Iterable, overload, Union, Optional + +@overload +def always_bytes(str_or_bytes: None) -> None: ... +@overload +def always_bytes(str_or_bytes: Union[str, bytes]) -> bytes: ... +def always_bytes(str_or_bytes: Union[None, str, bytes]) -> Optional[bytes]: + pass + +class Headers: + def __init__(self, iter: Iterable[tuple[bytes, bytes]]) -> None: ... + +headers: Union[Headers, dict[Union[str, bytes], Union[str, bytes]], Iterable[tuple[bytes, bytes]]] + +if isinstance(headers, dict): + headers = Headers( + (always_bytes(k), always_bytes(v)) for k, v in headers.items() + ) + +reveal_type(headers) # N: Revealed type is "Union[__main__.Headers, typing.Iterable[tuple[builtins.bytes, builtins.bytes]]]" +[builtins fixtures/isinstancelist.pyi] diff --git a/test-data/unit/fixtures/isinstancelist.pyi b/test-data/unit/fixtures/isinstancelist.pyi index 0ee5258ff74b0..2a43606f361a3 100644 --- a/test-data/unit/fixtures/isinstancelist.pyi +++ b/test-data/unit/fixtures/isinstancelist.pyi @@ -26,6 +26,7 @@ class bool(int): pass class str: def __add__(self, x: str) -> str: pass def __getitem__(self, x: int) -> str: pass +class bytes: pass T = TypeVar('T') KT = TypeVar('KT') @@ -52,6 +53,7 @@ class dict(Mapping[KT, VT]): def __setitem__(self, k: KT, v: VT) -> None: pass def __iter__(self) -> Iterator[KT]: pass def update(self, a: Mapping[KT, VT]) -> None: pass + def items(self) -> Iterable[Tuple[KT, VT]]: pass class set(Generic[T]): def __iter__(self) -> Iterator[T]: pass From ad76e1628e2d4fe52d89a80d51890bf84afad38f Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Mon, 28 Jul 2025 20:44:07 +0200 Subject: [PATCH 112/424] Move self argument checks to a later phase - after decorator application, if any (#19490) Fixes #19392. Fixes #18989. Fixes #18720. Fixes #13434 (correct support for `staticmethod` wrappers, but not for equivalent `classmethod`s reported in #18968). Deferring this check in presence of decorators allows decorators that perform non-trivial transformations (such as making methods from non-methods and vice versa). --- mypy/checker.py | 104 ++++++++----- mypy/semanal.py | 7 +- test-data/unit/check-classes.test | 230 ++++++++++++++++++++++++++++- test-data/unit/fine-grained.test | 2 +- test-data/unit/semanal-errors.test | 16 -- 5 files changed, 300 insertions(+), 59 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 97c1e10259c98..f201a767a860e 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -1373,49 +1373,19 @@ def check_func_def( ) # Store argument types. + found_self = False + if isinstance(defn, FuncDef) and not defn.is_decorated: + found_self = self.require_correct_self_argument(typ, defn) for i in range(len(typ.arg_types)): arg_type = typ.arg_types[i] - if ( - isinstance(defn, FuncDef) - and ref_type is not None - and i == 0 - and defn.has_self_or_cls_argument - and typ.arg_kinds[0] not in [nodes.ARG_STAR, nodes.ARG_STAR2] - ): - if defn.is_class or defn.name == "__new__": - ref_type = mypy.types.TypeType.make_normalized(ref_type) - if not is_same_type(arg_type, ref_type): - # This level of erasure matches the one in checkmember.check_self_arg(), - # better keep these two checks consistent. - erased = get_proper_type(erase_typevars(erase_to_bound(arg_type))) - if not is_subtype(ref_type, erased, ignore_type_params=True): - if ( - isinstance(erased, Instance) - and erased.type.is_protocol - or isinstance(erased, TypeType) - and isinstance(erased.item, Instance) - and erased.item.type.is_protocol - ): - # We allow the explicit self-type to be not a supertype of - # the current class if it is a protocol. For such cases - # the consistency check will be performed at call sites. - msg = None - elif typ.arg_names[i] in {"self", "cls"}: - msg = message_registry.ERASED_SELF_TYPE_NOT_SUPERTYPE.format( - erased.str_with_options(self.options), - ref_type.str_with_options(self.options), - ) - else: - msg = message_registry.MISSING_OR_INVALID_SELF_TYPE - if msg: - self.fail(msg, defn) - elif isinstance(arg_type, TypeVarType): + if isinstance(arg_type, TypeVarType): # Refuse covariant parameter type variables # TODO: check recursively for inner type variables if ( arg_type.variance == COVARIANT and defn.name not in ("__init__", "__new__", "__post_init__") and not is_private(defn.name) # private methods are not inherited + and (i != 0 or not found_self) ): ctx: Context = arg_type if ctx.line < 0: @@ -1565,6 +1535,69 @@ def check_func_def( self.binder = old_binder + def require_correct_self_argument(self, func: Type, defn: FuncDef) -> bool: + func = get_proper_type(func) + if not isinstance(func, CallableType): + return False + + # Do not report errors for untyped methods in classes nested in untyped funcs. + if not ( + self.options.check_untyped_defs + or len(self.dynamic_funcs) < 2 + or not self.dynamic_funcs[-2] + or not defn.is_dynamic() + ): + return bool(func.arg_types) + + with self.scope.push_function(defn): + # We temporary push the definition to get the self type as + # visible from *inside* of this function/method. + ref_type: Type | None = self.scope.active_self_type() + if ref_type is None: + return False + + if not defn.has_self_or_cls_argument or ( + func.arg_kinds and func.arg_kinds[0] in [nodes.ARG_STAR, nodes.ARG_STAR2] + ): + return False + + if not func.arg_types: + self.fail( + 'Method must have at least one argument. Did you forget the "self" argument?', defn + ) + return False + + arg_type = func.arg_types[0] + if defn.is_class or defn.name == "__new__": + ref_type = mypy.types.TypeType.make_normalized(ref_type) + if is_same_type(arg_type, ref_type): + return True + + # This level of erasure matches the one in checkmember.check_self_arg(), + # better keep these two checks consistent. + erased = get_proper_type(erase_typevars(erase_to_bound(arg_type))) + if not is_subtype(ref_type, erased, ignore_type_params=True): + if ( + isinstance(erased, Instance) + and erased.type.is_protocol + or isinstance(erased, TypeType) + and isinstance(erased.item, Instance) + and erased.item.type.is_protocol + ): + # We allow the explicit self-type to be not a supertype of + # the current class if it is a protocol. For such cases + # the consistency check will be performed at call sites. + msg = None + elif func.arg_names[0] in {"self", "cls"}: + msg = message_registry.ERASED_SELF_TYPE_NOT_SUPERTYPE.format( + erased.str_with_options(self.options), ref_type.str_with_options(self.options) + ) + else: + msg = message_registry.MISSING_OR_INVALID_SELF_TYPE + if msg: + self.fail(msg, defn) + return True + def is_var_redefined_in_outer_context(self, v: Var, after_line: int) -> bool: """Can the variable be assigned to at module top level or outer function? @@ -5306,6 +5339,7 @@ def visit_decorator_inner( ) if non_trivial_decorator: self.check_untyped_after_decorator(sig, e.func) + self.require_correct_self_argument(sig, e.func) sig = set_callable_name(sig, e.func) e.var.type = sig e.var.is_ready = True diff --git a/mypy/semanal.py b/mypy/semanal.py index 01b7f4989d800..1840e606af378 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1072,12 +1072,7 @@ def prepare_method_signature(self, func: FuncDef, info: TypeInfo, has_self_type: if func.has_self_or_cls_argument: if func.name in ["__init_subclass__", "__class_getitem__"]: func.is_class = True - if not func.arguments: - self.fail( - 'Method must have at least one argument. Did you forget the "self" argument?', - func, - ) - elif isinstance(functype, CallableType): + if func.arguments and isinstance(functype, CallableType): self_type = get_proper_type(functype.arg_types[0]) if isinstance(self_type, AnyType): if has_self_type: diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index ae91815d1e9ef..f713fe69bcd2a 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -3261,7 +3261,10 @@ b.bad = 'a' # E: Incompatible types in assignment (expression has type "str", v from typing import Any class Test: - def __setattr__() -> None: ... # E: Method must have at least one argument. Did you forget the "self" argument? # E: Invalid signature "Callable[[], None]" for "__setattr__" + def __setattr__() -> None: ... \ + # E: Invalid signature "Callable[[], None]" for "__setattr__" \ + # E: Method must have at least one argument. Did you forget the "self" argument? + t = Test() t.crash = 'test' # E: Attribute function "__setattr__" with type "Callable[[], None]" does not accept self argument \ # E: "Test" has no attribute "crash" @@ -7742,6 +7745,231 @@ class Foo: def bad(): # E: Method must have at least one argument. Did you forget the "self" argument? self.x = 0 # E: Name "self" is not defined +[case testMethodSelfArgumentChecks] +from typing import Callable, ParamSpec, TypeVar + +T = TypeVar("T") +P = ParamSpec("P") + +def to_number_1(fn: Callable[[], int]) -> int: + return 0 + +def to_number_2(fn: Callable[[int], int]) -> int: + return 0 + +def to_same_callable(fn: Callable[P, T]) -> Callable[P, T]: + return fn + +class A: + def undecorated() -> None: ... # E: Method must have at least one argument. Did you forget the "self" argument? + + def undecorated_not_self(x: int) -> None: ... # E: Self argument missing for a non-static method (or an invalid type for self) + + def undecorated_not_self_2(self: int) -> None: ... # E: The erased type of self "builtins.int" is not a supertype of its class "__main__.A" + + @to_number_1 + def fn1() -> int: + return 0 + + @to_number_1 # E: Argument 1 to "to_number_1" has incompatible type "Callable[[int], int]"; expected "Callable[[], int]" + def fn2(_x: int) -> int: + return 0 + + @to_number_2 # E: Argument 1 to "to_number_2" has incompatible type "Callable[[], int]"; expected "Callable[[int], int]" + def fn3() -> int: + return 0 + + @to_number_2 + def fn4(_x: int) -> int: + return 0 + + @to_number_2 # E: Argument 1 to "to_number_2" has incompatible type "Callable[[str], int]"; expected "Callable[[int], int]" + def fn5(_x: str) -> int: + return 0 + + @to_same_callable + def g1() -> None: ... # E: Method must have at least one argument. Did you forget the "self" argument? + + @to_same_callable + def g2(x: int) -> None: ... # E: Self argument missing for a non-static method (or an invalid type for self) + + @to_same_callable + def g3(self: int) -> None: ... # E: The erased type of self "builtins.int" is not a supertype of its class "__main__.A" + +reveal_type(A().fn1) # N: Revealed type is "builtins.int" +reveal_type(A().fn2) # N: Revealed type is "builtins.int" +reveal_type(A().fn3) # N: Revealed type is "builtins.int" +reveal_type(A().fn4) # N: Revealed type is "builtins.int" +reveal_type(A().fn5) # N: Revealed type is "builtins.int" + +reveal_type(A().g1) # E: Attribute function "g1" with type "Callable[[], None]" does not accept self argument \ + # N: Revealed type is "def ()" +reveal_type(A().g2) # E: Invalid self argument "A" to attribute function "g2" with type "Callable[[int], None]" \ + # N: Revealed type is "def ()" +reveal_type(A().g3) # E: Invalid self argument "A" to attribute function "g3" with type "Callable[[int], None]" \ + # N: Revealed type is "def ()" +[builtins fixtures/tuple.pyi] + +[case testMethodSelfArgumentChecksConcatenate] +from typing import Callable, ParamSpec, TypeVar +from typing_extensions import Concatenate + +T = TypeVar("T") +P = ParamSpec("P") +R = TypeVar("R") + +def to_same_callable(fn: Callable[Concatenate[T, P], R]) -> Callable[Concatenate[T, P], R]: + return fn + +def remove_first(fn: Callable[Concatenate[T, P], R]) -> Callable[P, R]: + ... + +def add_correct_first(fn: Callable[P, R]) -> Callable[Concatenate["C", P], R]: + ... + +def add_wrong_first(fn: Callable[P, R]) -> Callable[Concatenate[int, P], R]: + ... + +class A: + @to_same_callable # E: Argument 1 to "to_same_callable" has incompatible type "Callable[[], int]"; expected "Callable[[T], int]" + def fn1() -> int: + return 0 + + @to_same_callable + def fn2(_x: int) -> int: # E: Self argument missing for a non-static method (or an invalid type for self) + return 0 + + @to_same_callable + def fn3(self, _x: int) -> int: + return 0 + +reveal_type(A().fn1) # N: Revealed type is "def () -> builtins.int" +reveal_type(A().fn2) # E: Invalid self argument "A" to attribute function "fn2" with type "Callable[[int], int]" \ + # N: Revealed type is "def () -> builtins.int" +reveal_type(A().fn3) # N: Revealed type is "def (_x: builtins.int) -> builtins.int" + +class B: + @remove_first # E: Argument 1 to "remove_first" has incompatible type "Callable[[], int]"; expected "Callable[[T], int]" + def fn1() -> int: # E: Method must have at least one argument. Did you forget the "self" argument? + return 0 + + @remove_first + def fn2(_x: int) -> int: # E: Method must have at least one argument. Did you forget the "self" argument? + return 0 + + @remove_first + def fn3(self, _x: int) -> int: # E: Self argument missing for a non-static method (or an invalid type for self) + return 0 + + @remove_first + def fn4(self, new_self: 'B') -> int: + return 0 + +reveal_type(B().fn1) # E: Attribute function "fn1" with type "Callable[[], int]" does not accept self argument \ + # N: Revealed type is "def () -> builtins.int" +reveal_type(B().fn2) # E: Attribute function "fn2" with type "Callable[[], int]" does not accept self argument \ + # N: Revealed type is "def () -> builtins.int" +reveal_type(B().fn3) # E: Invalid self argument "B" to attribute function "fn3" with type "Callable[[int], int]" \ + # N: Revealed type is "def () -> builtins.int" +reveal_type(B().fn4) # N: Revealed type is "def () -> builtins.int" + +class C: + @add_correct_first + def fn1() -> int: + return 0 + + @add_correct_first + def fn2(_x: int) -> int: + return 0 + + @add_correct_first + def fn3(self, _x: int) -> int: + return 0 + +reveal_type(C().fn1) # N: Revealed type is "def () -> builtins.int" +reveal_type(C().fn2) # N: Revealed type is "def (_x: builtins.int) -> builtins.int" +reveal_type(C().fn3) # N: Revealed type is "def (self: __main__.C, _x: builtins.int) -> builtins.int" + +class D: + @add_wrong_first + def fn1() -> int: # E: Self argument missing for a non-static method (or an invalid type for self) + return 0 + + @add_wrong_first + def fn2(_x: int) -> int: # E: Self argument missing for a non-static method (or an invalid type for self) + return 0 + + @add_wrong_first + def fn3(self, _x: int) -> int: # E: Self argument missing for a non-static method (or an invalid type for self) + return 0 + +reveal_type(D().fn1) # E: Invalid self argument "D" to attribute function "fn1" with type "Callable[[int], int]" \ + # N: Revealed type is "def () -> builtins.int" +reveal_type(D().fn2) # E: Invalid self argument "D" to attribute function "fn2" with type "Callable[[int, int], int]" \ + # N: Revealed type is "def (_x: builtins.int) -> builtins.int" +reveal_type(D().fn3) # E: Invalid self argument "D" to attribute function "fn3" with type "Callable[[int, D, int], int]" \ + # N: Revealed type is "def (self: __main__.D, _x: builtins.int) -> builtins.int" +[builtins fixtures/tuple.pyi] + +[case testMethodSelfArgumentChecksInUntyped] +from typing import Callable, ParamSpec, TypeVar + +T = TypeVar("T") +P = ParamSpec("P") + +def to_same_callable(fn: Callable[P, T]) -> Callable[P, T]: + return fn + +def unchecked(): + class Bad: + def fn() -> None: ... # E: Method must have at least one argument. Did you forget the "self" argument? + def fn2(x: int) -> None: ... # E: Self argument missing for a non-static method (or an invalid type for self) + + # TODO: would be nice to make this error, but now we see the func + # being decorated as Any, not as a callable + @to_same_callable + def gaaa() -> None: ... + @to_same_callable + def gaaa2(x: int) -> None: ... + + class Ok: + def fn(): ... + def fn2(x): ... + + @to_same_callable + def g(): ... + @to_same_callable + def g2(x): ... + +def checked() -> None: + class Bad: + def fn() -> None: ... # E: Method must have at least one argument. Did you forget the "self" argument? + def fn2(x: int) -> None: ... # E: Self argument missing for a non-static method (or an invalid type for self) + + @to_same_callable + def g() -> None: ... # E: Method must have at least one argument. Did you forget the "self" argument? + @to_same_callable + def g2(x: int) -> None: ... # E: Self argument missing for a non-static method (or an invalid type for self) + + class AlsoBad: + def fn(): ... # E: Method must have at least one argument. Did you forget the "self" argument? + def fn2(x): ... + + @to_same_callable + def g(): ... # E: Method must have at least one argument. Did you forget the "self" argument? + @to_same_callable + def g2(x): ... + +class Ok: + def fn(): ... # E: Method must have at least one argument. Did you forget the "self" argument? + def fn2(x): ... + + @to_same_callable + def g(): ... # E: Method must have at least one argument. Did you forget the "self" argument? + @to_same_callable + def g2(x): ... +[builtins fixtures/tuple.pyi] + [case testTypeAfterAttributeAccessWithDisallowAnyExpr] # flags: --disallow-any-expr diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index 503135d901f89..c25ed79e73562 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -1720,7 +1720,7 @@ from typing import Iterator, Callable, List, Optional from a import f import a -def dec(f: Callable[['A'], Optional[Iterator[int]]]) -> Callable[[int], int]: pass +def dec(f: Callable[['A'], Optional[Iterator[int]]]) -> Callable[['A', int], int]: pass class A: @dec diff --git a/test-data/unit/semanal-errors.test b/test-data/unit/semanal-errors.test index 2d381644629b3..8053b33b94fdc 100644 --- a/test-data/unit/semanal-errors.test +++ b/test-data/unit/semanal-errors.test @@ -537,13 +537,6 @@ def f(y: t): x = t main:4: error: "t" is a type variable and only valid in type context main:5: error: "t" is a type variable and only valid in type context -[case testMissingSelf] -import typing -class A: - def f(): pass -[out] -main:3: error: Method must have at least one argument. Did you forget the "self" argument? - [case testInvalidBaseClass] import typing class A(B): pass @@ -558,15 +551,6 @@ def f() -> None: super().y main:2: error: "super" used outside class main:3: error: "super" used outside class -[case testMissingSelfInMethod] -import typing -class A: - def f() -> None: pass - def g(): pass -[out] -main:3: error: Method must have at least one argument. Did you forget the "self" argument? -main:4: error: Method must have at least one argument. Did you forget the "self" argument? - [case testMultipleMethodDefinition] import typing class A: From 703bee951e51083e01406f82e4835deda28e1672 Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Tue, 29 Jul 2025 03:00:16 +0200 Subject: [PATCH 113/424] perf: deduplicate `fast_container_type` and `fast_dict_type` items before joining (#19409) Vastly improves #14718. Some type joins are really heavy - especially joins between overloads. This does not fully remove the problem, a collection of different pairwise equivalent overloads differing only in their names is still slow to check, but at least we will not join every such monster callable with itself multiple times. --- mypy/checkexpr.py | 73 ++++++++++++++++++++++++----- test-data/unit/check-generics.test | 10 ++-- test-data/unit/check-redefine2.test | 2 +- test-data/unit/check-selftype.test | 2 +- 4 files changed, 68 insertions(+), 19 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index a8afebbd9923f..d6982a6bed435 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -5069,31 +5069,56 @@ def fast_container_type( module-level constant definitions. Limitations: + - no active type context + - at least one item - no star expressions - - the joined type of all entries must be an Instance or Tuple type + - not after deferral + - either exactly one distinct type inside, + or the joined type of all entries is an Instance or Tuple type, """ ctx = self.type_context[-1] - if ctx: + if ctx or not e.items: + return None + if self.chk.current_node_deferred: + # Guarantees that all items will be Any, we'll reject it anyway. return None rt = self.resolved_type.get(e, None) if rt is not None: return rt if isinstance(rt, Instance) else None values: list[Type] = [] + # Preserve join order while avoiding O(n) lookups at every iteration + values_set: set[Type] = set() for item in e.items: if isinstance(item, StarExpr): # fallback to slow path self.resolved_type[e] = NoneType() return None - values.append(self.accept(item)) - vt = join.join_type_list(values) - if not allow_fast_container_literal(vt): + + typ = self.accept(item) + if typ not in values_set: + values.append(typ) + values_set.add(typ) + + vt = self._first_or_join_fast_item(values) + if vt is None: self.resolved_type[e] = NoneType() return None ct = self.chk.named_generic_type(container_fullname, [vt]) self.resolved_type[e] = ct return ct + def _first_or_join_fast_item(self, items: list[Type]) -> Type | None: + if len(items) == 1 and not self.chk.current_node_deferred: + return items[0] + typ = join.join_type_list(items) + if not allow_fast_container_literal(typ): + # TODO: This is overly strict, many other types can be joined safely here. + # However, our join implementation isn't bug-free, and some joins may produce + # undesired `Any`s or even more surprising results. + return None + return typ + def check_lst_expr(self, e: ListExpr | SetExpr | TupleExpr, fullname: str, tag: str) -> Type: # fast path t = self.fast_container_type(e, fullname) @@ -5254,18 +5279,30 @@ def fast_dict_type(self, e: DictExpr) -> Type | None: module-level constant definitions. Limitations: + - no active type context + - at least one item - only supported star expressions are other dict instances - - the joined types of all keys and values must be Instance or Tuple types + - either exactly one distinct type (keys and values separately) inside, + or the joined type of all entries is an Instance or Tuple type """ ctx = self.type_context[-1] - if ctx: + if ctx or not e.items: return None + + if self.chk.current_node_deferred: + # Guarantees that all items will be Any, we'll reject it anyway. + return None + rt = self.resolved_type.get(e, None) if rt is not None: return rt if isinstance(rt, Instance) else None + keys: list[Type] = [] values: list[Type] = [] + # Preserve join order while avoiding O(n) lookups at every iteration + keys_set: set[Type] = set() + values_set: set[Type] = set() stargs: tuple[Type, Type] | None = None for key, value in e.items: if key is None: @@ -5280,13 +5317,25 @@ def fast_dict_type(self, e: DictExpr) -> Type | None: self.resolved_type[e] = NoneType() return None else: - keys.append(self.accept(key)) - values.append(self.accept(value)) - kt = join.join_type_list(keys) - vt = join.join_type_list(values) - if not (allow_fast_container_literal(kt) and allow_fast_container_literal(vt)): + key_t = self.accept(key) + if key_t not in keys_set: + keys.append(key_t) + keys_set.add(key_t) + value_t = self.accept(value) + if value_t not in values_set: + values.append(value_t) + values_set.add(value_t) + + kt = self._first_or_join_fast_item(keys) + if kt is None: self.resolved_type[e] = NoneType() return None + + vt = self._first_or_join_fast_item(values) + if vt is None: + self.resolved_type[e] = NoneType() + return None + if stargs and (stargs[0] != kt or stargs[1] != vt): self.resolved_type[e] = NoneType() return None diff --git a/test-data/unit/check-generics.test b/test-data/unit/check-generics.test index 78680684f69bf..abeb5face26fb 100644 --- a/test-data/unit/check-generics.test +++ b/test-data/unit/check-generics.test @@ -2929,8 +2929,8 @@ def mix(fs: List[Callable[[S], T]]) -> Callable[[S], List[T]]: def id(__x: U) -> U: ... fs = [id, id, id] -reveal_type(mix(fs)) # N: Revealed type is "def [S] (S`7) -> builtins.list[S`7]" -reveal_type(mix([id, id, id])) # N: Revealed type is "def [S] (S`9) -> builtins.list[S`9]" +reveal_type(mix(fs)) # N: Revealed type is "def [S] (S`2) -> builtins.list[S`2]" +reveal_type(mix([id, id, id])) # N: Revealed type is "def [S] (S`4) -> builtins.list[S`4]" [builtins fixtures/list.pyi] [case testInferenceAgainstGenericCurry] @@ -3118,11 +3118,11 @@ def dec4_bound(f: Callable[[I], List[T]]) -> Callable[[I], T]: reveal_type(dec1(lambda x: x)) # N: Revealed type is "def [T] (T`3) -> builtins.list[T`3]" reveal_type(dec2(lambda x: x)) # N: Revealed type is "def [S] (S`5) -> builtins.list[S`5]" reveal_type(dec3(lambda x: x[0])) # N: Revealed type is "def [S] (S`8) -> S`8" -reveal_type(dec4(lambda x: [x])) # N: Revealed type is "def [S] (S`12) -> S`12" +reveal_type(dec4(lambda x: [x])) # N: Revealed type is "def [S] (S`11) -> S`11" reveal_type(dec1(lambda x: 1)) # N: Revealed type is "def (builtins.int) -> builtins.list[builtins.int]" reveal_type(dec5(lambda x: x)) # N: Revealed type is "def (builtins.int) -> builtins.list[builtins.int]" -reveal_type(dec3(lambda x: x)) # N: Revealed type is "def [S] (S`20) -> builtins.list[S`20]" -reveal_type(dec4(lambda x: x)) # N: Revealed type is "def [T] (builtins.list[T`24]) -> T`24" +reveal_type(dec3(lambda x: x)) # N: Revealed type is "def [S] (S`19) -> builtins.list[S`19]" +reveal_type(dec4(lambda x: x)) # N: Revealed type is "def [T] (builtins.list[T`23]) -> T`23" dec4_bound(lambda x: x) # E: Value of type variable "I" of "dec4_bound" cannot be "list[T]" [builtins fixtures/list.pyi] diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 3523772611aab..1abe957240b5e 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -1073,7 +1073,7 @@ def f() -> None: while int(): x = [x] - reveal_type(x) # N: Revealed type is "Union[Any, builtins.list[Any], builtins.list[Union[Any, builtins.list[Any]]], builtins.list[Union[Any, builtins.list[Any], builtins.list[Union[Any, builtins.list[Any]]]]], builtins.list[Union[Any, builtins.list[Any], builtins.list[Union[Any, builtins.list[Any]]], builtins.list[Union[Any, builtins.list[Any], builtins.list[Union[Any, builtins.list[Any]]]]]]]]" + reveal_type(x) # N: Revealed type is "Union[Any, builtins.list[Any]]" [case testNewRedefinePartialNoneEmptyList] # flags: --allow-redefinition-new --local-partial-types diff --git a/test-data/unit/check-selftype.test b/test-data/unit/check-selftype.test index 88ca53c8ed66c..05c34eb707966 100644 --- a/test-data/unit/check-selftype.test +++ b/test-data/unit/check-selftype.test @@ -2018,7 +2018,7 @@ class Ben(Object): } @classmethod def doit(cls) -> Foo: - reveal_type(cls.MY_MAP) # N: Revealed type is "builtins.dict[builtins.str, def [Self <: __main__.Foo] (self: Self`4) -> Self`4]" + reveal_type(cls.MY_MAP) # N: Revealed type is "builtins.dict[builtins.str, def [Self <: __main__.Foo] (self: Self`1) -> Self`1]" foo_method = cls.MY_MAP["foo"] return foo_method(Foo()) [builtins fixtures/isinstancelist.pyi] From c53367f87efeebd9781042f0d0790fd11fe5031f Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 29 Jul 2025 02:12:20 +0100 Subject: [PATCH 114/424] Use cache for OpExpr (#19523) Fixes https://github.com/python/mypy/issues/14978 This is irrelevant for self-check (and for most code likely), but it is important for numerical code, where it avoids exponential slowdown in formulas involving arrays etc (see e.g. the original issue). Unless there will be fallout in `mypy_primer` and/or there are objections, I am going to go ahead and merge it. --- mypy/checkexpr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index d6982a6bed435..e954bbd671e67 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -6043,7 +6043,7 @@ def accept( # We cannot use cache inside lambdas, because they skip immediate type # context, and use enclosing one, see infer_lambda_type_using_context(). # TODO: consider using cache for more expression kinds. - elif isinstance(node, (CallExpr, ListExpr, TupleExpr)) and not ( + elif isinstance(node, (CallExpr, ListExpr, TupleExpr, OpExpr)) and not ( self.in_lambda_expr or self.chk.current_node_deferred ): if (node, type_context) in self.expr_cache: From 0f78f9c578bfbc6fbe4a20888353c20e9691178a Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 30 Jul 2025 10:12:29 +0100 Subject: [PATCH 115/424] Use cache for DictExpr as well (#19536) Fixes https://github.com/python/mypy/issues/14271 Fixes https://github.com/python/mypy/issues/14636 TBH examples in those issues are already sufficiently fast (probably because of combined effect of fast dict literals, and the fact that there are some lists and/or function calls in that examples, so some caching already kicks in). But this PR will probably make them even faster. This has ~0 effect on self-check. --- mypy/checkexpr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index e954bbd671e67..ecae451299d72 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -6043,7 +6043,7 @@ def accept( # We cannot use cache inside lambdas, because they skip immediate type # context, and use enclosing one, see infer_lambda_type_using_context(). # TODO: consider using cache for more expression kinds. - elif isinstance(node, (CallExpr, ListExpr, TupleExpr, OpExpr)) and not ( + elif isinstance(node, (CallExpr, ListExpr, TupleExpr, DictExpr, OpExpr)) and not ( self.in_lambda_expr or self.chk.current_node_deferred ): if (node, type_context) in self.expr_cache: From 5750690e625606a8c6875cff07a0ed299703646c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 30 Jul 2025 13:36:06 +0100 Subject: [PATCH 116/424] [mypyc] Make type objects immortal if using free threading (#19538) If they are not immortal, concurrent construction of objects by multiple threads can cause serious contention due to reference count updates. Making them immortal is similar to how both user-defined normal Python classes and built-in types in free-threaded builds are immortal. Fix the issue for both native and non-native classes. Dataclasses still have contention, and they may be harder to fix (this may require a fix in CPython). This speeds up a few micro-benchmarks that construct instances of classes in multiple threads by a big factor (5x+). --- mypyc/irbuild/builder.py | 8 ++++++++ mypyc/irbuild/classdef.py | 8 ++++++++ mypyc/irbuild/ll_builder.py | 8 ++++++++ mypyc/lib-rt/CPy.h | 4 ++++ mypyc/lib-rt/misc_ops.c | 12 +++++++++++- mypyc/lib-rt/mypyc_util.h | 3 ++- mypyc/primitives/misc_ops.py | 15 +++++++++++++++ 7 files changed, 56 insertions(+), 2 deletions(-) diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index 7e63d482c786e..ec3c1b1b1f3cf 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -424,6 +424,10 @@ def new_tuple(self, items: list[Value], line: int) -> Value: def debug_print(self, toprint: str | Value) -> None: return self.builder.debug_print(toprint) + def set_immortal_if_free_threaded(self, v: Value, line: int) -> None: + """Make an object immortal on free-threaded builds (to avoid contention).""" + self.builder.set_immortal_if_free_threaded(v, line) + # Helpers for IR building def add_to_non_ext_dict( @@ -433,6 +437,10 @@ def add_to_non_ext_dict( key_unicode = self.load_str(key) self.primitive_op(dict_set_item_op, [non_ext.dict, key_unicode, val], line) + # It's important that accessing class dictionary items from multiple threads + # doesn't cause contention. + self.builder.set_immortal_if_free_threaded(val, line) + def gen_import(self, id: str, line: int) -> None: self.imports[id] = None diff --git a/mypyc/irbuild/classdef.py b/mypyc/irbuild/classdef.py index 6b59750c7dec5..3282e836ac9e1 100644 --- a/mypyc/irbuild/classdef.py +++ b/mypyc/irbuild/classdef.py @@ -262,6 +262,9 @@ def finalize(self, ir: ClassIR) -> None: non_ext_class = load_non_ext_class(self.builder, ir, self.non_ext, self.cdef.line) non_ext_class = load_decorated_class(self.builder, self.cdef, non_ext_class) + # Try to avoid contention when using free threading. + self.builder.set_immortal_if_free_threaded(non_ext_class, self.cdef.line) + # Save the decorated class self.builder.add( InitStatic(non_ext_class, self.cdef.name, self.builder.module_name, NAMESPACE_TYPE) @@ -449,6 +452,11 @@ def allocate_class(builder: IRBuilder, cdef: ClassDef) -> Value: ) # Create the class tp = builder.call_c(pytype_from_template_op, [template, tp_bases, modname], cdef.line) + + # Set type object to be immortal if free threaded, as otherwise reference count contention + # can cause a big performance hit. + builder.set_immortal_if_free_threaded(tp, cdef.line) + # Immediately fix up the trait vtables, before doing anything with the class. ir = builder.mapper.type_to_ir[cdef.info] if not ir.is_trait and not ir.builtin_base: diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index 79ad4cc62822e..a5e28268efeda 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -6,6 +6,7 @@ from __future__ import annotations +import sys from collections.abc import Sequence from typing import Callable, Final, Optional @@ -16,6 +17,7 @@ from mypyc.common import ( BITMAP_BITS, FAST_ISINSTANCE_MAX_SUBCLASSES, + IS_FREE_THREADED, MAX_LITERAL_SHORT_INT, MAX_SHORT_INT, MIN_LITERAL_SHORT_INT, @@ -164,6 +166,7 @@ fast_isinstance_op, none_object_op, not_implemented_op, + set_immortal_op, var_object_size, ) from mypyc.primitives.registry import ( @@ -2322,6 +2325,11 @@ def new_tuple_with_length(self, length: Value, line: int) -> Value: def int_to_float(self, n: Value, line: int) -> Value: return self.primitive_op(int_to_float_op, [n], line) + def set_immortal_if_free_threaded(self, v: Value, line: int) -> None: + """Make an object immortal on free-threaded builds (to avoid contention).""" + if IS_FREE_THREADED and sys.version_info >= (3, 14): + self.primitive_op(set_immortal_op, [v], line) + # Internal helpers def decompose_union_helper( diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index e7a7f9a076265..1881aa97f3084 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -931,6 +931,10 @@ PyObject *CPy_GetANext(PyObject *aiter); void CPy_SetTypeAliasTypeComputeFunction(PyObject *alias, PyObject *compute_value); void CPyTrace_LogEvent(const char *location, const char *line, const char *op, const char *details); +#if CPY_3_14_FEATURES +void CPy_SetImmortal(PyObject *obj); +#endif + #ifdef __cplusplus } #endif diff --git a/mypyc/lib-rt/misc_ops.c b/mypyc/lib-rt/misc_ops.c index 8aa25cc11e023..3787ea553037b 100644 --- a/mypyc/lib-rt/misc_ops.c +++ b/mypyc/lib-rt/misc_ops.c @@ -1058,7 +1058,7 @@ void CPyTrace_LogEvent(const char *location, const char *line, const char *op, c #endif -#ifdef CPY_3_12_FEATURES +#if CPY_3_12_FEATURES // Copied from Python 3.12.3, since this struct is internal to CPython. It defines // the structure of typing.TypeAliasType objects. We need it since compute_value is @@ -1088,3 +1088,13 @@ void CPy_SetTypeAliasTypeComputeFunction(PyObject *alias, PyObject *compute_valu } #endif + +#if CPY_3_14_FEATURES + +#include "internal/pycore_object.h" + +void CPy_SetImmortal(PyObject *obj) { + _Py_SetImmortal(obj); +} + +#endif diff --git a/mypyc/lib-rt/mypyc_util.h b/mypyc/lib-rt/mypyc_util.h index 3d4eba3a3cdb4..f200d4f90defa 100644 --- a/mypyc/lib-rt/mypyc_util.h +++ b/mypyc/lib-rt/mypyc_util.h @@ -139,8 +139,9 @@ static inline CPyTagged CPyTagged_ShortFromSsize_t(Py_ssize_t x) { return x << 1; } -// Are we targeting Python 3.12 or newer? +// Are we targeting Python 3.X or newer? #define CPY_3_12_FEATURES (PY_VERSION_HEX >= 0x030c0000) +#define CPY_3_14_FEATURES (PY_VERSION_HEX >= 0x030e0000) #if CPY_3_12_FEATURES diff --git a/mypyc/primitives/misc_ops.py b/mypyc/primitives/misc_ops.py index e2a1aea1a8d6a..e3d59f53ed761 100644 --- a/mypyc/primitives/misc_ops.py +++ b/mypyc/primitives/misc_ops.py @@ -311,3 +311,18 @@ return_type=void_rtype, error_kind=ERR_NEVER, ) + +# Mark object as immortal -- it won't be freed via reference counting, as +# the reference count won't be updated any longer. Immortal objects support +# fast concurrent read-only access from multiple threads when using free +# threading, since this eliminates contention from concurrent reference count +# updates. +# +# Needs at least Python 3.14. +set_immortal_op = custom_primitive_op( + name="set_immmortal", + c_function_name="CPy_SetImmortal", + arg_types=[object_rprimitive], + return_type=void_rtype, + error_kind=ERR_NEVER, +) From 43364c1be9d0e53d4e715ad02b7b71628d40db79 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 30 Jul 2025 17:20:01 +0100 Subject: [PATCH 117/424] [mypyc] Enable free threading when compiling multiple modules (#19541) Previously we used multi-phase initialization to enable free threading on Python builds that support it, but only if a single module was compiled in a group. Implements it also for multiple modules in a group. Add support for multi-phase initialization in module shims and the shared library. It's still only used on free-threaded builds, and we fall back to the old approach on other Python versions/builds. This enables compiling mypy and mypyc on free-threaded Python builds. At least almost all mypy and mypyc tests now pass when compiled and on 3.14.0b4 with free threading (only tested on macOS so far). --- mypyc/build.py | 10 +- mypyc/codegen/emitmodule.py | 134 +++++++++++++----- .../lib-rt/module_shim_no_gil_multiphase.tmpl | 41 ++++++ 3 files changed, 146 insertions(+), 39 deletions(-) create mode 100644 mypyc/lib-rt/module_shim_no_gil_multiphase.tmpl diff --git a/mypyc/build.py b/mypyc/build.py index b7d3e1b25366a..4a2d703b9f108 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -36,7 +36,7 @@ from mypy.util import write_junit_xml from mypyc.annotate import generate_annotated_html from mypyc.codegen import emitmodule -from mypyc.common import RUNTIME_C_FILES, shared_lib_name +from mypyc.common import IS_FREE_THREADED, RUNTIME_C_FILES, shared_lib_name from mypyc.errors import Errors from mypyc.ir.pprint import format_modules from mypyc.namegen import exported_name @@ -176,9 +176,15 @@ def generate_c_extension_shim( cname = "%s.c" % full_module_name.replace(".", os.sep) cpath = os.path.join(dir_name, cname) + if IS_FREE_THREADED: + # We use multi-phase init in free-threaded builds to enable free threading. + shim_name = "module_shim_no_gil_multiphase.tmpl" + else: + shim_name = "module_shim.tmpl" + # We load the C extension shim template from a file. # (So that the file could be reused as a bazel template also.) - with open(os.path.join(include_dir(), "module_shim.tmpl")) as f: + with open(os.path.join(include_dir(), shim_name)) as f: shim_template = f.read() write_file( diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 047309ec71e38..de34ed9fc7da4 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -784,28 +784,15 @@ def generate_shared_lib_init(self, emitter: Emitter) -> None: assert self.group_name is not None emitter.emit_line() + + short_name = shared_lib_name(self.group_name).split(".")[-1] + emitter.emit_lines( - "PyMODINIT_FUNC PyInit_{}(void)".format( - shared_lib_name(self.group_name).split(".")[-1] - ), + f"static int exec_{short_name}(PyObject *module)", "{", - ( - 'static PyModuleDef def = {{ PyModuleDef_HEAD_INIT, "{}", NULL, -1, NULL, NULL }};'.format( - shared_lib_name(self.group_name) - ) - ), "int res;", "PyObject *capsule;", "PyObject *tmp;", - "static PyObject *module;", - "if (module) {", - "Py_INCREF(module);", - "return module;", - "}", - "module = PyModule_Create(&def);", - "if (!module) {", - "goto fail;", - "}", "", ) @@ -827,15 +814,26 @@ def generate_shared_lib_init(self, emitter: Emitter) -> None: for mod in self.modules: name = exported_name(mod) + if self.multi_phase_init: + capsule_func_prefix = "CPyExec_" + capsule_name_prefix = "exec_" + emitter.emit_line(f"extern int CPyExec_{name}(PyObject *);") + else: + capsule_func_prefix = "CPyInit_" + capsule_name_prefix = "init_" + emitter.emit_line(f"extern PyObject *CPyInit_{name}(void);") emitter.emit_lines( - f"extern PyObject *CPyInit_{name}(void);", - 'capsule = PyCapsule_New((void *)CPyInit_{}, "{}.init_{}", NULL);'.format( - name, shared_lib_name(self.group_name), name + 'capsule = PyCapsule_New((void *){}{}, "{}.{}{}", NULL);'.format( + capsule_func_prefix, + name, + shared_lib_name(self.group_name), + capsule_name_prefix, + name, ), "if (!capsule) {", "goto fail;", "}", - f'res = PyObject_SetAttrString(module, "init_{name}", capsule);', + f'res = PyObject_SetAttrString(module, "{capsule_name_prefix}{name}", capsule);', "Py_DECREF(capsule);", "if (res < 0) {", "goto fail;", @@ -861,7 +859,56 @@ def generate_shared_lib_init(self, emitter: Emitter) -> None: "", ) - emitter.emit_lines("return module;", "fail:", "Py_XDECREF(module);", "return NULL;", "}") + emitter.emit_lines("return 0;", "fail:", "return -1;", "}") + + if self.multi_phase_init: + emitter.emit_lines( + f"static PyModuleDef_Slot slots_{short_name}[] = {{", + f"{{Py_mod_exec, exec_{short_name}}},", + "{Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED},", + "{Py_mod_gil, Py_MOD_GIL_NOT_USED},", + "{0, NULL},", + "};", + ) + + size = 0 if self.multi_phase_init else -1 + emitter.emit_lines( + f"static PyModuleDef module_def_{short_name} = {{", + "PyModuleDef_HEAD_INIT,", + f'.m_name = "{shared_lib_name(self.group_name)}",', + ".m_doc = NULL,", + f".m_size = {size},", + ".m_methods = NULL,", + ) + if self.multi_phase_init: + emitter.emit_line(f".m_slots = slots_{short_name},") + emitter.emit_line("};") + + if self.multi_phase_init: + emitter.emit_lines( + f"PyMODINIT_FUNC PyInit_{short_name}(void) {{", + f"return PyModuleDef_Init(&module_def_{short_name});", + "}", + ) + else: + emitter.emit_lines( + f"PyMODINIT_FUNC PyInit_{short_name}(void) {{", + "static PyObject *module = NULL;", + "if (module) {", + "Py_INCREF(module);", + "return module;", + "}", + f"module = PyModule_Create(&module_def_{short_name});", + "if (!module) {", + "return NULL;", + "}", + f"if (exec_{short_name}(module) < 0) {{", + "Py_DECREF(module);", + "return NULL;", + "}", + "return module;", + "}", + ) def generate_globals_init(self, emitter: Emitter) -> None: emitter.emit_lines( @@ -887,16 +934,22 @@ def generate_globals_init(self, emitter: Emitter) -> None: def generate_module_def(self, emitter: Emitter, module_name: str, module: ModuleIR) -> None: """Emit the PyModuleDef struct for a module and the module init function.""" module_prefix = emitter.names.private_name(module_name) - self.emit_module_exec_func(emitter, module_name, module_prefix, module) - if self.multi_phase_init: - self.emit_module_def_slots(emitter, module_prefix) self.emit_module_methods(emitter, module_name, module_prefix, module) - self.emit_module_def_struct(emitter, module_name, module_prefix) - self.emit_module_init_func(emitter, module_name, module_prefix) + self.emit_module_exec_func(emitter, module_name, module_prefix, module) - def emit_module_def_slots(self, emitter: Emitter, module_prefix: str) -> None: + # If using multi-phase init and a shared lib, parts of module definition + # will happen in the shim modules, so we skip some steps here. + if not (self.multi_phase_init and self.use_shared_lib): + if self.multi_phase_init: + self.emit_module_def_slots(emitter, module_prefix, module_name) + self.emit_module_def_struct(emitter, module_name, module_prefix) + self.emit_module_init_func(emitter, module_name, module_prefix) + + def emit_module_def_slots( + self, emitter: Emitter, module_prefix: str, module_name: str + ) -> None: name = f"{module_prefix}_slots" - exec_name = f"{module_prefix}_exec" + exec_name = f"CPyExec_{exported_name(module_name)}" emitter.emit_line(f"static PyModuleDef_Slot {name}[] = {{") emitter.emit_line(f"{{Py_mod_exec, {exec_name}}},") @@ -951,7 +1004,7 @@ def emit_module_def_struct( "0, /* size of per-interpreter state of the module */", f"{module_prefix}module_methods,", ) - if self.multi_phase_init: + if self.multi_phase_init and not self.use_shared_lib: slots_name = f"{module_prefix}_slots" emitter.emit_line(f"{slots_name}, /* m_slots */") else: @@ -962,15 +1015,16 @@ def emit_module_def_struct( def emit_module_exec_func( self, emitter: Emitter, module_name: str, module_prefix: str, module: ModuleIR ) -> None: - """Emit the module init function. + """Emit the module exec function. - If we are compiling just one module, this will be the C API init - function. If we are compiling 2+ modules, we generate a shared + If we are compiling just one module, this will be the normal C API + exec function. If we are compiling 2+ modules, we generate a shared library for the modules and shims that call into the shared - library, and in this case we use an internal module initialized - function that will be called by the shim. + library, and in this case the shared module defines an internal + exec function for each module and these will be called by the shims + via Capsules. """ - declaration = f"static int {module_prefix}_exec(PyObject *module)" + declaration = f"int CPyExec_{exported_name(module_name)}(PyObject *module)" module_static = self.module_internal_static_name(module_name, emitter) emitter.emit_lines(declaration, "{") emitter.emit_line("PyObject* modname = NULL;") @@ -987,6 +1041,12 @@ def emit_module_exec_func( " goto fail;", ) + if self.multi_phase_init: + emitter.emit_lines( + f"if (PyModule_AddFunctions(module, {module_prefix}module_methods) < 0)", + " goto fail;", + ) + # HACK: Manually instantiate generated classes here type_structs: list[str] = [] for cl in module.classes: @@ -1038,7 +1098,7 @@ def emit_module_init_func( emitter.emit_line("}") return - exec_func = f"{module_prefix}_exec" + exec_func = f"CPyExec_{exported_name(module_name)}" # Store the module reference in a static and return it when necessary. # This is separate from the *global* reference to the module that will diff --git a/mypyc/lib-rt/module_shim_no_gil_multiphase.tmpl b/mypyc/lib-rt/module_shim_no_gil_multiphase.tmpl new file mode 100644 index 0000000000000..b9bfe9c91962e --- /dev/null +++ b/mypyc/lib-rt/module_shim_no_gil_multiphase.tmpl @@ -0,0 +1,41 @@ +#include + +static int {modname}_exec(PyObject *module) +{{ + PyObject *tmp; + if (!(tmp = PyImport_ImportModule("{libname}"))) return -1; + PyObject *capsule = PyObject_GetAttrString(tmp, "exec_{full_modname}"); + Py_DECREF(tmp); + if (capsule == NULL) return -1; + void *exec_func = PyCapsule_GetPointer(capsule, "{libname}.exec_{full_modname}"); + Py_DECREF(capsule); + if (!exec_func) return -1; + if (((int (*)(PyObject *))exec_func)(module) != 0) return -1; + return 0; +}} + +static PyModuleDef_Slot {modname}_slots[] = {{ + {{Py_mod_exec, {modname}_exec}}, + {{Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}}, + {{Py_mod_gil, Py_MOD_GIL_NOT_USED}}, + {{0, NULL}}, +}}; + +static struct PyModuleDef {modname}_module = {{ + PyModuleDef_HEAD_INIT, + .m_name = "{modname}", + .m_doc = NULL, + .m_methods = NULL, + .m_size = 0, + .m_slots = {modname}_slots, +}}; + +PyMODINIT_FUNC +PyInit_{modname}(void) +{{ + return PyModuleDef_Init(&{modname}_module); +}} + +// distutils sometimes spuriously tells cl to export CPyInit___init__, +// so provide that so it chills out +PyMODINIT_FUNC PyInit___init__(void) {{ return PyInit_{modname}(); }} From e727ea69d7ff2b2e1a758b80983f5962404b4e9d Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Thu, 31 Jul 2025 01:24:08 +0200 Subject: [PATCH 118/424] Update test requirements (#19539) --- test-requirements.txt | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index bcdf02319306b..11ac675eca15e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -8,9 +8,9 @@ attrs==25.3.0 # via -r test-requirements.in cfgv==3.4.0 # via pre-commit -coverage==7.8.2 +coverage==7.10.1 # via pytest-cov -distlib==0.3.9 +distlib==0.4.0 # via virtualenv execnet==2.1.1 # via pytest-xdist @@ -22,7 +22,7 @@ identify==2.6.12 # via pre-commit iniconfig==2.1.0 # via pytest -lxml==5.4.0 ; python_version < "3.14" +lxml==6.0.0 ; python_version < "3.14" # via -r test-requirements.in mypy-extensions==1.1.0 # via -r mypy-requirements.txt @@ -35,31 +35,35 @@ pathspec==0.12.1 platformdirs==4.3.8 # via virtualenv pluggy==1.6.0 - # via pytest + # via + # pytest + # pytest-cov pre-commit==4.2.0 # via -r test-requirements.in psutil==7.0.0 # via -r test-requirements.in -pytest==8.3.5 +pygments==2.19.2 + # via pytest +pytest==8.4.1 # via # -r test-requirements.in # pytest-cov # pytest-xdist -pytest-cov==6.1.1 +pytest-cov==6.2.1 # via -r test-requirements.in -pytest-xdist==3.7.0 +pytest-xdist==3.8.0 # via -r test-requirements.in pyyaml==6.0.2 # via pre-commit tomli==2.2.1 # via -r test-requirements.in -types-psutil==7.0.0.20250516 +types-psutil==7.0.0.20250601 # via -r build-requirements.txt -types-setuptools==80.8.0.20250521 +types-setuptools==80.9.0.20250529 # via -r build-requirements.txt -typing-extensions==4.13.2 +typing-extensions==4.14.1 # via -r mypy-requirements.txt -virtualenv==20.31.2 +virtualenv==20.32.0 # via pre-commit # The following packages are considered to be unsafe in a requirements file: From 270142f90c76f9137cbd2c2ac57568d1f2e17a58 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Wed, 30 Jul 2025 23:55:04 -0700 Subject: [PATCH 119/424] Update changelog for 1.17.1 (#19550) --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a74fb46aba6b6..5bdb888ff9d6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -170,6 +170,11 @@ Related PRs: * Enable ANSI color codes for dmypy client in Windows (wyattscarpenter, PR [19088](https://github.com/python/mypy/pull/19088)) * Extend special case for context-based type variable inference to unions in return position (Stanislav Terliakov, PR [18976](https://github.com/python/mypy/pull/18976)) +### Mypy 1.17.1 +* Retain `None` as constraints bottom if no bottoms were provided (Stanislav Terliakov, PR [19485](https://github.com/python/mypy/pull/19485)) +* Fix "ignored exception in `hasattr`" in dmypy (Stanislav Terliakov, PR [19428](https://github.com/python/mypy/pull/19428)) +* Prevent a crash when InitVar is redefined with a method in a subclass (Stanislav Terliakov, PR [19453](https://github.com/python/mypy/pull/19453)) + ### Acknowledgements Thanks to all mypy contributors who contributed to this release: From 7534898319cb7f16738c11e4bc1bdcef0eb13c38 Mon Sep 17 00:00:00 2001 From: Emily Date: Thu, 31 Jul 2025 08:57:41 +0100 Subject: [PATCH 120/424] =?UTF-8?q?Explicitly=20check=20case=E2=80=90sensi?= =?UTF-8?q?tivity=20of=20file=20system=20for=20tests=20(#19540)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both macOS and Windows support using case‐sensitive file systems; this fixes the test suite in those situations. --- mypy/test/testcheck.py | 11 +++++++---- mypy/test/testfscache.py | 3 +-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index fb2eb3a75b9b8..206eab726a216 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -5,6 +5,8 @@ import os import re import sys +import tempfile +from pathlib import Path from mypy import build from mypy.build import Graph @@ -46,15 +48,16 @@ if sys.version_info < (3, 13): typecheck_files.remove("check-python313.test") -# Special tests for platforms with case-insensitive filesystems. -if sys.platform not in ("darwin", "win32"): - typecheck_files.remove("check-modules-case.test") - class TypeCheckSuite(DataSuite): files = typecheck_files def run_case(self, testcase: DataDrivenTestCase) -> None: + if os.path.basename(testcase.file) == "check-modules-case.test": + with tempfile.NamedTemporaryFile(prefix="test", dir=".") as temp_file: + temp_path = Path(temp_file.name) + if not temp_path.with_name(temp_path.name.upper()).exists(): + pytest.skip("File system is not case‐insensitive") if lxml is None and os.path.basename(testcase.file) == "check-reports.test": pytest.skip("Cannot import lxml. Is it installed?") incremental = ( diff --git a/mypy/test/testfscache.py b/mypy/test/testfscache.py index 44b0d32f57977..529402dade96c 100644 --- a/mypy/test/testfscache.py +++ b/mypy/test/testfscache.py @@ -4,7 +4,6 @@ import os import shutil -import sys import tempfile import unittest @@ -83,7 +82,7 @@ def test_isfile_case_other_directory(self) -> None: assert self.isfile_case(os.path.join(other, "other_dir.py")) assert not self.isfile_case(os.path.join(other, "Other_Dir.py")) assert not self.isfile_case(os.path.join(other, "bar.py")) - if sys.platform in ("win32", "darwin"): + if os.path.exists(os.path.join(other, "PKG/other_dir.py")): # We only check case for directories under our prefix, and since # this path is not under the prefix, case difference is fine. assert self.isfile_case(os.path.join(other, "PKG/other_dir.py")) From 07d4a1bf9c73f39d6c05d6665c1fafc8f12c3e77 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Thu, 31 Jul 2025 14:21:01 -0400 Subject: [PATCH 121/424] feat: new mypyc primitive for weakref.ref (#19099) This PR adds a new mypyc primitive for `weakref.ref` I wasn't able to figure out what name mypyc expects for `weakref.proxy`, so I took that out and will keep that for a separate PR later on. ref is more commonly used than proxy anyway. for later, I tried: - weakref.proxy - weakref.ProxyType - weakref.weakproxy no luck with those also for later, I'll need to finish #19145 to add a primitive for `weakref.ref.__call__` --- mypyc/primitives/registry.py | 3 +- mypyc/primitives/weakref_ops.py | 22 ++++++++++++ mypyc/test-data/irbuild-weakref.test | 51 ++++++++++++++++++++++++++++ mypyc/test-data/run-weakref.test | 30 ++++++++++++++++ mypyc/test/test_irbuild.py | 1 + mypyc/test/test_run.py | 1 + test-data/unit/lib-stub/weakref.pyi | 11 ++++++ 7 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 mypyc/primitives/weakref_ops.py create mode 100644 mypyc/test-data/irbuild-weakref.test create mode 100644 mypyc/test-data/run-weakref.test create mode 100644 test-data/unit/lib-stub/weakref.pyi diff --git a/mypyc/primitives/registry.py b/mypyc/primitives/registry.py index 5e7ecb70f55d8..07546663d08ed 100644 --- a/mypyc/primitives/registry.py +++ b/mypyc/primitives/registry.py @@ -371,4 +371,5 @@ def load_address_op(name: str, type: RType, src: str) -> LoadAddressDescription: import mypyc.primitives.list_ops import mypyc.primitives.misc_ops import mypyc.primitives.str_ops -import mypyc.primitives.tuple_ops # noqa: F401 +import mypyc.primitives.tuple_ops +import mypyc.primitives.weakref_ops # noqa: F401 diff --git a/mypyc/primitives/weakref_ops.py b/mypyc/primitives/weakref_ops.py new file mode 100644 index 0000000000000..a7ac035b22a4e --- /dev/null +++ b/mypyc/primitives/weakref_ops.py @@ -0,0 +1,22 @@ +from mypyc.ir.ops import ERR_MAGIC +from mypyc.ir.rtypes import object_rprimitive, pointer_rprimitive +from mypyc.primitives.registry import function_op + +# Weakref operations + +new_ref_op = function_op( + name="weakref.ReferenceType", + arg_types=[object_rprimitive], + return_type=object_rprimitive, + c_function_name="PyWeakref_NewRef", + extra_int_constants=[(0, pointer_rprimitive)], + error_kind=ERR_MAGIC, +) + +new_ref__with_callback_op = function_op( + name="weakref.ReferenceType", + arg_types=[object_rprimitive, object_rprimitive], + return_type=object_rprimitive, + c_function_name="PyWeakref_NewRef", + error_kind=ERR_MAGIC, +) diff --git a/mypyc/test-data/irbuild-weakref.test b/mypyc/test-data/irbuild-weakref.test new file mode 100644 index 0000000000000..58ac6417d2970 --- /dev/null +++ b/mypyc/test-data/irbuild-weakref.test @@ -0,0 +1,51 @@ +[case testWeakrefRef] +import weakref +from typing import Any, Callable +def f(x: object) -> object: + return weakref.ref(x) + +[out] +def f(x): + x, r0 :: object +L0: + r0 = PyWeakref_NewRef(x, 0) + return r0 + +[case testWeakrefRefCallback] +import weakref +from typing import Any, Callable +def f(x: object, cb: Callable[[object], Any]) -> object: + return weakref.ref(x, cb) + +[out] +def f(x, cb): + x, cb, r0 :: object +L0: + r0 = PyWeakref_NewRef(x, cb) + return r0 + +[case testFromWeakrefRef] +from typing import Any, Callable +from weakref import ref +def f(x: object) -> object: + return ref(x) + +[out] +def f(x): + x, r0 :: object +L0: + r0 = PyWeakref_NewRef(x, 0) + return r0 + +[case testFromWeakrefRefCallback] +from typing import Any, Callable +from weakref import ref +def f(x: object, cb: Callable[[object], Any]) -> object: + return ref(x, cb) + +[out] +def f(x, cb): + x, cb, r0 :: object +L0: + r0 = PyWeakref_NewRef(x, cb) + return r0 diff --git a/mypyc/test-data/run-weakref.test b/mypyc/test-data/run-weakref.test new file mode 100644 index 0000000000000..902c9e407ff45 --- /dev/null +++ b/mypyc/test-data/run-weakref.test @@ -0,0 +1,30 @@ +# Test cases for weakrefs (compile and run) + +[case testWeakrefRef] +from weakref import ref +from mypy_extensions import mypyc_attr + +@mypyc_attr(native_class=False) +class Object: + """some random weakreffable object""" + pass + +def test_weakref_ref(): + obj = Object() + r = ref(obj) + assert r() is obj + obj = None + assert r() is None, r() + +def test_weakref_ref_with_callback(): + obj = Object() + r = ref(obj, lambda x: x) + assert r() is obj + obj = None + assert r() is None, r() + +[file driver.py] +from native import test_weakref_ref, test_weakref_ref_with_callback + +test_weakref_ref() +test_weakref_ref_with_callback() diff --git a/mypyc/test/test_irbuild.py b/mypyc/test/test_irbuild.py index 9c0ad06416a7e..d8f974ef201b6 100644 --- a/mypyc/test/test_irbuild.py +++ b/mypyc/test/test_irbuild.py @@ -53,6 +53,7 @@ "irbuild-constant-fold.test", "irbuild-glue-methods.test", "irbuild-math.test", + "irbuild-weakref.test", ] if sys.version_info >= (3, 10): diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py index fcc24403df8ec..5078426b977f5 100644 --- a/mypyc/test/test_run.py +++ b/mypyc/test/test_run.py @@ -70,6 +70,7 @@ "run-singledispatch.test", "run-attrs.test", "run-signatures.test", + "run-weakref.test", "run-python37.test", "run-python38.test", ] diff --git a/test-data/unit/lib-stub/weakref.pyi b/test-data/unit/lib-stub/weakref.pyi new file mode 100644 index 0000000000000..34e01f4d48f1f --- /dev/null +++ b/test-data/unit/lib-stub/weakref.pyi @@ -0,0 +1,11 @@ +from collections.abc import Callable +from typing import Any, Generic, TypeVar +from typing_extensions import Self + +_T = TypeVar("_T") + +class ReferenceType(Generic[_T]): # "weakref" + __callback__: Callable[[Self], Any] + def __new__(cls, o: _T, callback: Callable[[Self], Any] | None = ..., /) -> Self: ... + +ref = ReferenceType From 7c4ec520f4b03d95d4bf26801f8541022728c43c Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 1 Aug 2025 18:15:42 +0100 Subject: [PATCH 122/424] Keep trivial instances and aliases during expansion (#19543) This weirdly looking change consistently shows 1% performance improvement on my machine (Python 3.12, compiled). The key here is to _not_ create new objects for trivial instances and aliases (i.e. those with no `.args`). The problem however is that some callers modify expanded type aliases _in place_ (for better error locations). I updated couple places discovered by tests, but there may be more. I think we should go ahead, and then fix bugs exposed by this using following strategy: * Always use original aliases (not theirs expansions) as error locations (see change in `semanal.py`). * If above is not possible (see unpacked tuple change), then call `t.copy_modified()` followed by `t.set_line()` manually. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- mypy/expandtype.py | 5 ++-- mypy/meet.py | 11 ++++---- mypy/plugins/proper_plugin.py | 1 + mypy/semanal.py | 8 +++--- mypy/types.py | 51 +++++++++++++++-------------------- 5 files changed, 35 insertions(+), 41 deletions(-) diff --git a/mypy/expandtype.py b/mypy/expandtype.py index f704df3b010e3..8433708eda44a 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -210,8 +210,7 @@ def visit_erased_type(self, t: ErasedType) -> Type: def visit_instance(self, t: Instance) -> Type: if len(t.args) == 0: - # TODO: Why do we need to create a copy here? - return t.copy_modified() + return t args = self.expand_type_tuple_with_unpack(t.args) @@ -525,6 +524,8 @@ def visit_type_type(self, t: TypeType) -> Type: def visit_type_alias_type(self, t: TypeAliasType) -> Type: # Target of the type alias cannot contain type variables (not bound by the type # alias itself), so we just expand the arguments. + if len(t.args) == 0: + return t args = self.expand_type_list_with_unpack(t.args) # TODO: normalize if target is Tuple, and args are [*tuple[X, ...]]? return t.copy_modified(args=args) diff --git a/mypy/meet.py b/mypy/meet.py index 2e238be7765e2..fb35bce438ab1 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -116,8 +116,11 @@ def meet_types(s: Type, t: Type) -> ProperType: def narrow_declared_type(declared: Type, narrowed: Type) -> Type: """Return the declared type narrowed down to another type.""" # TODO: check infinite recursion for aliases here. - if isinstance(narrowed, TypeGuardedType): # type: ignore[misc] - # A type guard forces the new type even if it doesn't overlap the old. + if isinstance(narrowed, TypeGuardedType): + # A type guard forces the new type even if it doesn't overlap the old... + if is_proper_subtype(declared, narrowed.type_guard, ignore_promotions=True): + # ...unless it is a proper supertype of declared type. + return declared return narrowed.type_guard original_declared = declared @@ -308,9 +311,7 @@ def is_overlapping_types( positives), for example: None only overlaps with explicitly optional types, Any doesn't overlap with anything except object, we don't ignore positional argument names. """ - if isinstance(left, TypeGuardedType) or isinstance( # type: ignore[misc] - right, TypeGuardedType - ): + if isinstance(left, TypeGuardedType) or isinstance(right, TypeGuardedType): # A type guard forces the new type even if it doesn't overlap the old. return True diff --git a/mypy/plugins/proper_plugin.py b/mypy/plugins/proper_plugin.py index f51685c80afad..0189bfbd22fcd 100644 --- a/mypy/plugins/proper_plugin.py +++ b/mypy/plugins/proper_plugin.py @@ -107,6 +107,7 @@ def is_special_target(right: ProperType) -> bool: "mypy.types.DeletedType", "mypy.types.RequiredType", "mypy.types.ReadOnlyType", + "mypy.types.TypeGuardedType", ): # Special case: these are not valid targets for a type alias and thus safe. # TODO: introduce a SyntheticType base to simplify this? diff --git a/mypy/semanal.py b/mypy/semanal.py index 1840e606af378..7cca406b661b6 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1046,12 +1046,12 @@ def remove_unpack_kwargs(self, defn: FuncDef, typ: CallableType) -> CallableType last_type = typ.arg_types[-1] if not isinstance(last_type, UnpackType): return typ - last_type = get_proper_type(last_type.type) - if not isinstance(last_type, TypedDictType): + p_last_type = get_proper_type(last_type.type) + if not isinstance(p_last_type, TypedDictType): self.fail("Unpack item in ** argument must be a TypedDict", last_type) new_arg_types = typ.arg_types[:-1] + [AnyType(TypeOfAny.from_error)] return typ.copy_modified(arg_types=new_arg_types) - overlap = set(typ.arg_names) & set(last_type.items) + overlap = set(typ.arg_names) & set(p_last_type.items) # It is OK for TypedDict to have a key named 'kwargs'. overlap.discard(typ.arg_names[-1]) if overlap: @@ -1060,7 +1060,7 @@ def remove_unpack_kwargs(self, defn: FuncDef, typ: CallableType) -> CallableType new_arg_types = typ.arg_types[:-1] + [AnyType(TypeOfAny.from_error)] return typ.copy_modified(arg_types=new_arg_types) # OK, everything looks right now, mark the callable type as using unpack. - new_arg_types = typ.arg_types[:-1] + [last_type] + new_arg_types = typ.arg_types[:-1] + [p_last_type] return typ.copy_modified(arg_types=new_arg_types, unpack_kwargs=True) def prepare_method_signature(self, func: FuncDef, info: TypeInfo, has_self_type: bool) -> None: diff --git a/mypy/types.py b/mypy/types.py index e9d299dbc8fc3..4b5ef332ccf90 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -355,11 +355,7 @@ def _expand_once(self) -> Type: ): mapping[tvar.id] = sub - new_tp = self.alias.target.accept(InstantiateAliasVisitor(mapping)) - new_tp.accept(LocationSetter(self.line, self.column)) - new_tp.line = self.line - new_tp.column = self.column - return new_tp + return self.alias.target.accept(InstantiateAliasVisitor(mapping)) def _partial_expansion(self, nothing_args: bool = False) -> tuple[ProperType, bool]: # Private method mostly for debugging and testing. @@ -3214,7 +3210,8 @@ def get_proper_type(typ: Type | None) -> ProperType | None: """ if typ is None: return None - if isinstance(typ, TypeGuardedType): # type: ignore[misc] + # TODO: this is an ugly hack, remove. + if isinstance(typ, TypeGuardedType): typ = typ.type_guard while isinstance(typ, TypeAliasType): typ = typ._expand_once() @@ -3238,9 +3235,7 @@ def get_proper_types( if isinstance(types, list): typelist = types # Optimize for the common case so that we don't need to allocate anything - if not any( - isinstance(t, (TypeAliasType, TypeGuardedType)) for t in typelist # type: ignore[misc] - ): + if not any(isinstance(t, (TypeAliasType, TypeGuardedType)) for t in typelist): return cast("list[ProperType]", typelist) return [get_proper_type(t) for t in typelist] else: @@ -3260,7 +3255,6 @@ def get_proper_types( TypeTranslator as TypeTranslator, TypeVisitor as TypeVisitor, ) -from mypy.typetraverser import TypeTraverserVisitor class TypeStrVisitor(SyntheticTypeVisitor[str]): @@ -3598,23 +3592,6 @@ def is_named_instance(t: Type, fullnames: str | tuple[str, ...]) -> TypeGuard[In return isinstance(t, Instance) and t.type.fullname in fullnames -class LocationSetter(TypeTraverserVisitor): - # TODO: Should we update locations of other Type subclasses? - def __init__(self, line: int, column: int) -> None: - self.line = line - self.column = column - - def visit_instance(self, typ: Instance) -> None: - typ.line = self.line - typ.column = self.column - super().visit_instance(typ) - - def visit_type_alias_type(self, typ: TypeAliasType) -> None: - typ.line = self.line - typ.column = self.column - super().visit_type_alias_type(typ) - - class HasTypeVars(BoolTypeQuery): """Visitor for querying whether a type has a type variable component.""" @@ -3709,8 +3686,8 @@ def flatten_nested_unions( flat_items: list[Type] = [] for t in typelist: - if handle_type_alias_type: - if not handle_recursive and isinstance(t, TypeAliasType) and t.is_recursive: + if handle_type_alias_type and isinstance(t, TypeAliasType): + if not handle_recursive and t.is_recursive: tp: Type = t else: tp = get_proper_type(t) @@ -3757,7 +3734,21 @@ def flatten_nested_tuples(types: Iterable[Type]) -> list[Type]: if not isinstance(p_type, TupleType): res.append(typ) continue - res.extend(flatten_nested_tuples(p_type.items)) + if isinstance(typ.type, TypeAliasType): + items = [] + for item in p_type.items: + if ( + isinstance(item, ProperType) + and isinstance(item, Instance) + or isinstance(item, TypeAliasType) + ): + if len(item.args) == 0: + item = item.copy_modified() + item.set_line(typ) + items.append(item) + else: + items = p_type.items + res.extend(flatten_nested_tuples(items)) return res From 45ee5a34a025e7b10be458bb6b6271e4b03c1138 Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Sat, 2 Aug 2025 02:27:01 +0200 Subject: [PATCH 123/424] fix: prevent crash on dataclass with PEP695 TypeVarTuple on py3.13 (#19565) Fixes #19559. Replaces ad-hoc Instance creation with regular `fill_typevars`. Huge thanks to @A5rocks for the pointer! --- mypy/plugins/dataclasses.py | 3 +-- test-data/unit/check-python312.test | 11 +++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/mypy/plugins/dataclasses.py b/mypy/plugins/dataclasses.py index ee6f8889b8946..e916ded01dd2a 100644 --- a/mypy/plugins/dataclasses.py +++ b/mypy/plugins/dataclasses.py @@ -410,13 +410,12 @@ def _add_dunder_replace(self, attributes: list[DataclassAttribute]) -> None: for attr in attributes if attr.is_in_init ] - type_vars = [tv for tv in self._cls.type_vars] add_method_to_class( self._api, self._cls, "__replace__", args=args, - return_type=Instance(self._cls.info, type_vars), + return_type=fill_typevars(self._cls.info), ) def _add_internal_replace_method(self, attributes: list[DataclassAttribute]) -> None: diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index bfd6334b50772..d275503dc411b 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -2084,3 +2084,14 @@ reveal_type(A1().x) # N: Revealed type is "builtins.int" reveal_type(A2().x) # N: Revealed type is "builtins.int" reveal_type(A3().x) # N: Revealed type is "builtins.list[builtins.int]" [builtins fixtures/tuple.pyi] + +[case testDataclassWithTypeVarTuple] +# flags: --python-version 3.13 +# https://github.com/python/mypy/issues/19559 +from typing import Callable +from dataclasses import dataclass + +@dataclass +class Test[*Ts, R]: + a: Callable[[*Ts], R] +[builtins fixtures/dict.pyi] From 8e02657be8217146142fd374a7229f239452f871 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 2 Aug 2025 15:53:34 +0100 Subject: [PATCH 124/424] Optimize bind_self() and deprecation checks (#19556) This contains two related optimizations: * Simplify deprecation check by removing several `bind_self()` calls and instead restoring callable type definitions in `fixup.py`, and using them during overload item matching. * Consequently, best effort filtering in `bind_self()` should be not needed anymore, since all non-trivial calls to `bind_self()` are now after `check_self_arg()` calls. There are also few micro-optimizations I noticed when looking at relevant code. In total this gives around 1% on self-check. Note: this may be not a no-op in some corner cases. If so, I will adjust overload filtering in `check_self_arg()`. --- mypy/checker.py | 30 ++---------- mypy/checker_shared.py | 6 --- mypy/checkexpr.py | 27 +++-------- mypy/checkmember.py | 74 +++++++++++++++++------------ mypy/erasetype.py | 19 +++++--- mypy/fixup.py | 2 + mypy/meet.py | 24 +++++----- mypy/nodes.py | 6 ++- mypy/typeops.py | 62 ++---------------------- test-data/unit/check-selftype.test | 27 +++++++++++ test-data/unit/check-serialize.test | 1 + 11 files changed, 114 insertions(+), 164 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index f201a767a860e..35a67d1883111 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -726,6 +726,7 @@ def _visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: assert isinstance(item, Decorator) item_type = self.extract_callable_type(item.var.type, item) if item_type is not None: + item_type.definition = item item_types.append(item_type) if item_types: defn.type = Overloaded(item_types) @@ -4927,17 +4928,7 @@ def visit_operator_assignment_stmt(self, s: OperatorAssignmentStmt) -> None: inplace, method = infer_operator_assignment_method(lvalue_type, s.op) if inplace: # There is __ifoo__, treat as x = x.__ifoo__(y) - rvalue_type, method_type = self.expr_checker.check_op(method, lvalue_type, s.rvalue, s) - if isinstance(inst := get_proper_type(lvalue_type), Instance) and isinstance( - defn := inst.type.get_method(method), OverloadedFuncDef - ): - for item in defn.items: - if ( - isinstance(item, Decorator) - and isinstance(typ := item.func.type, CallableType) - and (bind_self(typ) == method_type) - ): - self.warn_deprecated(item.func, s) + rvalue_type, _ = self.expr_checker.check_op(method, lvalue_type, s.rvalue, s) if not is_subtype(rvalue_type, lvalue_type): self.msg.incompatible_operator_assignment(s.op, s) else: @@ -7962,7 +7953,7 @@ def warn_deprecated(self, node: Node | None, context: Context) -> None: node = node.func if ( isinstance(node, (FuncDef, OverloadedFuncDef, TypeInfo)) - and ((deprecated := node.deprecated) is not None) + and (deprecated := node.deprecated) is not None and not self.is_typeshed_stub and not any( node.fullname == p or node.fullname.startswith(f"{p}.") @@ -7972,21 +7963,6 @@ def warn_deprecated(self, node: Node | None, context: Context) -> None: warn = self.msg.note if self.options.report_deprecated_as_note else self.msg.fail warn(deprecated, context, code=codes.DEPRECATED) - def warn_deprecated_overload_item( - self, node: Node | None, context: Context, *, target: Type, selftype: Type | None = None - ) -> None: - """Warn if the overload item corresponding to the given callable is deprecated.""" - target = get_proper_type(target) - if isinstance(node, OverloadedFuncDef) and isinstance(target, CallableType): - for item in node.items: - if isinstance(item, Decorator) and isinstance( - candidate := item.func.type, CallableType - ): - if selftype is not None and not node.is_static: - candidate = bind_self(candidate, selftype) - if candidate == target: - self.warn_deprecated(item.func, context) - # leafs def visit_pass_stmt(self, o: PassStmt, /) -> None: diff --git a/mypy/checker_shared.py b/mypy/checker_shared.py index 7a5e9cb52c70f..2a8fbdb0c9f19 100644 --- a/mypy/checker_shared.py +++ b/mypy/checker_shared.py @@ -253,12 +253,6 @@ def check_deprecated(self, node: Node | None, context: Context) -> None: def warn_deprecated(self, node: Node | None, context: Context) -> None: raise NotImplementedError - @abstractmethod - def warn_deprecated_overload_item( - self, node: Node | None, context: Context, *, target: Type, selftype: Type | None = None - ) -> None: - raise NotImplementedError - @abstractmethod def type_is_iterable(self, type: Type) -> bool: raise NotImplementedError diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index ecae451299d72..bec34eef49838 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -129,7 +129,6 @@ validate_instance, ) from mypy.typeops import ( - bind_self, callable_type, custom_special_method, erase_to_union_or_bound, @@ -1517,15 +1516,6 @@ def check_call_expr_with_callee_type( object_type=object_type, ) proper_callee = get_proper_type(callee_type) - if isinstance(e.callee, (NameExpr, MemberExpr)): - node = e.callee.node - if node is None and member is not None and isinstance(object_type, Instance): - if (symbol := object_type.type.get(member)) is not None: - node = symbol.node - self.chk.check_deprecated(node, e) - self.chk.warn_deprecated_overload_item( - node, e, target=callee_type, selftype=object_type - ) if isinstance(e.callee, RefExpr) and isinstance(proper_callee, CallableType): # Cache it for find_isinstance_check() if proper_callee.type_guard is not None: @@ -2943,6 +2933,8 @@ def infer_overload_return_type( # check for ambiguity due to 'Any' below. if not args_contain_any: self.chk.store_types(m) + if isinstance(infer_type, ProperType) and isinstance(infer_type, CallableType): + self.chk.check_deprecated(infer_type.definition, context) return ret_type, infer_type p_infer_type = get_proper_type(infer_type) if isinstance(p_infer_type, CallableType): @@ -2979,6 +2971,11 @@ def infer_overload_return_type( else: # Success! No ambiguity; return the first match. self.chk.store_types(type_maps[0]) + inferred_callable = inferred_types[0] + if isinstance(inferred_callable, ProperType) and isinstance( + inferred_callable, CallableType + ): + self.chk.check_deprecated(inferred_callable.definition, context) return return_types[0], inferred_types[0] def overload_erased_call_targets( @@ -4103,16 +4100,6 @@ def lookup_definer(typ: Instance, attr_name: str) -> str | None: errors.append(local_errors.filtered_errors()) results.append(result) else: - if isinstance(obj, Instance) and isinstance( - defn := obj.type.get_method(name), OverloadedFuncDef - ): - for item in defn.items: - if ( - isinstance(item, Decorator) - and isinstance(typ := item.func.type, CallableType) - and bind_self(typ) == result[1] - ): - self.chk.check_deprecated(item.func, context) return result # We finish invoking above operators and no early return happens. Therefore, diff --git a/mypy/checkmember.py b/mypy/checkmember.py index 7eedab2e399a9..8447dfc7fe647 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -5,7 +5,7 @@ from collections.abc import Sequence from typing import Callable, TypeVar, cast -from mypy import message_registry, state, subtypes +from mypy import message_registry, state from mypy.checker_shared import TypeCheckerSharedApi from mypy.erasetype import erase_typevars from mypy.expandtype import ( @@ -14,6 +14,7 @@ freshen_all_functions_type_vars, ) from mypy.maptype import map_instance_to_supertype +from mypy.meet import is_overlapping_types from mypy.messages import MessageBuilder from mypy.nodes import ( ARG_POS, @@ -39,6 +40,7 @@ is_final_node, ) from mypy.plugin import AttributeContext +from mypy.subtypes import is_subtype from mypy.typeops import ( bind_self, erase_to_bound, @@ -745,10 +747,8 @@ def analyze_descriptor_access(descriptor_type: Type, mx: MemberContext) -> Type: callable_name=callable_name, ) - mx.chk.check_deprecated(dunder_get, mx.context) - mx.chk.warn_deprecated_overload_item( - dunder_get, mx.context, target=inferred_dunder_get_type, selftype=descriptor_type - ) + # Search for possible deprecations: + mx.chk.warn_deprecated(dunder_get, mx.context) inferred_dunder_get_type = get_proper_type(inferred_dunder_get_type) if isinstance(inferred_dunder_get_type, AnyType): @@ -825,10 +825,7 @@ def analyze_descriptor_assign(descriptor_type: Instance, mx: MemberContext) -> T ) # Search for possible deprecations: - mx.chk.check_deprecated(dunder_set, mx.context) - mx.chk.warn_deprecated_overload_item( - dunder_set, mx.context, target=inferred_dunder_set_type, selftype=descriptor_type - ) + mx.chk.warn_deprecated(dunder_set, mx.context) # In the following cases, a message already will have been recorded in check_call. if (not isinstance(inferred_dunder_set_type, CallableType)) or ( @@ -1053,6 +1050,7 @@ def f(self: S) -> T: ... new_items = [] if is_classmethod: dispatched_arg_type = TypeType.make_normalized(dispatched_arg_type) + p_dispatched_arg_type = get_proper_type(dispatched_arg_type) for item in items: if not item.arg_types or item.arg_kinds[0] not in (ARG_POS, ARG_STAR): @@ -1061,28 +1059,42 @@ def f(self: S) -> T: ... # This is pretty bad, so just return the original signature if # there is at least one such error. return functype - else: - selfarg = get_proper_type(item.arg_types[0]) - # This matches similar special-casing in bind_self(), see more details there. - self_callable = name == "__call__" and isinstance(selfarg, CallableType) - if self_callable or subtypes.is_subtype( - dispatched_arg_type, - # This level of erasure matches the one in checker.check_func_def(), - # better keep these two checks consistent. - erase_typevars(erase_to_bound(selfarg)), - # This is to work around the fact that erased ParamSpec and TypeVarTuple - # callables are not always compatible with non-erased ones both ways. - always_covariant=any( - not isinstance(tv, TypeVarType) for tv in get_all_type_vars(selfarg) - ), - ignore_pos_arg_names=True, - ): - new_items.append(item) - elif isinstance(selfarg, ParamSpecType): - # TODO: This is not always right. What's the most reasonable thing to do here? - new_items.append(item) - elif isinstance(selfarg, TypeVarTupleType): - raise NotImplementedError + selfarg = get_proper_type(item.arg_types[0]) + if isinstance(selfarg, Instance) and isinstance(p_dispatched_arg_type, Instance): + if selfarg.type is p_dispatched_arg_type.type and selfarg.args: + if not is_overlapping_types(p_dispatched_arg_type, selfarg): + # This special casing is needed since `actual <: erased(template)` + # logic below doesn't always work, and a more correct approach may + # be tricky. + continue + new_items.append(item) + + if new_items: + items = new_items + new_items = [] + + for item in items: + selfarg = get_proper_type(item.arg_types[0]) + # This matches similar special-casing in bind_self(), see more details there. + self_callable = name == "__call__" and isinstance(selfarg, CallableType) + if self_callable or is_subtype( + dispatched_arg_type, + # This level of erasure matches the one in checker.check_func_def(), + # better keep these two checks consistent. + erase_typevars(erase_to_bound(selfarg)), + # This is to work around the fact that erased ParamSpec and TypeVarTuple + # callables are not always compatible with non-erased ones both ways. + always_covariant=any( + not isinstance(tv, TypeVarType) for tv in get_all_type_vars(selfarg) + ), + ignore_pos_arg_names=True, + ): + new_items.append(item) + elif isinstance(selfarg, ParamSpecType): + # TODO: This is not always right. What's the most reasonable thing to do here? + new_items.append(item) + elif isinstance(selfarg, TypeVarTupleType): + raise NotImplementedError if not new_items: # Choose first item for the message (it may be not very helpful for overloads). msg.incompatible_self_argument( diff --git a/mypy/erasetype.py b/mypy/erasetype.py index 6c47670d6687f..3f33ea1648f08 100644 --- a/mypy/erasetype.py +++ b/mypy/erasetype.py @@ -145,29 +145,34 @@ def erase_typevars(t: Type, ids_to_erase: Container[TypeVarId] | None = None) -> or just the ones in the provided collection. """ + if ids_to_erase is None: + return t.accept(TypeVarEraser(None, AnyType(TypeOfAny.special_form))) + def erase_id(id: TypeVarId) -> bool: - if ids_to_erase is None: - return True return id in ids_to_erase return t.accept(TypeVarEraser(erase_id, AnyType(TypeOfAny.special_form))) +def erase_meta_id(id: TypeVarId) -> bool: + return id.is_meta_var() + + def replace_meta_vars(t: Type, target_type: Type) -> Type: """Replace unification variables in a type with the target type.""" - return t.accept(TypeVarEraser(lambda id: id.is_meta_var(), target_type)) + return t.accept(TypeVarEraser(erase_meta_id, target_type)) class TypeVarEraser(TypeTranslator): """Implementation of type erasure""" - def __init__(self, erase_id: Callable[[TypeVarId], bool], replacement: Type) -> None: + def __init__(self, erase_id: Callable[[TypeVarId], bool] | None, replacement: Type) -> None: super().__init__() self.erase_id = erase_id self.replacement = replacement def visit_type_var(self, t: TypeVarType) -> Type: - if self.erase_id(t.id): + if self.erase_id is None or self.erase_id(t.id): return self.replacement return t @@ -212,12 +217,12 @@ def visit_callable_type(self, t: CallableType) -> Type: return result def visit_type_var_tuple(self, t: TypeVarTupleType) -> Type: - if self.erase_id(t.id): + if self.erase_id is None or self.erase_id(t.id): return t.tuple_fallback.copy_modified(args=[self.replacement]) return t def visit_param_spec(self, t: ParamSpecType) -> Type: - if self.erase_id(t.id): + if self.erase_id is None or self.erase_id(t.id): return self.replacement return t diff --git a/mypy/fixup.py b/mypy/fixup.py index 0e9c186fd42a7..c0f8e401777ca 100644 --- a/mypy/fixup.py +++ b/mypy/fixup.py @@ -165,6 +165,8 @@ def visit_func_def(self, func: FuncDef) -> None: func.info = self.current_info if func.type is not None: func.type.accept(self.type_fixer) + if isinstance(func.type, CallableType): + func.type.definition = func def visit_overloaded_func_def(self, o: OverloadedFuncDef) -> None: if self.current_info is not None: diff --git a/mypy/meet.py b/mypy/meet.py index fb35bce438ab1..349c15e668c3c 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -294,6 +294,18 @@ def is_object(t: ProperType) -> bool: return isinstance(t, Instance) and t.type.fullname == "builtins.object" +def is_none_typevarlike_overlap(t1: ProperType, t2: ProperType) -> bool: + return isinstance(t1, NoneType) and isinstance(t2, TypeVarLikeType) + + +def is_none_object_overlap(t1: ProperType, t2: ProperType) -> bool: + return ( + isinstance(t1, NoneType) + and isinstance(t2, Instance) + and t2.type.fullname == "builtins.object" + ) + + def is_overlapping_types( left: Type, right: Type, @@ -383,14 +395,6 @@ def _is_overlapping_types(left: Type, right: Type) -> bool: ): return True - def is_none_object_overlap(t1: Type, t2: Type) -> bool: - t1, t2 = get_proper_types((t1, t2)) - return ( - isinstance(t1, NoneType) - and isinstance(t2, Instance) - and t2.type.fullname == "builtins.object" - ) - if overlap_for_overloads: if is_none_object_overlap(left, right) or is_none_object_overlap(right, left): return False @@ -420,10 +424,6 @@ def _is_subtype(left: Type, right: Type) -> bool: # If both types are singleton variants (and are not TypeVarLikes), we've hit the base case: # we skip these checks to avoid infinitely recursing. - def is_none_typevarlike_overlap(t1: Type, t2: Type) -> bool: - t1, t2 = get_proper_types((t1, t2)) - return isinstance(t1, NoneType) and isinstance(t2, TypeVarLikeType) - if prohibit_none_typevar_overlap: if is_none_typevarlike_overlap(left, right) or is_none_typevarlike_overlap(right, left): return False diff --git a/mypy/nodes.py b/mypy/nodes.py index 921620866a06e..011e4e703a0c2 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -595,12 +595,14 @@ def is_trivial_self(self) -> bool: """ if self._is_trivial_self is not None: return self._is_trivial_self - for item in self.items: + for i, item in enumerate(self.items): + # Note: bare @property is removed in visit_decorator(). + trivial = 1 if i > 0 or not self.is_property else 0 if isinstance(item, FuncDef): if not item.is_trivial_self: self._is_trivial_self = False return False - elif item.decorators or not item.func.is_trivial_self: + elif len(item.decorators) > trivial or not item.func.is_trivial_self: self._is_trivial_self = False return False self._is_trivial_self = True diff --git a/mypy/typeops.py b/mypy/typeops.py index 1c22a1711944a..75213bd936744 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -16,7 +16,6 @@ from mypy.expandtype import expand_type, expand_type_by_instance from mypy.maptype import map_instance_to_supertype from mypy.nodes import ( - ARG_OPT, ARG_POS, ARG_STAR, ARG_STAR2, @@ -421,27 +420,9 @@ class B(A): pass """ if isinstance(method, Overloaded): - items = [] - original_type = get_proper_type(original_type) - for c in method.items: - if isinstance(original_type, Instance): - # Filter based on whether declared self type can match actual object type. - # For example, if self has type C[int] and method is accessed on a C[str] value, - # omit this item. This is best effort since bind_self can be called in many - # contexts, and doing complete validation might trigger infinite recursion. - # - # Note that overload item filtering normally happens elsewhere. This is needed - # at least during constraint inference. - keep = is_valid_self_type_best_effort(c, original_type) - else: - keep = True - if keep: - items.append(bind_self(c, original_type, is_classmethod, ignore_instances)) - if len(items) == 0: - # If no item matches, returning all items helps avoid some spurious errors - items = [ - bind_self(c, original_type, is_classmethod, ignore_instances) for c in method.items - ] + items = [ + bind_self(c, original_type, is_classmethod, ignore_instances) for c in method.items + ] return cast(F, Overloaded(items)) assert isinstance(method, CallableType) func: CallableType = method @@ -510,43 +491,6 @@ class B(A): pass return cast(F, res) -def is_valid_self_type_best_effort(c: CallableType, self_type: Instance) -> bool: - """Quickly check if self_type might match the self in a callable. - - Avoid performing any complex type operations. This is performance-critical. - - Default to returning True if we don't know (or it would be too expensive). - """ - if ( - self_type.args - and c.arg_types - and isinstance((arg_type := get_proper_type(c.arg_types[0])), Instance) - and c.arg_kinds[0] in (ARG_POS, ARG_OPT) - and arg_type.args - and self_type.type.fullname != "functools._SingleDispatchCallable" - ): - if self_type.type is not arg_type.type: - # We can't map to supertype, since it could trigger expensive checks for - # protocol types, so we consevatively assume this is fine. - return True - - # Fast path: no explicit annotation on self - if all( - ( - type(arg) is TypeVarType - and type(arg.upper_bound) is Instance - and arg.upper_bound.type.fullname == "builtins.object" - ) - for arg in arg_type.args - ): - return True - - from mypy.meet import is_overlapping_types - - return is_overlapping_types(self_type, c.arg_types[0]) - return True - - def erase_to_bound(t: Type) -> Type: # TODO: use value restrictions to produce a union? t = get_proper_type(t) diff --git a/test-data/unit/check-selftype.test b/test-data/unit/check-selftype.test index 05c34eb707966..d99c5ee354a4e 100644 --- a/test-data/unit/check-selftype.test +++ b/test-data/unit/check-selftype.test @@ -2282,3 +2282,30 @@ class Check: reveal_type(Check.foo()) # N: Revealed type is "def () -> __main__.Check" reveal_type(Check().foo()) # N: Revealed type is "__main__.Check" [builtins fixtures/tuple.pyi] + +[case testSelfTypeUpperBoundFiler] +from typing import Generic, TypeVar, overload, Sequence + +class B: ... +class C(B): ... + +TB = TypeVar("TB", bound=B) +TC = TypeVar("TC", bound=C) + +class G(Generic[TB]): + @overload + def test(self: G[TC]) -> list[TC]: ... + @overload + def test(self: G[TB]) -> Sequence[TB]: ... + def test(self): + ... + +class D1(B): ... +class D2(C): ... + +gb: G[D1] +gc: G[D2] + +reveal_type(gb.test()) # N: Revealed type is "typing.Sequence[__main__.D1]" +reveal_type(gc.test()) # N: Revealed type is "builtins.list[__main__.D2]" +[builtins fixtures/list.pyi] diff --git a/test-data/unit/check-serialize.test b/test-data/unit/check-serialize.test index 63d9ccfc80cbf..5265832f5f274 100644 --- a/test-data/unit/check-serialize.test +++ b/test-data/unit/check-serialize.test @@ -158,6 +158,7 @@ def f(__x: int) -> None: pass [out2] tmp/a.py:3: error: Argument 1 to "f" has incompatible type "str"; expected "int" tmp/a.py:4: error: Unexpected keyword argument "__x" for "f" +tmp/b.py: note: "f" defined here [case testSerializeArgumentKindsErrors] import a From 2ce57952f7f0bb63d24dff5eab3f3933a7fcf28f Mon Sep 17 00:00:00 2001 From: Brian Schubert Date: Sat, 2 Aug 2025 11:04:56 -0400 Subject: [PATCH 125/424] Update crash issue template to use syntax highlighting in code blocks (#19527) People often forget or don't know how to add syntax highlighting to markdown code blocks (especially for tracebacks!). Let's help them out by pre-filling the template with code blocks that have the right language hints. --- .github/ISSUE_TEMPLATE/crash.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/crash.md b/.github/ISSUE_TEMPLATE/crash.md index fed16a8d28acd..ea82a9a28642d 100644 --- a/.github/ISSUE_TEMPLATE/crash.md +++ b/.github/ISSUE_TEMPLATE/crash.md @@ -15,7 +15,7 @@ labels: "crash" **Traceback** -``` +```python-traceback (Insert traceback and other messages from mypy here -- use `--show-traceback`.) ``` @@ -25,6 +25,11 @@ labels: "crash" appreciated. We also very much appreciate it if you try to narrow the source down to a small stand-alone example.) +```python +# Ideally, a small sample program that demonstrates the problem. +# Or even better, a reproducible playground link https://mypy-play.net/ (use the "Gist" button) +``` + **Your Environment** From ac3e240029d74ee0975f4b64131e79e48f3c0d07 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 2 Aug 2025 17:47:38 +0100 Subject: [PATCH 126/424] Add internal flag to disable expression cache (#19569) --- mypy/checkexpr.py | 6 ++++-- mypy/main.py | 7 ++++--- mypy/options.py | 2 ++ 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index bec34eef49838..740efb0d2ee4c 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -6030,8 +6030,10 @@ def accept( # We cannot use cache inside lambdas, because they skip immediate type # context, and use enclosing one, see infer_lambda_type_using_context(). # TODO: consider using cache for more expression kinds. - elif isinstance(node, (CallExpr, ListExpr, TupleExpr, DictExpr, OpExpr)) and not ( - self.in_lambda_expr or self.chk.current_node_deferred + elif ( + isinstance(node, (CallExpr, ListExpr, TupleExpr, DictExpr, OpExpr)) + and not (self.in_lambda_expr or self.chk.current_node_deferred) + and not self.chk.options.disable_expression_cache ): if (node, type_context) in self.expr_cache: binder_version, typ, messages, type_map = self.expr_cache[(node, type_context)] diff --git a/mypy/main.py b/mypy/main.py index a407a88d3ac14..6e307ab25c487 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -1090,14 +1090,15 @@ def add_invertible_flag( help="Use a custom typing module", ) internals_group.add_argument( - "--old-type-inference", - action="store_true", - help="Disable new experimental type inference algorithm", + "--old-type-inference", action="store_true", help=argparse.SUPPRESS ) # Deprecated reverse variant of the above. internals_group.add_argument( "--new-type-inference", action="store_true", help=argparse.SUPPRESS ) + internals_group.add_argument( + "--disable-expression-cache", action="store_true", help=argparse.SUPPRESS + ) parser.add_argument( "--enable-incomplete-feature", action="append", diff --git a/mypy/options.py b/mypy/options.py index 4a89ef529c071..be61059be5c69 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -395,6 +395,8 @@ def __init__(self) -> None: self.old_type_inference = False # Deprecated reverse version of the above, do not use. self.new_type_inference = False + # Disable expression cache (for debugging). + self.disable_expression_cache = False # Export line-level, limited, fine-grained dependency information in cache data # (undocumented feature). self.export_ref_info = False From c5a4efe008f943e379c5b3ee7334e28d9475eb43 Mon Sep 17 00:00:00 2001 From: Ali Hamdan Date: Sat, 2 Aug 2025 19:05:46 +0200 Subject: [PATCH 127/424] Add stubtest test for property aliases (#19567) Closes #19509 Bisect show that the issue was fixed by #19297 --- mypy/test/teststubtest.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mypy/test/teststubtest.py b/mypy/test/teststubtest.py index 7925f2a6bd3ec..3b19063f08c8e 100644 --- a/mypy/test/teststubtest.py +++ b/mypy/test/teststubtest.py @@ -850,11 +850,13 @@ def test_property(self) -> Iterator[Case]: class Good: @property def read_only_attr(self) -> int: ... + read_only_attr_alias = read_only_attr """, runtime=""" class Good: @property def read_only_attr(self): return 1 + read_only_attr_alias = read_only_attr """, error=None, ) @@ -916,6 +918,7 @@ class Z: def read_write_attr(self) -> int: ... @read_write_attr.setter def read_write_attr(self, val: int) -> None: ... + read_write_attr_alias = read_write_attr """, runtime=""" class Z: @@ -923,6 +926,7 @@ class Z: def read_write_attr(self): return self._val @read_write_attr.setter def read_write_attr(self, val): self._val = val + read_write_attr_alias = read_write_attr """, error=None, ) From e8147f2bdcd8899109fe1a9cffc770c436de95dc Mon Sep 17 00:00:00 2001 From: Brian Schubert Date: Sat, 2 Aug 2025 13:10:22 -0400 Subject: [PATCH 128/424] [stubtest] Allow runtime-existing aliases of types marked as `@type_check_only` (#19568) In typeshed, there's a few cases of stubs like this: ```python class _DoesNotExist: ... # does not exist at runtime if sys.version_info >= (3, X): Exists = _DoesNotExist ``` Ideally, it would be nice to mark `_DoesNotExit` as `@type_check_only` to make it clear that this type isn't available at runtime. However, this currently can't be done, because doing so will make stubtest think that `Exists` is also `@type_check_only`, which sets off alarm bells due to `Exists` being available at runtime. This PR makes it so stubtest doesn't consider `@type_check_only`-status when checking type alias targets, making it possible to mark types like the above as `@type_check_only`. --- mypy/stubtest.py | 10 +++++++--- mypy/test/teststubtest.py | 11 +++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index d16e491fb1ab1..ef8c8dc318e1a 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -506,9 +506,13 @@ def _verify_metaclass( @verify.register(nodes.TypeInfo) def verify_typeinfo( - stub: nodes.TypeInfo, runtime: MaybeMissing[type[Any]], object_path: list[str] + stub: nodes.TypeInfo, + runtime: MaybeMissing[type[Any]], + object_path: list[str], + *, + is_alias_target: bool = False, ) -> Iterator[Error]: - if stub.is_type_check_only: + if stub.is_type_check_only and not is_alias_target: # This type only exists in stubs, we only check that the runtime part # is missing. Other checks are not required. if not isinstance(runtime, Missing): @@ -1449,7 +1453,7 @@ def verify_typealias( # Okay, either we couldn't construct a fullname # or the fullname of the stub didn't match the fullname of the runtime. # Fallback to a full structural check of the runtime vis-a-vis the stub. - yield from verify(stub_origin, runtime_origin, object_path) + yield from verify_typeinfo(stub_origin, runtime_origin, object_path, is_alias_target=True) return if isinstance(stub_target, mypy.types.UnionType): # complain if runtime is not a Union or UnionType diff --git a/mypy/test/teststubtest.py b/mypy/test/teststubtest.py index 3b19063f08c8e..b071c0ee8ab6e 100644 --- a/mypy/test/teststubtest.py +++ b/mypy/test/teststubtest.py @@ -2472,6 +2472,17 @@ def func2() -> None: ... runtime="def func2() -> None: ...", error="func2", ) + # A type that exists at runtime is allowed to alias a type marked + # as '@type_check_only' in the stubs. + yield Case( + stub=""" + @type_check_only + class _X1: ... + X2 = _X1 + """, + runtime="class X2: ...", + error=None, + ) def remove_color_code(s: str) -> str: From 82ad62cfed5293e377984c1783c8367aed9fd4ed Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Sat, 2 Aug 2025 14:43:36 -0700 Subject: [PATCH 129/424] [docs] Include a real listing of the flags strict enables in the online documentation (#19062) Fixes #19061 Currently, https://mypy.readthedocs.io/en/stable/command_line.html just says > You can see the list of flags enabled by strict mode in the full [mypy --help](https://mypy.readthedocs.io/en/stable/command_line.html#cmdoption-mypy-h) output. Which makes cross-referencing the documentation difficult. Instead, there should be the same list there as appears when running `mypy --help`: eg > --warn-unused-configs, --disallow-any-generics, --disallow-subclassing-any, --disallow-untyped- calls, --disallow-untyped-defs, --disallow- incomplete-defs, --check-untyped-defs, --disallow- untyped-decorators, --warn-redundant-casts, --warn-unused-ignores, --warn-return-any, --no- implicit-reexport, --strict-equality, --extra- checks Ideally this section would be automatically generated from the code. --- docs/source/command_line.rst | 8 ++++++ docs/source/html_builder.py | 22 ++++++++++++++ mypy/main.py | 56 ++++++++++++++++++++++++------------ 3 files changed, 67 insertions(+), 19 deletions(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index 697e0fb69eed5..db2407e17df8b 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -812,6 +812,14 @@ of the above sections. Note: the exact list of flags enabled by running :option:`--strict` may change over time. + .. include:: strict_list.rst + .. + The above file is autogenerated and included during html generation. + (That's an include directive, and this is a comment.) + It would be fine to generate it at some other time instead, + theoretically, but we already had a convenient hook during html gen. + + .. option:: --disable-error-code This flag allows disabling one or multiple error codes globally. diff --git a/docs/source/html_builder.py b/docs/source/html_builder.py index ea3594e0617b6..387f7f13b4c29 100644 --- a/docs/source/html_builder.py +++ b/docs/source/html_builder.py @@ -11,16 +11,37 @@ from sphinx.builders.html import StandaloneHTMLBuilder from sphinx.environment import BuildEnvironment +from mypy.main import define_options + class MypyHTMLBuilder(StandaloneHTMLBuilder): + strict_file: Path + def __init__(self, app: Sphinx, env: BuildEnvironment) -> None: super().__init__(app, env) self._ref_to_doc = {} + self.strict_file = Path(self.srcdir) / "strict_list.rst" + self._add_strict_list() def write_doc(self, docname: str, doctree: document) -> None: super().write_doc(docname, doctree) self._ref_to_doc.update({_id: docname for _id in doctree.ids}) + def _add_strict_list(self) -> None: + strict_flags: list[str] + _, strict_flags, _ = define_options() + strict_part = ", ".join(f":option:`{s} `" for s in strict_flags) + if ( + not strict_part + or strict_part.isspace() + or len(strict_part) < 20 + or len(strict_part) > 2000 + ): + raise ValueError(f"{strict_part=}, which doesn't look right (by a simple heuristic).") + self.strict_file.write_text( + "For this version of mypy, the list of flags enabled by strict is: " + strict_part + ) + def _verify_error_codes(self) -> None: from mypy.errorcodes import error_codes @@ -55,6 +76,7 @@ def _write_ref_redirector(self) -> None: def finish(self) -> None: super().finish() self._write_ref_redirector() + self.strict_file.unlink() def setup(app: Sphinx) -> dict[str, Any]: diff --git a/mypy/main.py b/mypy/main.py index 6e307ab25c487..2fbb9671e721e 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -462,24 +462,18 @@ def __call__( parser.exit() -def process_options( - args: list[str], - stdout: TextIO | None = None, - stderr: TextIO | None = None, - require_targets: bool = True, - server_options: bool = False, - fscache: FileSystemCache | None = None, +def define_options( program: str = "mypy", header: str = HEADER, -) -> tuple[list[BuildSource], Options]: - """Parse command line arguments. - - If a FileSystemCache is passed in, and package_root options are given, - call fscache.set_package_root() to set the cache's package root. - """ - stdout = stdout or sys.stdout - stderr = stderr or sys.stderr - + stdout: TextIO = sys.stdout, + stderr: TextIO = sys.stderr, + server_options: bool = False, +) -> tuple[CapturableArgumentParser, list[str], list[tuple[str, bool]]]: + """Define the options in the parser (by calling a bunch of methods that express/build our desired command-line flags). + Returns a tuple of: + a parser object, that can parse command line arguments to mypy (expected consumer: main's process_options), + a list of what flags are strict (expected consumer: docs' html_builder's _add_strict_list), + strict_flag_assignments (expected consumer: main's process_options).""" parser = CapturableArgumentParser( prog=program, usage=header, @@ -1342,6 +1336,32 @@ def add_invertible_flag( dest="special-opts:files", help="Type-check given files or directories", ) + return parser, strict_flag_names, strict_flag_assignments + + +def process_options( + args: list[str], + stdout: TextIO | None = None, + stderr: TextIO | None = None, + require_targets: bool = True, + server_options: bool = False, + fscache: FileSystemCache | None = None, + program: str = "mypy", + header: str = HEADER, +) -> tuple[list[BuildSource], Options]: + """Parse command line arguments. + + If a FileSystemCache is passed in, and package_root options are given, + call fscache.set_package_root() to set the cache's package root. + + Returns a tuple of: a list of source files, an Options collected from flags. + """ + stdout = stdout if stdout is not None else sys.stdout + stderr = stderr if stderr is not None else sys.stderr + + parser, _, strict_flag_assignments = define_options( + program, header, stdout, stderr, server_options + ) # Parse arguments once into a dummy namespace so we can get the # filename for the config file and know if the user requested all strict options. @@ -1526,11 +1546,9 @@ def set_strict_flags() -> None: targets.extend(p_targets) for m in special_opts.modules: targets.append(BuildSource(None, m, None)) - return targets, options elif special_opts.command: options.build_type = BuildType.PROGRAM_TEXT targets = [BuildSource(None, None, "\n".join(special_opts.command))] - return targets, options else: try: targets = create_source_list(special_opts.files, options, fscache) @@ -1539,7 +1557,7 @@ def set_strict_flags() -> None: # exceptions of different types. except InvalidSourceList as e2: fail(str(e2), stderr, options) - return targets, options + return targets, options def process_package_roots( From b4102f2e8e2aa0634c4249161081eaba078dcb1b Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Sat, 2 Aug 2025 14:47:22 -0700 Subject: [PATCH 130/424] [docs] Update common_issues.rst: mention orjson in the Mypy slow section (#19058) This mostly just copies the verbiage from the release blog post. --- docs/source/common_issues.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/source/common_issues.rst b/docs/source/common_issues.rst index 96d73e5f03992..266d0c5b2c80c 100644 --- a/docs/source/common_issues.rst +++ b/docs/source/common_issues.rst @@ -218,6 +218,14 @@ daemon `, which can speed up incremental mypy runtimes by a factor of 10 or more. :ref:`Remote caching ` can make cold mypy runs several times faster. +Furthermore: as of `mypy 1.13 `_, +mypy allows use of the orjson library for handling the cache instead of the stdlib json, for +improved performance. You can ensure the presence of orjson using the faster-cache extra: + + python3 -m pip install -U mypy[faster-cache] + +Mypy may depend on orjson by default in the future. + Types of empty collections -------------------------- From c962993db73fc90081c5a06381b6032acf7c0e66 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Sat, 2 Aug 2025 14:56:00 -0700 Subject: [PATCH 131/424] Mention in the Any documentation how object is preferable (#19103) This implements a suggestion in https://github.com/python/mypy/issues/9153#issuecomment-1837446187, which I thought was a good idea. --------- Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> --- docs/source/dynamic_typing.rst | 3 ++- docs/source/kinds_of_types.rst | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/source/dynamic_typing.rst b/docs/source/dynamic_typing.rst index 304e25c085a83..da40142d377d4 100644 --- a/docs/source/dynamic_typing.rst +++ b/docs/source/dynamic_typing.rst @@ -1,6 +1,5 @@ .. _dynamic-typing: - Dynamically typed code ====================== @@ -94,6 +93,8 @@ third party libraries that mypy does not know about. This is particularly the ca when using the :option:`--ignore-missing-imports ` flag. See :ref:`fix-missing-imports` for more information about this. +.. _any-vs-object: + Any vs. object -------------- diff --git a/docs/source/kinds_of_types.rst b/docs/source/kinds_of_types.rst index 54693cddf953b..8e721c0fb3218 100644 --- a/docs/source/kinds_of_types.rst +++ b/docs/source/kinds_of_types.rst @@ -41,6 +41,11 @@ operations are permitted on the value, and the operations are only checked at runtime. You can use ``Any`` as an "escape hatch" when you can't use a more precise type for some reason. +This should not be confused with the +:py:class:`object` type, which represents the set of all values. +Unlike ``object``, ``Any`` introduces type unsafety — see +:ref:`any-vs-object` for more. + ``Any`` is compatible with every other type, and vice versa. You can freely assign a value of type ``Any`` to a variable with a more precise type: From 813b4d11152e77bf3c96e7f762df2dcd041667db Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Sat, 2 Aug 2025 15:01:46 -0700 Subject: [PATCH 132/424] Fix a bug where inline configurations of error codes would lose their values if accompanied by another inline configuration. (#19075) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The following code produces a name-defined code error, despite our instructions. How can it be?! ```py3 # mypy: disable-error-code=name-defined # mypy: strict-equality a ``` The answer is, there was a bug that caused all inline configurations, even those that didn't specify any enable/disables, to overwrite the lists of enable/disables. I have now fixed that. I've also added some tests. Closes #12342 — I discovered this problem while investigating the last foible of issue #12342 (itself of tangential interest to something else I was doing), which can now be closed. --------- Co-authored-by: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> --- mypy/config_parser.py | 23 +++- mypy/errorcodes.py | 4 + mypy/options.py | 1 - test-data/unit/check-inline-config.test | 150 +++++++++++++++++++++++- 4 files changed, 171 insertions(+), 7 deletions(-) diff --git a/mypy/config_parser.py b/mypy/config_parser.py index e5c0dc893c768..208c12adafbe7 100644 --- a/mypy/config_parser.py +++ b/mypy/config_parser.py @@ -650,9 +650,8 @@ def parse_mypy_comments( Returns a dictionary of options to be applied and a list of error messages generated. """ - errors: list[tuple[int, str]] = [] - sections = {} + sections: dict[str, object] = {"enable_error_code": [], "disable_error_code": []} for lineno, line in args: # In order to easily match the behavior for bools, we abuse configparser. @@ -660,7 +659,6 @@ def parse_mypy_comments( # method is to create a config parser. parser = configparser.RawConfigParser() options, parse_errors = mypy_comments_to_config_map(line, template) - if "python_version" in options: errors.append((lineno, "python_version not supported in inline configuration")) del options["python_version"] @@ -690,9 +688,24 @@ def set_strict_flags() -> None: '(see "mypy -h" for the list of flags enabled in strict mode)', ) ) - + # Because this is currently special-cased + # (the new_sections for an inline config *always* includes 'disable_error_code' and + # 'enable_error_code' fields, usually empty, which overwrite the old ones), + # we have to manipulate them specially. + # This could use a refactor, but so could the whole subsystem. + if ( + "enable_error_code" in new_sections + and isinstance(neec := new_sections["enable_error_code"], list) + and isinstance(eec := sections.get("enable_error_code", []), list) + ): + new_sections["enable_error_code"] = sorted(set(neec + eec)) + if ( + "disable_error_code" in new_sections + and isinstance(ndec := new_sections["disable_error_code"], list) + and isinstance(dec := sections.get("disable_error_code", []), list) + ): + new_sections["disable_error_code"] = sorted(set(ndec + dec)) sections.update(new_sections) - return sections, errors diff --git a/mypy/errorcodes.py b/mypy/errorcodes.py index 8f85a6f6351a9..bcfdbf6edc2bf 100644 --- a/mypy/errorcodes.py +++ b/mypy/errorcodes.py @@ -37,6 +37,10 @@ def __init__( def __str__(self) -> str: return f"" + def __repr__(self) -> str: + """This doesn't fulfill the goals of repr but it's better than the default view.""" + return f"" + def __eq__(self, other: object) -> bool: if not isinstance(other, ErrorCode): return False diff --git a/mypy/options.py b/mypy/options.py index be61059be5c69..573be14c4b1b3 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -507,7 +507,6 @@ def apply_changes(self, changes: dict[str, object]) -> Options: code = error_codes[code_str] new_options.enabled_error_codes.add(code) new_options.disabled_error_codes.discard(code) - return new_options def compare_stable(self, other_snapshot: dict[str, object]) -> bool: diff --git a/test-data/unit/check-inline-config.test b/test-data/unit/check-inline-config.test index 8a306b1dfac09..37d59a84c873d 100644 --- a/test-data/unit/check-inline-config.test +++ b/test-data/unit/check-inline-config.test @@ -211,6 +211,99 @@ enable_error_code = ignore-without-code, truthy-bool \[mypy-tests.*] disable_error_code = ignore-without-code +[case testInlineErrorCodesOverrideConfigSmall] +# flags: --config-file tmp/mypy.ini +import tests.baz +[file tests/__init__.py] +[file tests/baz.py] +42 + "no" # type: ignore + +[file mypy.ini] +\[mypy] +enable_error_code = ignore-without-code, truthy-bool + +\[mypy-tests.*] +disable_error_code = ignore-without-code + +[case testInlineErrorCodesOverrideConfigSmall2] +# flags: --config-file tmp/mypy.ini +import tests.bar +import tests.baz +[file tests/__init__.py] +[file tests/baz.py] +42 + "no" # type: ignore +[file tests/bar.py] +# mypy: enable-error-code="ignore-without-code" + +def foo() -> int: ... +if foo: ... # E: Function "foo" could always be true in boolean context +42 + "no" # type: ignore # E: "type: ignore" comment without error code (consider "type: ignore[operator]" instead) + +[file mypy.ini] +\[mypy] +enable_error_code = ignore-without-code, truthy-bool + +\[mypy-tests.*] +disable_error_code = ignore-without-code + + +[case testInlineErrorCodesOverrideConfigSmallBackward] +# flags: --config-file tmp/mypy.ini +import tests.bar +import tests.baz +[file tests/__init__.py] +[file tests/baz.py] +42 + "no" # type: ignore # E: "type: ignore" comment without error code (consider "type: ignore[operator]" instead) +[file tests/bar.py] +# mypy: disable-error-code="ignore-without-code" +42 + "no" # type: ignore + +[file mypy.ini] +\[mypy] +enable_error_code = ignore-without-code, truthy-bool + +\[mypy-tests.*] +enable_error_code = ignore-without-code + +[case testInlineOverrideConfig] +# flags: --config-file tmp/mypy.ini +import foo +import tests.bar +import tests.baz +[file foo.py] +# mypy: disable-error-code="truthy-bool" +class Foo: + pass + +foo = Foo() +if foo: ... +42 # type: ignore # E: Unused "type: ignore" comment + +[file tests/__init__.py] +[file tests/bar.py] +# mypy: warn_unused_ignores + +def foo() -> int: ... +if foo: ... # E: Function "foo" could always be true in boolean context +42 # type: ignore # E: Unused "type: ignore" comment + +[file tests/baz.py] +# mypy: disable-error-code="truthy-bool" +class Foo: + pass + +foo = Foo() +if foo: ... +42 # type: ignore + +[file mypy.ini] +\[mypy] +warn_unused_ignores = True + +\[mypy-tests.*] +warn_unused_ignores = False + + [case testIgnoreErrorsSimple] # mypy: ignore-errors=True @@ -324,6 +417,61 @@ foo = Foo() if foo: ... 42 + "no" # type: ignore - [case testInlinePythonVersion] # mypy: python-version=3.10 # E: python_version not supported in inline configuration + +[case testInlineErrorCodesArentRuinedByOthersBaseCase] +# mypy: disable-error-code=name-defined +a + +[case testInlineErrorCodesArentRuinedByOthersInvalid] +# mypy: disable-error-code=name-defined +# mypy: AMONGUS +a +[out] +main:2: error: Unrecognized option: amongus = True + +[case testInlineErrorCodesArentRuinedByOthersInvalidBefore] +# mypy: AMONGUS +# mypy: disable-error-code=name-defined +a +[out] +main:1: error: Unrecognized option: amongus = True + +[case testInlineErrorCodesArentRuinedByOthersSe] +# mypy: disable-error-code=name-defined +# mypy: strict-equality +def is_magic(x: bytes) -> bool: + y + return x == 'magic' # E: Unsupported left operand type for == ("bytes") + +[case testInlineConfigErrorCodesOffAndOn] +# mypy: disable-error-code=name-defined +# mypy: enable-error-code=name-defined +a # E: Name "a" is not defined + +[case testInlineConfigErrorCodesOnAndOff] +# mypy: enable-error-code=name-defined +# mypy: disable-error-code=name-defined +a # E: Name "a" is not defined + +[case testConfigFileErrorCodesOnAndOff] +# flags: --config-file tmp/mypy.ini +import foo +[file foo.py] +42 + "no" # type: ignore # E: "type: ignore" comment without error code (consider "type: ignore[operator]" instead) +[file mypy.ini] +\[mypy] +enable_error_code = ignore-without-code +disable_error_code = ignore-without-code + +[case testInlineConfigBaseCaseWui] +# mypy: warn_unused_ignores +x = 1 # type: ignore # E: Unused "type: ignore" comment + +[case testInlineConfigIsntRuinedByOthersInvalidWui] +# mypy: warn_unused_ignores +# mypy: AMONGUS +x = 1 # type: ignore # E: Unused "type: ignore" comment +[out] +main:2: error: Unrecognized option: amongus = True From c213db074d319a222464ec82846c908fcadc2bc8 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Sat, 2 Aug 2025 15:12:54 -0700 Subject: [PATCH 133/424] [docs] update information about reveal type & locals in common_issues (#19059) Previously it was impossible to have these in at runtime (more or less), but now you can just import one of them. --------- Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> --- docs/source/common_issues.rst | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/source/common_issues.rst b/docs/source/common_issues.rst index 266d0c5b2c80c..aa325dd3b05c4 100644 --- a/docs/source/common_issues.rst +++ b/docs/source/common_issues.rst @@ -513,11 +513,15 @@ to see the types of all local variables at once. Example: # b: builtins.str .. note:: - ``reveal_type`` and ``reveal_locals`` are only understood by mypy and - don't exist in Python. If you try to run your program, you'll have to - remove any ``reveal_type`` and ``reveal_locals`` calls before you can - run your code. Both are always available and you don't need to import - them. + ``reveal_type`` and ``reveal_locals`` are handled specially by mypy during + type checking, and don't have to be defined or imported. + + However, if you want to run your code, + you'll have to remove any ``reveal_type`` and ``reveal_locals`` + calls from your program or else Python will give you an error at runtime. + + Alternatively, you can import ``reveal_type`` from ``typing_extensions`` + or ``typing`` (on Python 3.11 and newer) .. _silencing-linters: From 5b28b62d9f87ac72e8dd5b65cf2cb9fa84fa4be5 Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Sun, 3 Aug 2025 02:35:06 +0200 Subject: [PATCH 134/424] Check slots assignments on self types (#19332) Fixes #19331. --- mypy/checker.py | 3 +++ test-data/unit/check-slots.test | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/mypy/checker.py b/mypy/checker.py index 35a67d1883111..dfbfa753d5f2c 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -3752,6 +3752,9 @@ def check_assignment_to_slots(self, lvalue: Lvalue) -> None: return inst = get_proper_type(self.expr_checker.accept(lvalue.expr)) + if isinstance(inst, TypeVarType) and inst.id.is_self(): + # Unwrap self type + inst = get_proper_type(inst.upper_bound) if not isinstance(inst, Instance): return if inst.type.slots is None: diff --git a/test-data/unit/check-slots.test b/test-data/unit/check-slots.test index e924ac9e5f57e..10b664bffb119 100644 --- a/test-data/unit/check-slots.test +++ b/test-data/unit/check-slots.test @@ -544,3 +544,19 @@ x = X() X.a # E: "a" in __slots__ conflicts with class variable access x.a [builtins fixtures/tuple.pyi] + +[case testSlotsOnSelfType] +from typing_extensions import Self + +class X: + __slots__ = ("foo",) + foo: int + + def method1(self: Self) -> Self: + self.bar = 0 # E: Trying to assign name "bar" that is not in "__slots__" of type "__main__.X" + return self + + def method2(self) -> Self: + self.bar = 0 # E: Trying to assign name "bar" that is not in "__slots__" of type "__main__.X" + return self +[builtins fixtures/tuple.pyi] From 06e28f8707baf8eab69eac84a2e4202060420e04 Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Sun, 3 Aug 2025 15:01:59 +0200 Subject: [PATCH 135/424] Follow-up after #19025: test and cleanup (#19294) As requested by @ilevkivskyi in #19025 --------- Co-authored-by: Ivan Levkivskyi --- mypy/checkmember.py | 5 +--- test-data/unit/check-selftype.test | 37 ++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/mypy/checkmember.py b/mypy/checkmember.py index 8447dfc7fe647..d261b3156a0bc 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -1249,9 +1249,6 @@ def analyze_class_attribute_access( or isinstance(node.node, Var) and node.node.is_classmethod ) - is_staticmethod = (is_decorated and cast(Decorator, node.node).func.is_static) or ( - isinstance(node.node, SYMBOL_FUNCBASE_TYPES) and node.node.is_static - ) t = get_proper_type(t) is_trivial_self = False if isinstance(node.node, Decorator): @@ -1274,7 +1271,7 @@ def analyze_class_attribute_access( original_vars=original_vars, is_trivial_self=is_trivial_self, ) - if is_decorated and not is_staticmethod: + if is_decorated: t = expand_self_type_if_needed( t, mx, cast(Decorator, node.node).var, itype, is_class=is_classmethod ) diff --git a/test-data/unit/check-selftype.test b/test-data/unit/check-selftype.test index d99c5ee354a4e..6481a17669445 100644 --- a/test-data/unit/check-selftype.test +++ b/test-data/unit/check-selftype.test @@ -2283,6 +2283,43 @@ reveal_type(Check.foo()) # N: Revealed type is "def () -> __main__.Check" reveal_type(Check().foo()) # N: Revealed type is "__main__.Check" [builtins fixtures/tuple.pyi] +[case testSelfInClassmethodWithOtherSelfMethod] +from typing import Any, Callable, Self, TypeVar + +_C = TypeVar("_C", bound=Callable[..., Any]) + +def identity(func: _C, /) -> _C: + return func + +class A: + def meth(self) -> Self: ... + + @classmethod + def other_meth(cls) -> Self: + reveal_type(cls.meth) # N: Revealed type is "def [Self <: __main__.A] (self: Self`1) -> Self`1" + reveal_type(A.meth) # N: Revealed type is "def [Self <: __main__.A] (self: Self`2) -> Self`2" + return cls().meth() + +class B: + @identity + def meth(self) -> Self: ... + + @classmethod + def other_meth(cls) -> Self: + reveal_type(cls.meth) # N: Revealed type is "def [Self <: __main__.B] (self: Self`5) -> Self`5" + reveal_type(B.meth) # N: Revealed type is "def [Self <: __main__.B] (self: Self`6) -> Self`6" + return cls().meth() + +class C: + @classmethod + def other_meth(cls) -> Self: ... + + def meth(self) -> Self: + reveal_type(self.other_meth) # N: Revealed type is "def () -> Self`0" + reveal_type(type(self).other_meth) # N: Revealed type is "def () -> Self`0" + return self.other_meth() +[builtins fixtures/tuple.pyi] + [case testSelfTypeUpperBoundFiler] from typing import Generic, TypeVar, overload, Sequence From 9d3a0524b32a54858d8efd52d06157908acec348 Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Sun, 3 Aug 2025 18:47:36 +0200 Subject: [PATCH 136/424] Support attribute access on enum members correctly (#19422) Fixes #11368 (apparently canonical). Fixes #10910. Fixes #12107. Fixes #13841. Fixes #15186. Fixes #15454. Fixes #19418. `mypy` now understands attribute access on enum members - the "recursive" behaviour of supporting access of almost-all enum members from members. "Almost", because `.name` and `.value` take precedence even if a member of the same name exists. ```python from enum import Enum class E(Enum): FOO = 1 BAR = 1 # The following is still a `E.BAR` instance: E.FOO.FOO.BAR.BAR ``` Looks like this is a much wanted feature. --- mypy/checkmember.py | 12 ++++++ mypy/plugins/enums.py | 2 +- test-data/unit/check-enum.test | 56 ++++++++++++++++++++++++--- test-data/unit/check-incremental.test | 2 + 4 files changed, 66 insertions(+), 6 deletions(-) diff --git a/mypy/checkmember.py b/mypy/checkmember.py index d261b3156a0bc..da67591a4553f 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -921,6 +921,18 @@ def analyze_var( result = AnyType(TypeOfAny.special_form) fullname = f"{var.info.fullname}.{name}" hook = mx.chk.plugin.get_attribute_hook(fullname) + + if var.info.is_enum and not mx.is_lvalue: + if name in var.info.enum_members and name not in {"name", "value"}: + enum_literal = LiteralType(name, fallback=itype) + result = itype.copy_modified(last_known_value=enum_literal) + elif ( + isinstance(p_result := get_proper_type(result), Instance) + and p_result.type.fullname == "enum.nonmember" + and p_result.args + ): + # Unwrap nonmember similar to class-level access + result = p_result.args[0] if result and not (implicit or var.info.is_protocol and is_instance_var(var)): result = analyze_descriptor_access(result, mx) if hook: diff --git a/mypy/plugins/enums.py b/mypy/plugins/enums.py index d21b21fb39f84..0be2e083b6ddf 100644 --- a/mypy/plugins/enums.py +++ b/mypy/plugins/enums.py @@ -144,7 +144,7 @@ def _implements_new(info: TypeInfo) -> bool: def enum_member_callback(ctx: mypy.plugin.FunctionContext) -> Type: """By default `member(1)` will be inferred as `member[int]`, we want to improve the inference to be `Literal[1]` here.""" - if ctx.arg_types or ctx.arg_types[0]: + if ctx.arg_types and ctx.arg_types[0]: arg = get_proper_type(ctx.arg_types[0][0]) proper_return = get_proper_type(ctx.default_return_type) if ( diff --git a/test-data/unit/check-enum.test b/test-data/unit/check-enum.test index d034fe1a6f5f7..3bcf9745a801c 100644 --- a/test-data/unit/check-enum.test +++ b/test-data/unit/check-enum.test @@ -1953,7 +1953,8 @@ class A(Enum): x: int def method(self) -> int: pass class B(A): - x = 1 # E: Cannot override writable attribute "x" with a final one + x = 1 # E: Cannot override writable attribute "x" with a final one \ + # E: Incompatible types in assignment (expression has type "B", base class "A" defined the type as "int") class A1(Enum): x: int = 1 # E: Enum members must be left unannotated \ @@ -1971,8 +1972,8 @@ class B2(A2): # E: Cannot extend enum with existing members: "A2" class A3(Enum): x: Final[int] # type: ignore class B3(A3): - x = 1 # E: Cannot override final attribute "x" (previously declared in base class "A3") - + x = 1 # E: Cannot override final attribute "x" (previously declared in base class "A3") \ + # E: Incompatible types in assignment (expression has type "B3", base class "A3" defined the type as "int") [builtins fixtures/bool.pyi] [case testEnumNotFinalWithMethodsAndUninitializedValuesStub] @@ -1984,14 +1985,16 @@ class A(Enum): # E: Detected enum "lib.A" in a type stub with zero members. The # N: See https://typing.readthedocs.io/en/latest/spec/enums.html#defining-members x: int class B(A): - x = 1 # E: Cannot override writable attribute "x" with a final one + x = 1 # E: Cannot override writable attribute "x" with a final one \ + # E: Incompatible types in assignment (expression has type "B", base class "A" defined the type as "int") class C(Enum): x = 1 class D(C): # E: Cannot extend enum with existing members: "C" \ # E: Detected enum "lib.D" in a type stub with zero members. There is a chance this is due to a recent change in the semantics of enum membership. If so, use `member = value` to mark an enum member, instead of `member: type` \ # N: See https://typing.readthedocs.io/en/latest/spec/enums.html#defining-members - x: int # E: Cannot assign to final name "x" + x: int # E: Incompatible types in assignment (expression has type "int", base class "C" defined the type as "C") \ + # E: Cannot assign to final name "x" [builtins fixtures/bool.pyi] [case testEnumNotFinalWithMethodsAndUninitializedValuesStubMember] @@ -2419,6 +2422,49 @@ def some_a(a: A): reveal_type(a) # N: Revealed type is "Literal[__main__.A.x]" [builtins fixtures/dict.pyi] +[case testEnumAccessFromInstance] +# flags: --python-version 3.11 --warn-unreachable +# This was added in 3.11 +from enum import Enum, member, nonmember + +class A(Enum): + x = 1 + y = member(2) + z = nonmember(3) + +reveal_type(A.x) # N: Revealed type is "Literal[__main__.A.x]?" +reveal_type(A.y) # N: Revealed type is "Literal[__main__.A.y]?" +reveal_type(A.z) # N: Revealed type is "builtins.int" + +reveal_type(A.x.x) # N: Revealed type is "Literal[__main__.A.x]?" +reveal_type(A.x.x.x) # N: Revealed type is "Literal[__main__.A.x]?" +reveal_type(A.x.y) # N: Revealed type is "Literal[__main__.A.y]?" +reveal_type(A.x.y.y) # N: Revealed type is "Literal[__main__.A.y]?" +reveal_type(A.x.z) # N: Revealed type is "builtins.int" + +reveal_type(A.y.x) # N: Revealed type is "Literal[__main__.A.x]?" +reveal_type(A.y.y) # N: Revealed type is "Literal[__main__.A.y]?" +reveal_type(A.y.z) # N: Revealed type is "builtins.int" + +A.z.x # E: "int" has no attribute "x" + +class B(Enum): + x = 1 + value = 2 + +reveal_type(B.x) # N: Revealed type is "Literal[__main__.B.x]?" +reveal_type(B.x.value) # N: Revealed type is "Literal[2]?" +reveal_type(B.x.x.value) # N: Revealed type is "Literal[2]?" +B.x.value.value # E: "int" has no attribute "value" +B.x.value.value.value # E: "int" has no attribute "value" +reveal_type(B.value) # N: Revealed type is "Literal[__main__.B.value]?" +reveal_type(B.value.x) # N: Revealed type is "Literal[__main__.B.x]?" +reveal_type(B.value.x.x) # N: Revealed type is "Literal[__main__.B.x]?" +reveal_type(B.value.x.value) # N: Revealed type is "Literal[2]?" +B.value.x.value.value # E: "int" has no attribute "value" +B.value.value.value # E: "int" has no attribute "value" +[builtins fixtures/dict.pyi] + [case testErrorOnAnnotatedMember] from enum import Enum diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 4c170ec4753fb..5f2194114d5e5 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -5675,10 +5675,12 @@ class FinalEnum(Enum): [builtins fixtures/isinstance.pyi] [out] main:3: error: Cannot override writable attribute "x" with a final one +main:3: error: Incompatible types in assignment (expression has type "Ok", base class "RegularEnum" defined the type as "int") main:4: error: Cannot extend enum with existing members: "FinalEnum" main:5: error: Cannot override final attribute "x" (previously declared in base class "FinalEnum") [out2] main:3: error: Cannot override writable attribute "x" with a final one +main:3: error: Incompatible types in assignment (expression has type "Ok", base class "RegularEnum" defined the type as "int") main:4: error: Cannot extend enum with existing members: "FinalEnum" main:5: error: Cannot override final attribute "x" (previously declared in base class "FinalEnum") From 68657d2431714394e81c6420e41d6c3bc8a2cdf5 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 3 Aug 2025 18:47:35 +0100 Subject: [PATCH 137/424] Interpret bare ClassVar as inferred, not Any (#19573) Fixes https://github.com/python/mypy/issues/5587 Apparently, it is part of _the spec_ now. --- mypy/semanal.py | 7 +++++++ test-data/unit/check-classvar.test | 20 ++++++++++++++++++++ test-data/unit/check-incremental.test | 2 +- test-data/unit/semanal-classvar.test | 2 +- 4 files changed, 29 insertions(+), 2 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 7cca406b661b6..ab9075cd06ce1 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -5094,6 +5094,7 @@ def check_classvar(self, s: AssignmentStmt) -> None: return if not s.type or not self.is_classvar(s.type): return + assert isinstance(s.type, UnboundType) if self.is_class_scope() and isinstance(lvalue, NameExpr): node = lvalue.node if isinstance(node, Var): @@ -5110,6 +5111,12 @@ def check_classvar(self, s: AssignmentStmt) -> None: # In case of member access, report error only when assigning to self # Other kinds of member assignments should be already reported self.fail_invalid_classvar(lvalue) + if not s.type.args: + if isinstance(s.rvalue, TempNode) and s.rvalue.no_rhs: + if self.options.disallow_any_generics: + self.fail("ClassVar without type argument becomes Any", s, code=codes.TYPE_ARG) + return + s.type = None def is_classvar(self, typ: Type) -> bool: if not isinstance(typ, UnboundType): diff --git a/test-data/unit/check-classvar.test b/test-data/unit/check-classvar.test index 8384e56247934..7918ccded2fe0 100644 --- a/test-data/unit/check-classvar.test +++ b/test-data/unit/check-classvar.test @@ -360,3 +360,23 @@ reveal_type(C.x) # E: Access to generic instance variables via class is ambiguo # N: Revealed type is "Any" reveal_type(C.y) # E: Access to generic class variables is ambiguous \ # N: Revealed type is "Any" + +[case testClassVarBareAnnotation] +from typing import ClassVar + +class C: + x: ClassVar = 1 + y: ClassVar + +reveal_type(C.x) # N: Revealed type is "builtins.int" +reveal_type(C().x) # N: Revealed type is "builtins.int" +reveal_type(C.y) # N: Revealed type is "Any" +reveal_type(C().y) # N: Revealed type is "Any" + +[case testClassVarBareAnnotationDisabled] +# flags: --disallow-any-generics +from typing import ClassVar + +class C: + x: ClassVar = 1 + y: ClassVar # E: ClassVar without type argument becomes Any diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 5f2194114d5e5..7d791319537f8 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -2051,7 +2051,7 @@ warn_no_return = True [case testIncrementalClassVar] from typing import ClassVar class A: - x = None # type: ClassVar + x: ClassVar A().x = 0 [out1] main:4: error: Cannot assign to class variable "x" via instance diff --git a/test-data/unit/semanal-classvar.test b/test-data/unit/semanal-classvar.test index 8add559bdd27e..62151666c011a 100644 --- a/test-data/unit/semanal-classvar.test +++ b/test-data/unit/semanal-classvar.test @@ -52,7 +52,7 @@ MypyFile:1( AssignmentStmt:3( NameExpr(x [m]) IntExpr(1) - Any))) + builtins.int))) [case testClassVarWithTypeVar] From 268c837ce30ca46b82b52be21a6edc96a8a8a8b7 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Sun, 3 Aug 2025 18:40:25 -0400 Subject: [PATCH 138/424] [mypyc] feat: unwrap NewType types to their base types for optimized code paths (#19497) This PR adds special case logic for unwrapping NewType types to their actual type. This logic is currently working. a `NewType("name", str)` now generates the same code as a `str`. I wasn't entirely sure of the best way to test this, so I just tweaked the str tests to use a union of str and newtype str and validated that the IR still uses `str` and not `object` Almost all of my tests are running fine, but I get a strange mypy error in the str.count tests saying that `str` objects do not have a .count method? Of course a str object has a .count method. Do you think this might related to the typeshed stuff again? --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- mypyc/irbuild/mapper.py | 4 + mypyc/test-data/fixtures/ir.py | 1 + mypyc/test-data/irbuild-str.test | 149 ++++++++++++++++++++++--------- 3 files changed, 113 insertions(+), 41 deletions(-) diff --git a/mypyc/irbuild/mapper.py b/mypyc/irbuild/mapper.py index 4a01255e2d5d5..815688d90fb6f 100644 --- a/mypyc/irbuild/mapper.py +++ b/mypyc/irbuild/mapper.py @@ -73,6 +73,10 @@ def type_to_rtype(self, typ: Type | None) -> RType: typ = get_proper_type(typ) if isinstance(typ, Instance): + if typ.type.is_newtype: + # Unwrap NewType to its base type for rprimitive mapping + assert len(typ.type.bases) == 1, typ.type.bases + return self.type_to_rtype(typ.type.bases[0]) if typ.type.fullname == "builtins.int": return int_rprimitive elif typ.type.fullname == "builtins.float": diff --git a/mypyc/test-data/fixtures/ir.py b/mypyc/test-data/fixtures/ir.py index 3776a3dcc79a1..76afc1ea58ccb 100644 --- a/mypyc/test-data/fixtures/ir.py +++ b/mypyc/test-data/fixtures/ir.py @@ -122,6 +122,7 @@ def rpartition(self, sep: str, /) -> Tuple[str, str, str]: ... def removeprefix(self, prefix: str, /) -> str: ... def removesuffix(self, suffix: str, /) -> str: ... def islower(self) -> bool: ... + def count(self, substr: str, start: Optional[int] = None, end: Optional[int] = None) -> int: pass class float: def __init__(self, x: object) -> None: pass diff --git a/mypyc/test-data/irbuild-str.test b/mypyc/test-data/irbuild-str.test index 4a4992d41a5d0..3e69325a454bf 100644 --- a/mypyc/test-data/irbuild-str.test +++ b/mypyc/test-data/irbuild-str.test @@ -1,13 +1,15 @@ [case testStrSplit] -from typing import Optional, List +from typing import NewType, Optional, List, Union +NewStr = NewType("NewStr", str) -def do_split(s: str, sep: Optional[str] = None, max_split: Optional[int] = None) -> List[str]: +def do_split(s: Union[str, NewStr], sep: Optional[str] = None, max_split: Optional[int] = None) -> List[str]: if sep is not None: if max_split is not None: return s.split(sep, max_split) else: return s.split(sep) return s.split() +[typing fixtures/typing-full.pyi] [out] def do_split(s, sep, max_split): s :: str @@ -56,12 +58,15 @@ L9: [case testStrEquality] +from typing import NewType, Union +NewStr = NewType("NewStr", str) def eq(x: str, y: str) -> bool: return x == y -def neq(x: str, y: str) -> bool: +def neq(x: str, y: Union[str, NewStr]) -> bool: return x != y +[typing fixtures/typing-full.pyi] [out] def eq(x, y): x, y :: str @@ -79,13 +84,14 @@ L0: return r1 [case testStrReplace] -from typing import Optional - -def do_replace(s: str, old_substr: str, new_substr: str, max_count: Optional[int] = None) -> str: +from typing import NewType, Optional, Union +NewStr = NewType("NewStr", str) +def do_replace(s: Union[str, NewStr], old_substr: str, new_substr: str, max_count: Optional[int] = None) -> str: if max_count is not None: return s.replace(old_substr, new_substr, max_count) else: return s.replace(old_substr, new_substr) +[typing fixtures/typing-full.pyi] [out] def do_replace(s, old_substr, new_substr, max_count): s, old_substr, new_substr :: str @@ -114,17 +120,19 @@ L5: unreachable [case testStrStartswithEndswithTuple] -from typing import Tuple +from typing import NewType, Tuple, Union +NewStr = NewType("NewStr", str) -def do_startswith(s1: str, s2: Tuple[str, ...]) -> bool: +def do_startswith(s1: Union[str, NewStr], s2: Tuple[str, ...]) -> bool: return s1.startswith(s2) -def do_endswith(s1: str, s2: Tuple[str, ...]) -> bool: +def do_endswith(s1: Union[str, NewStr], s2: Tuple[str, ...]) -> bool: return s1.endswith(s2) -def do_tuple_literal_args(s1: str) -> None: +def do_tuple_literal_args(s1: Union[str, NewStr]) -> None: x = s1.startswith(("a", "b")) y = s1.endswith(("a", "b")) +[typing fixtures/typing-full.pyi] [out] def do_startswith(s1, s2): s1 :: str @@ -165,11 +173,14 @@ L0: return 1 [case testStrToBool] -def is_true(x: str) -> bool: +from typing import NewType, Union +NewStr = NewType("NewStr", str) +def is_true(x: Union[str, NewStr]) -> bool: if x: return True else: return False +[typing fixtures/typing-full.pyi] [out] def is_true(x): x :: str @@ -185,11 +196,14 @@ L3: unreachable [case testStringFormatMethod] -def f(s: str, num: int) -> None: +from typing import NewType, Union +NewStr = NewType("NewStr", str) +def f(s: Union[str, NewStr], num: int) -> None: s1 = "Hi! I'm {}, and I'm {} years old.".format(s, num) s2 = ''.format() s3 = 'abc'.format() s4 = '}}{}{{{}}}{{{}'.format(num, num, num) +[typing fixtures/typing-full.pyi] [out] def f(s, num): s :: str @@ -217,11 +231,14 @@ L0: return 1 [case testFStrings_64bit] -def f(var: str, num: int) -> None: +from typing import NewType, Union +NewStr = NewType("NewStr", str) +def f(var: Union[str, NewStr], num: int) -> None: s1 = f"Hi! I'm {var}. I am {num} years old." s2 = f'Hello {var:>{num}}' s3 = f'' s4 = f'abc' +[typing fixtures/typing-full.pyi] [out] def f(var, num): var :: str @@ -267,7 +284,9 @@ L0: return 1 [case testStringFormattingCStyle] -def f(var: str, num: int) -> None: +from typing import NewType, Union +NewStr = NewType("NewStr", str) +def f(var: Union[str, NewStr], num: int) -> None: s1 = "Hi! I'm %s." % var s2 = "I am %d years old." % num s3 = "Hi! I'm %s. I am %d years old." % (var, num) @@ -322,7 +341,9 @@ L0: return 1 [case testEncode_64bit] -def f(s: str) -> None: +from typing import NewType, Union +NewStr = NewType("NewStr", str) +def f(s: Union[str, NewStr]) -> None: s.encode() s.encode('utf-8') s.encode('utf8', 'strict') @@ -340,6 +361,7 @@ def f(s: str) -> None: s.encode(encoding=encoding, errors=errors) s.encode('latin2') +[typing fixtures/typing-full.pyi] [out] def f(s): s :: str @@ -410,7 +432,9 @@ L0: return 1 [case testOrd] -def str_ord(x: str) -> int: +from typing import NewType, Union +NewStr = NewType("NewStr", str) +def str_ord(x: Union[str, NewStr]) -> int: return ord(x) def str_ord_literal() -> int: return ord("a") @@ -420,6 +444,7 @@ def bytes_ord_literal() -> int: return ord(b"a") def any_ord(x) -> int: return ord(x) +[typing fixtures/typing-full.pyi] [out] def str_ord(x): x :: str @@ -459,13 +484,16 @@ L0: return r6 [case testStrip] -def do_strip(s: str) -> None: +from typing import NewType, Union +NewStr = NewType("NewStr", str) +def do_strip(s: Union[str, NewStr]) -> None: s.lstrip("x") s.strip("y") s.rstrip("z") s.lstrip() s.strip() s.rstrip() +[typing fixtures/typing-full.pyi] [out] def do_strip(s): s, r0, r1, r2, r3, r4, r5, r6, r7, r8 :: str @@ -481,60 +509,99 @@ L0: r8 = CPyStr_RStrip(s, 0) return 1 -[case testCountAll] +[case testCountAll_64bit] +from typing import NewType, Union +NewStr = NewType("NewStr", str) def do_count(s: str) -> int: - return s.count("x") # type: ignore [attr-defined] + return s.count("x") +[typing fixtures/typing-full.pyi] [out] def do_count(s): s, r0 :: str r1 :: native_int - r2 :: bit - r3 :: object - r4 :: int + r2, r3, r4 :: bit + r5, r6, r7 :: int L0: r0 = 'x' r1 = CPyStr_Count(s, r0, 0) r2 = r1 >= 0 :: signed - r3 = box(native_int, r1) - r4 = unbox(int, r3) - return r4 + r3 = r1 <= 4611686018427387903 :: signed + if r3 goto L1 else goto L2 :: bool +L1: + r4 = r1 >= -4611686018427387904 :: signed + if r4 goto L3 else goto L2 :: bool +L2: + r5 = CPyTagged_FromInt64(r1) + r6 = r5 + goto L4 +L3: + r7 = r1 << 1 + r6 = r7 +L4: + return r6 -[case testCountStart] +[case testCountStart_64bit] +from typing import NewType, Union +NewStr = NewType("NewStr", str) def do_count(s: str, start: int) -> int: - return s.count("x", start) # type: ignore [attr-defined] + return s.count("x", start) +[typing fixtures/typing-full.pyi] [out] def do_count(s, start): s :: str start :: int r0 :: str r1 :: native_int - r2 :: bit - r3 :: object - r4 :: int + r2, r3, r4 :: bit + r5, r6, r7 :: int L0: r0 = 'x' r1 = CPyStr_Count(s, r0, start) r2 = r1 >= 0 :: signed - r3 = box(native_int, r1) - r4 = unbox(int, r3) - return r4 + r3 = r1 <= 4611686018427387903 :: signed + if r3 goto L1 else goto L2 :: bool +L1: + r4 = r1 >= -4611686018427387904 :: signed + if r4 goto L3 else goto L2 :: bool +L2: + r5 = CPyTagged_FromInt64(r1) + r6 = r5 + goto L4 +L3: + r7 = r1 << 1 + r6 = r7 +L4: + return r6 -[case testCountStartEnd] +[case testCountStartEnd_64bit] +from typing import NewType, Union +NewStr = NewType("NewStr", str) def do_count(s: str, start: int, end: int) -> int: - return s.count("x", start, end) # type: ignore [attr-defined] + return s.count("x", start, end) +[typing fixtures/typing-full.pyi] [out] def do_count(s, start, end): s :: str start, end :: int r0 :: str r1 :: native_int - r2 :: bit - r3 :: object - r4 :: int + r2, r3, r4 :: bit + r5, r6, r7 :: int L0: r0 = 'x' r1 = CPyStr_CountFull(s, r0, start, end) r2 = r1 >= 0 :: signed - r3 = box(native_int, r1) - r4 = unbox(int, r3) - return r4 + r3 = r1 <= 4611686018427387903 :: signed + if r3 goto L1 else goto L2 :: bool +L1: + r4 = r1 >= -4611686018427387904 :: signed + if r4 goto L3 else goto L2 :: bool +L2: + r5 = CPyTagged_FromInt64(r1) + r6 = r5 + goto L4 +L3: + r7 = r1 << 1 + r6 = r7 +L4: + return r6 From 555edf36a8483042013a7b3c4e858c3f2c0aadcb Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Mon, 4 Aug 2025 00:42:53 +0200 Subject: [PATCH 139/424] Introduce temporary named expressions for `match` subjects (#18446) Fixes #18440. Fixes #17230. Fixes #16650. Improves behavior in #14731 (but still thinks that match is non-exhaustive, "missing return" false positive remains). #16503 did this specifically for `CallExpr`, but that isn't the only kind of such statements. I propose to expand this for more general expressions and believe that a blacklist is more reasonable here: we do **not** want to introduce a temporary name only for certain expressions that either are already named or can be used to infer contained variables (inline tuple/list/dict/set literals). Writing logic to generate a name for every other kind of expression would be quite cumbersome - I circumvent this by using a simple counter to generate unique names on demand. --------- Co-authored-by: Ivan Levkivskyi --- mypy/binder.py | 20 +++++- mypy/checker.py | 54 ++++++++++---- test-data/unit/check-python310.test | 108 +++++++++++++++++++++++++++- 3 files changed, 165 insertions(+), 17 deletions(-) diff --git a/mypy/binder.py b/mypy/binder.py index 2ae58dad1fe07..c95481329a571 100644 --- a/mypy/binder.py +++ b/mypy/binder.py @@ -8,7 +8,16 @@ from mypy.erasetype import remove_instance_last_known_values from mypy.literals import Key, extract_var_from_literal_hash, literal, literal_hash, subkeys -from mypy.nodes import Expression, IndexExpr, MemberExpr, NameExpr, RefExpr, TypeInfo, Var +from mypy.nodes import ( + LITERAL_NO, + Expression, + IndexExpr, + MemberExpr, + NameExpr, + RefExpr, + TypeInfo, + Var, +) from mypy.options import Options from mypy.subtypes import is_same_type, is_subtype from mypy.typeops import make_simplified_union @@ -173,6 +182,15 @@ def _get(self, key: Key, index: int = -1) -> CurrentType | None: return self.frames[i].types[key] return None + @classmethod + def can_put_directly(cls, expr: Expression) -> bool: + """Will `.put()` on this expression be successful? + + This is inlined in `.put()` because the logic is rather hot and must be kept + in sync. + """ + return isinstance(expr, (IndexExpr, MemberExpr, NameExpr)) and literal(expr) > LITERAL_NO + def put(self, expr: Expression, typ: Type, *, from_assignment: bool = True) -> None: """Directly set the narrowed type of expression (if it supports it). diff --git a/mypy/checker.py b/mypy/checker.py index dfbfa753d5f2c..c1ff29aa33d9f 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -358,6 +358,9 @@ class TypeChecker(NodeVisitor[None], TypeCheckerSharedApi): # functions such as open(), etc. plugin: Plugin + # A helper state to produce unique temporary names on demand. + _unique_id: int + def __init__( self, errors: Errors, @@ -428,6 +431,7 @@ def __init__( self, self.msg, self.plugin, per_line_checking_time_ns ) self.pattern_checker = PatternChecker(self, self.msg, self.plugin, options) + self._unique_id = 0 @property def expr_checker(self) -> mypy.checkexpr.ExpressionChecker: @@ -5476,21 +5480,10 @@ def visit_continue_stmt(self, s: ContinueStmt) -> None: return def visit_match_stmt(self, s: MatchStmt) -> None: - named_subject: Expression - if isinstance(s.subject, CallExpr): - # Create a dummy subject expression to handle cases where a match statement's subject - # is not a literal value. This lets us correctly narrow types and check exhaustivity - # This is hack! - if s.subject_dummy is None: - id = s.subject.callee.fullname if isinstance(s.subject.callee, RefExpr) else "" - name = "dummy-match-" + id - v = Var(name) - s.subject_dummy = NameExpr(name) - s.subject_dummy.node = v - named_subject = s.subject_dummy - else: - named_subject = s.subject - + named_subject = self._make_named_statement_for_match(s) + # In sync with similar actions elsewhere, narrow the target if + # we are matching an AssignmentExpr + unwrapped_subject = collapse_walrus(s.subject) with self.binder.frame_context(can_skip=False, fall_through=0): subject_type = get_proper_type(self.expr_checker.accept(s.subject)) @@ -5523,6 +5516,12 @@ def visit_match_stmt(self, s: MatchStmt) -> None: pattern_map, else_map = conditional_types_to_typemaps( named_subject, pattern_type.type, pattern_type.rest_type ) + # Maybe the subject type can be inferred from constraints on + # its attribute/item? + if pattern_map and named_subject in pattern_map: + pattern_map[unwrapped_subject] = pattern_map[named_subject] + if else_map and named_subject in else_map: + else_map[unwrapped_subject] = else_map[named_subject] pattern_map = self.propagate_up_typemap_info(pattern_map) else_map = self.propagate_up_typemap_info(else_map) self.remove_capture_conflicts(pattern_type.captures, inferred_types) @@ -5575,6 +5574,25 @@ def visit_match_stmt(self, s: MatchStmt) -> None: with self.binder.frame_context(can_skip=False, fall_through=2): pass + def _make_named_statement_for_match(self, s: MatchStmt) -> Expression: + """Construct a fake NameExpr for inference if a match clause is complex.""" + subject = s.subject + if self.binder.can_put_directly(subject): + # Already named - we should infer type of it as given + return subject + elif s.subject_dummy is not None: + return s.subject_dummy + else: + # Create a dummy subject expression to handle cases where a match statement's subject + # is not a literal value. This lets us correctly narrow types and check exhaustivity + # This is hack! + name = self.new_unique_dummy_name("match") + v = Var(name) + named_subject = NameExpr(name) + named_subject.node = v + s.subject_dummy = named_subject + return named_subject + def _get_recursive_sub_patterns_map( self, expr: Expression, typ: Type ) -> dict[Expression, Type]: @@ -7966,6 +7984,12 @@ def warn_deprecated(self, node: Node | None, context: Context) -> None: warn = self.msg.note if self.options.report_deprecated_as_note else self.msg.fail warn(deprecated, context, code=codes.DEPRECATED) + def new_unique_dummy_name(self, namespace: str) -> str: + """Generate a name that is guaranteed to be unique for this TypeChecker instance.""" + name = f"dummy-{namespace}-{self._unique_id}" + self._unique_id += 1 + return name + # leafs def visit_pass_stmt(self, o: PassStmt, /) -> None: diff --git a/test-data/unit/check-python310.test b/test-data/unit/check-python310.test index 80fd64fa3569a..f264167cb0670 100644 --- a/test-data/unit/check-python310.test +++ b/test-data/unit/check-python310.test @@ -1302,7 +1302,7 @@ def main() -> None: case a: reveal_type(a) # N: Revealed type is "builtins.int" -[case testMatchCapturePatternFromAsyncFunctionReturningUnion-xfail] +[case testMatchCapturePatternFromAsyncFunctionReturningUnion] async def func1(arg: bool) -> str | int: ... async def func2(arg: bool) -> bytes | int: ... @@ -2179,9 +2179,11 @@ def f() -> None: match x := returns_a_or_none(): case A(): reveal_type(x.a) # N: Revealed type is "builtins.int" + reveal_type(x) # N: Revealed type is "Union[__main__.A, None]" match x := returns_a(): case A(): reveal_type(x.a) # N: Revealed type is "builtins.int" + reveal_type(x) # N: Revealed type is "__main__.A" y = returns_a_or_none() match y: case A(): @@ -2586,6 +2588,110 @@ def fn2(x: Some | int | str) -> None: pass [builtins fixtures/dict.pyi] +[case testMatchFunctionCall] +# flags: --warn-unreachable + +def fn() -> int | str: ... + +match fn(): + case str(s): + reveal_type(s) # N: Revealed type is "builtins.str" + case int(i): + reveal_type(i) # N: Revealed type is "builtins.int" + case other: + other # E: Statement is unreachable + +[case testMatchAttribute] +# flags: --warn-unreachable + +class A: + foo: int | str + +match A().foo: + case str(s): + reveal_type(s) # N: Revealed type is "builtins.str" + case int(i): + reveal_type(i) # N: Revealed type is "builtins.int" + case other: + other # E: Statement is unreachable + +[case testMatchLiteral] +# flags: --warn-unreachable + +def int_literal() -> None: + match 12: + case 1 as s: + reveal_type(s) # N: Revealed type is "Literal[1]" + case int(i): + reveal_type(i) # N: Revealed type is "Literal[12]?" + case other: + other # E: Statement is unreachable + +def str_literal() -> None: + match 'foo': + case 'a' as s: + reveal_type(s) # N: Revealed type is "Literal['a']" + case str(i): + reveal_type(i) # N: Revealed type is "Literal['foo']?" + case other: + other # E: Statement is unreachable + +[case testMatchOperations] +# flags: --warn-unreachable + +x: int +match -x: + case -1 as s: + reveal_type(s) # N: Revealed type is "Literal[-1]" + case int(s): + reveal_type(s) # N: Revealed type is "builtins.int" + case other: + other # E: Statement is unreachable + +match 1 + 2: + case 3 as s: + reveal_type(s) # N: Revealed type is "Literal[3]" + case int(s): + reveal_type(s) # N: Revealed type is "builtins.int" + case other: + other # E: Statement is unreachable + +match 1 > 2: + case True as s: + reveal_type(s) # N: Revealed type is "Literal[True]" + case False as s: + reveal_type(s) # N: Revealed type is "Literal[False]" + case other: + other # E: Statement is unreachable +[builtins fixtures/ops.pyi] + +[case testMatchDictItem] +# flags: --warn-unreachable + +m: dict[str, int | str] +k: str + +match m[k]: + case str(s): + reveal_type(s) # N: Revealed type is "builtins.str" + case int(i): + reveal_type(i) # N: Revealed type is "builtins.int" + case other: + other # E: Statement is unreachable + +[builtins fixtures/dict.pyi] + +[case testMatchLiteralValuePathological] +# flags: --warn-unreachable + +match 0: + case 0 as i: + reveal_type(i) # N: Revealed type is "Literal[0]?" + case int(i): + i # E: Statement is unreachable + case other: + other # E: Statement is unreachable + [case testMatchNamedTupleSequence] from typing import Any, NamedTuple From 0c922ce8c0fe28e9ea713e1787b2d9e044c75cd0 Mon Sep 17 00:00:00 2001 From: Brian Schubert Date: Sun, 3 Aug 2025 22:33:45 -0400 Subject: [PATCH 140/424] Recognize buffer protocol special methods (#19581) The special methods `__buffer__` and `__release_buffer__` were [added in Python 3.12](https://docs.python.org/3/reference/datamodel.html#emulating-buffer-types). Both are [implemented with slot descriptors](https://docs.python.org/3/c-api/typeobj.html#:~:text=__buffer__), and so should be treated as being implicitly positional-only like other slot special methods. --- mypy/sharedparse.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mypy/sharedparse.py b/mypy/sharedparse.py index ef2e4f7206649..71d1dee8f7d6e 100644 --- a/mypy/sharedparse.py +++ b/mypy/sharedparse.py @@ -10,6 +10,7 @@ "__call__", "__complex__", "__contains__", + "__buffer__", "__del__", "__delattr__", "__delitem__", @@ -31,6 +32,7 @@ "__new__", "__oct__", "__pos__", + "__release_buffer__", "__repr__", "__reversed__", "__setattr__", From 5b03024e829940cf3c3e3d99fc6625f569d02728 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 4 Aug 2025 04:09:40 +0100 Subject: [PATCH 141/424] Remove --new-type-inference flag (#19570) It was on deprecated almost two years ago (it is on by default). --- mypy/main.py | 10 ------ mypy/options.py | 4 +-- test-data/unit/check-generics.test | 35 ------------------- test-data/unit/check-inference-context.test | 1 - test-data/unit/check-inference.test | 2 -- test-data/unit/check-overloading.test | 1 - .../unit/check-parameter-specification.test | 4 --- test-data/unit/check-plugin-attrs.test | 1 - test-data/unit/check-typevar-tuple.test | 1 - test-data/unit/cmdline.test | 8 ----- test-data/unit/pythoneval.test | 14 +++----- 11 files changed, 6 insertions(+), 75 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index 2fbb9671e721e..fd50c7677a112 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -1086,10 +1086,6 @@ def add_invertible_flag( internals_group.add_argument( "--old-type-inference", action="store_true", help=argparse.SUPPRESS ) - # Deprecated reverse variant of the above. - internals_group.add_argument( - "--new-type-inference", action="store_true", help=argparse.SUPPRESS - ) internals_group.add_argument( "--disable-expression-cache", action="store_true", help=argparse.SUPPRESS ) @@ -1507,12 +1503,6 @@ def set_strict_flags() -> None: if options.logical_deps: options.cache_fine_grained = True - if options.new_type_inference: - print( - "Warning: --new-type-inference flag is deprecated;" - " new type inference algorithm is already enabled by default" - ) - if options.strict_concatenate and not strict_option_set: print("Warning: --strict-concatenate is deprecated; use --extra-checks instead") diff --git a/mypy/options.py b/mypy/options.py index 573be14c4b1b3..6d7eca7728883 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -391,10 +391,8 @@ def __init__(self) -> None: # skip most errors after this many messages have been reported. # -1 means unlimited. self.many_errors_threshold = defaults.MANY_ERRORS_THRESHOLD - # Disable new experimental type inference algorithm. + # Disable new type inference algorithm. self.old_type_inference = False - # Deprecated reverse version of the above, do not use. - self.new_type_inference = False # Disable expression cache (for debugging). self.disable_expression_cache = False # Export line-level, limited, fine-grained dependency information in cache data diff --git a/test-data/unit/check-generics.test b/test-data/unit/check-generics.test index abeb5face26fb..3b535ab4a1c04 100644 --- a/test-data/unit/check-generics.test +++ b/test-data/unit/check-generics.test @@ -2750,7 +2750,6 @@ reveal_type(func(1)) # N: Revealed type is "builtins.int" [builtins fixtures/tuple.pyi] [case testGenericLambdaGenericMethodNoCrash] -# flags: --new-type-inference from typing import TypeVar, Union, Callable, Generic S = TypeVar("S") @@ -2789,7 +2788,6 @@ reveal_type(dict2) # N: Revealed type is "builtins.dict[Any, __main__.B]" -- ------------------------------------------------------------------ [case testInferenceAgainstGenericCallable] -# flags: --new-type-inference from typing import TypeVar, Callable, List X = TypeVar('X') @@ -2807,7 +2805,6 @@ reveal_type(bar(id)) # N: Revealed type is "builtins.list[builtins.int]" [builtins fixtures/list.pyi] [case testInferenceAgainstGenericCallableNoLeak] -# flags: --new-type-inference from typing import TypeVar, Callable T = TypeVar('T') @@ -2823,7 +2820,6 @@ reveal_type(f(tpl)) # N: Revealed type is "Any" [out] [case testInferenceAgainstGenericCallableChain] -# flags: --new-type-inference from typing import TypeVar, Callable, List X = TypeVar('X') @@ -2836,7 +2832,6 @@ reveal_type(chain(id, id)) # N: Revealed type is "def (builtins.int) -> builtin [builtins fixtures/list.pyi] [case testInferenceAgainstGenericCallableGeneric] -# flags: --new-type-inference from typing import TypeVar, Callable, List S = TypeVar('S') @@ -2857,7 +2852,6 @@ reveal_type(same(42)) # N: Revealed type is "builtins.list[builtins.int]" [builtins fixtures/list.pyi] [case testInferenceAgainstGenericCallableGenericReverse] -# flags: --new-type-inference from typing import TypeVar, Callable, List S = TypeVar('S') @@ -2878,7 +2872,6 @@ reveal_type(same([42])) # N: Revealed type is "builtins.int" [builtins fixtures/list.pyi] [case testInferenceAgainstGenericCallableGenericArg] -# flags: --new-type-inference from typing import TypeVar, Callable, List S = TypeVar('S') @@ -2899,7 +2892,6 @@ reveal_type(single(42)) # N: Revealed type is "builtins.list[builtins.int]" [builtins fixtures/list.pyi] [case testInferenceAgainstGenericCallableGenericChain] -# flags: --new-type-inference from typing import TypeVar, Callable, List S = TypeVar('S') @@ -2913,7 +2905,6 @@ reveal_type(comb(id, id)) # N: Revealed type is "def [T] (T`1) -> T`1" [builtins fixtures/list.pyi] [case testInferenceAgainstGenericCallableGenericNonLinear] -# flags: --new-type-inference from typing import TypeVar, Callable, List S = TypeVar('S') @@ -2934,7 +2925,6 @@ reveal_type(mix([id, id, id])) # N: Revealed type is "def [S] (S`4) -> builtins [builtins fixtures/list.pyi] [case testInferenceAgainstGenericCurry] -# flags: --new-type-inference from typing import Callable, List, TypeVar S = TypeVar("S") @@ -2953,7 +2943,6 @@ reveal_type(dec2(test2)) # N: Revealed type is "def [T] (T`3) -> def (T`3) -> T [builtins fixtures/list.pyi] [case testInferenceAgainstGenericCallableNewVariable] -# flags: --new-type-inference from typing import TypeVar, Callable, List S = TypeVar('S') @@ -2968,7 +2957,6 @@ reveal_type(dec(test)) # N: Revealed type is "def [U] (builtins.list[U`-1]) -> [builtins fixtures/list.pyi] [case testInferenceAgainstGenericCallableGenericAlias] -# flags: --new-type-inference from typing import TypeVar, Callable, List S = TypeVar('S') @@ -2986,7 +2974,6 @@ reveal_type(dec(id)) # N: Revealed type is "def [S] (S`1) -> builtins.list[S`1] [builtins fixtures/list.pyi] [case testInferenceAgainstGenericCallableGenericProtocol] -# flags: --new-type-inference from typing import TypeVar, Protocol, Generic, Optional T = TypeVar('T') @@ -3002,7 +2989,6 @@ reveal_type(lift(g)) # N: Revealed type is "def [T] (Union[T`1, None]) -> Union [builtins fixtures/list.pyi] [case testInferenceAgainstGenericSplitOrder] -# flags: --new-type-inference from typing import TypeVar, Callable, List S = TypeVar('S') @@ -3017,7 +3003,6 @@ reveal_type(dec(id, id)) # N: Revealed type is "def (builtins.int) -> builtins. [builtins fixtures/list.pyi] [case testInferenceAgainstGenericSplitOrderGeneric] -# flags: --new-type-inference from typing import TypeVar, Callable, Tuple S = TypeVar('S') @@ -3048,7 +3033,6 @@ reveal_type(id) # N: Revealed type is "def (builtins.int) -> builtins.int" [builtins fixtures/tuple.pyi] [case testInferenceAgainstGenericEllipsisSelfSpecialCase] -# flags: --new-type-inference from typing import Self, Callable, TypeVar T = TypeVar("T") @@ -3062,7 +3046,6 @@ c: C reveal_type(c.test()) # N: Revealed type is "__main__.C" [case testInferenceAgainstGenericBoundsAndValues] -# flags: --new-type-inference from typing import TypeVar, Callable, List class B: ... @@ -3090,7 +3073,6 @@ reveal_type(dec2(id2)) # N: Revealed type is "def (Never) -> builtins.list[Neve # E: Argument 1 to "dec2" has incompatible type "Callable[[V], V]"; expected "Callable[[Never], Never]" [case testInferenceAgainstGenericLambdas] -# flags: --new-type-inference from typing import TypeVar, Callable, List S = TypeVar('S') @@ -3127,7 +3109,6 @@ dec4_bound(lambda x: x) # E: Value of type variable "I" of "dec4_bound" cannot [builtins fixtures/list.pyi] [case testInferenceAgainstGenericParamSpecBasicInList] -# flags: --new-type-inference from typing import TypeVar, Callable, List, Tuple from typing_extensions import ParamSpec @@ -3146,7 +3127,6 @@ reveal_type(dec(pair)) # N: Revealed type is "def [U, V] (x: U`-1, y: V`-2) -> [builtins fixtures/list.pyi] [case testInferenceAgainstGenericParamSpecBasicDeList] -# flags: --new-type-inference from typing import TypeVar, Callable, List, Tuple from typing_extensions import ParamSpec @@ -3163,7 +3143,6 @@ reveal_type(dec(either)) # N: Revealed type is "def [T] (x: builtins.list[T`5], [builtins fixtures/list.pyi] [case testInferenceAgainstGenericParamSpecPopOff] -# flags: --new-type-inference from typing import TypeVar, Callable, List, Tuple from typing_extensions import ParamSpec, Concatenate @@ -3184,7 +3163,6 @@ reveal_type(dec(dec)) # N: Revealed type is "def () -> def [T, P, S] (def (T`-1 [builtins fixtures/list.pyi] [case testInferenceAgainstGenericParamSpecPopOn] -# flags: --new-type-inference from typing import TypeVar, Callable, List, Tuple from typing_extensions import ParamSpec, Concatenate @@ -3206,7 +3184,6 @@ reveal_type(dec(dec)) # N: Revealed type is "def [T, S] (T`13, f: def () -> def [builtins fixtures/list.pyi] [case testInferenceAgainstGenericParamSpecVsParamSpec] -# flags: --new-type-inference from typing import TypeVar, Callable, List, Tuple, Generic from typing_extensions import ParamSpec, Concatenate @@ -3227,7 +3204,6 @@ reveal_type(dec(h)) # N: Revealed type is "def [T, Q] (T`-1, *Q.args, **Q.kwarg [builtins fixtures/list.pyi] [case testInferenceAgainstGenericParamSpecVsParamSpecConcatenate] -# flags: --new-type-inference from typing import TypeVar, Callable, List, Tuple, Generic from typing_extensions import ParamSpec, Concatenate @@ -3245,7 +3221,6 @@ reveal_type(dec(h)) # N: Revealed type is "def [T, Q] (T`-1, *Q.args, **Q.kwarg [builtins fixtures/list.pyi] [case testInferenceAgainstGenericParamSpecSecondary] -# flags: --new-type-inference from typing import TypeVar, Callable, List, Tuple, Generic from typing_extensions import ParamSpec, Concatenate @@ -3263,7 +3238,6 @@ reveal_type(dec(g)) # N: Revealed type is "def (builtins.int) -> __main__.Foo[[ [builtins fixtures/list.pyi] [case testInferenceAgainstGenericParamSpecSecondOrder] -# flags: --new-type-inference from typing import TypeVar, Callable from typing_extensions import ParamSpec, Concatenate @@ -3285,7 +3259,6 @@ reveal_type(transform(dec2)) # N: Revealed type is "def [W, T] (def (builtins.i [builtins fixtures/tuple.pyi] [case testNoAccidentalVariableClashInNestedGeneric] -# flags: --new-type-inference from typing import TypeVar, Callable, Generic, Tuple T = TypeVar('T') @@ -3302,7 +3275,6 @@ def apply(a: S, b: T) -> None: [builtins fixtures/tuple.pyi] [case testInferenceAgainstGenericParamSpecSpuriousBoundsNotUsed] -# flags: --new-type-inference from typing import TypeVar, Callable, Generic from typing_extensions import ParamSpec, Concatenate @@ -3321,7 +3293,6 @@ reveal_type(test) # N: Revealed type is "def () -> def [Q] (__main__.Foo[Q`-1]) [builtins fixtures/tuple.pyi] [case testInferenceAgainstGenericVariadicBasicInList] -# flags: --new-type-inference from typing import Tuple, TypeVar, List, Callable from typing_extensions import Unpack, TypeVarTuple @@ -3341,7 +3312,6 @@ reveal_type(dec(pair)) # N: Revealed type is "def [U, V] (U`-1, V`-2) -> builti [builtins fixtures/tuple.pyi] [case testInferenceAgainstGenericVariadicBasicDeList] -# flags: --new-type-inference from typing import Tuple, TypeVar, List, Callable from typing_extensions import Unpack, TypeVarTuple @@ -3359,7 +3329,6 @@ reveal_type(dec(either)) # N: Revealed type is "def [T] (builtins.list[T`5], bu [builtins fixtures/tuple.pyi] [case testInferenceAgainstGenericVariadicPopOff] -# flags: --new-type-inference from typing import TypeVar, Callable, List, Tuple from typing_extensions import Unpack, TypeVarTuple @@ -3381,7 +3350,6 @@ reveal_type(dec(dec)) # N: Revealed type is "def () -> def [T, Ts, S] (def (T`- [builtins fixtures/list.pyi] [case testInferenceAgainstGenericVariadicPopOn] -# flags: --new-type-inference from typing import TypeVar, Callable, List, Tuple from typing_extensions import Unpack, TypeVarTuple @@ -3404,7 +3372,6 @@ reveal_type(dec(dec)) # N: Revealed type is "def [T, S] (T`13, def () -> def (T [builtins fixtures/list.pyi] [case testInferenceAgainstGenericVariadicVsVariadic] -# flags: --new-type-inference from typing import TypeVar, Callable, List, Generic from typing_extensions import Unpack, TypeVarTuple @@ -3424,7 +3391,6 @@ reveal_type(dec(g)) # N: Revealed type is "def [Ts] (*Unpack[Ts`4]) -> builtins [builtins fixtures/list.pyi] [case testInferenceAgainstGenericVariadicVsVariadicConcatenate] -# flags: --new-type-inference from typing import TypeVar, Callable, Generic from typing_extensions import Unpack, TypeVarTuple @@ -3443,7 +3409,6 @@ reveal_type(dec(h)) # N: Revealed type is "def [T, Us] (T`-1, *Unpack[Us`-2]) - [builtins fixtures/list.pyi] [case testInferenceAgainstGenericVariadicSecondary] -# flags: --new-type-inference from typing import TypeVar, Callable, Generic from typing_extensions import Unpack, TypeVarTuple diff --git a/test-data/unit/check-inference-context.test b/test-data/unit/check-inference-context.test index ff726530cf9f0..5a674cca09da3 100644 --- a/test-data/unit/check-inference-context.test +++ b/test-data/unit/check-inference-context.test @@ -693,7 +693,6 @@ f(lambda: None) g(lambda: None) [case testIsinstanceInInferredLambda] -# flags: --new-type-inference from typing import TypeVar, Callable, Optional T = TypeVar('T') S = TypeVar('S') diff --git a/test-data/unit/check-inference.test b/test-data/unit/check-inference.test index 6564fb3192d04..53efcc0d22e39 100644 --- a/test-data/unit/check-inference.test +++ b/test-data/unit/check-inference.test @@ -1405,14 +1405,12 @@ class B: pass [builtins fixtures/list.pyi] [case testUninferableLambda] -# flags: --new-type-inference from typing import TypeVar, Callable X = TypeVar('X') def f(x: Callable[[X], X]) -> X: pass y = f(lambda x: x) # E: Need type annotation for "y" [case testUninferableLambdaWithTypeError] -# flags: --new-type-inference from typing import TypeVar, Callable X = TypeVar('X') def f(x: Callable[[X], X], y: str) -> X: pass diff --git a/test-data/unit/check-overloading.test b/test-data/unit/check-overloading.test index 22221416f151d..e7f6ff04c13ee 100644 --- a/test-data/unit/check-overloading.test +++ b/test-data/unit/check-overloading.test @@ -6637,7 +6637,6 @@ reveal_type(Snafu.snafu('123')) # N: Revealed type is "builtins.str" [builtins fixtures/staticmethod.pyi] [case testOverloadedWithInternalTypeVars] -# flags: --new-type-inference import m [file m.pyi] diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index 0835ba7ac57d1..2b4f92c7c8195 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -1121,7 +1121,6 @@ reveal_type(jf(1)) # N: Revealed type is "None" [builtins fixtures/paramspec.pyi] [case testGenericsInInferredParamspecReturn] -# flags: --new-type-inference from typing import Callable, TypeVar, Generic from typing_extensions import ParamSpec @@ -1646,7 +1645,6 @@ def f(f: Callable[P, None], *args: P.args, **kwargs: P.kwargs) -> None: ... [builtins fixtures/paramspec.pyi] [case testParamSpecDecoratorAppliedToGeneric] -# flags: --new-type-inference from typing import Callable, List, TypeVar from typing_extensions import ParamSpec @@ -1684,7 +1682,6 @@ reveal_type(test(*ints)) # N: Revealed type is "__main__.Foo[[*builtins.int]]" [builtins fixtures/paramspec.pyi] [case testParamSpecArgumentParamInferenceGeneric] -# flags: --new-type-inference from typing import Callable, TypeVar from typing_extensions import ParamSpec @@ -1702,7 +1699,6 @@ y: int = call(identity, 2) [builtins fixtures/paramspec.pyi] [case testParamSpecNestedApplyNoCrash] -# flags: --new-type-inference from typing import Callable, TypeVar from typing_extensions import ParamSpec diff --git a/test-data/unit/check-plugin-attrs.test b/test-data/unit/check-plugin-attrs.test index 00bec13ab16d5..42f21e945ef0d 100644 --- a/test-data/unit/check-plugin-attrs.test +++ b/test-data/unit/check-plugin-attrs.test @@ -1201,7 +1201,6 @@ class A: [builtins fixtures/bool.pyi] [case testAttrsFactoryBadReturn] -# flags: --new-type-inference import attr def my_factory() -> int: return 7 diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index e931c0c01aeeb..2de2e45f0a960 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -886,7 +886,6 @@ reveal_type(z) # N: Revealed type is "tuple[builtins.int, Unpack[builtins.tuple [builtins fixtures/tuple.pyi] [case testInferenceAgainstGenericVariadicWithBadType] -# flags: --new-type-inference from typing import TypeVar, Callable, Generic from typing_extensions import Unpack, TypeVarTuple diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test index aa0c8916ba0f6..ff60c24b72a57 100644 --- a/test-data/unit/cmdline.test +++ b/test-data/unit/cmdline.test @@ -1249,14 +1249,6 @@ note: A user-defined top-level module with name "typing" is not supported Failed to find builtin module mypy_extensions, perhaps typeshed is broken? == Return code: 2 -[case testNewTypeInferenceFlagDeprecated] -# cmd: mypy --new-type-inference a.py -[file a.py] -pass -[out] -Warning: --new-type-inference flag is deprecated; new type inference algorithm is already enabled by default -== Return code: 0 - [case testCustomTypeshedDirFilePassedExplicitly] # cmd: mypy --custom-typeshed-dir dir m.py dir/stdlib/foo.pyi [file m.py] diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index 72c00a3b9b1c6..4bd94dfce03e5 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -797,7 +797,6 @@ _program.py:3: error: Dict entry 1 has incompatible type "str": "str"; expected _program.py:5: error: "dict[str, int]" has no attribute "xyz" [case testDefaultDict] -# flags: --new-type-inference import typing as t from collections import defaultdict @@ -823,11 +822,11 @@ class MyDDict(t.DefaultDict[int,T], t.Generic[T]): MyDDict(dict)['0'] MyDDict(dict)[0] [out] -_program.py:7: error: Argument 1 to "defaultdict" has incompatible type "type[list[_T]]"; expected "Optional[Callable[[], str]]" -_program.py:10: error: Invalid index type "str" for "defaultdict[int, str]"; expected type "int" -_program.py:10: error: Incompatible types in assignment (expression has type "int", target has type "str") -_program.py:20: error: Argument 1 to "tst" has incompatible type "defaultdict[str, list[Never]]"; expected "defaultdict[int, list[Never]]" -_program.py:24: error: Invalid index type "str" for "MyDDict[dict[Never, Never]]"; expected type "int" +_program.py:6: error: Argument 1 to "defaultdict" has incompatible type "type[list[_T]]"; expected "Optional[Callable[[], str]]" +_program.py:9: error: Invalid index type "str" for "defaultdict[int, str]"; expected type "int" +_program.py:9: error: Incompatible types in assignment (expression has type "int", target has type "str") +_program.py:19: error: Argument 1 to "tst" has incompatible type "defaultdict[str, list[Never]]"; expected "defaultdict[int, list[Never]]" +_program.py:23: error: Invalid index type "str" for "MyDDict[dict[Never, Never]]"; expected type "int" [case testCollectionsAliases] import typing as t @@ -1977,7 +1976,6 @@ _testDataclassReplace.py:11: error: Argument "x" to "replace" of "A" has incompa _testDataclassReplace.py:12: error: Unexpected keyword argument "q" for "replace" of "A" [case testGenericInferenceWithTuple] -# flags: --new-type-inference from typing import TypeVar, Callable, Tuple T = TypeVar("T") @@ -1989,7 +1987,6 @@ x: Tuple[str, ...] = f(tuple) [out] [case testGenericInferenceWithDataclass] -# flags: --new-type-inference from typing import Any, Collection, List from dataclasses import dataclass, field @@ -2002,7 +1999,6 @@ class A: [out] [case testGenericInferenceWithItertools] -# flags: --new-type-inference from typing import TypeVar, Tuple from itertools import groupby K = TypeVar("K") From 5051a4831e174b2a42c8e9fcbd1e6954a86542ae Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 4 Aug 2025 13:25:32 +0100 Subject: [PATCH 142/424] Do not use dictionary in CallableType (#19580) This is mypyc-specific optimization: `dict` creation is quite slow, so we should not create a dict in the `CallableType` constructor. Instead, I store the required info on the relevant `FuncDef` object (and restore the link to definition in `fixup.py`). Quite surprisingly, this gives 2% speedup on my desktop. --- mypy/checker.py | 2 +- mypy/checkexpr.py | 4 ++-- mypy/fixup.py | 8 +++++++ mypy/messages.py | 10 +++++++-- mypy/nodes.py | 9 ++++++++ mypy/types.py | 34 +---------------------------- test-data/unit/check-serialize.test | 1 + 7 files changed, 30 insertions(+), 38 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index c1ff29aa33d9f..4d82214157549 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -730,7 +730,7 @@ def _visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: assert isinstance(item, Decorator) item_type = self.extract_callable_type(item.var.type, item) if item_type is not None: - item_type.definition = item + item_type.definition = item.func item_types.append(item_type) if item_types: defn.type = Overloaded(item_types) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 740efb0d2ee4c..1b10370b08cb4 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -2934,7 +2934,7 @@ def infer_overload_return_type( if not args_contain_any: self.chk.store_types(m) if isinstance(infer_type, ProperType) and isinstance(infer_type, CallableType): - self.chk.check_deprecated(infer_type.definition, context) + self.chk.warn_deprecated(infer_type.definition, context) return ret_type, infer_type p_infer_type = get_proper_type(infer_type) if isinstance(p_infer_type, CallableType): @@ -2975,7 +2975,7 @@ def infer_overload_return_type( if isinstance(inferred_callable, ProperType) and isinstance( inferred_callable, CallableType ): - self.chk.check_deprecated(inferred_callable.definition, context) + self.chk.warn_deprecated(inferred_callable.definition, context) return return_types[0], inferred_types[0] def overload_erased_call_targets( diff --git a/mypy/fixup.py b/mypy/fixup.py index c0f8e401777ca..0007fe8faabf1 100644 --- a/mypy/fixup.py +++ b/mypy/fixup.py @@ -29,6 +29,7 @@ Overloaded, Parameters, ParamSpecType, + ProperType, TupleType, TypeAliasType, TypedDictType, @@ -177,6 +178,11 @@ def visit_overloaded_func_def(self, o: OverloadedFuncDef) -> None: item.accept(self) if o.impl: o.impl.accept(self) + if isinstance(o.type, Overloaded): + # For error messages we link the original definition for each item. + for typ, item in zip(o.type.items, o.items): + if isinstance(item, Decorator): + typ.definition = item.func def visit_decorator(self, d: Decorator) -> None: if self.current_info is not None: @@ -187,6 +193,8 @@ def visit_decorator(self, d: Decorator) -> None: d.var.accept(self) for node in d.decorators: node.accept(self) + if isinstance(d.var.type, ProperType) and isinstance(d.var.type, CallableType): + d.var.type.definition = d.func def visit_class_def(self, c: ClassDef) -> None: for v in c.type_vars: diff --git a/mypy/messages.py b/mypy/messages.py index 44ed25a195179..6b55da59d183a 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -1003,7 +1003,7 @@ def maybe_note_about_special_args(self, callee: CallableType, context: Context) if self.prefer_simple_messages(): return # https://github.com/python/mypy/issues/11309 - first_arg = callee.def_extras.get("first_arg") + first_arg = get_first_arg(callee) if first_arg and first_arg not in {"self", "cls", "mcs"}: self.note( "Looks like the first special argument in a method " @@ -3007,7 +3007,7 @@ def [T <: int] f(self, x: int, y: T) -> None s = definition_arg_names[0] + s s = f"{tp.definition.name}({s})" elif tp.name: - first_arg = tp.def_extras.get("first_arg") + first_arg = get_first_arg(tp) if first_arg: if s: s = ", " + s @@ -3050,6 +3050,12 @@ def [T <: int] f(self, x: int, y: T) -> None return f"def {s}" +def get_first_arg(tp: CallableType) -> str | None: + if not isinstance(tp.definition, FuncDef) or not tp.definition.info or tp.definition.is_static: + return None + return tp.definition.original_first_arg + + def variance_string(variance: int) -> str: if variance == COVARIANT: return "covariant" diff --git a/mypy/nodes.py b/mypy/nodes.py index 011e4e703a0c2..6ffe579efe71f 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -823,6 +823,7 @@ class FuncDef(FuncItem, SymbolNode, Statement): "dataclass_transform_spec", "docstring", "deprecated", + "original_first_arg", ) __match_args__ = ("name", "arguments", "type", "body") @@ -855,6 +856,12 @@ def __init__( # the majority). In cases where self is not annotated and there are no Self # in the signature we can simply drop the first argument. self.is_trivial_self = False + # This is needed because for positional-only arguments the name is set to None, + # but we sometimes still want to show it in error messages. + if arguments: + self.original_first_arg: str | None = arguments[0].variable.name + else: + self.original_first_arg = None @property def name(self) -> str: @@ -886,6 +893,7 @@ def serialize(self) -> JsonDict: else self.dataclass_transform_spec.serialize() ), "deprecated": self.deprecated, + "original_first_arg": self.original_first_arg, } @classmethod @@ -906,6 +914,7 @@ def deserialize(cls, data: JsonDict) -> FuncDef: set_flags(ret, data["flags"]) # NOTE: ret.info is set in the fixup phase. ret.arg_names = data["arg_names"] + ret.original_first_arg = data.get("original_first_arg") ret.arg_kinds = [ArgKind(x) for x in data["arg_kinds"]] ret.abstract_status = data["abstract_status"] ret.dataclass_transform_spec = ( diff --git a/mypy/types.py b/mypy/types.py index 4b5ef332ccf90..029477c1d5c47 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -21,16 +21,7 @@ import mypy.nodes from mypy.bogus_type import Bogus -from mypy.nodes import ( - ARG_POS, - ARG_STAR, - ARG_STAR2, - INVARIANT, - ArgKind, - FakeInfo, - FuncDef, - SymbolNode, -) +from mypy.nodes import ARG_POS, ARG_STAR, ARG_STAR2, INVARIANT, ArgKind, FakeInfo, SymbolNode from mypy.options import Options from mypy.state import state from mypy.util import IdMapper @@ -1841,8 +1832,6 @@ class CallableType(FunctionLike): "from_type_type", # Was this callable generated by analyzing Type[...] # instantiation? "is_bound", # Is this a bound method? - "def_extras", # Information about original definition we want to serialize. - # This is used for more detailed error messages. "type_guard", # T, if -> TypeGuard[T] (ret_type is bool in this case). "type_is", # T, if -> TypeIs[T] (ret_type is bool in this case). "from_concatenate", # whether this callable is from a concatenate object @@ -1869,7 +1858,6 @@ def __init__( special_sig: str | None = None, from_type_type: bool = False, is_bound: bool = False, - def_extras: dict[str, Any] | None = None, type_guard: Type | None = None, type_is: Type | None = None, from_concatenate: bool = False, @@ -1902,22 +1890,6 @@ def __init__( self.from_concatenate = from_concatenate self.imprecise_arg_kinds = imprecise_arg_kinds self.is_bound = is_bound - if def_extras: - self.def_extras = def_extras - elif isinstance(definition, FuncDef): - # This information would be lost if we don't have definition - # after serialization, but it is useful in error messages. - # TODO: decide how to add more info here (file, line, column) - # without changing interface hash. - first_arg: str | None = None - if definition.arg_names and definition.info and not definition.is_static: - if getattr(definition, "arguments", None): - first_arg = definition.arguments[0].variable.name - else: - first_arg = definition.arg_names[0] - self.def_extras = {"first_arg": first_arg} - else: - self.def_extras = {} self.type_guard = type_guard self.type_is = type_is self.unpack_kwargs = unpack_kwargs @@ -1939,7 +1911,6 @@ def copy_modified( special_sig: Bogus[str | None] = _dummy, from_type_type: Bogus[bool] = _dummy, is_bound: Bogus[bool] = _dummy, - def_extras: Bogus[dict[str, Any]] = _dummy, type_guard: Bogus[Type | None] = _dummy, type_is: Bogus[Type | None] = _dummy, from_concatenate: Bogus[bool] = _dummy, @@ -1964,7 +1935,6 @@ def copy_modified( special_sig=special_sig if special_sig is not _dummy else self.special_sig, from_type_type=from_type_type if from_type_type is not _dummy else self.from_type_type, is_bound=is_bound if is_bound is not _dummy else self.is_bound, - def_extras=def_extras if def_extras is not _dummy else dict(self.def_extras), type_guard=type_guard if type_guard is not _dummy else self.type_guard, type_is=type_is if type_is is not _dummy else self.type_is, from_concatenate=( @@ -2291,7 +2261,6 @@ def serialize(self) -> JsonDict: "is_ellipsis_args": self.is_ellipsis_args, "implicit": self.implicit, "is_bound": self.is_bound, - "def_extras": dict(self.def_extras), "type_guard": self.type_guard.serialize() if self.type_guard is not None else None, "type_is": (self.type_is.serialize() if self.type_is is not None else None), "from_concatenate": self.from_concatenate, @@ -2314,7 +2283,6 @@ def deserialize(cls, data: JsonDict) -> CallableType: is_ellipsis_args=data["is_ellipsis_args"], implicit=data["implicit"], is_bound=data["is_bound"], - def_extras=data["def_extras"], type_guard=( deserialize_type(data["type_guard"]) if data["type_guard"] is not None else None ), diff --git a/test-data/unit/check-serialize.test b/test-data/unit/check-serialize.test index 5265832f5f274..03c185a5694b3 100644 --- a/test-data/unit/check-serialize.test +++ b/test-data/unit/check-serialize.test @@ -224,6 +224,7 @@ def f(x: int) -> int: pass [out2] tmp/a.py:2: note: Revealed type is "builtins.str" tmp/a.py:3: error: Unexpected keyword argument "x" for "f" +tmp/b.py: note: "f" defined here [case testSerializeTypeGuardFunction] import a From 1c48286eb22551d8fe2dc557e387595bcf8335b5 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Mon, 4 Aug 2025 16:23:45 +0200 Subject: [PATCH 143/424] Sync typeshed (#19585) Source commit: https://github.com/python/typeshed/commit/e16c23d3768c84433b62e0461085f79c740ade57 --------- Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Co-authored-by: AlexWaygood --- ...ially-revert-Clean-up-argparse-hacks.patch | 10 +- ...redundant-inheritances-from-Iterator.patch | 30 +- mypy/typeshed/stdlib/VERSIONS | 1 + mypy/typeshed/stdlib/_compression.pyi | 3 +- mypy/typeshed/stdlib/_ctypes.pyi | 5 +- mypy/typeshed/stdlib/_gdbm.pyi | 3 +- mypy/typeshed/stdlib/_heapq.pyi | 9 +- mypy/typeshed/stdlib/_interpreters.pyi | 11 +- mypy/typeshed/stdlib/_io.pyi | 1 + mypy/typeshed/stdlib/_json.pyi | 2 +- mypy/typeshed/stdlib/_msi.pyi | 5 + mypy/typeshed/stdlib/_operator.pyi | 9 +- mypy/typeshed/stdlib/_pickle.pyi | 1 + mypy/typeshed/stdlib/_sqlite3.pyi | 357 ++++++------ mypy/typeshed/stdlib/_ssl.pyi | 4 +- mypy/typeshed/stdlib/_thread.pyi | 2 +- mypy/typeshed/stdlib/_typeshed/__init__.pyi | 18 +- mypy/typeshed/stdlib/_winapi.pyi | 3 + mypy/typeshed/stdlib/argparse.pyi | 67 ++- mypy/typeshed/stdlib/ast.pyi | 13 +- mypy/typeshed/stdlib/asyncio/__init__.pyi | 7 - mypy/typeshed/stdlib/asyncio/base_futures.pyi | 8 +- mypy/typeshed/stdlib/asyncio/events.pyi | 9 +- .../stdlib/asyncio/format_helpers.pyi | 3 +- mypy/typeshed/stdlib/asyncio/futures.pyi | 8 +- mypy/typeshed/stdlib/asyncio/queues.pyi | 3 +- mypy/typeshed/stdlib/asyncio/streams.pyi | 3 +- mypy/typeshed/stdlib/asyncio/tasks.pyi | 5 +- mypy/typeshed/stdlib/asyncio/unix_events.pyi | 2 +- mypy/typeshed/stdlib/bdb.pyi | 4 +- mypy/typeshed/stdlib/builtins.pyi | 18 +- mypy/typeshed/stdlib/bz2.pyi | 4 +- mypy/typeshed/stdlib/cgi.pyi | 4 +- mypy/typeshed/stdlib/codecs.pyi | 12 +- mypy/typeshed/stdlib/collections/__init__.pyi | 5 +- mypy/typeshed/stdlib/compileall.pyi | 3 +- .../stdlib/concurrent/futures/__init__.pyi | 4 +- .../stdlib/concurrent/futures/_base.pyi | 3 +- .../concurrent/interpreters/__init__.pyi | 68 +++ .../concurrent/interpreters/_crossinterp.pyi | 29 + .../concurrent/interpreters/_queues.pyi | 58 ++ mypy/typeshed/stdlib/configparser.pyi | 16 +- mypy/typeshed/stdlib/contextlib.pyi | 5 +- mypy/typeshed/stdlib/copy.pyi | 3 +- mypy/typeshed/stdlib/ctypes/__init__.pyi | 3 +- mypy/typeshed/stdlib/dataclasses.pyi | 1 + mypy/typeshed/stdlib/datetime.pyi | 48 +- mypy/typeshed/stdlib/dbm/__init__.pyi | 1 + mypy/typeshed/stdlib/difflib.pyi | 9 +- mypy/typeshed/stdlib/email/headerregistry.pyi | 3 +- mypy/typeshed/stdlib/email/message.pyi | 4 +- mypy/typeshed/stdlib/encodings/__init__.pyi | 4 + mypy/typeshed/stdlib/enum.pyi | 5 + mypy/typeshed/stdlib/fileinput.pyi | 3 +- mypy/typeshed/stdlib/fractions.pyi | 3 +- mypy/typeshed/stdlib/functools.pyi | 6 +- mypy/typeshed/stdlib/gettext.pyi | 3 +- mypy/typeshed/stdlib/gzip.pyi | 4 +- mypy/typeshed/stdlib/hashlib.pyi | 4 +- mypy/typeshed/stdlib/html/parser.pyi | 12 +- mypy/typeshed/stdlib/imghdr.pyi | 3 +- mypy/typeshed/stdlib/imp.pyi | 3 +- .../stdlib/importlib/resources/abc.pyi | 24 +- mypy/typeshed/stdlib/inspect.pyi | 5 +- mypy/typeshed/stdlib/ipaddress.pyi | 4 +- mypy/typeshed/stdlib/logging/__init__.pyi | 4 +- mypy/typeshed/stdlib/logging/handlers.pyi | 3 +- mypy/typeshed/stdlib/mailbox.pyi | 5 +- mypy/typeshed/stdlib/math.pyi | 7 +- mypy/typeshed/stdlib/mmap.pyi | 105 ++-- mypy/typeshed/stdlib/multiprocessing/heap.pyi | 3 +- .../stdlib/multiprocessing/sharedctypes.pyi | 4 +- mypy/typeshed/stdlib/nt.pyi | 3 + mypy/typeshed/stdlib/numbers.pyi | 5 +- mypy/typeshed/stdlib/optparse.pyi | 3 +- mypy/typeshed/stdlib/os/__init__.pyi | 12 +- mypy/typeshed/stdlib/pathlib/__init__.pyi | 10 +- mypy/typeshed/stdlib/platform.pyi | 23 +- mypy/typeshed/stdlib/pprint.pyi | 69 ++- mypy/typeshed/stdlib/pydoc.pyi | 3 +- mypy/typeshed/stdlib/queue.pyi | 5 +- mypy/typeshed/stdlib/quopri.pyi | 3 +- mypy/typeshed/stdlib/re.pyi | 4 +- mypy/typeshed/stdlib/shutil.pyi | 3 +- mypy/typeshed/stdlib/signal.pyi | 98 ++-- mypy/typeshed/stdlib/smtplib.pyi | 4 +- mypy/typeshed/stdlib/socket.pyi | 3 +- mypy/typeshed/stdlib/sqlite3/__init__.pyi | 32 +- mypy/typeshed/stdlib/sqlite3/dbapi2.pyi | 3 +- mypy/typeshed/stdlib/ssl.pyi | 97 ++-- mypy/typeshed/stdlib/string/templatelib.pyi | 11 +- mypy/typeshed/stdlib/sys/__init__.pyi | 10 +- mypy/typeshed/stdlib/sys/_monitoring.pyi | 64 ++- mypy/typeshed/stdlib/tarfile.pyi | 96 ++-- mypy/typeshed/stdlib/tempfile.pyi | 3 +- mypy/typeshed/stdlib/termios.pyi | 533 +++++++++--------- mypy/typeshed/stdlib/threading.pyi | 2 +- mypy/typeshed/stdlib/time.pyi | 3 +- mypy/typeshed/stdlib/tkinter/__init__.pyi | 5 + mypy/typeshed/stdlib/tkinter/dnd.pyi | 3 +- mypy/typeshed/stdlib/tkinter/font.pyi | 4 +- mypy/typeshed/stdlib/tkinter/ttk.pyi | 6 +- mypy/typeshed/stdlib/tty.pyi | 14 +- mypy/typeshed/stdlib/turtle.pyi | 14 +- mypy/typeshed/stdlib/typing.pyi | 2 + mypy/typeshed/stdlib/typing_extensions.pyi | 12 +- mypy/typeshed/stdlib/unittest/case.pyi | 17 +- mypy/typeshed/stdlib/unittest/main.pyi | 3 +- mypy/typeshed/stdlib/unittest/mock.pyi | 43 +- mypy/typeshed/stdlib/unittest/runner.pyi | 4 +- mypy/typeshed/stdlib/urllib/request.pyi | 3 +- mypy/typeshed/stdlib/uuid.pyi | 2 +- mypy/typeshed/stdlib/xml/dom/minidom.pyi | 4 +- .../stdlib/xml/etree/ElementInclude.pyi | 3 +- .../typeshed/stdlib/xml/etree/ElementTree.pyi | 1 + mypy/typeshed/stdlib/xml/sax/__init__.pyi | 3 +- mypy/typeshed/stdlib/xmlrpc/client.pyi | 3 +- mypy/typeshed/stdlib/xmlrpc/server.pyi | 8 +- mypy/typeshed/stdlib/zipfile/__init__.pyi | 13 +- mypy/typeshed/stdlib/zoneinfo/_common.pyi | 3 +- 120 files changed, 1419 insertions(+), 974 deletions(-) create mode 100644 mypy/typeshed/stdlib/concurrent/interpreters/__init__.pyi create mode 100644 mypy/typeshed/stdlib/concurrent/interpreters/_crossinterp.pyi create mode 100644 mypy/typeshed/stdlib/concurrent/interpreters/_queues.pyi diff --git a/misc/typeshed_patches/0001-Partially-revert-Clean-up-argparse-hacks.patch b/misc/typeshed_patches/0001-Partially-revert-Clean-up-argparse-hacks.patch index f76818d10cba2..5c31569711e5a 100644 --- a/misc/typeshed_patches/0001-Partially-revert-Clean-up-argparse-hacks.patch +++ b/misc/typeshed_patches/0001-Partially-revert-Clean-up-argparse-hacks.patch @@ -1,4 +1,4 @@ -From 05f351f6a37fe8b73c698c348bf6aa5108363049 Mon Sep 17 00:00:00 2001 +From 84a9d586544a0408d4654f57f83a93cb048070fb Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Sat, 15 Feb 2025 20:11:06 +0100 Subject: [PATCH] Partially revert Clean up argparse hacks @@ -8,15 +8,15 @@ Subject: [PATCH] Partially revert Clean up argparse hacks 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mypy/typeshed/stdlib/argparse.pyi b/mypy/typeshed/stdlib/argparse.pyi -index 95ad6c7da..79e6cfde1 100644 +index b9fa31139..3c3ba116a 100644 --- a/mypy/typeshed/stdlib/argparse.pyi +++ b/mypy/typeshed/stdlib/argparse.pyi @@ -2,7 +2,7 @@ import sys from _typeshed import SupportsWrite, sentinel from collections.abc import Callable, Generator, Iterable, Sequence from re import Pattern --from typing import IO, Any, ClassVar, Final, Generic, NoReturn, Protocol, TypeVar, overload -+from typing import IO, Any, ClassVar, Final, Generic, NewType, NoReturn, Protocol, TypeVar, overload +-from typing import IO, Any, ClassVar, Final, Generic, NoReturn, Protocol, TypeVar, overload, type_check_only ++from typing import IO, Any, ClassVar, Final, Generic, NewType, NoReturn, Protocol, TypeVar, overload, type_check_only from typing_extensions import Self, TypeAlias, deprecated __all__ = [ @@ -41,5 +41,5 @@ index 95ad6c7da..79e6cfde1 100644 default: Any = ..., type: _ActionType = ..., -- -2.49.0 +2.50.1 diff --git a/misc/typeshed_patches/0001-Revert-Remove-redundant-inheritances-from-Iterator.patch b/misc/typeshed_patches/0001-Revert-Remove-redundant-inheritances-from-Iterator.patch index 5b30a63f1318a..d3f49a4eef3ef 100644 --- a/misc/typeshed_patches/0001-Revert-Remove-redundant-inheritances-from-Iterator.patch +++ b/misc/typeshed_patches/0001-Revert-Remove-redundant-inheritances-from-Iterator.patch @@ -1,4 +1,4 @@ -From 363d69b366695fea117631d30c348e36b9a5a99d Mon Sep 17 00:00:00 2001 +From c217544146d36899d50e828d627652a0d8f63bb7 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Sat, 21 Dec 2024 22:36:38 +0100 Subject: [PATCH] Revert Remove redundant inheritances from Iterator in @@ -15,7 +15,7 @@ Subject: [PATCH] Revert Remove redundant inheritances from Iterator in 7 files changed, 34 insertions(+), 34 deletions(-) diff --git a/mypy/typeshed/stdlib/_asyncio.pyi b/mypy/typeshed/stdlib/_asyncio.pyi -index 4544680cc..19a2d12d8 100644 +index ed56f33af..5253e967e 100644 --- a/mypy/typeshed/stdlib/_asyncio.pyi +++ b/mypy/typeshed/stdlib/_asyncio.pyi @@ -1,6 +1,6 @@ @@ -36,10 +36,10 @@ index 4544680cc..19a2d12d8 100644 @property def _exception(self) -> BaseException | None: ... diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi -index ea77a730f..900c4c93f 100644 +index 0575be3c8..d9be595fe 100644 --- a/mypy/typeshed/stdlib/builtins.pyi +++ b/mypy/typeshed/stdlib/builtins.pyi -@@ -1170,7 +1170,7 @@ class frozenset(AbstractSet[_T_co]): +@@ -1186,7 +1186,7 @@ class frozenset(AbstractSet[_T_co]): def __hash__(self) -> int: ... def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... @@ -48,7 +48,7 @@ index ea77a730f..900c4c93f 100644 def __new__(cls, iterable: Iterable[_T], start: int = 0) -> Self: ... def __iter__(self) -> Self: ... def __next__(self) -> tuple[int, _T]: ... -@@ -1366,7 +1366,7 @@ else: +@@ -1380,7 +1380,7 @@ else: exit: _sitebuiltins.Quitter @@ -57,7 +57,7 @@ index ea77a730f..900c4c93f 100644 @overload def __new__(cls, function: None, iterable: Iterable[_T | None], /) -> Self: ... @overload -@@ -1431,7 +1431,7 @@ license: _sitebuiltins._Printer +@@ -1444,7 +1444,7 @@ license: _sitebuiltins._Printer def locals() -> dict[str, Any]: ... @@ -66,7 +66,7 @@ index ea77a730f..900c4c93f 100644 # 3.14 adds `strict` argument. if sys.version_info >= (3, 14): @overload -@@ -1734,7 +1734,7 @@ def pow(base: _SupportsSomeKindOfPow, exp: complex, mod: None = None) -> complex +@@ -1750,7 +1750,7 @@ def pow(base: _SupportsSomeKindOfPow, exp: complex, mod: None = None) -> complex quit: _sitebuiltins.Quitter @@ -75,7 +75,7 @@ index ea77a730f..900c4c93f 100644 @overload def __new__(cls, sequence: Reversible[_T], /) -> Iterator[_T]: ... # type: ignore[misc] @overload -@@ -1795,7 +1795,7 @@ def vars(object: type, /) -> types.MappingProxyType[str, Any]: ... +@@ -1814,7 +1814,7 @@ def vars(object: type, /) -> types.MappingProxyType[str, Any]: ... @overload def vars(object: Any = ..., /) -> dict[str, Any]: ... @@ -107,7 +107,7 @@ index 2c8e7109c..4ed0ab1d8 100644 restkey: _T | None restval: str | Any | None diff --git a/mypy/typeshed/stdlib/fileinput.pyi b/mypy/typeshed/stdlib/fileinput.pyi -index 948b39ea1..1d5f9cf00 100644 +index 910d63814..eb942bc55 100644 --- a/mypy/typeshed/stdlib/fileinput.pyi +++ b/mypy/typeshed/stdlib/fileinput.pyi @@ -1,8 +1,8 @@ @@ -116,12 +116,12 @@ index 948b39ea1..1d5f9cf00 100644 -from collections.abc import Callable, Iterable +from collections.abc import Callable, Iterable, Iterator from types import GenericAlias, TracebackType --from typing import IO, Any, AnyStr, Generic, Literal, Protocol, overload -+from typing import IO, Any, AnyStr, Literal, Protocol, overload +-from typing import IO, Any, AnyStr, Generic, Literal, Protocol, overload, type_check_only ++from typing import IO, Any, AnyStr, Literal, Protocol, overload, type_check_only from typing_extensions import Self, TypeAlias __all__ = [ -@@ -104,7 +104,7 @@ def fileno() -> int: ... +@@ -105,7 +105,7 @@ def fileno() -> int: ... def isfirstline() -> bool: ... def isstdin() -> bool: ... @@ -307,10 +307,10 @@ index b79f9e773..f276372d0 100644 def __iter__(self) -> Self: ... def next(self, timeout: float | None = None) -> _T: ... diff --git a/mypy/typeshed/stdlib/sqlite3/__init__.pyi b/mypy/typeshed/stdlib/sqlite3/__init__.pyi -index 5d3c2330b..ab783dbde 100644 +index bcfea3a13..5a659deac 100644 --- a/mypy/typeshed/stdlib/sqlite3/__init__.pyi +++ b/mypy/typeshed/stdlib/sqlite3/__init__.pyi -@@ -399,7 +399,7 @@ class Connection: +@@ -405,7 +405,7 @@ class Connection: self, type: type[BaseException] | None, value: BaseException | None, traceback: TracebackType | None, / ) -> Literal[False]: ... @@ -320,5 +320,5 @@ index 5d3c2330b..ab783dbde 100644 @property def connection(self) -> Connection: ... -- -2.49.0 +2.50.1 diff --git a/mypy/typeshed/stdlib/VERSIONS b/mypy/typeshed/stdlib/VERSIONS index 8baf207ad7b85..6fcf0161790d6 100644 --- a/mypy/typeshed/stdlib/VERSIONS +++ b/mypy/typeshed/stdlib/VERSIONS @@ -124,6 +124,7 @@ compileall: 3.0- compression: 3.14- concurrent: 3.2- concurrent.futures.interpreter: 3.14- +concurrent.interpreters: 3.14- configparser: 3.0- contextlib: 3.0- contextvars: 3.7- diff --git a/mypy/typeshed/stdlib/_compression.pyi b/mypy/typeshed/stdlib/_compression.pyi index 80d38b4db824b..aa67df2ab4787 100644 --- a/mypy/typeshed/stdlib/_compression.pyi +++ b/mypy/typeshed/stdlib/_compression.pyi @@ -3,10 +3,11 @@ from _typeshed import Incomplete, WriteableBuffer from collections.abc import Callable from io import DEFAULT_BUFFER_SIZE, BufferedIOBase, RawIOBase -from typing import Any, Protocol +from typing import Any, Protocol, type_check_only BUFFER_SIZE = DEFAULT_BUFFER_SIZE +@type_check_only class _Reader(Protocol): def read(self, n: int, /) -> bytes: ... def seekable(self) -> bool: ... diff --git a/mypy/typeshed/stdlib/_ctypes.pyi b/mypy/typeshed/stdlib/_ctypes.pyi index e134066f0bcfa..bfd0f910f4822 100644 --- a/mypy/typeshed/stdlib/_ctypes.pyi +++ b/mypy/typeshed/stdlib/_ctypes.pyi @@ -103,7 +103,10 @@ class _SimpleCData(_CData, Generic[_T], metaclass=_PyCSimpleType): def __init__(self, value: _T = ...) -> None: ... # pyright: ignore[reportInvalidTypeVarUse] def __ctypes_from_outparam__(self, /) -> _T: ... # type: ignore[override] +@type_check_only class _CanCastTo(_CData): ... + +@type_check_only class _PointerLike(_CanCastTo): ... # This type is not exposed. It calls itself _ctypes.PyCPointerType. @@ -114,7 +117,7 @@ class _PyCPointerType(_CTypeBaseType): def from_buffer_copy(self: type[_typeshed.Self], buffer: ReadableBuffer, offset: int = 0, /) -> _typeshed.Self: ... def from_param(self: type[_typeshed.Self], value: Any, /) -> _typeshed.Self | _CArgObject: ... def in_dll(self: type[_typeshed.Self], dll: CDLL, name: str, /) -> _typeshed.Self: ... - def set_type(self, type: Any, /) -> None: ... + def set_type(self, type: _CTypeBaseType, /) -> None: ... if sys.version_info < (3, 13): # Inherited from CType_Type starting on 3.13 def __mul__(cls: type[_CT], other: int) -> type[Array[_CT]]: ... # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] diff --git a/mypy/typeshed/stdlib/_gdbm.pyi b/mypy/typeshed/stdlib/_gdbm.pyi index 1d1d541f54770..2cb5fba29dfa1 100644 --- a/mypy/typeshed/stdlib/_gdbm.pyi +++ b/mypy/typeshed/stdlib/_gdbm.pyi @@ -1,7 +1,7 @@ import sys from _typeshed import ReadOnlyBuffer, StrOrBytesPath from types import TracebackType -from typing import TypeVar, overload +from typing import TypeVar, overload, type_check_only from typing_extensions import Self, TypeAlias if sys.platform != "win32": @@ -13,6 +13,7 @@ if sys.platform != "win32": class error(OSError): ... # Actual typename gdbm, not exposed by the implementation + @type_check_only class _gdbm: def firstkey(self) -> bytes | None: ... def nextkey(self, key: _KeyType) -> bytes | None: ... diff --git a/mypy/typeshed/stdlib/_heapq.pyi b/mypy/typeshed/stdlib/_heapq.pyi index 3363fbcd7e740..4d7d6aba32418 100644 --- a/mypy/typeshed/stdlib/_heapq.pyi +++ b/mypy/typeshed/stdlib/_heapq.pyi @@ -1,18 +1,17 @@ import sys -from typing import Any, Final, TypeVar - -_T = TypeVar("_T") # list items must be comparable +from _typeshed import SupportsRichComparisonT as _T # All type variable use in this module requires comparability. +from typing import Final __about__: Final[str] -def heapify(heap: list[Any], /) -> None: ... # list items must be comparable +def heapify(heap: list[_T], /) -> None: ... def heappop(heap: list[_T], /) -> _T: ... def heappush(heap: list[_T], item: _T, /) -> None: ... def heappushpop(heap: list[_T], item: _T, /) -> _T: ... def heapreplace(heap: list[_T], item: _T, /) -> _T: ... if sys.version_info >= (3, 14): - def heapify_max(heap: list[Any], /) -> None: ... # list items must be comparable + def heapify_max(heap: list[_T], /) -> None: ... def heappop_max(heap: list[_T], /) -> _T: ... def heappush_max(heap: list[_T], item: _T, /) -> None: ... def heappushpop_max(heap: list[_T], item: _T, /) -> _T: ... diff --git a/mypy/typeshed/stdlib/_interpreters.pyi b/mypy/typeshed/stdlib/_interpreters.pyi index ad8eccbe33284..54fc0e39d239d 100644 --- a/mypy/typeshed/stdlib/_interpreters.pyi +++ b/mypy/typeshed/stdlib/_interpreters.pyi @@ -1,8 +1,10 @@ import types from collections.abc import Callable -from typing import Any, Final, Literal, SupportsIndex +from typing import Any, Final, Literal, SupportsIndex, TypeVar from typing_extensions import TypeAlias +_R = TypeVar("_R") + _Configs: TypeAlias = Literal["default", "isolated", "legacy", "empty", ""] _SharedDict: TypeAlias = dict[str, Any] # many objects can be shared @@ -21,7 +23,7 @@ def get_current() -> tuple[int, int]: ... def get_main() -> tuple[int, int]: ... def is_running(id: SupportsIndex, *, restrict: bool = False) -> bool: ... def get_config(id: SupportsIndex, *, restrict: bool = False) -> types.SimpleNamespace: ... -def whence(id: SupportsIndex) -> int: ... +def whence(id: SupportsIndex) -> _Whence: ... def exec( id: SupportsIndex, code: str | types.CodeType | Callable[[], object], @@ -31,12 +33,12 @@ def exec( ) -> None | types.SimpleNamespace: ... def call( id: SupportsIndex, - callable: Callable[..., object], + callable: Callable[..., _R], args: tuple[object, ...] | None = None, kwargs: dict[str, object] | None = None, *, restrict: bool = False, -) -> object: ... +) -> tuple[_R, types.SimpleNamespace]: ... def run_string( id: SupportsIndex, script: str | types.CodeType | Callable[[], object], @@ -53,6 +55,7 @@ def decref(id: SupportsIndex, *, restrict: bool = False) -> None: ... def is_shareable(obj: object) -> bool: ... def capture_exception(exc: BaseException | None = None) -> types.SimpleNamespace: ... +_Whence: TypeAlias = Literal[0, 1, 2, 3, 4, 5] WHENCE_UNKNOWN: Final = 0 WHENCE_RUNTIME: Final = 1 WHENCE_LEGACY_CAPI: Final = 2 diff --git a/mypy/typeshed/stdlib/_io.pyi b/mypy/typeshed/stdlib/_io.pyi index c77d75287c25a..e368ddef7f4e6 100644 --- a/mypy/typeshed/stdlib/_io.pyi +++ b/mypy/typeshed/stdlib/_io.pyi @@ -88,6 +88,7 @@ class BytesIO(BufferedIOBase, _BufferedIOBase, BinaryIO): # type: ignore[misc] def readlines(self, size: int | None = None, /) -> list[bytes]: ... def seek(self, pos: int, whence: int = 0, /) -> int: ... +@type_check_only class _BufferedReaderStream(Protocol): def read(self, n: int = ..., /) -> bytes: ... # Optional: def readall(self) -> bytes: ... diff --git a/mypy/typeshed/stdlib/_json.pyi b/mypy/typeshed/stdlib/_json.pyi index cc59146ed982b..4a77e5be594ab 100644 --- a/mypy/typeshed/stdlib/_json.pyi +++ b/mypy/typeshed/stdlib/_json.pyi @@ -48,4 +48,4 @@ class make_scanner: def encode_basestring(s: str, /) -> str: ... def encode_basestring_ascii(s: str, /) -> str: ... -def scanstring(string: str, end: int, strict: bool = ...) -> tuple[str, int]: ... +def scanstring(string: str, end: int, strict: bool = True) -> tuple[str, int]: ... diff --git a/mypy/typeshed/stdlib/_msi.pyi b/mypy/typeshed/stdlib/_msi.pyi index 779fda3b67fe9..ef45ff6dc3c82 100644 --- a/mypy/typeshed/stdlib/_msi.pyi +++ b/mypy/typeshed/stdlib/_msi.pyi @@ -1,8 +1,10 @@ import sys +from typing import type_check_only if sys.platform == "win32": class MSIError(Exception): ... # Actual typename View, not exposed by the implementation + @type_check_only class _View: def Execute(self, params: _Record | None = ...) -> None: ... def GetColumnInfo(self, kind: int) -> _Record: ... @@ -14,6 +16,7 @@ if sys.platform == "win32": __init__: None # type: ignore[assignment] # Actual typename SummaryInformation, not exposed by the implementation + @type_check_only class _SummaryInformation: def GetProperty(self, field: int) -> int | bytes | None: ... def GetPropertyCount(self) -> int: ... @@ -24,6 +27,7 @@ if sys.platform == "win32": __init__: None # type: ignore[assignment] # Actual typename Database, not exposed by the implementation + @type_check_only class _Database: def OpenView(self, sql: str) -> _View: ... def Commit(self) -> None: ... @@ -34,6 +38,7 @@ if sys.platform == "win32": __init__: None # type: ignore[assignment] # Actual typename Record, not exposed by the implementation + @type_check_only class _Record: def GetFieldCount(self) -> int: ... def GetInteger(self, field: int) -> int: ... diff --git a/mypy/typeshed/stdlib/_operator.pyi b/mypy/typeshed/stdlib/_operator.pyi index 967215d8fa211..cb1c1bcfc4aab 100644 --- a/mypy/typeshed/stdlib/_operator.pyi +++ b/mypy/typeshed/stdlib/_operator.pyi @@ -2,7 +2,7 @@ import sys from _typeshed import SupportsGetItem from collections.abc import Callable, Container, Iterable, MutableMapping, MutableSequence, Sequence from operator import attrgetter as attrgetter, itemgetter as itemgetter, methodcaller as methodcaller -from typing import Any, AnyStr, Protocol, SupportsAbs, SupportsIndex, TypeVar, overload +from typing import Any, AnyStr, Protocol, SupportsAbs, SupportsIndex, TypeVar, overload, type_check_only from typing_extensions import ParamSpec, TypeAlias, TypeIs _R = TypeVar("_R") @@ -16,26 +16,33 @@ _P = ParamSpec("_P") # operators can be overloaded to return an arbitrary object. For example, # the numpy.array comparison dunders return another numpy.array. +@type_check_only class _SupportsDunderLT(Protocol): def __lt__(self, other: Any, /) -> Any: ... +@type_check_only class _SupportsDunderGT(Protocol): def __gt__(self, other: Any, /) -> Any: ... +@type_check_only class _SupportsDunderLE(Protocol): def __le__(self, other: Any, /) -> Any: ... +@type_check_only class _SupportsDunderGE(Protocol): def __ge__(self, other: Any, /) -> Any: ... _SupportsComparison: TypeAlias = _SupportsDunderLE | _SupportsDunderGE | _SupportsDunderGT | _SupportsDunderLT +@type_check_only class _SupportsInversion(Protocol[_T_co]): def __invert__(self) -> _T_co: ... +@type_check_only class _SupportsNeg(Protocol[_T_co]): def __neg__(self) -> _T_co: ... +@type_check_only class _SupportsPos(Protocol[_T_co]): def __pos__(self) -> _T_co: ... diff --git a/mypy/typeshed/stdlib/_pickle.pyi b/mypy/typeshed/stdlib/_pickle.pyi index 8e8afb600efac..03051bb09d3cf 100644 --- a/mypy/typeshed/stdlib/_pickle.pyi +++ b/mypy/typeshed/stdlib/_pickle.pyi @@ -4,6 +4,7 @@ from pickle import PickleBuffer as PickleBuffer from typing import Any, Protocol, type_check_only from typing_extensions import TypeAlias +@type_check_only class _ReadableFileobj(Protocol): def read(self, n: int, /) -> bytes: ... def readline(self) -> bytes: ... diff --git a/mypy/typeshed/stdlib/_sqlite3.pyi b/mypy/typeshed/stdlib/_sqlite3.pyi index 6f06542c1ba71..50006dcf4032d 100644 --- a/mypy/typeshed/stdlib/_sqlite3.pyi +++ b/mypy/typeshed/stdlib/_sqlite3.pyi @@ -16,6 +16,7 @@ from sqlite3 import ( ProgrammingError as ProgrammingError, Row as Row, Warning as Warning, + _IsolationLevel, ) from typing import Any, Final, Literal, TypeVar, overload from typing_extensions import TypeAlias @@ -29,45 +30,45 @@ _SqliteData: TypeAlias = str | ReadableBuffer | int | float | None _Adapter: TypeAlias = Callable[[_T], _SqliteData] _Converter: TypeAlias = Callable[[bytes], Any] -PARSE_COLNAMES: Final[int] -PARSE_DECLTYPES: Final[int] -SQLITE_ALTER_TABLE: Final[int] -SQLITE_ANALYZE: Final[int] -SQLITE_ATTACH: Final[int] -SQLITE_CREATE_INDEX: Final[int] -SQLITE_CREATE_TABLE: Final[int] -SQLITE_CREATE_TEMP_INDEX: Final[int] -SQLITE_CREATE_TEMP_TABLE: Final[int] -SQLITE_CREATE_TEMP_TRIGGER: Final[int] -SQLITE_CREATE_TEMP_VIEW: Final[int] -SQLITE_CREATE_TRIGGER: Final[int] -SQLITE_CREATE_VIEW: Final[int] -SQLITE_CREATE_VTABLE: Final[int] -SQLITE_DELETE: Final[int] -SQLITE_DENY: Final[int] -SQLITE_DETACH: Final[int] -SQLITE_DONE: Final[int] -SQLITE_DROP_INDEX: Final[int] -SQLITE_DROP_TABLE: Final[int] -SQLITE_DROP_TEMP_INDEX: Final[int] -SQLITE_DROP_TEMP_TABLE: Final[int] -SQLITE_DROP_TEMP_TRIGGER: Final[int] -SQLITE_DROP_TEMP_VIEW: Final[int] -SQLITE_DROP_TRIGGER: Final[int] -SQLITE_DROP_VIEW: Final[int] -SQLITE_DROP_VTABLE: Final[int] -SQLITE_FUNCTION: Final[int] -SQLITE_IGNORE: Final[int] -SQLITE_INSERT: Final[int] -SQLITE_OK: Final[int] -SQLITE_PRAGMA: Final[int] -SQLITE_READ: Final[int] -SQLITE_RECURSIVE: Final[int] -SQLITE_REINDEX: Final[int] -SQLITE_SAVEPOINT: Final[int] -SQLITE_SELECT: Final[int] -SQLITE_TRANSACTION: Final[int] -SQLITE_UPDATE: Final[int] +PARSE_COLNAMES: Final = 2 +PARSE_DECLTYPES: Final = 1 +SQLITE_ALTER_TABLE: Final = 26 +SQLITE_ANALYZE: Final = 28 +SQLITE_ATTACH: Final = 24 +SQLITE_CREATE_INDEX: Final = 1 +SQLITE_CREATE_TABLE: Final = 2 +SQLITE_CREATE_TEMP_INDEX: Final = 3 +SQLITE_CREATE_TEMP_TABLE: Final = 4 +SQLITE_CREATE_TEMP_TRIGGER: Final = 5 +SQLITE_CREATE_TEMP_VIEW: Final = 6 +SQLITE_CREATE_TRIGGER: Final = 7 +SQLITE_CREATE_VIEW: Final = 8 +SQLITE_CREATE_VTABLE: Final = 29 +SQLITE_DELETE: Final = 9 +SQLITE_DENY: Final = 1 +SQLITE_DETACH: Final = 25 +SQLITE_DONE: Final = 101 +SQLITE_DROP_INDEX: Final = 10 +SQLITE_DROP_TABLE: Final = 11 +SQLITE_DROP_TEMP_INDEX: Final = 12 +SQLITE_DROP_TEMP_TABLE: Final = 13 +SQLITE_DROP_TEMP_TRIGGER: Final = 14 +SQLITE_DROP_TEMP_VIEW: Final = 15 +SQLITE_DROP_TRIGGER: Final = 16 +SQLITE_DROP_VIEW: Final = 17 +SQLITE_DROP_VTABLE: Final = 30 +SQLITE_FUNCTION: Final = 31 +SQLITE_IGNORE: Final = 2 +SQLITE_INSERT: Final = 18 +SQLITE_OK: Final = 0 +SQLITE_PRAGMA: Final = 19 +SQLITE_READ: Final = 20 +SQLITE_RECURSIVE: Final = 33 +SQLITE_REINDEX: Final = 27 +SQLITE_SAVEPOINT: Final = 32 +SQLITE_SELECT: Final = 21 +SQLITE_TRANSACTION: Final = 22 +SQLITE_UPDATE: Final = 23 adapters: dict[tuple[type[Any], type[Any]], _Adapter[Any]] converters: dict[str, _Converter] sqlite_version: str @@ -76,141 +77,141 @@ if sys.version_info < (3, 12): version: str if sys.version_info >= (3, 12): - LEGACY_TRANSACTION_CONTROL: Final[int] - SQLITE_DBCONFIG_DEFENSIVE: Final[int] - SQLITE_DBCONFIG_DQS_DDL: Final[int] - SQLITE_DBCONFIG_DQS_DML: Final[int] - SQLITE_DBCONFIG_ENABLE_FKEY: Final[int] - SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER: Final[int] - SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION: Final[int] - SQLITE_DBCONFIG_ENABLE_QPSG: Final[int] - SQLITE_DBCONFIG_ENABLE_TRIGGER: Final[int] - SQLITE_DBCONFIG_ENABLE_VIEW: Final[int] - SQLITE_DBCONFIG_LEGACY_ALTER_TABLE: Final[int] - SQLITE_DBCONFIG_LEGACY_FILE_FORMAT: Final[int] - SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE: Final[int] - SQLITE_DBCONFIG_RESET_DATABASE: Final[int] - SQLITE_DBCONFIG_TRIGGER_EQP: Final[int] - SQLITE_DBCONFIG_TRUSTED_SCHEMA: Final[int] - SQLITE_DBCONFIG_WRITABLE_SCHEMA: Final[int] + LEGACY_TRANSACTION_CONTROL: Final = -1 + SQLITE_DBCONFIG_DEFENSIVE: Final = 1010 + SQLITE_DBCONFIG_DQS_DDL: Final = 1014 + SQLITE_DBCONFIG_DQS_DML: Final = 1013 + SQLITE_DBCONFIG_ENABLE_FKEY: Final = 1002 + SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER: Final = 1004 + SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION: Final = 1005 + SQLITE_DBCONFIG_ENABLE_QPSG: Final = 1007 + SQLITE_DBCONFIG_ENABLE_TRIGGER: Final = 1003 + SQLITE_DBCONFIG_ENABLE_VIEW: Final = 1015 + SQLITE_DBCONFIG_LEGACY_ALTER_TABLE: Final = 1012 + SQLITE_DBCONFIG_LEGACY_FILE_FORMAT: Final = 1016 + SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE: Final = 1006 + SQLITE_DBCONFIG_RESET_DATABASE: Final = 1009 + SQLITE_DBCONFIG_TRIGGER_EQP: Final = 1008 + SQLITE_DBCONFIG_TRUSTED_SCHEMA: Final = 1017 + SQLITE_DBCONFIG_WRITABLE_SCHEMA: Final = 1011 if sys.version_info >= (3, 11): - SQLITE_ABORT: Final[int] - SQLITE_ABORT_ROLLBACK: Final[int] - SQLITE_AUTH: Final[int] - SQLITE_AUTH_USER: Final[int] - SQLITE_BUSY: Final[int] - SQLITE_BUSY_RECOVERY: Final[int] - SQLITE_BUSY_SNAPSHOT: Final[int] - SQLITE_BUSY_TIMEOUT: Final[int] - SQLITE_CANTOPEN: Final[int] - SQLITE_CANTOPEN_CONVPATH: Final[int] - SQLITE_CANTOPEN_DIRTYWAL: Final[int] - SQLITE_CANTOPEN_FULLPATH: Final[int] - SQLITE_CANTOPEN_ISDIR: Final[int] - SQLITE_CANTOPEN_NOTEMPDIR: Final[int] - SQLITE_CANTOPEN_SYMLINK: Final[int] - SQLITE_CONSTRAINT: Final[int] - SQLITE_CONSTRAINT_CHECK: Final[int] - SQLITE_CONSTRAINT_COMMITHOOK: Final[int] - SQLITE_CONSTRAINT_FOREIGNKEY: Final[int] - SQLITE_CONSTRAINT_FUNCTION: Final[int] - SQLITE_CONSTRAINT_NOTNULL: Final[int] - SQLITE_CONSTRAINT_PINNED: Final[int] - SQLITE_CONSTRAINT_PRIMARYKEY: Final[int] - SQLITE_CONSTRAINT_ROWID: Final[int] - SQLITE_CONSTRAINT_TRIGGER: Final[int] - SQLITE_CONSTRAINT_UNIQUE: Final[int] - SQLITE_CONSTRAINT_VTAB: Final[int] - SQLITE_CORRUPT: Final[int] - SQLITE_CORRUPT_INDEX: Final[int] - SQLITE_CORRUPT_SEQUENCE: Final[int] - SQLITE_CORRUPT_VTAB: Final[int] - SQLITE_EMPTY: Final[int] - SQLITE_ERROR: Final[int] - SQLITE_ERROR_MISSING_COLLSEQ: Final[int] - SQLITE_ERROR_RETRY: Final[int] - SQLITE_ERROR_SNAPSHOT: Final[int] - SQLITE_FORMAT: Final[int] - SQLITE_FULL: Final[int] - SQLITE_INTERNAL: Final[int] - SQLITE_INTERRUPT: Final[int] - SQLITE_IOERR: Final[int] - SQLITE_IOERR_ACCESS: Final[int] - SQLITE_IOERR_AUTH: Final[int] - SQLITE_IOERR_BEGIN_ATOMIC: Final[int] - SQLITE_IOERR_BLOCKED: Final[int] - SQLITE_IOERR_CHECKRESERVEDLOCK: Final[int] - SQLITE_IOERR_CLOSE: Final[int] - SQLITE_IOERR_COMMIT_ATOMIC: Final[int] - SQLITE_IOERR_CONVPATH: Final[int] - SQLITE_IOERR_CORRUPTFS: Final[int] - SQLITE_IOERR_DATA: Final[int] - SQLITE_IOERR_DELETE: Final[int] - SQLITE_IOERR_DELETE_NOENT: Final[int] - SQLITE_IOERR_DIR_CLOSE: Final[int] - SQLITE_IOERR_DIR_FSYNC: Final[int] - SQLITE_IOERR_FSTAT: Final[int] - SQLITE_IOERR_FSYNC: Final[int] - SQLITE_IOERR_GETTEMPPATH: Final[int] - SQLITE_IOERR_LOCK: Final[int] - SQLITE_IOERR_MMAP: Final[int] - SQLITE_IOERR_NOMEM: Final[int] - SQLITE_IOERR_RDLOCK: Final[int] - SQLITE_IOERR_READ: Final[int] - SQLITE_IOERR_ROLLBACK_ATOMIC: Final[int] - SQLITE_IOERR_SEEK: Final[int] - SQLITE_IOERR_SHMLOCK: Final[int] - SQLITE_IOERR_SHMMAP: Final[int] - SQLITE_IOERR_SHMOPEN: Final[int] - SQLITE_IOERR_SHMSIZE: Final[int] - SQLITE_IOERR_SHORT_READ: Final[int] - SQLITE_IOERR_TRUNCATE: Final[int] - SQLITE_IOERR_UNLOCK: Final[int] - SQLITE_IOERR_VNODE: Final[int] - SQLITE_IOERR_WRITE: Final[int] - SQLITE_LIMIT_ATTACHED: Final[int] - SQLITE_LIMIT_COLUMN: Final[int] - SQLITE_LIMIT_COMPOUND_SELECT: Final[int] - SQLITE_LIMIT_EXPR_DEPTH: Final[int] - SQLITE_LIMIT_FUNCTION_ARG: Final[int] - SQLITE_LIMIT_LENGTH: Final[int] - SQLITE_LIMIT_LIKE_PATTERN_LENGTH: Final[int] - SQLITE_LIMIT_SQL_LENGTH: Final[int] - SQLITE_LIMIT_TRIGGER_DEPTH: Final[int] - SQLITE_LIMIT_VARIABLE_NUMBER: Final[int] - SQLITE_LIMIT_VDBE_OP: Final[int] - SQLITE_LIMIT_WORKER_THREADS: Final[int] - SQLITE_LOCKED: Final[int] - SQLITE_LOCKED_SHAREDCACHE: Final[int] - SQLITE_LOCKED_VTAB: Final[int] - SQLITE_MISMATCH: Final[int] - SQLITE_MISUSE: Final[int] - SQLITE_NOLFS: Final[int] - SQLITE_NOMEM: Final[int] - SQLITE_NOTADB: Final[int] - SQLITE_NOTFOUND: Final[int] - SQLITE_NOTICE: Final[int] - SQLITE_NOTICE_RECOVER_ROLLBACK: Final[int] - SQLITE_NOTICE_RECOVER_WAL: Final[int] - SQLITE_OK_LOAD_PERMANENTLY: Final[int] - SQLITE_OK_SYMLINK: Final[int] - SQLITE_PERM: Final[int] - SQLITE_PROTOCOL: Final[int] - SQLITE_RANGE: Final[int] - SQLITE_READONLY: Final[int] - SQLITE_READONLY_CANTINIT: Final[int] - SQLITE_READONLY_CANTLOCK: Final[int] - SQLITE_READONLY_DBMOVED: Final[int] - SQLITE_READONLY_DIRECTORY: Final[int] - SQLITE_READONLY_RECOVERY: Final[int] - SQLITE_READONLY_ROLLBACK: Final[int] - SQLITE_ROW: Final[int] - SQLITE_SCHEMA: Final[int] - SQLITE_TOOBIG: Final[int] - SQLITE_WARNING: Final[int] - SQLITE_WARNING_AUTOINDEX: Final[int] - threadsafety: Final[int] + SQLITE_ABORT: Final = 4 + SQLITE_ABORT_ROLLBACK: Final = 516 + SQLITE_AUTH: Final = 23 + SQLITE_AUTH_USER: Final = 279 + SQLITE_BUSY: Final = 5 + SQLITE_BUSY_RECOVERY: Final = 261 + SQLITE_BUSY_SNAPSHOT: Final = 517 + SQLITE_BUSY_TIMEOUT: Final = 773 + SQLITE_CANTOPEN: Final = 14 + SQLITE_CANTOPEN_CONVPATH: Final = 1038 + SQLITE_CANTOPEN_DIRTYWAL: Final = 1294 + SQLITE_CANTOPEN_FULLPATH: Final = 782 + SQLITE_CANTOPEN_ISDIR: Final = 526 + SQLITE_CANTOPEN_NOTEMPDIR: Final = 270 + SQLITE_CANTOPEN_SYMLINK: Final = 1550 + SQLITE_CONSTRAINT: Final = 19 + SQLITE_CONSTRAINT_CHECK: Final = 275 + SQLITE_CONSTRAINT_COMMITHOOK: Final = 531 + SQLITE_CONSTRAINT_FOREIGNKEY: Final = 787 + SQLITE_CONSTRAINT_FUNCTION: Final = 1043 + SQLITE_CONSTRAINT_NOTNULL: Final = 1299 + SQLITE_CONSTRAINT_PINNED: Final = 2835 + SQLITE_CONSTRAINT_PRIMARYKEY: Final = 1555 + SQLITE_CONSTRAINT_ROWID: Final = 2579 + SQLITE_CONSTRAINT_TRIGGER: Final = 1811 + SQLITE_CONSTRAINT_UNIQUE: Final = 2067 + SQLITE_CONSTRAINT_VTAB: Final = 2323 + SQLITE_CORRUPT: Final = 11 + SQLITE_CORRUPT_INDEX: Final = 779 + SQLITE_CORRUPT_SEQUENCE: Final = 523 + SQLITE_CORRUPT_VTAB: Final = 267 + SQLITE_EMPTY: Final = 16 + SQLITE_ERROR: Final = 1 + SQLITE_ERROR_MISSING_COLLSEQ: Final = 257 + SQLITE_ERROR_RETRY: Final = 513 + SQLITE_ERROR_SNAPSHOT: Final = 769 + SQLITE_FORMAT: Final = 24 + SQLITE_FULL: Final = 13 + SQLITE_INTERNAL: Final = 2 + SQLITE_INTERRUPT: Final = 9 + SQLITE_IOERR: Final = 10 + SQLITE_IOERR_ACCESS: Final = 3338 + SQLITE_IOERR_AUTH: Final = 7178 + SQLITE_IOERR_BEGIN_ATOMIC: Final = 7434 + SQLITE_IOERR_BLOCKED: Final = 2826 + SQLITE_IOERR_CHECKRESERVEDLOCK: Final = 3594 + SQLITE_IOERR_CLOSE: Final = 4106 + SQLITE_IOERR_COMMIT_ATOMIC: Final = 7690 + SQLITE_IOERR_CONVPATH: Final = 6666 + SQLITE_IOERR_CORRUPTFS: Final = 8458 + SQLITE_IOERR_DATA: Final = 8202 + SQLITE_IOERR_DELETE: Final = 2570 + SQLITE_IOERR_DELETE_NOENT: Final = 5898 + SQLITE_IOERR_DIR_CLOSE: Final = 4362 + SQLITE_IOERR_DIR_FSYNC: Final = 1290 + SQLITE_IOERR_FSTAT: Final = 1802 + SQLITE_IOERR_FSYNC: Final = 1034 + SQLITE_IOERR_GETTEMPPATH: Final = 6410 + SQLITE_IOERR_LOCK: Final = 3850 + SQLITE_IOERR_MMAP: Final = 6154 + SQLITE_IOERR_NOMEM: Final = 3082 + SQLITE_IOERR_RDLOCK: Final = 2314 + SQLITE_IOERR_READ: Final = 266 + SQLITE_IOERR_ROLLBACK_ATOMIC: Final = 7946 + SQLITE_IOERR_SEEK: Final = 5642 + SQLITE_IOERR_SHMLOCK: Final = 5130 + SQLITE_IOERR_SHMMAP: Final = 5386 + SQLITE_IOERR_SHMOPEN: Final = 4618 + SQLITE_IOERR_SHMSIZE: Final = 4874 + SQLITE_IOERR_SHORT_READ: Final = 522 + SQLITE_IOERR_TRUNCATE: Final = 1546 + SQLITE_IOERR_UNLOCK: Final = 2058 + SQLITE_IOERR_VNODE: Final = 6922 + SQLITE_IOERR_WRITE: Final = 778 + SQLITE_LIMIT_ATTACHED: Final = 7 + SQLITE_LIMIT_COLUMN: Final = 22 + SQLITE_LIMIT_COMPOUND_SELECT: Final = 4 + SQLITE_LIMIT_EXPR_DEPTH: Final = 3 + SQLITE_LIMIT_FUNCTION_ARG: Final = 6 + SQLITE_LIMIT_LENGTH: Final = 0 + SQLITE_LIMIT_LIKE_PATTERN_LENGTH: Final = 8 + SQLITE_LIMIT_SQL_LENGTH: Final = 1 + SQLITE_LIMIT_TRIGGER_DEPTH: Final = 10 + SQLITE_LIMIT_VARIABLE_NUMBER: Final = 9 + SQLITE_LIMIT_VDBE_OP: Final = 5 + SQLITE_LIMIT_WORKER_THREADS: Final = 11 + SQLITE_LOCKED: Final = 6 + SQLITE_LOCKED_SHAREDCACHE: Final = 262 + SQLITE_LOCKED_VTAB: Final = 518 + SQLITE_MISMATCH: Final = 20 + SQLITE_MISUSE: Final = 21 + SQLITE_NOLFS: Final = 22 + SQLITE_NOMEM: Final = 7 + SQLITE_NOTADB: Final = 26 + SQLITE_NOTFOUND: Final = 12 + SQLITE_NOTICE: Final = 27 + SQLITE_NOTICE_RECOVER_ROLLBACK: Final = 539 + SQLITE_NOTICE_RECOVER_WAL: Final = 283 + SQLITE_OK_LOAD_PERMANENTLY: Final = 256 + SQLITE_OK_SYMLINK: Final = 512 + SQLITE_PERM: Final = 3 + SQLITE_PROTOCOL: Final = 15 + SQLITE_RANGE: Final = 25 + SQLITE_READONLY: Final = 8 + SQLITE_READONLY_CANTINIT: Final = 1288 + SQLITE_READONLY_CANTLOCK: Final = 520 + SQLITE_READONLY_DBMOVED: Final = 1032 + SQLITE_READONLY_DIRECTORY: Final = 1544 + SQLITE_READONLY_RECOVERY: Final = 264 + SQLITE_READONLY_ROLLBACK: Final = 776 + SQLITE_ROW: Final = 100 + SQLITE_SCHEMA: Final = 17 + SQLITE_TOOBIG: Final = 18 + SQLITE_WARNING: Final = 28 + SQLITE_WARNING_AUTOINDEX: Final = 284 + threadsafety: Literal[0, 1, 3] # Can take or return anything depending on what's in the registry. @overload @@ -225,7 +226,7 @@ if sys.version_info >= (3, 12): database: StrOrBytesPath, timeout: float = 5.0, detect_types: int = 0, - isolation_level: Literal["DEFERRED", "EXCLUSIVE", "IMMEDIATE"] | None = "DEFERRED", + isolation_level: _IsolationLevel = "DEFERRED", check_same_thread: bool = True, cached_statements: int = 128, uri: bool = False, @@ -237,7 +238,7 @@ if sys.version_info >= (3, 12): database: StrOrBytesPath, timeout: float, detect_types: int, - isolation_level: Literal["DEFERRED", "EXCLUSIVE", "IMMEDIATE"] | None, + isolation_level: _IsolationLevel, check_same_thread: bool, factory: type[_ConnectionT], cached_statements: int = 128, @@ -250,7 +251,7 @@ if sys.version_info >= (3, 12): database: StrOrBytesPath, timeout: float = 5.0, detect_types: int = 0, - isolation_level: Literal["DEFERRED", "EXCLUSIVE", "IMMEDIATE"] | None = "DEFERRED", + isolation_level: _IsolationLevel = "DEFERRED", check_same_thread: bool = True, *, factory: type[_ConnectionT], @@ -265,7 +266,7 @@ else: database: StrOrBytesPath, timeout: float = 5.0, detect_types: int = 0, - isolation_level: Literal["DEFERRED", "EXCLUSIVE", "IMMEDIATE"] | None = "DEFERRED", + isolation_level: _IsolationLevel = "DEFERRED", check_same_thread: bool = True, cached_statements: int = 128, uri: bool = False, @@ -275,7 +276,7 @@ else: database: StrOrBytesPath, timeout: float, detect_types: int, - isolation_level: Literal["DEFERRED", "EXCLUSIVE", "IMMEDIATE"] | None, + isolation_level: _IsolationLevel, check_same_thread: bool, factory: type[_ConnectionT], cached_statements: int = 128, @@ -286,7 +287,7 @@ else: database: StrOrBytesPath, timeout: float = 5.0, detect_types: int = 0, - isolation_level: Literal["DEFERRED", "EXCLUSIVE", "IMMEDIATE"] | None = "DEFERRED", + isolation_level: _IsolationLevel = "DEFERRED", check_same_thread: bool = True, *, factory: type[_ConnectionT], diff --git a/mypy/typeshed/stdlib/_ssl.pyi b/mypy/typeshed/stdlib/_ssl.pyi index 7ab880e4def74..88dd067809046 100644 --- a/mypy/typeshed/stdlib/_ssl.pyi +++ b/mypy/typeshed/stdlib/_ssl.pyi @@ -12,7 +12,7 @@ from ssl import ( SSLWantWriteError as SSLWantWriteError, SSLZeroReturnError as SSLZeroReturnError, ) -from typing import Any, ClassVar, Literal, TypedDict, final, overload +from typing import Any, ClassVar, Literal, TypedDict, final, overload, type_check_only from typing_extensions import NotRequired, Self, TypeAlias _PasswordType: TypeAlias = Callable[[], str | bytes | bytearray] | str | bytes | bytearray @@ -20,6 +20,7 @@ _PCTRTT: TypeAlias = tuple[tuple[str, str], ...] _PCTRTTT: TypeAlias = tuple[_PCTRTT, ...] _PeerCertRetDictType: TypeAlias = dict[str, str | _PCTRTTT | _PCTRTT] +@type_check_only class _Cipher(TypedDict): aead: bool alg_bits: int @@ -33,6 +34,7 @@ class _Cipher(TypedDict): strength_bits: int symmetric: str +@type_check_only class _CertInfo(TypedDict): subject: tuple[tuple[tuple[str, str], ...], ...] issuer: tuple[tuple[tuple[str, str], ...], ...] diff --git a/mypy/typeshed/stdlib/_thread.pyi b/mypy/typeshed/stdlib/_thread.pyi index 9cfbe55b4fe31..970130dfb09c3 100644 --- a/mypy/typeshed/stdlib/_thread.pyi +++ b/mypy/typeshed/stdlib/_thread.pyi @@ -73,7 +73,7 @@ def start_new(function: Callable[[Unpack[_Ts]], object], args: tuple[Unpack[_Ts] def start_new(function: Callable[..., object], args: tuple[Any, ...], kwargs: dict[str, Any], /) -> int: ... if sys.version_info >= (3, 10): - def interrupt_main(signum: signal.Signals = ..., /) -> None: ... + def interrupt_main(signum: signal.Signals = signal.SIGINT, /) -> None: ... else: def interrupt_main() -> None: ... diff --git a/mypy/typeshed/stdlib/_typeshed/__init__.pyi b/mypy/typeshed/stdlib/_typeshed/__init__.pyi index f322244016dd0..98a369dfc5893 100644 --- a/mypy/typeshed/stdlib/_typeshed/__init__.pyi +++ b/mypy/typeshed/stdlib/_typeshed/__init__.pyi @@ -65,10 +65,10 @@ MaybeNone: TypeAlias = Any # stable # In cases where the sentinel object is exported and can be used by user code, # a construct like this is better: # -# _SentinelType = NewType("_SentinelType", object) -# sentinel: _SentinelType +# _SentinelType = NewType("_SentinelType", object) # does not exist at runtime +# sentinel: Final[_SentinelType] # def foo(x: int | None | _SentinelType = ...) -> None: ... -sentinel: Any +sentinel: Any # stable # stable class IdentityFunction(Protocol): @@ -82,19 +82,21 @@ class SupportsNext(Protocol[_T_co]): class SupportsAnext(Protocol[_T_co]): def __anext__(self) -> Awaitable[_T_co]: ... -# Comparison protocols +class SupportsBool(Protocol): + def __bool__(self) -> bool: ... +# Comparison protocols class SupportsDunderLT(Protocol[_T_contra]): - def __lt__(self, other: _T_contra, /) -> bool: ... + def __lt__(self, other: _T_contra, /) -> SupportsBool: ... class SupportsDunderGT(Protocol[_T_contra]): - def __gt__(self, other: _T_contra, /) -> bool: ... + def __gt__(self, other: _T_contra, /) -> SupportsBool: ... class SupportsDunderLE(Protocol[_T_contra]): - def __le__(self, other: _T_contra, /) -> bool: ... + def __le__(self, other: _T_contra, /) -> SupportsBool: ... class SupportsDunderGE(Protocol[_T_contra]): - def __ge__(self, other: _T_contra, /) -> bool: ... + def __ge__(self, other: _T_contra, /) -> SupportsBool: ... class SupportsAllComparisons( SupportsDunderLT[Any], SupportsDunderGT[Any], SupportsDunderLE[Any], SupportsDunderGE[Any], Protocol diff --git a/mypy/typeshed/stdlib/_winapi.pyi b/mypy/typeshed/stdlib/_winapi.pyi index 0f71a06877481..6083ea4ae57a6 100644 --- a/mypy/typeshed/stdlib/_winapi.pyi +++ b/mypy/typeshed/stdlib/_winapi.pyi @@ -172,6 +172,9 @@ if sys.platform == "win32": ERROR_ACCESS_DENIED: Final = 5 ERROR_PRIVILEGE_NOT_HELD: Final = 1314 + if sys.version_info >= (3, 14): + COPY_FILE_DIRECTORY: Final = 0x00000080 + def CloseHandle(handle: int, /) -> None: ... @overload def ConnectNamedPipe(handle: int, overlapped: Literal[True]) -> Overlapped: ... diff --git a/mypy/typeshed/stdlib/argparse.pyi b/mypy/typeshed/stdlib/argparse.pyi index c22777e45436d..3c3ba116a692e 100644 --- a/mypy/typeshed/stdlib/argparse.pyi +++ b/mypy/typeshed/stdlib/argparse.pyi @@ -2,7 +2,7 @@ import sys from _typeshed import SupportsWrite, sentinel from collections.abc import Callable, Generator, Iterable, Sequence from re import Pattern -from typing import IO, Any, ClassVar, Final, Generic, NewType, NoReturn, Protocol, TypeVar, overload +from typing import IO, Any, ClassVar, Final, Generic, NewType, NoReturn, Protocol, TypeVar, overload, type_check_only from typing_extensions import Self, TypeAlias, deprecated __all__ = [ @@ -114,6 +114,7 @@ class _ActionsContainer: def _handle_conflict_error(self, action: Action, conflicting_actions: Iterable[tuple[str, Action]]) -> NoReturn: ... def _handle_conflict_resolve(self, action: Action, conflicting_actions: Iterable[tuple[str, Action]]) -> None: ... +@type_check_only class _FormatterClass(Protocol): def __call__(self, *, prog: str) -> HelpFormatter: ... @@ -283,7 +284,7 @@ class HelpFormatter: if sys.version_info >= (3, 14): def __init__( - self, prog: str, indent_increment: int = 2, max_help_position: int = 24, width: int | None = None, color: bool = False + self, prog: str, indent_increment: int = 2, max_help_position: int = 24, width: int | None = None, color: bool = True ) -> None: ... else: def __init__( @@ -497,16 +498,40 @@ else: class _ArgumentGroup(_ActionsContainer): title: str | None _group_actions: list[Action] - def __init__( - self, - container: _ActionsContainer, - title: str | None = None, - description: str | None = None, - *, - prefix_chars: str = ..., - argument_default: Any = ..., - conflict_handler: str = ..., - ) -> None: ... + if sys.version_info >= (3, 14): + @overload + def __init__( + self, + container: _ActionsContainer, + title: str | None = None, + description: str | None = None, + *, + argument_default: Any = ..., + conflict_handler: str = ..., + ) -> None: ... + @overload + @deprecated("Undocumented `prefix_chars` parameter is deprecated since Python 3.14.") + def __init__( + self, + container: _ActionsContainer, + title: str | None = None, + description: str | None = None, + *, + prefix_chars: str, + argument_default: Any = ..., + conflict_handler: str = ..., + ) -> None: ... + else: + def __init__( + self, + container: _ActionsContainer, + title: str | None = None, + description: str | None = None, + *, + prefix_chars: str = ..., + argument_default: Any = ..., + conflict_handler: str = ..., + ) -> None: ... # undocumented class _MutuallyExclusiveGroup(_ArgumentGroup): @@ -740,9 +765,9 @@ class _SubParsersAction(Action, Generic[_ArgumentParserT]): fromfile_prefix_chars: str | None = ..., argument_default: Any = ..., conflict_handler: str = ..., - add_help: bool = ..., - allow_abbrev: bool = ..., - exit_on_error: bool = ..., + add_help: bool = True, + allow_abbrev: bool = True, + exit_on_error: bool = True, suggest_on_error: bool = False, color: bool = False, **kwargs: Any, # Accepting any additional kwargs for custom parser classes @@ -766,9 +791,9 @@ class _SubParsersAction(Action, Generic[_ArgumentParserT]): fromfile_prefix_chars: str | None = ..., argument_default: Any = ..., conflict_handler: str = ..., - add_help: bool = ..., - allow_abbrev: bool = ..., - exit_on_error: bool = ..., + add_help: bool = True, + allow_abbrev: bool = True, + exit_on_error: bool = True, **kwargs: Any, # Accepting any additional kwargs for custom parser classes ) -> _ArgumentParserT: ... else: @@ -789,9 +814,9 @@ class _SubParsersAction(Action, Generic[_ArgumentParserT]): fromfile_prefix_chars: str | None = ..., argument_default: Any = ..., conflict_handler: str = ..., - add_help: bool = ..., - allow_abbrev: bool = ..., - exit_on_error: bool = ..., + add_help: bool = True, + allow_abbrev: bool = True, + exit_on_error: bool = True, **kwargs: Any, # Accepting any additional kwargs for custom parser classes ) -> _ArgumentParserT: ... diff --git a/mypy/typeshed/stdlib/ast.pyi b/mypy/typeshed/stdlib/ast.pyi index fcd6e8b01e743..3ba56f55932ab 100644 --- a/mypy/typeshed/stdlib/ast.pyi +++ b/mypy/typeshed/stdlib/ast.pyi @@ -10,7 +10,7 @@ from _ast import ( ) from _typeshed import ReadableBuffer, Unused from collections.abc import Iterable, Iterator, Sequence -from typing import Any, ClassVar, Generic, Literal, TypedDict, TypeVar as _TypeVar, overload +from typing import Any, ClassVar, Generic, Literal, TypedDict, TypeVar as _TypeVar, overload, type_check_only from typing_extensions import Self, Unpack, deprecated if sys.version_info >= (3, 13): @@ -20,6 +20,7 @@ if sys.version_info >= (3, 13): _EndPositionT = typing_extensions.TypeVar("_EndPositionT", int, int | None, default=int | None) # Corresponds to the names in the `_attributes` class variable which is non-empty in certain AST nodes +@type_check_only class _Attributes(TypedDict, Generic[_EndPositionT], total=False): lineno: int col_offset: int @@ -1698,8 +1699,14 @@ if sys.version_info >= (3, 12): self, *, name: str = ..., default_value: expr | None = ..., **kwargs: Unpack[_Attributes[int]] ) -> Self: ... -class _ABC(type): - def __init__(cls, *args: Unused) -> None: ... +if sys.version_info >= (3, 14): + @type_check_only + class _ABC(type): + def __init__(cls, *args: Unused) -> None: ... + +else: + class _ABC(type): + def __init__(cls, *args: Unused) -> None: ... if sys.version_info < (3, 14): @deprecated("Replaced by ast.Constant; removed in Python 3.14") diff --git a/mypy/typeshed/stdlib/asyncio/__init__.pyi b/mypy/typeshed/stdlib/asyncio/__init__.pyi index 58739816a67eb..23cf57aaac335 100644 --- a/mypy/typeshed/stdlib/asyncio/__init__.pyi +++ b/mypy/typeshed/stdlib/asyncio/__init__.pyi @@ -41,14 +41,11 @@ if sys.platform == "win32": "Server", # from base_events "iscoroutinefunction", # from coroutines "iscoroutine", # from coroutines - "_AbstractEventLoopPolicy", # from events "AbstractEventLoop", # from events "AbstractServer", # from events "Handle", # from events "TimerHandle", # from events - "_get_event_loop_policy", # from events "get_event_loop_policy", # from events - "_set_event_loop_policy", # from events "set_event_loop_policy", # from events "get_event_loop", # from events "set_event_loop", # from events @@ -517,14 +514,11 @@ else: "Server", # from base_events "iscoroutinefunction", # from coroutines "iscoroutine", # from coroutines - "_AbstractEventLoopPolicy", # from events "AbstractEventLoop", # from events "AbstractServer", # from events "Handle", # from events "TimerHandle", # from events - "_get_event_loop_policy", # from events "get_event_loop_policy", # from events - "_set_event_loop_policy", # from events "set_event_loop_policy", # from events "get_event_loop", # from events "set_event_loop", # from events @@ -610,7 +604,6 @@ else: "DatagramTransport", # from transports "SubprocessTransport", # from transports "SelectorEventLoop", # from unix_events - "_DefaultEventLoopPolicy", # from unix_events "EventLoop", # from unix_events ) elif sys.version_info >= (3, 13): diff --git a/mypy/typeshed/stdlib/asyncio/base_futures.pyi b/mypy/typeshed/stdlib/asyncio/base_futures.pyi index 55d2fbdbdb627..2cd0f2e3a7e4a 100644 --- a/mypy/typeshed/stdlib/asyncio/base_futures.pyi +++ b/mypy/typeshed/stdlib/asyncio/base_futures.pyi @@ -1,19 +1,17 @@ +from _asyncio import Future from collections.abc import Callable, Sequence from contextvars import Context from typing import Any, Final +from typing_extensions import TypeIs from . import futures __all__ = () -# asyncio defines 'isfuture()' in base_futures.py and re-imports it in futures.py -# but it leads to circular import error in pytype tool. -# That's why the import order is reversed. -from .futures import isfuture as isfuture - _PENDING: Final = "PENDING" # undocumented _CANCELLED: Final = "CANCELLED" # undocumented _FINISHED: Final = "FINISHED" # undocumented +def isfuture(obj: object) -> TypeIs[Future[Any]]: ... def _format_callbacks(cb: Sequence[tuple[Callable[[futures.Future[Any]], None], Context]]) -> str: ... # undocumented def _future_repr_info(future: futures.Future[Any]) -> list[str]: ... # undocumented diff --git a/mypy/typeshed/stdlib/asyncio/events.pyi b/mypy/typeshed/stdlib/asyncio/events.pyi index 688ef3ed08794..a37f6f697b9a1 100644 --- a/mypy/typeshed/stdlib/asyncio/events.pyi +++ b/mypy/typeshed/stdlib/asyncio/events.pyi @@ -12,7 +12,7 @@ from collections.abc import Callable, Sequence from concurrent.futures import Executor from contextvars import Context from socket import AddressFamily, SocketKind, _Address, _RetAddress, socket -from typing import IO, Any, Literal, Protocol, TypeVar, overload +from typing import IO, Any, Literal, Protocol, TypeVar, overload, type_check_only from typing_extensions import Self, TypeAlias, TypeVarTuple, Unpack, deprecated from . import _AwaitableLike, _CoroutineLike @@ -28,14 +28,11 @@ if sys.version_info < (3, 14): # Keep asyncio.__all__ updated with any changes to __all__ here if sys.version_info >= (3, 14): __all__ = ( - "_AbstractEventLoopPolicy", "AbstractEventLoop", "AbstractServer", "Handle", "TimerHandle", - "_get_event_loop_policy", "get_event_loop_policy", - "_set_event_loop_policy", "set_event_loop_policy", "get_event_loop", "set_event_loop", @@ -71,6 +68,7 @@ _ExceptionHandler: TypeAlias = Callable[[AbstractEventLoop, _Context], object] _ProtocolFactory: TypeAlias = Callable[[], BaseProtocol] _SSLContext: TypeAlias = bool | None | ssl.SSLContext +@type_check_only class _TaskFactory(Protocol): def __call__(self, loop: AbstractEventLoop, factory: _CoroutineLike[_T], /) -> Future[_T]: ... @@ -602,6 +600,9 @@ class AbstractEventLoop: @abstractmethod async def shutdown_default_executor(self) -> None: ... +# This class does not exist at runtime, but stubtest complains if it's marked as +# @type_check_only because it has an alias that does exist at runtime. See mypy#19568. +# @type_check_only class _AbstractEventLoopPolicy: @abstractmethod def get_event_loop(self) -> AbstractEventLoop: ... diff --git a/mypy/typeshed/stdlib/asyncio/format_helpers.pyi b/mypy/typeshed/stdlib/asyncio/format_helpers.pyi index 41505b14cd087..597eb9e56e1a1 100644 --- a/mypy/typeshed/stdlib/asyncio/format_helpers.pyi +++ b/mypy/typeshed/stdlib/asyncio/format_helpers.pyi @@ -3,9 +3,10 @@ import sys import traceback from collections.abc import Iterable from types import FrameType, FunctionType -from typing import Any, overload +from typing import Any, overload, type_check_only from typing_extensions import TypeAlias +@type_check_only class _HasWrapper: __wrapper__: _HasWrapper | FunctionType diff --git a/mypy/typeshed/stdlib/asyncio/futures.pyi b/mypy/typeshed/stdlib/asyncio/futures.pyi index 644d2d0e94cab..c907c7036b040 100644 --- a/mypy/typeshed/stdlib/asyncio/futures.pyi +++ b/mypy/typeshed/stdlib/asyncio/futures.pyi @@ -1,9 +1,9 @@ import sys from _asyncio import Future as Future from concurrent.futures._base import Future as _ConcurrentFuture -from typing import Any, TypeVar -from typing_extensions import TypeIs +from typing import TypeVar +from .base_futures import isfuture as isfuture from .events import AbstractEventLoop # Keep asyncio.__all__ updated with any changes to __all__ here @@ -16,8 +16,4 @@ else: _T = TypeVar("_T") -# asyncio defines 'isfuture()' in base_futures.py and re-imports it in futures.py -# but it leads to circular import error in pytype tool. -# That's why the import order is reversed. -def isfuture(obj: object) -> TypeIs[Future[Any]]: ... def wrap_future(future: _ConcurrentFuture[_T] | Future[_T], *, loop: AbstractEventLoop | None = None) -> Future[_T]: ... diff --git a/mypy/typeshed/stdlib/asyncio/queues.pyi b/mypy/typeshed/stdlib/asyncio/queues.pyi index 63cd98f53da3f..2fa2226d0e6ae 100644 --- a/mypy/typeshed/stdlib/asyncio/queues.pyi +++ b/mypy/typeshed/stdlib/asyncio/queues.pyi @@ -1,4 +1,5 @@ import sys +from _typeshed import SupportsRichComparisonT from asyncio.events import AbstractEventLoop from types import GenericAlias from typing import Any, Generic, TypeVar @@ -50,5 +51,5 @@ class Queue(Generic[_T], _LoopBoundMixin): # noqa: Y059 if sys.version_info >= (3, 13): def shutdown(self, immediate: bool = False) -> None: ... -class PriorityQueue(Queue[_T]): ... +class PriorityQueue(Queue[SupportsRichComparisonT]): ... class LifoQueue(Queue[_T]): ... diff --git a/mypy/typeshed/stdlib/asyncio/streams.pyi b/mypy/typeshed/stdlib/asyncio/streams.pyi index 43df5ae2d0c81..bf8db0246ee21 100644 --- a/mypy/typeshed/stdlib/asyncio/streams.pyi +++ b/mypy/typeshed/stdlib/asyncio/streams.pyi @@ -3,7 +3,7 @@ import sys from _typeshed import ReadableBuffer, StrPath from collections.abc import Awaitable, Callable, Iterable, Sequence, Sized from types import ModuleType -from typing import Any, Protocol, SupportsIndex +from typing import Any, Protocol, SupportsIndex, type_check_only from typing_extensions import Self, TypeAlias from . import events, protocols, transports @@ -25,6 +25,7 @@ else: _ClientConnectedCallback: TypeAlias = Callable[[StreamReader, StreamWriter], Awaitable[None] | None] +@type_check_only class _ReaduntilBuffer(ReadableBuffer, Sized, Protocol): ... if sys.version_info >= (3, 10): diff --git a/mypy/typeshed/stdlib/asyncio/tasks.pyi b/mypy/typeshed/stdlib/asyncio/tasks.pyi index a088e95af653d..4104b3ecfeee4 100644 --- a/mypy/typeshed/stdlib/asyncio/tasks.pyi +++ b/mypy/typeshed/stdlib/asyncio/tasks.pyi @@ -8,7 +8,7 @@ from _asyncio import ( _unregister_task as _unregister_task, ) from collections.abc import AsyncIterator, Awaitable, Coroutine, Generator, Iterable, Iterator -from typing import Any, Literal, Protocol, TypeVar, overload +from typing import Any, Literal, Protocol, TypeVar, overload, type_check_only from typing_extensions import TypeAlias from . import _CoroutineLike @@ -87,6 +87,7 @@ FIRST_EXCEPTION = concurrent.futures.FIRST_EXCEPTION ALL_COMPLETED = concurrent.futures.ALL_COMPLETED if sys.version_info >= (3, 13): + @type_check_only class _SyncAndAsyncIterator(Iterator[_T_co], AsyncIterator[_T_co], Protocol[_T_co]): ... def as_completed(fs: Iterable[_FutureLike[_T]], *, timeout: float | None = None) -> _SyncAndAsyncIterator[Future[_T]]: ... @@ -445,6 +446,7 @@ elif sys.version_info >= (3, 12): if sys.version_info >= (3, 12): _TaskT_co = TypeVar("_TaskT_co", bound=Task[Any], covariant=True) + @type_check_only class _CustomTaskConstructor(Protocol[_TaskT_co]): def __call__( self, @@ -457,6 +459,7 @@ if sys.version_info >= (3, 12): eager_start: bool, ) -> _TaskT_co: ... + @type_check_only class _EagerTaskFactoryType(Protocol[_TaskT_co]): def __call__( self, diff --git a/mypy/typeshed/stdlib/asyncio/unix_events.pyi b/mypy/typeshed/stdlib/asyncio/unix_events.pyi index 49f200dcdcae7..b2bf22a27677a 100644 --- a/mypy/typeshed/stdlib/asyncio/unix_events.pyi +++ b/mypy/typeshed/stdlib/asyncio/unix_events.pyi @@ -16,7 +16,7 @@ _Ts = TypeVarTuple("_Ts") # Keep asyncio.__all__ updated with any changes to __all__ here if sys.platform != "win32": if sys.version_info >= (3, 14): - __all__ = ("SelectorEventLoop", "_DefaultEventLoopPolicy", "EventLoop") + __all__ = ("SelectorEventLoop", "EventLoop") elif sys.version_info >= (3, 13): # Adds EventLoop __all__ = ( diff --git a/mypy/typeshed/stdlib/bdb.pyi b/mypy/typeshed/stdlib/bdb.pyi index b73f894093ce5..b6be2210ffe2e 100644 --- a/mypy/typeshed/stdlib/bdb.pyi +++ b/mypy/typeshed/stdlib/bdb.pyi @@ -81,8 +81,8 @@ class Bdb: def get_bpbynumber(self, arg: SupportsInt) -> Breakpoint: ... def get_break(self, filename: str, lineno: int) -> bool: ... def get_breaks(self, filename: str, lineno: int) -> list[Breakpoint]: ... - def get_file_breaks(self, filename: str) -> list[Breakpoint]: ... - def get_all_breaks(self) -> list[Breakpoint]: ... + def get_file_breaks(self, filename: str) -> list[int]: ... + def get_all_breaks(self) -> dict[str, list[int]]: ... def get_stack(self, f: FrameType | None, t: TracebackType | None) -> tuple[list[tuple[FrameType, int]], int]: ... def format_stack_entry(self, frame_lineno: tuple[FrameType, int], lprefix: str = ": ") -> str: ... def run( diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi index b853330b18fba..baf399e7bb774 100644 --- a/mypy/typeshed/stdlib/builtins.pyi +++ b/mypy/typeshed/stdlib/builtins.pyi @@ -451,9 +451,11 @@ class complex: @classmethod def from_number(cls, number: complex | SupportsComplex | SupportsFloat | SupportsIndex, /) -> Self: ... +@type_check_only class _FormatMapMapping(Protocol): def __getitem__(self, key: str, /) -> Any: ... +@type_check_only class _TranslateTable(Protocol): def __getitem__(self, key: int, /) -> str | int | None: ... @@ -1240,6 +1242,9 @@ class property: def __set__(self, instance: Any, value: Any, /) -> None: ... def __delete__(self, instance: Any, /) -> None: ... +# This class does not exist at runtime, but stubtest complains if it's marked as +# @type_check_only because it has an alias that does exist at runtime. See mypy#19568. +# @type_check_only @final class _NotImplementedType(Any): __call__: None @@ -1257,7 +1262,7 @@ def chr(i: int | SupportsIndex, /) -> str: ... if sys.version_info >= (3, 10): def aiter(async_iterable: SupportsAiter[_SupportsAnextT_co], /) -> _SupportsAnextT_co: ... - + @type_check_only class _SupportsSynchronousAnext(Protocol[_AwaitableT_co]): def __anext__(self) -> _AwaitableT_co: ... @@ -1413,7 +1418,7 @@ help: _sitebuiltins._Helper def hex(number: int | SupportsIndex, /) -> str: ... def id(obj: object, /) -> int: ... def input(prompt: object = "", /) -> str: ... - +@type_check_only class _GetItemIterable(Protocol[_T_co]): def __getitem__(self, i: int, /) -> _T_co: ... @@ -1426,7 +1431,6 @@ def iter(object: Callable[[], _T | None], sentinel: None, /) -> Iterator[_T]: .. @overload def iter(object: Callable[[], _T], sentinel: object, /) -> Iterator[_T]: ... -# Keep this alias in sync with unittest.case._ClassInfo if sys.version_info >= (3, 10): _ClassInfo: TypeAlias = type | types.UnionType | tuple[_ClassInfo, ...] else: @@ -1669,7 +1673,7 @@ def open( opener: _Opener | None = None, ) -> IO[Any]: ... def ord(c: str | bytes | bytearray, /) -> int: ... - +@type_check_only class _SupportsWriteAndFlush(SupportsWrite[_T_contra], SupportsFlush, Protocol[_T_contra]): ... @overload @@ -1688,12 +1692,15 @@ def print( _E_contra = TypeVar("_E_contra", contravariant=True) _M_contra = TypeVar("_M_contra", contravariant=True) +@type_check_only class _SupportsPow2(Protocol[_E_contra, _T_co]): def __pow__(self, other: _E_contra, /) -> _T_co: ... +@type_check_only class _SupportsPow3NoneOnly(Protocol[_E_contra, _T_co]): def __pow__(self, other: _E_contra, modulo: None = None, /) -> _T_co: ... +@type_check_only class _SupportsPow3(Protocol[_E_contra, _M_contra, _T_co]): def __pow__(self, other: _E_contra, modulo: _M_contra, /) -> _T_co: ... @@ -1758,9 +1765,11 @@ def repr(obj: object, /) -> str: ... # and https://github.com/python/typeshed/pull/9151 # on why we don't use `SupportsRound` from `typing.pyi` +@type_check_only class _SupportsRound1(Protocol[_T_co]): def __round__(self) -> _T_co: ... +@type_check_only class _SupportsRound2(Protocol[_T_co]): def __round__(self, ndigits: int, /) -> _T_co: ... @@ -1782,6 +1791,7 @@ def sorted(iterable: Iterable[_T], /, *, key: Callable[[_T], SupportsRichCompari _AddableT1 = TypeVar("_AddableT1", bound=SupportsAdd[Any, Any]) _AddableT2 = TypeVar("_AddableT2", bound=SupportsAdd[Any, Any]) +@type_check_only class _SupportsSumWithNoDefaultGiven(SupportsAdd[Any, Any], SupportsRAdd[int, Any], Protocol): ... _SupportsSumNoDefaultT = TypeVar("_SupportsSumNoDefaultT", bound=_SupportsSumWithNoDefaultGiven) diff --git a/mypy/typeshed/stdlib/bz2.pyi b/mypy/typeshed/stdlib/bz2.pyi index dce6187a2da10..7bd829d040cb8 100644 --- a/mypy/typeshed/stdlib/bz2.pyi +++ b/mypy/typeshed/stdlib/bz2.pyi @@ -3,7 +3,7 @@ from _bz2 import BZ2Compressor as BZ2Compressor, BZ2Decompressor as BZ2Decompres from _typeshed import ReadableBuffer, StrOrBytesPath, WriteableBuffer from collections.abc import Iterable from io import TextIOWrapper -from typing import IO, Literal, Protocol, SupportsIndex, overload +from typing import IO, Literal, Protocol, SupportsIndex, overload, type_check_only from typing_extensions import Self, TypeAlias if sys.version_info >= (3, 14): @@ -16,8 +16,10 @@ __all__ = ["BZ2File", "BZ2Compressor", "BZ2Decompressor", "open", "compress", "d # The following attributes and methods are optional: # def fileno(self) -> int: ... # def close(self) -> object: ... +@type_check_only class _ReadableFileobj(_Reader, Protocol): ... +@type_check_only class _WritableFileobj(Protocol): def write(self, b: bytes, /) -> object: ... # The following attributes and methods are optional: diff --git a/mypy/typeshed/stdlib/cgi.pyi b/mypy/typeshed/stdlib/cgi.pyi index 3a2e2a91b2419..a7a95a1393300 100644 --- a/mypy/typeshed/stdlib/cgi.pyi +++ b/mypy/typeshed/stdlib/cgi.pyi @@ -3,7 +3,7 @@ from builtins import list as _list, type as _type from collections.abc import Iterable, Iterator, Mapping from email.message import Message from types import TracebackType -from typing import IO, Any, Protocol +from typing import IO, Any, Protocol, type_check_only from typing_extensions import Self __all__ = [ @@ -31,7 +31,7 @@ def parse( def parse_multipart( fp: IO[Any], pdict: SupportsGetItem[str, bytes], encoding: str = "utf-8", errors: str = "replace", separator: str = "&" ) -> dict[str, list[Any]]: ... - +@type_check_only class _Environ(Protocol): def __getitem__(self, k: str, /) -> str: ... def keys(self) -> Iterable[str]: ... diff --git a/mypy/typeshed/stdlib/codecs.pyi b/mypy/typeshed/stdlib/codecs.pyi index 579d09c66a1bc..15e184fc10388 100644 --- a/mypy/typeshed/stdlib/codecs.pyi +++ b/mypy/typeshed/stdlib/codecs.pyi @@ -3,7 +3,7 @@ from _codecs import * from _typeshed import ReadableBuffer from abc import abstractmethod from collections.abc import Callable, Generator, Iterable -from typing import Any, BinaryIO, ClassVar, Final, Literal, Protocol, TextIO, overload +from typing import Any, BinaryIO, ClassVar, Final, Literal, Protocol, TextIO, overload, type_check_only from typing_extensions import Self, TypeAlias __all__ = [ @@ -73,16 +73,19 @@ _BufferedEncoding: TypeAlias = Literal[ "utf-8-sig", ] +@type_check_only class _WritableStream(Protocol): def write(self, data: bytes, /) -> object: ... def seek(self, offset: int, whence: int, /) -> object: ... def close(self) -> object: ... +@type_check_only class _ReadableStream(Protocol): def read(self, size: int = ..., /) -> bytes: ... def seek(self, offset: int, whence: int, /) -> object: ... def close(self) -> object: ... +@type_check_only class _Stream(_WritableStream, _ReadableStream, Protocol): ... # TODO: this only satisfies the most common interface, where @@ -91,24 +94,31 @@ class _Stream(_WritableStream, _ReadableStream, Protocol): ... # There *are* bytes->bytes and str->str encodings in the standard library. # They were much more common in Python 2 than in Python 3. +@type_check_only class _Encoder(Protocol): def __call__(self, input: str, errors: str = ..., /) -> tuple[bytes, int]: ... # signature of Codec().encode +@type_check_only class _Decoder(Protocol): def __call__(self, input: ReadableBuffer, errors: str = ..., /) -> tuple[str, int]: ... # signature of Codec().decode +@type_check_only class _StreamReader(Protocol): def __call__(self, stream: _ReadableStream, errors: str = ..., /) -> StreamReader: ... +@type_check_only class _StreamWriter(Protocol): def __call__(self, stream: _WritableStream, errors: str = ..., /) -> StreamWriter: ... +@type_check_only class _IncrementalEncoder(Protocol): def __call__(self, errors: str = ...) -> IncrementalEncoder: ... +@type_check_only class _IncrementalDecoder(Protocol): def __call__(self, errors: str = ...) -> IncrementalDecoder: ... +@type_check_only class _BufferedIncrementalDecoder(Protocol): def __call__(self, errors: str = ...) -> BufferedIncrementalDecoder: ... diff --git a/mypy/typeshed/stdlib/collections/__init__.pyi b/mypy/typeshed/stdlib/collections/__init__.pyi index bc33d91caa1d0..df9449ef4c9b0 100644 --- a/mypy/typeshed/stdlib/collections/__init__.pyi +++ b/mypy/typeshed/stdlib/collections/__init__.pyi @@ -2,7 +2,7 @@ import sys from _collections_abc import dict_items, dict_keys, dict_values from _typeshed import SupportsItems, SupportsKeysAndGetItem, SupportsRichComparison, SupportsRichComparisonT from types import GenericAlias -from typing import Any, ClassVar, Generic, NoReturn, SupportsIndex, TypeVar, final, overload +from typing import Any, ClassVar, Generic, NoReturn, SupportsIndex, TypeVar, final, overload, type_check_only from typing_extensions import Self if sys.version_info >= (3, 10): @@ -342,14 +342,17 @@ class _OrderedDictValuesView(ValuesView[_VT_co]): # but they are not exposed anywhere) # pyright doesn't have a specific error code for subclassing error! @final +@type_check_only class _odict_keys(dict_keys[_KT_co, _VT_co]): # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] def __reversed__(self) -> Iterator[_KT_co]: ... @final +@type_check_only class _odict_items(dict_items[_KT_co, _VT_co]): # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] def __reversed__(self) -> Iterator[tuple[_KT_co, _VT_co]]: ... @final +@type_check_only class _odict_values(dict_values[_KT_co, _VT_co]): # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] def __reversed__(self) -> Iterator[_VT_co]: ... diff --git a/mypy/typeshed/stdlib/compileall.pyi b/mypy/typeshed/stdlib/compileall.pyi index a599b1b235402..8972d50a4a634 100644 --- a/mypy/typeshed/stdlib/compileall.pyi +++ b/mypy/typeshed/stdlib/compileall.pyi @@ -1,10 +1,11 @@ import sys from _typeshed import StrPath from py_compile import PycInvalidationMode -from typing import Any, Protocol +from typing import Any, Protocol, type_check_only __all__ = ["compile_dir", "compile_file", "compile_path"] +@type_check_only class _SupportsSearch(Protocol): def search(self, string: str, /) -> Any: ... diff --git a/mypy/typeshed/stdlib/concurrent/futures/__init__.pyi b/mypy/typeshed/stdlib/concurrent/futures/__init__.pyi index dd1f6da80c4d6..ad4d20ea54453 100644 --- a/mypy/typeshed/stdlib/concurrent/futures/__init__.pyi +++ b/mypy/typeshed/stdlib/concurrent/futures/__init__.pyi @@ -19,7 +19,7 @@ from .thread import ThreadPoolExecutor as ThreadPoolExecutor if sys.version_info >= (3, 14): from .interpreter import InterpreterPoolExecutor as InterpreterPoolExecutor - __all__ = ( + __all__ = [ "FIRST_COMPLETED", "FIRST_EXCEPTION", "ALL_COMPLETED", @@ -34,7 +34,7 @@ if sys.version_info >= (3, 14): "ProcessPoolExecutor", "ThreadPoolExecutor", "InterpreterPoolExecutor", - ) + ] elif sys.version_info >= (3, 13): __all__ = ( diff --git a/mypy/typeshed/stdlib/concurrent/futures/_base.pyi b/mypy/typeshed/stdlib/concurrent/futures/_base.pyi index fbf07a3fc78f9..4063027f3eed3 100644 --- a/mypy/typeshed/stdlib/concurrent/futures/_base.pyi +++ b/mypy/typeshed/stdlib/concurrent/futures/_base.pyi @@ -4,7 +4,7 @@ from _typeshed import Unused from collections.abc import Callable, Iterable, Iterator from logging import Logger from types import GenericAlias, TracebackType -from typing import Any, Final, Generic, NamedTuple, Protocol, TypeVar +from typing import Any, Final, Generic, NamedTuple, Protocol, TypeVar, type_check_only from typing_extensions import ParamSpec, Self FIRST_COMPLETED: Final = "FIRST_COMPLETED" @@ -74,6 +74,7 @@ class Executor: self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None ) -> bool | None: ... +@type_check_only class _AsCompletedFuture(Protocol[_T_co]): # as_completed only mutates non-generic aspects of passed Futures and does not do any nominal # checks. Therefore, we can use a Protocol here to allow as_completed to act covariantly. diff --git a/mypy/typeshed/stdlib/concurrent/interpreters/__init__.pyi b/mypy/typeshed/stdlib/concurrent/interpreters/__init__.pyi new file mode 100644 index 0000000000000..3839e6bef09b6 --- /dev/null +++ b/mypy/typeshed/stdlib/concurrent/interpreters/__init__.pyi @@ -0,0 +1,68 @@ +import sys +import threading +import types +from collections.abc import Callable +from typing import Any, Literal, TypeVar +from typing_extensions import ParamSpec, Self + +if sys.version_info >= (3, 13): # needed to satisfy pyright checks for Python <3.13 + from _interpreters import ( + InterpreterError as InterpreterError, + InterpreterNotFoundError as InterpreterNotFoundError, + NotShareableError as NotShareableError, + _SharedDict, + _Whence, + is_shareable as is_shareable, + ) + + from ._queues import Queue as Queue, QueueEmpty as QueueEmpty, QueueFull as QueueFull, create as create_queue + + __all__ = [ + "ExecutionFailed", + "Interpreter", + "InterpreterError", + "InterpreterNotFoundError", + "NotShareableError", + "Queue", + "QueueEmpty", + "QueueFull", + "create", + "create_queue", + "get_current", + "get_main", + "is_shareable", + "list_all", + ] + + _R = TypeVar("_R") + _P = ParamSpec("_P") + + class ExecutionFailed(InterpreterError): + excinfo: types.SimpleNamespace + + def __init__(self, excinfo: types.SimpleNamespace) -> None: ... + + def create() -> Interpreter: ... + def list_all() -> list[Interpreter]: ... + def get_current() -> Interpreter: ... + def get_main() -> Interpreter: ... + + class Interpreter: + def __new__(cls, id: int, /, _whence: _Whence | None = None, _ownsref: bool | None = None) -> Self: ... + def __reduce__(self) -> tuple[type[Self], int]: ... + def __hash__(self) -> int: ... + def __del__(self) -> None: ... + @property + def id(self) -> int: ... + @property + def whence( + self, + ) -> Literal["unknown", "runtime init", "legacy C-API", "C-API", "cross-interpreter C-API", "_interpreters module"]: ... + def is_running(self) -> bool: ... + def close(self) -> None: ... + def prepare_main( + self, ns: _SharedDict | None = None, /, **kwargs: Any + ) -> None: ... # kwargs has same value restrictions as _SharedDict + def exec(self, code: str | types.CodeType | Callable[[], object], /) -> None: ... + def call(self, callable: Callable[_P, _R], /, *args: _P.args, **kwargs: _P.kwargs) -> _R: ... + def call_in_thread(self, callable: Callable[_P, object], /, *args: _P.args, **kwargs: _P.kwargs) -> threading.Thread: ... diff --git a/mypy/typeshed/stdlib/concurrent/interpreters/_crossinterp.pyi b/mypy/typeshed/stdlib/concurrent/interpreters/_crossinterp.pyi new file mode 100644 index 0000000000000..b073aefa7ca7e --- /dev/null +++ b/mypy/typeshed/stdlib/concurrent/interpreters/_crossinterp.pyi @@ -0,0 +1,29 @@ +import sys +from collections.abc import Callable +from typing import Final, NewType +from typing_extensions import Never, Self, TypeAlias + +if sys.version_info >= (3, 13): # needed to satisfy pyright checks for Python <3.13 + from _interpqueues import _UnboundOp + + class ItemInterpreterDestroyed(Exception): ... + # Actually a descriptor that behaves similarly to classmethod but prevents + # access from instances. + classonly = classmethod + + class UnboundItem: + def __new__(cls) -> Never: ... + @classonly + def singleton(cls, kind: str, module: str, name: str = "UNBOUND") -> Self: ... + + # Sentinel types and alias that don't exist at runtime. + _UnboundErrorType = NewType("_UnboundErrorType", object) + _UnboundRemoveType = NewType("_UnboundRemoveType", object) + _AnyUnbound: TypeAlias = _UnboundErrorType | _UnboundRemoveType | UnboundItem + + UNBOUND_ERROR: Final[_UnboundErrorType] + UNBOUND_REMOVE: Final[_UnboundRemoveType] + UNBOUND: Final[UnboundItem] # analogous to UNBOUND_REPLACE in C + + def serialize_unbound(unbound: _AnyUnbound) -> tuple[_UnboundOp]: ... + def resolve_unbound(flag: _UnboundOp, exctype_destroyed: Callable[[str], BaseException]) -> UnboundItem: ... diff --git a/mypy/typeshed/stdlib/concurrent/interpreters/_queues.pyi b/mypy/typeshed/stdlib/concurrent/interpreters/_queues.pyi new file mode 100644 index 0000000000000..39a057ee9a7bc --- /dev/null +++ b/mypy/typeshed/stdlib/concurrent/interpreters/_queues.pyi @@ -0,0 +1,58 @@ +import queue +import sys +from typing import Final, SupportsIndex +from typing_extensions import Self + +if sys.version_info >= (3, 13): # needed to satisfy pyright checks for Python <3.13 + from _interpqueues import QueueError as QueueError, QueueNotFoundError as QueueNotFoundError + + from . import _crossinterp + from ._crossinterp import UNBOUND_ERROR as UNBOUND_ERROR, UNBOUND_REMOVE as UNBOUND_REMOVE, UnboundItem, _AnyUnbound + + __all__ = [ + "UNBOUND", + "UNBOUND_ERROR", + "UNBOUND_REMOVE", + "ItemInterpreterDestroyed", + "Queue", + "QueueEmpty", + "QueueError", + "QueueFull", + "QueueNotFoundError", + "create", + "list_all", + ] + + class QueueEmpty(QueueError, queue.Empty): ... + class QueueFull(QueueError, queue.Full): ... + class ItemInterpreterDestroyed(QueueError, _crossinterp.ItemInterpreterDestroyed): ... + UNBOUND: Final[UnboundItem] + + def create(maxsize: int = 0, *, unbounditems: _AnyUnbound = ...) -> Queue: ... + def list_all() -> list[Queue]: ... + + class Queue: + def __new__(cls, id: int, /) -> Self: ... + def __del__(self) -> None: ... + def __hash__(self) -> int: ... + def __reduce__(self) -> tuple[type[Self], int]: ... + @property + def id(self) -> int: ... + @property + def unbounditems(self) -> _AnyUnbound: ... + @property + def maxsize(self) -> int: ... + def empty(self) -> bool: ... + def full(self) -> bool: ... + def qsize(self) -> int: ... + def put( + self, + obj: object, + timeout: SupportsIndex | None = None, + *, + unbounditems: _AnyUnbound | None = None, + _delay: float = ..., + ) -> None: ... + def put_nowait(self, obj: object, *, unbounditems: _AnyUnbound | None = None) -> None: ... + def get(self, timeout: SupportsIndex | None = None, *, _delay: float = ...) -> object: ... + def get_nowait(self) -> object: ... diff --git a/mypy/typeshed/stdlib/configparser.pyi b/mypy/typeshed/stdlib/configparser.pyi index 15c564c025897..fb02701e3711d 100644 --- a/mypy/typeshed/stdlib/configparser.pyi +++ b/mypy/typeshed/stdlib/configparser.pyi @@ -2,7 +2,7 @@ import sys from _typeshed import MaybeNone, StrOrBytesPath, SupportsWrite from collections.abc import Callable, ItemsView, Iterable, Iterator, Mapping, MutableMapping, Sequence from re import Pattern -from typing import Any, ClassVar, Final, Literal, TypeVar, overload +from typing import Any, ClassVar, Final, Literal, TypeVar, overload, type_check_only from typing_extensions import TypeAlias if sys.version_info >= (3, 14): @@ -104,7 +104,9 @@ else: ] if sys.version_info >= (3, 13): + @type_check_only class _UNNAMED_SECTION: ... + UNNAMED_SECTION: _UNNAMED_SECTION _SectionName: TypeAlias = str | _UNNAMED_SECTION @@ -369,17 +371,17 @@ class SectionProxy(MutableMapping[str, str]): # These are partially-applied version of the methods with the same names in # RawConfigParser; the stubs should be kept updated together @overload - def getint(self, option: str, *, raw: bool = ..., vars: _Section | None = ...) -> int | None: ... + def getint(self, option: str, *, raw: bool = False, vars: _Section | None = None) -> int | None: ... @overload - def getint(self, option: str, fallback: _T = ..., *, raw: bool = ..., vars: _Section | None = ...) -> int | _T: ... + def getint(self, option: str, fallback: _T = ..., *, raw: bool = False, vars: _Section | None = None) -> int | _T: ... @overload - def getfloat(self, option: str, *, raw: bool = ..., vars: _Section | None = ...) -> float | None: ... + def getfloat(self, option: str, *, raw: bool = False, vars: _Section | None = None) -> float | None: ... @overload - def getfloat(self, option: str, fallback: _T = ..., *, raw: bool = ..., vars: _Section | None = ...) -> float | _T: ... + def getfloat(self, option: str, fallback: _T = ..., *, raw: bool = False, vars: _Section | None = None) -> float | _T: ... @overload - def getboolean(self, option: str, *, raw: bool = ..., vars: _Section | None = ...) -> bool | None: ... + def getboolean(self, option: str, *, raw: bool = False, vars: _Section | None = None) -> bool | None: ... @overload - def getboolean(self, option: str, fallback: _T = ..., *, raw: bool = ..., vars: _Section | None = ...) -> bool | _T: ... + def getboolean(self, option: str, fallback: _T = ..., *, raw: bool = False, vars: _Section | None = None) -> bool | _T: ... # SectionProxy can have arbitrary attributes when custom converters are used def __getattr__(self, key: str) -> Callable[..., Any]: ... diff --git a/mypy/typeshed/stdlib/contextlib.pyi b/mypy/typeshed/stdlib/contextlib.pyi index 4663b448c79c8..c616c1f5bf19f 100644 --- a/mypy/typeshed/stdlib/contextlib.pyi +++ b/mypy/typeshed/stdlib/contextlib.pyi @@ -4,7 +4,7 @@ from _typeshed import FileDescriptorOrPath, Unused from abc import ABC, abstractmethod from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Generator, Iterator from types import TracebackType -from typing import IO, Any, Generic, Protocol, TypeVar, overload, runtime_checkable +from typing import IO, Any, Generic, Protocol, TypeVar, overload, runtime_checkable, type_check_only from typing_extensions import ParamSpec, Self, TypeAlias __all__ = [ @@ -112,7 +112,7 @@ else: ) -> bool | None: ... def asynccontextmanager(func: Callable[_P, AsyncIterator[_T_co]]) -> Callable[_P, _AsyncGeneratorContextManager[_T_co]]: ... - +@type_check_only class _SupportsClose(Protocol): def close(self) -> object: ... @@ -123,6 +123,7 @@ class closing(AbstractContextManager[_SupportsCloseT, None]): def __exit__(self, *exc_info: Unused) -> None: ... if sys.version_info >= (3, 10): + @type_check_only class _SupportsAclose(Protocol): def aclose(self) -> Awaitable[object]: ... diff --git a/mypy/typeshed/stdlib/copy.pyi b/mypy/typeshed/stdlib/copy.pyi index 2cceec6a22509..10d2f0ae37103 100644 --- a/mypy/typeshed/stdlib/copy.pyi +++ b/mypy/typeshed/stdlib/copy.pyi @@ -1,5 +1,5 @@ import sys -from typing import Any, Protocol, TypeVar +from typing import Any, Protocol, TypeVar, type_check_only from typing_extensions import Self __all__ = ["Error", "copy", "deepcopy"] @@ -7,6 +7,7 @@ __all__ = ["Error", "copy", "deepcopy"] _T = TypeVar("_T") _SR = TypeVar("_SR", bound=_SupportsReplace) +@type_check_only class _SupportsReplace(Protocol): # In reality doesn't support args, but there's no other great way to express this. def __replace__(self, *args: Any, **kwargs: Any) -> Self: ... diff --git a/mypy/typeshed/stdlib/ctypes/__init__.pyi b/mypy/typeshed/stdlib/ctypes/__init__.pyi index 52288d011e984..15649da9ff733 100644 --- a/mypy/typeshed/stdlib/ctypes/__init__.pyi +++ b/mypy/typeshed/stdlib/ctypes/__init__.pyi @@ -11,6 +11,7 @@ from _ctypes import ( _CData as _CData, _CDataType as _CDataType, _CField as _CField, + _CTypeBaseType, _Pointer as _Pointer, _PointerLike as _PointerLike, _SimpleCData as _SimpleCData, @@ -162,7 +163,7 @@ c_buffer = create_string_buffer def create_unicode_buffer(init: int | str, size: int | None = None) -> Array[c_wchar]: ... @deprecated("Deprecated in Python 3.13; removal scheduled for Python 3.15") -def SetPointerType(pointer: type[_Pointer[Any]], cls: Any) -> None: ... +def SetPointerType(pointer: type[_Pointer[Any]], cls: _CTypeBaseType) -> None: ... def ARRAY(typ: _CT, len: int) -> Array[_CT]: ... # Soft Deprecated, no plans to remove if sys.platform == "win32": diff --git a/mypy/typeshed/stdlib/dataclasses.pyi b/mypy/typeshed/stdlib/dataclasses.pyi index c76b0b0e61e27..b3183f57ebd2a 100644 --- a/mypy/typeshed/stdlib/dataclasses.pyi +++ b/mypy/typeshed/stdlib/dataclasses.pyi @@ -165,6 +165,7 @@ else: ) -> Callable[[type[_T]], type[_T]]: ... # See https://github.com/python/mypy/issues/10750 +@type_check_only class _DefaultFactory(Protocol[_T_co]): def __call__(self) -> _T_co: ... diff --git a/mypy/typeshed/stdlib/datetime.pyi b/mypy/typeshed/stdlib/datetime.pyi index 37d6a06dfff95..c54de6159b514 100644 --- a/mypy/typeshed/stdlib/datetime.pyi +++ b/mypy/typeshed/stdlib/datetime.pyi @@ -118,13 +118,13 @@ class time: resolution: ClassVar[timedelta] def __new__( cls, - hour: SupportsIndex = ..., - minute: SupportsIndex = ..., - second: SupportsIndex = ..., - microsecond: SupportsIndex = ..., - tzinfo: _TzInfo | None = ..., + hour: SupportsIndex = 0, + minute: SupportsIndex = 0, + second: SupportsIndex = 0, + microsecond: SupportsIndex = 0, + tzinfo: _TzInfo | None = None, *, - fold: int = ..., + fold: int = 0, ) -> Self: ... @property def hour(self) -> int: ... @@ -144,7 +144,7 @@ class time: def __gt__(self, value: time, /) -> bool: ... def __eq__(self, value: object, /) -> bool: ... def __hash__(self) -> int: ... - def isoformat(self, timespec: str = ...) -> str: ... + def isoformat(self, timespec: str = "auto") -> str: ... @classmethod def fromisoformat(cls, time_string: str, /) -> Self: ... @@ -197,13 +197,13 @@ class timedelta: resolution: ClassVar[timedelta] def __new__( cls, - days: float = ..., - seconds: float = ..., - microseconds: float = ..., - milliseconds: float = ..., - minutes: float = ..., - hours: float = ..., - weeks: float = ..., + days: float = 0, + seconds: float = 0, + microseconds: float = 0, + milliseconds: float = 0, + minutes: float = 0, + hours: float = 0, + weeks: float = 0, ) -> Self: ... @property def days(self) -> int: ... @@ -247,13 +247,13 @@ class datetime(date): year: SupportsIndex, month: SupportsIndex, day: SupportsIndex, - hour: SupportsIndex = ..., - minute: SupportsIndex = ..., - second: SupportsIndex = ..., - microsecond: SupportsIndex = ..., - tzinfo: _TzInfo | None = ..., + hour: SupportsIndex = 0, + minute: SupportsIndex = 0, + second: SupportsIndex = 0, + microsecond: SupportsIndex = 0, + tzinfo: _TzInfo | None = None, *, - fold: int = ..., + fold: int = 0, ) -> Self: ... @property def hour(self) -> int: ... @@ -272,10 +272,10 @@ class datetime(date): # meaning it is only *safe* to pass it as a keyword argument on 3.12+ if sys.version_info >= (3, 12): @classmethod - def fromtimestamp(cls, timestamp: float, tz: _TzInfo | None = ...) -> Self: ... + def fromtimestamp(cls, timestamp: float, tz: _TzInfo | None = None) -> Self: ... else: @classmethod - def fromtimestamp(cls, timestamp: float, /, tz: _TzInfo | None = ...) -> Self: ... + def fromtimestamp(cls, timestamp: float, /, tz: _TzInfo | None = None) -> Self: ... @classmethod @deprecated("Use timezone-aware objects to represent datetimes in UTC; e.g. by calling .fromtimestamp(datetime.timezone.utc)") @@ -321,8 +321,8 @@ class datetime(date): *, fold: int = ..., ) -> Self: ... - def astimezone(self, tz: _TzInfo | None = ...) -> Self: ... - def isoformat(self, sep: str = ..., timespec: str = ...) -> str: ... + def astimezone(self, tz: _TzInfo | None = None) -> Self: ... + def isoformat(self, sep: str = "T", timespec: str = "auto") -> str: ... @classmethod def strptime(cls, date_string: str, format: str, /) -> Self: ... def utcoffset(self) -> timedelta | None: ... diff --git a/mypy/typeshed/stdlib/dbm/__init__.pyi b/mypy/typeshed/stdlib/dbm/__init__.pyi index 7f344060f9ab5..7cbb63cf2f06e 100644 --- a/mypy/typeshed/stdlib/dbm/__init__.pyi +++ b/mypy/typeshed/stdlib/dbm/__init__.pyi @@ -76,6 +76,7 @@ _TFlags: TypeAlias = Literal[ "nusf", ] +@type_check_only class _Database(MutableMapping[_KeyType, bytes]): def close(self) -> None: ... def __getitem__(self, key: _KeyType) -> bytes: ... diff --git a/mypy/typeshed/stdlib/difflib.pyi b/mypy/typeshed/stdlib/difflib.pyi index 18583a3acfe9d..6efe68322bb65 100644 --- a/mypy/typeshed/stdlib/difflib.pyi +++ b/mypy/typeshed/stdlib/difflib.pyi @@ -1,3 +1,5 @@ +import re +import sys from collections.abc import Callable, Iterable, Iterator, Sequence from types import GenericAlias from typing import Any, AnyStr, Generic, Literal, NamedTuple, TypeVar, overload @@ -60,7 +62,12 @@ class Differ: def __init__(self, linejunk: Callable[[str], bool] | None = None, charjunk: Callable[[str], bool] | None = None) -> None: ... def compare(self, a: Sequence[str], b: Sequence[str]) -> Iterator[str]: ... -def IS_LINE_JUNK(line: str, pat: Any = ...) -> bool: ... # pat is undocumented +if sys.version_info >= (3, 14): + def IS_LINE_JUNK(line: str, pat: Callable[[str], re.Match[str] | None] | None = None) -> bool: ... + +else: + def IS_LINE_JUNK(line: str, pat: Callable[[str], re.Match[str] | None] = ...) -> bool: ... + def IS_CHARACTER_JUNK(ch: str, ws: str = " \t") -> bool: ... # ws is undocumented def unified_diff( a: Sequence[str], diff --git a/mypy/typeshed/stdlib/email/headerregistry.pyi b/mypy/typeshed/stdlib/email/headerregistry.pyi index dc641c8c952b3..dff9593b731f5 100644 --- a/mypy/typeshed/stdlib/email/headerregistry.pyi +++ b/mypy/typeshed/stdlib/email/headerregistry.pyi @@ -13,7 +13,7 @@ from email._header_value_parser import ( ) from email.errors import MessageDefect from email.policy import Policy -from typing import Any, ClassVar, Literal, Protocol +from typing import Any, ClassVar, Literal, Protocol, type_check_only from typing_extensions import Self class BaseHeader(str): @@ -137,6 +137,7 @@ class MessageIDHeader: @staticmethod def value_parser(value: str) -> MessageID: ... +@type_check_only class _HeaderParser(Protocol): max_count: ClassVar[Literal[1] | None] @staticmethod diff --git a/mypy/typeshed/stdlib/email/message.pyi b/mypy/typeshed/stdlib/email/message.pyi index e4d14992168a1..794882b140e61 100644 --- a/mypy/typeshed/stdlib/email/message.pyi +++ b/mypy/typeshed/stdlib/email/message.pyi @@ -5,7 +5,7 @@ from email.charset import Charset from email.contentmanager import ContentManager from email.errors import MessageDefect from email.policy import Policy -from typing import Any, Generic, Literal, Protocol, TypeVar, overload +from typing import Any, Generic, Literal, Protocol, TypeVar, overload, type_check_only from typing_extensions import Self, TypeAlias __all__ = ["Message", "EmailMessage"] @@ -24,9 +24,11 @@ _EncodedPayloadType: TypeAlias = Message | bytes _MultipartPayloadType: TypeAlias = list[_PayloadType] _CharsetType: TypeAlias = Charset | str | None +@type_check_only class _SupportsEncodeToPayload(Protocol): def encode(self, encoding: str, /) -> _PayloadType | _MultipartPayloadType | _SupportsDecodeToPayload: ... +@type_check_only class _SupportsDecodeToPayload(Protocol): def decode(self, encoding: str, errors: str, /) -> _PayloadType | _MultipartPayloadType: ... diff --git a/mypy/typeshed/stdlib/encodings/__init__.pyi b/mypy/typeshed/stdlib/encodings/__init__.pyi index 12ec6792d49b5..61f86d243c720 100644 --- a/mypy/typeshed/stdlib/encodings/__init__.pyi +++ b/mypy/typeshed/stdlib/encodings/__init__.pyi @@ -1,3 +1,4 @@ +import sys from codecs import CodecInfo class CodecRegistryError(LookupError, SystemError): ... @@ -5,5 +6,8 @@ class CodecRegistryError(LookupError, SystemError): ... def normalize_encoding(encoding: str | bytes) -> str: ... def search_function(encoding: str) -> CodecInfo | None: ... +if sys.version_info >= (3, 14) and sys.platform == "win32": + def win32_code_page_search_function(encoding: str) -> CodecInfo | None: ... + # Needed for submodules def __getattr__(name: str): ... # incomplete module diff --git a/mypy/typeshed/stdlib/enum.pyi b/mypy/typeshed/stdlib/enum.pyi index 327b135459a00..eb7d2e3819fd8 100644 --- a/mypy/typeshed/stdlib/enum.pyi +++ b/mypy/typeshed/stdlib/enum.pyi @@ -219,6 +219,11 @@ class Enum(metaclass=EnumMeta): if sys.version_info >= (3, 12) and sys.version_info < (3, 14): @classmethod def __signature__(cls) -> str: ... + if sys.version_info >= (3, 13): + # Value may be any type, even in special enums. Enabling Enum parsing from + # multiple value types + def _add_value_alias_(self, value: Any) -> None: ... + def _add_alias_(self, name: str) -> None: ... if sys.version_info >= (3, 11): class ReprEnum(Enum): ... diff --git a/mypy/typeshed/stdlib/fileinput.pyi b/mypy/typeshed/stdlib/fileinput.pyi index 1d5f9cf00f368..eb942bc551770 100644 --- a/mypy/typeshed/stdlib/fileinput.pyi +++ b/mypy/typeshed/stdlib/fileinput.pyi @@ -2,7 +2,7 @@ import sys from _typeshed import AnyStr_co, StrOrBytesPath from collections.abc import Callable, Iterable, Iterator from types import GenericAlias, TracebackType -from typing import IO, Any, AnyStr, Literal, Protocol, overload +from typing import IO, Any, AnyStr, Literal, Protocol, overload, type_check_only from typing_extensions import Self, TypeAlias __all__ = [ @@ -25,6 +25,7 @@ if sys.version_info >= (3, 11): else: _TextMode: TypeAlias = Literal["r", "rU", "U"] +@type_check_only class _HasReadlineAndFileno(Protocol[AnyStr_co]): def readline(self) -> AnyStr_co: ... def fileno(self) -> int: ... diff --git a/mypy/typeshed/stdlib/fractions.pyi b/mypy/typeshed/stdlib/fractions.pyi index 16259fcfadc7c..e81fbaf5dad78 100644 --- a/mypy/typeshed/stdlib/fractions.pyi +++ b/mypy/typeshed/stdlib/fractions.pyi @@ -2,13 +2,14 @@ import sys from collections.abc import Callable from decimal import Decimal from numbers import Rational, Real -from typing import Any, Literal, Protocol, SupportsIndex, overload +from typing import Any, Literal, Protocol, SupportsIndex, overload, type_check_only from typing_extensions import Self, TypeAlias _ComparableNum: TypeAlias = int | float | Decimal | Real __all__ = ["Fraction"] +@type_check_only class _ConvertibleToIntegerRatio(Protocol): def as_integer_ratio(self) -> tuple[int | Rational, int | Rational]: ... diff --git a/mypy/typeshed/stdlib/functools.pyi b/mypy/typeshed/stdlib/functools.pyi index e31399fb87054..6e17ba7d35dc7 100644 --- a/mypy/typeshed/stdlib/functools.pyi +++ b/mypy/typeshed/stdlib/functools.pyi @@ -3,7 +3,7 @@ import types from _typeshed import SupportsAllComparisons, SupportsItems from collections.abc import Callable, Hashable, Iterable, Sized from types import GenericAlias -from typing import Any, Final, Generic, Literal, NamedTuple, TypedDict, TypeVar, final, overload +from typing import Any, Final, Generic, Literal, NamedTuple, TypedDict, TypeVar, final, overload, type_check_only from typing_extensions import ParamSpec, Self, TypeAlias __all__ = [ @@ -48,6 +48,7 @@ class _CacheInfo(NamedTuple): maxsize: int | None currsize: int +@type_check_only class _CacheParameters(TypedDict): maxsize: int typed: bool @@ -96,6 +97,7 @@ else: WRAPPER_UPDATES: tuple[Literal["__dict__"]] +@type_check_only class _Wrapped(Generic[_PWrapped, _RWrapped, _PWrapper, _RWrapper]): __wrapped__: Callable[_PWrapped, _RWrapped] def __call__(self, *args: _PWrapper.args, **kwargs: _PWrapper.kwargs) -> _RWrapper: ... @@ -103,6 +105,7 @@ class _Wrapped(Generic[_PWrapped, _RWrapped, _PWrapper, _RWrapper]): __name__: str __qualname__: str +@type_check_only class _Wrapper(Generic[_PWrapped, _RWrapped]): def __call__(self, f: Callable[_PWrapper, _RWrapper]) -> _Wrapped[_PWrapped, _RWrapped, _PWrapper, _RWrapper]: ... @@ -180,6 +183,7 @@ if sys.version_info >= (3, 11): else: _RegType: TypeAlias = type[Any] +@type_check_only class _SingleDispatchCallable(Generic[_T]): registry: types.MappingProxyType[Any, Callable[..., _T]] def dispatch(self, cls: Any) -> Callable[..., _T]: ... diff --git a/mypy/typeshed/stdlib/gettext.pyi b/mypy/typeshed/stdlib/gettext.pyi index d8fd92a00e132..5ff98b052cdbe 100644 --- a/mypy/typeshed/stdlib/gettext.pyi +++ b/mypy/typeshed/stdlib/gettext.pyi @@ -2,7 +2,7 @@ import io import sys from _typeshed import StrPath from collections.abc import Callable, Container, Iterable, Sequence -from typing import Any, Final, Literal, Protocol, TypeVar, overload +from typing import Any, Final, Literal, Protocol, TypeVar, overload, type_check_only __all__ = [ "NullTranslations", @@ -26,6 +26,7 @@ __all__ = [ if sys.version_info < (3, 11): __all__ += ["bind_textdomain_codeset", "ldgettext", "ldngettext", "lgettext", "lngettext"] +@type_check_only class _TranslationsReader(Protocol): def read(self) -> bytes: ... # optional: diff --git a/mypy/typeshed/stdlib/gzip.pyi b/mypy/typeshed/stdlib/gzip.pyi index 34ae92b4d8ed6..06f5e2880bd5a 100644 --- a/mypy/typeshed/stdlib/gzip.pyi +++ b/mypy/typeshed/stdlib/gzip.pyi @@ -2,7 +2,7 @@ import sys import zlib from _typeshed import ReadableBuffer, SizedBuffer, StrOrBytesPath, WriteableBuffer from io import FileIO, TextIOWrapper -from typing import Final, Literal, Protocol, overload +from typing import Final, Literal, Protocol, overload, type_check_only from typing_extensions import TypeAlias if sys.version_info >= (3, 14): @@ -25,6 +25,7 @@ FEXTRA: Final[int] # actually Literal[4] # undocumented FNAME: Final[int] # actually Literal[8] # undocumented FCOMMENT: Final[int] # actually Literal[16] # undocumented +@type_check_only class _ReadableFileobj(Protocol): def read(self, n: int, /) -> bytes: ... def seek(self, n: int, /) -> object: ... @@ -33,6 +34,7 @@ class _ReadableFileobj(Protocol): # mode: str # def fileno() -> int: ... +@type_check_only class _WritableFileobj(Protocol): def write(self, b: bytes, /) -> object: ... def flush(self) -> object: ... diff --git a/mypy/typeshed/stdlib/hashlib.pyi b/mypy/typeshed/stdlib/hashlib.pyi index b32c0e9925740..924136301b215 100644 --- a/mypy/typeshed/stdlib/hashlib.pyi +++ b/mypy/typeshed/stdlib/hashlib.pyi @@ -20,7 +20,7 @@ from _hashlib import ( ) from _typeshed import ReadableBuffer from collections.abc import Callable, Set as AbstractSet -from typing import Protocol +from typing import Protocol, type_check_only if sys.version_info >= (3, 11): __all__ = ( @@ -72,9 +72,11 @@ algorithms_guaranteed: AbstractSet[str] algorithms_available: AbstractSet[str] if sys.version_info >= (3, 11): + @type_check_only class _BytesIOLike(Protocol): def getbuffer(self) -> ReadableBuffer: ... + @type_check_only class _FileDigestFileObj(Protocol): def readinto(self, buf: bytearray, /) -> int: ... def readable(self) -> bool: ... diff --git a/mypy/typeshed/stdlib/html/parser.pyi b/mypy/typeshed/stdlib/html/parser.pyi index 5d38c9c0d800c..45336f03aaa7a 100644 --- a/mypy/typeshed/stdlib/html/parser.pyi +++ b/mypy/typeshed/stdlib/html/parser.pyi @@ -1,9 +1,15 @@ +import sys from _markupbase import ParserBase from re import Pattern +from typing import Final __all__ = ["HTMLParser"] class HTMLParser(ParserBase): + CDATA_CONTENT_ELEMENTS: Final[tuple[str, ...]] + if sys.version_info >= (3, 14): + RCDATA_CONTENT_ELEMENTS: Final[tuple[str, ...]] + def __init__(self, *, convert_charrefs: bool = True) -> None: ... def feed(self, data: str) -> None: ... def close(self) -> None: ... @@ -17,7 +23,6 @@ class HTMLParser(ParserBase): def handle_comment(self, data: str) -> None: ... def handle_decl(self, decl: str) -> None: ... def handle_pi(self, data: str) -> None: ... - CDATA_CONTENT_ELEMENTS: tuple[str, ...] def check_for_whole_start_tag(self, i: int) -> int: ... # undocumented def clear_cdata_mode(self) -> None: ... # undocumented def goahead(self, end: bool) -> None: ... # undocumented @@ -26,7 +31,10 @@ class HTMLParser(ParserBase): def parse_html_declaration(self, i: int) -> int: ... # undocumented def parse_pi(self, i: int) -> int: ... # undocumented def parse_starttag(self, i: int) -> int: ... # undocumented - def set_cdata_mode(self, elem: str) -> None: ... # undocumented + if sys.version_info >= (3, 14): + def set_cdata_mode(self, elem: str, *, escapable: bool = False) -> None: ... # undocumented + else: + def set_cdata_mode(self, elem: str) -> None: ... # undocumented rawdata: str # undocumented cdata_elem: str | None # undocumented convert_charrefs: bool # undocumented diff --git a/mypy/typeshed/stdlib/imghdr.pyi b/mypy/typeshed/stdlib/imghdr.pyi index 6e1b858b8f320..e45ca3eb5bdbc 100644 --- a/mypy/typeshed/stdlib/imghdr.pyi +++ b/mypy/typeshed/stdlib/imghdr.pyi @@ -1,9 +1,10 @@ from _typeshed import StrPath from collections.abc import Callable -from typing import Any, BinaryIO, Protocol, overload +from typing import Any, BinaryIO, Protocol, overload, type_check_only __all__ = ["what"] +@type_check_only class _ReadableBinary(Protocol): def tell(self) -> int: ... def read(self, size: int, /) -> bytes: ... diff --git a/mypy/typeshed/stdlib/imp.pyi b/mypy/typeshed/stdlib/imp.pyi index ee5a0cd7bc726..f045fd969b27d 100644 --- a/mypy/typeshed/stdlib/imp.pyi +++ b/mypy/typeshed/stdlib/imp.pyi @@ -13,7 +13,7 @@ from _imp import ( from _typeshed import StrPath from os import PathLike from types import TracebackType -from typing import IO, Any, Protocol +from typing import IO, Any, Protocol, type_check_only SEARCH_ERROR: int PY_SOURCE: int @@ -39,6 +39,7 @@ class NullImporter: # Technically, a text file has to support a slightly different set of operations than a binary file, # but we ignore that here. +@type_check_only class _FileLike(Protocol): closed: bool mode: str diff --git a/mypy/typeshed/stdlib/importlib/resources/abc.pyi b/mypy/typeshed/stdlib/importlib/resources/abc.pyi index fe0fe64dba0df..80d92a608604e 100644 --- a/mypy/typeshed/stdlib/importlib/resources/abc.pyi +++ b/mypy/typeshed/stdlib/importlib/resources/abc.pyi @@ -10,13 +10,8 @@ if sys.version_info >= (3, 11): def open_resource(self, resource: str) -> IO[bytes]: ... @abstractmethod def resource_path(self, resource: str) -> str: ... - if sys.version_info >= (3, 10): - @abstractmethod - def is_resource(self, path: str) -> bool: ... - else: - @abstractmethod - def is_resource(self, name: str) -> bool: ... - + @abstractmethod + def is_resource(self, path: str) -> bool: ... @abstractmethod def contents(self) -> Iterator[str]: ... @@ -28,12 +23,8 @@ if sys.version_info >= (3, 11): def is_file(self) -> bool: ... @abstractmethod def iterdir(self) -> Iterator[Traversable]: ... - if sys.version_info >= (3, 11): - @abstractmethod - def joinpath(self, *descendants: str) -> Traversable: ... - else: - @abstractmethod - def joinpath(self, child: str, /) -> Traversable: ... + @abstractmethod + def joinpath(self, *descendants: str) -> Traversable: ... # The documentation and runtime protocol allows *args, **kwargs arguments, # but this would mean that all implementers would have to support them, @@ -47,12 +38,7 @@ if sys.version_info >= (3, 11): @property @abstractmethod def name(self) -> str: ... - if sys.version_info >= (3, 10): - def __truediv__(self, child: str, /) -> Traversable: ... - else: - @abstractmethod - def __truediv__(self, child: str, /) -> Traversable: ... - + def __truediv__(self, child: str, /) -> Traversable: ... @abstractmethod def read_bytes(self) -> bytes: ... @abstractmethod diff --git a/mypy/typeshed/stdlib/inspect.pyi b/mypy/typeshed/stdlib/inspect.pyi index e19c2a634aa09..e73f9e75838d0 100644 --- a/mypy/typeshed/stdlib/inspect.pyi +++ b/mypy/typeshed/stdlib/inspect.pyi @@ -25,7 +25,7 @@ from types import ( TracebackType, WrapperDescriptorType, ) -from typing import Any, ClassVar, Final, Literal, NamedTuple, Protocol, TypeVar, overload +from typing import Any, ClassVar, Final, Literal, NamedTuple, Protocol, TypeVar, overload, type_check_only from typing_extensions import ParamSpec, Self, TypeAlias, TypeGuard, TypeIs if sys.version_info >= (3, 14): @@ -240,10 +240,11 @@ def isasyncgenfunction(obj: Callable[..., AsyncGenerator[Any, Any]]) -> bool: .. def isasyncgenfunction(obj: Callable[_P, Any]) -> TypeGuard[Callable[_P, AsyncGeneratorType[Any, Any]]]: ... @overload def isasyncgenfunction(obj: object) -> TypeGuard[Callable[..., AsyncGeneratorType[Any, Any]]]: ... - +@type_check_only class _SupportsSet(Protocol[_T_contra, _V_contra]): def __set__(self, instance: _T_contra, value: _V_contra, /) -> None: ... +@type_check_only class _SupportsDelete(Protocol[_T_contra]): def __delete__(self, instance: _T_contra, /) -> None: ... diff --git a/mypy/typeshed/stdlib/ipaddress.pyi b/mypy/typeshed/stdlib/ipaddress.pyi index 9df6bab7c167b..6d49eb8bd94ac 100644 --- a/mypy/typeshed/stdlib/ipaddress.pyi +++ b/mypy/typeshed/stdlib/ipaddress.pyi @@ -137,7 +137,7 @@ class IPv4Address(_BaseV4, _BaseAddress): def ipv6_mapped(self) -> IPv6Address: ... class IPv4Network(_BaseV4, _BaseNetwork[IPv4Address]): - def __init__(self, address: object, strict: bool = ...) -> None: ... + def __init__(self, address: object, strict: bool = True) -> None: ... class IPv4Interface(IPv4Address): netmask: IPv4Address @@ -197,7 +197,7 @@ class IPv6Address(_BaseV6, _BaseAddress): def __eq__(self, other: object) -> bool: ... class IPv6Network(_BaseV6, _BaseNetwork[IPv6Address]): - def __init__(self, address: object, strict: bool = ...) -> None: ... + def __init__(self, address: object, strict: bool = True) -> None: ... @property def is_site_local(self) -> bool: ... diff --git a/mypy/typeshed/stdlib/logging/__init__.pyi b/mypy/typeshed/stdlib/logging/__init__.pyi index 24529bd48d6a7..03c79cc3e2658 100644 --- a/mypy/typeshed/stdlib/logging/__init__.pyi +++ b/mypy/typeshed/stdlib/logging/__init__.pyi @@ -7,7 +7,7 @@ from re import Pattern from string import Template from time import struct_time from types import FrameType, GenericAlias, TracebackType -from typing import Any, ClassVar, Final, Generic, Literal, Protocol, TextIO, TypeVar, overload +from typing import Any, ClassVar, Final, Generic, Literal, Protocol, TextIO, TypeVar, overload, type_check_only from typing_extensions import Self, TypeAlias, deprecated __all__ = [ @@ -67,11 +67,13 @@ _Level: TypeAlias = int | str _FormatStyle: TypeAlias = Literal["%", "{", "$"] if sys.version_info >= (3, 12): + @type_check_only class _SupportsFilter(Protocol): def filter(self, record: LogRecord, /) -> bool | LogRecord: ... _FilterType: TypeAlias = Filter | Callable[[LogRecord], bool | LogRecord] | _SupportsFilter else: + @type_check_only class _SupportsFilter(Protocol): def filter(self, record: LogRecord, /) -> bool: ... diff --git a/mypy/typeshed/stdlib/logging/handlers.pyi b/mypy/typeshed/stdlib/logging/handlers.pyi index 9636b81dc4f3c..e231d1de3fb59 100644 --- a/mypy/typeshed/stdlib/logging/handlers.pyi +++ b/mypy/typeshed/stdlib/logging/handlers.pyi @@ -9,7 +9,7 @@ from re import Pattern from socket import SocketKind, socket from threading import Thread from types import TracebackType -from typing import Any, ClassVar, Final, Protocol, TypeVar +from typing import Any, ClassVar, Final, Protocol, TypeVar, type_check_only from typing_extensions import Self _T = TypeVar("_T") @@ -225,6 +225,7 @@ class HTTPHandler(Handler): def mapLogRecord(self, record: LogRecord) -> dict[str, Any]: ... def getConnection(self, host: str, secure: bool) -> http.client.HTTPConnection: ... # undocumented +@type_check_only class _QueueLike(Protocol[_T]): def get(self) -> _T: ... def put_nowait(self, item: _T, /) -> None: ... diff --git a/mypy/typeshed/stdlib/mailbox.pyi b/mypy/typeshed/stdlib/mailbox.pyi index ff605c0661fb1..89bd998b4dfeb 100644 --- a/mypy/typeshed/stdlib/mailbox.pyi +++ b/mypy/typeshed/stdlib/mailbox.pyi @@ -6,7 +6,7 @@ from abc import ABCMeta, abstractmethod from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from email._policybase import _MessageT from types import GenericAlias, TracebackType -from typing import IO, Any, AnyStr, Generic, Literal, Protocol, TypeVar, overload +from typing import IO, Any, AnyStr, Generic, Literal, Protocol, TypeVar, overload, type_check_only from typing_extensions import Self, TypeAlias __all__ = [ @@ -31,13 +31,16 @@ __all__ = [ _T = TypeVar("_T") +@type_check_only class _SupportsReadAndReadline(SupportsRead[bytes], SupportsNoArgReadline[bytes], Protocol): ... _MessageData: TypeAlias = email.message.Message | bytes | str | io.StringIO | _SupportsReadAndReadline +@type_check_only class _HasIteritems(Protocol): def iteritems(self) -> Iterator[tuple[str, _MessageData]]: ... +@type_check_only class _HasItems(Protocol): def items(self) -> Iterator[tuple[str, _MessageData]]: ... diff --git a/mypy/typeshed/stdlib/math.pyi b/mypy/typeshed/stdlib/math.pyi index 9e77f0cd7e068..1903d488f7bb3 100644 --- a/mypy/typeshed/stdlib/math.pyi +++ b/mypy/typeshed/stdlib/math.pyi @@ -1,7 +1,7 @@ import sys from _typeshed import SupportsMul, SupportsRMul from collections.abc import Iterable -from typing import Any, Final, Literal, Protocol, SupportsFloat, SupportsIndex, TypeVar, overload +from typing import Any, Final, Literal, Protocol, SupportsFloat, SupportsIndex, TypeVar, overload, type_check_only from typing_extensions import TypeAlias _T = TypeVar("_T") @@ -26,6 +26,7 @@ def atanh(x: _SupportsFloatOrIndex, /) -> float: ... if sys.version_info >= (3, 11): def cbrt(x: _SupportsFloatOrIndex, /) -> float: ... +@type_check_only class _SupportsCeil(Protocol[_T_co]): def __ceil__(self) -> _T_co: ... @@ -49,7 +50,7 @@ if sys.version_info >= (3, 11): def expm1(x: _SupportsFloatOrIndex, /) -> float: ... def fabs(x: _SupportsFloatOrIndex, /) -> float: ... def factorial(x: SupportsIndex, /) -> int: ... - +@type_check_only class _SupportsFloor(Protocol[_T_co]): def __floor__(self) -> _T_co: ... @@ -99,6 +100,7 @@ _LiteralInteger = _PositiveInteger | _NegativeInteger | Literal[0] # noqa: Y026 _MultiplicableT1 = TypeVar("_MultiplicableT1", bound=SupportsMul[Any, Any]) _MultiplicableT2 = TypeVar("_MultiplicableT2", bound=SupportsMul[Any, Any]) +@type_check_only class _SupportsProdWithNoDefaultGiven(SupportsMul[Any, Any], SupportsRMul[int, Any], Protocol): ... _SupportsProdNoDefaultT = TypeVar("_SupportsProdNoDefaultT", bound=_SupportsProdWithNoDefaultGiven) @@ -127,6 +129,7 @@ def tan(x: _SupportsFloatOrIndex, /) -> float: ... def tanh(x: _SupportsFloatOrIndex, /) -> float: ... # Is different from `_typeshed.SupportsTrunc`, which is not generic +@type_check_only class _SupportsTrunc(Protocol[_T_co]): def __trunc__(self) -> _T_co: ... diff --git a/mypy/typeshed/stdlib/mmap.pyi b/mypy/typeshed/stdlib/mmap.pyi index c9b8358cde6cb..261a2bfdfc449 100644 --- a/mypy/typeshed/stdlib/mmap.pyi +++ b/mypy/typeshed/stdlib/mmap.pyi @@ -1,38 +1,39 @@ +import os import sys from _typeshed import ReadableBuffer, Unused from collections.abc import Iterator from typing import Final, Literal, NoReturn, overload from typing_extensions import Self -ACCESS_DEFAULT: int -ACCESS_READ: int -ACCESS_WRITE: int -ACCESS_COPY: int +ACCESS_DEFAULT: Final = 0 +ACCESS_READ: Final = 1 +ACCESS_WRITE: Final = 2 +ACCESS_COPY: Final = 3 -ALLOCATIONGRANULARITY: int +ALLOCATIONGRANULARITY: Final[int] if sys.platform == "linux": - MAP_DENYWRITE: int - MAP_EXECUTABLE: int + MAP_DENYWRITE: Final[int] + MAP_EXECUTABLE: Final[int] if sys.version_info >= (3, 10): - MAP_POPULATE: int + MAP_POPULATE: Final[int] if sys.version_info >= (3, 11) and sys.platform != "win32" and sys.platform != "darwin": - MAP_STACK: int + MAP_STACK: Final[int] if sys.platform != "win32": - MAP_ANON: int - MAP_ANONYMOUS: int - MAP_PRIVATE: int - MAP_SHARED: int - PROT_EXEC: int - PROT_READ: int - PROT_WRITE: int + MAP_ANON: Final[int] + MAP_ANONYMOUS: Final[int] + MAP_PRIVATE: Final[int] + MAP_SHARED: Final[int] + PROT_EXEC: Final[int] + PROT_READ: Final[int] + PROT_WRITE: Final[int] -PAGESIZE: int +PAGESIZE: Final[int] class mmap: if sys.platform == "win32": - def __init__(self, fileno: int, length: int, tagname: str | None = ..., access: int = ..., offset: int = ...) -> None: ... + def __init__(self, fileno: int, length: int, tagname: str | None = None, access: int = 0, offset: int = 0) -> None: ... else: if sys.version_info >= (3, 13): def __new__( @@ -41,34 +42,38 @@ class mmap: length: int, flags: int = ..., prot: int = ..., - access: int = ..., - offset: int = ..., + access: int = 0, + offset: int = 0, *, trackfd: bool = True, ) -> Self: ... else: def __new__( - cls, fileno: int, length: int, flags: int = ..., prot: int = ..., access: int = ..., offset: int = ... + cls, fileno: int, length: int, flags: int = ..., prot: int = ..., access: int = 0, offset: int = 0 ) -> Self: ... def close(self) -> None: ... - def flush(self, offset: int = ..., size: int = ...) -> None: ... + def flush(self, offset: int = 0, size: int = ...) -> None: ... def move(self, dest: int, src: int, count: int) -> None: ... def read_byte(self) -> int: ... def readline(self) -> bytes: ... def resize(self, newsize: int) -> None: ... - def seek(self, pos: int, whence: int = ...) -> None: ... + if sys.platform != "win32": + def seek(self, pos: int, whence: Literal[0, 1, 2, 3, 4] = os.SEEK_SET) -> None: ... + else: + def seek(self, pos: int, whence: Literal[0, 1, 2] = os.SEEK_SET) -> None: ... + def size(self) -> int: ... def tell(self) -> int: ... def write_byte(self, byte: int) -> None: ... def __len__(self) -> int: ... closed: bool if sys.platform != "win32": - def madvise(self, option: int, start: int = ..., length: int = ...) -> None: ... + def madvise(self, option: int, start: int = 0, length: int = ...) -> None: ... def find(self, sub: ReadableBuffer, start: int = ..., stop: int = ...) -> int: ... def rfind(self, sub: ReadableBuffer, start: int = ..., stop: int = ...) -> int: ... - def read(self, n: int | None = ...) -> bytes: ... + def read(self, n: int | None = None) -> bytes: ... def write(self, bytes: ReadableBuffer) -> int: ... @overload def __getitem__(self, key: int, /) -> int: ... @@ -93,42 +98,42 @@ class mmap: def seekable(self) -> Literal[True]: ... if sys.platform != "win32": - MADV_NORMAL: int - MADV_RANDOM: int - MADV_SEQUENTIAL: int - MADV_WILLNEED: int - MADV_DONTNEED: int - MADV_FREE: int + MADV_NORMAL: Final[int] + MADV_RANDOM: Final[int] + MADV_SEQUENTIAL: Final[int] + MADV_WILLNEED: Final[int] + MADV_DONTNEED: Final[int] + MADV_FREE: Final[int] if sys.platform == "linux": - MADV_REMOVE: int - MADV_DONTFORK: int - MADV_DOFORK: int - MADV_HWPOISON: int - MADV_MERGEABLE: int - MADV_UNMERGEABLE: int + MADV_REMOVE: Final[int] + MADV_DONTFORK: Final[int] + MADV_DOFORK: Final[int] + MADV_HWPOISON: Final[int] + MADV_MERGEABLE: Final[int] + MADV_UNMERGEABLE: Final[int] # Seems like this constant is not defined in glibc. # See https://github.com/python/typeshed/pull/5360 for details - # MADV_SOFT_OFFLINE: int - MADV_HUGEPAGE: int - MADV_NOHUGEPAGE: int - MADV_DONTDUMP: int - MADV_DODUMP: int + # MADV_SOFT_OFFLINE: Final[int] + MADV_HUGEPAGE: Final[int] + MADV_NOHUGEPAGE: Final[int] + MADV_DONTDUMP: Final[int] + MADV_DODUMP: Final[int] # This Values are defined for FreeBSD but type checkers do not support conditions for these if sys.platform != "linux" and sys.platform != "darwin" and sys.platform != "win32": - MADV_NOSYNC: int - MADV_AUTOSYNC: int - MADV_NOCORE: int - MADV_CORE: int - MADV_PROTECT: int + MADV_NOSYNC: Final[int] + MADV_AUTOSYNC: Final[int] + MADV_NOCORE: Final[int] + MADV_CORE: Final[int] + MADV_PROTECT: Final[int] if sys.version_info >= (3, 10) and sys.platform == "darwin": - MADV_FREE_REUSABLE: int - MADV_FREE_REUSE: int + MADV_FREE_REUSABLE: Final[int] + MADV_FREE_REUSE: Final[int] if sys.version_info >= (3, 13) and sys.platform != "win32": - MAP_32BIT: Final = 32768 + MAP_32BIT: Final[int] if sys.version_info >= (3, 13) and sys.platform == "darwin": MAP_NORESERVE: Final = 64 diff --git a/mypy/typeshed/stdlib/multiprocessing/heap.pyi b/mypy/typeshed/stdlib/multiprocessing/heap.pyi index b5e2ced5e8ee5..38191a099f1ec 100644 --- a/mypy/typeshed/stdlib/multiprocessing/heap.pyi +++ b/mypy/typeshed/stdlib/multiprocessing/heap.pyi @@ -2,7 +2,7 @@ import sys from _typeshed import Incomplete from collections.abc import Callable from mmap import mmap -from typing import Protocol +from typing import Protocol, type_check_only from typing_extensions import TypeAlias __all__ = ["BufferWrapper"] @@ -20,6 +20,7 @@ class Arena: _Block: TypeAlias = tuple[Arena, int, int] if sys.platform != "win32": + @type_check_only class _SupportsDetach(Protocol): def detach(self) -> int: ... diff --git a/mypy/typeshed/stdlib/multiprocessing/sharedctypes.pyi b/mypy/typeshed/stdlib/multiprocessing/sharedctypes.pyi index 5283445d8545b..e2ec15f05ea23 100644 --- a/mypy/typeshed/stdlib/multiprocessing/sharedctypes.pyi +++ b/mypy/typeshed/stdlib/multiprocessing/sharedctypes.pyi @@ -5,7 +5,7 @@ from ctypes import _SimpleCData, c_char from multiprocessing.context import BaseContext from multiprocessing.synchronize import _LockLike from types import TracebackType -from typing import Any, Generic, Literal, Protocol, TypeVar, overload +from typing import Any, Generic, Literal, Protocol, TypeVar, overload, type_check_only __all__ = ["RawValue", "RawArray", "Value", "Array", "copy", "synchronized"] @@ -81,7 +81,7 @@ def synchronized( ) -> SynchronizedArray[_T]: ... @overload def synchronized(obj: _CT, lock: _LockLike | None = None, ctx: Any | None = None) -> SynchronizedBase[_CT]: ... - +@type_check_only class _AcquireFunc(Protocol): def __call__(self, block: bool = ..., timeout: float | None = ..., /) -> bool: ... diff --git a/mypy/typeshed/stdlib/nt.pyi b/mypy/typeshed/stdlib/nt.pyi index 3ed8f8af379b8..0c87444d18f44 100644 --- a/mypy/typeshed/stdlib/nt.pyi +++ b/mypy/typeshed/stdlib/nt.pyi @@ -110,4 +110,7 @@ if sys.platform == "win32": if sys.version_info >= (3, 13): from os import fchmod as fchmod, lchmod as lchmod + if sys.version_info >= (3, 14): + from os import readinto as readinto + environ: dict[str, str] diff --git a/mypy/typeshed/stdlib/numbers.pyi b/mypy/typeshed/stdlib/numbers.pyi index 02d469ce0ee54..b24591719cfff 100644 --- a/mypy/typeshed/stdlib/numbers.pyi +++ b/mypy/typeshed/stdlib/numbers.pyi @@ -8,7 +8,7 @@ # nor `float` as a subtype of `numbers.Real`, etc.) from abc import ABCMeta, abstractmethod -from typing import ClassVar, Literal, Protocol, overload +from typing import ClassVar, Literal, Protocol, overload, type_check_only __all__ = ["Number", "Complex", "Real", "Rational", "Integral"] @@ -22,6 +22,7 @@ __all__ = ["Number", "Complex", "Real", "Rational", "Integral"] # NOTE: We can't include `__complex__` here, # as we want `int` to be seen as a subtype of `_ComplexLike`, # and `int.__complex__` does not exist :( +@type_check_only class _ComplexLike(Protocol): def __neg__(self) -> _ComplexLike: ... def __pos__(self) -> _ComplexLike: ... @@ -29,6 +30,7 @@ class _ComplexLike(Protocol): # _RealLike is a structural-typing approximation # of the `Real` ABC, which is not (and cannot be) a protocol +@type_check_only class _RealLike(_ComplexLike, Protocol): def __trunc__(self) -> _IntegralLike: ... def __floor__(self) -> _IntegralLike: ... @@ -41,6 +43,7 @@ class _RealLike(_ComplexLike, Protocol): # _IntegralLike is a structural-typing approximation # of the `Integral` ABC, which is not (and cannot be) a protocol +@type_check_only class _IntegralLike(_RealLike, Protocol): def __invert__(self) -> _IntegralLike: ... def __int__(self) -> int: ... diff --git a/mypy/typeshed/stdlib/optparse.pyi b/mypy/typeshed/stdlib/optparse.pyi index 8b7fcd82e5a55..c522917992800 100644 --- a/mypy/typeshed/stdlib/optparse.pyi +++ b/mypy/typeshed/stdlib/optparse.pyi @@ -24,8 +24,7 @@ __all__ = [ "BadOptionError", "check_choice", ] -# pytype is not happy with `NO_DEFAULT: Final = ("NO", "DEFAULT")` -NO_DEFAULT: Final[tuple[Literal["NO"], Literal["DEFAULT"]]] +NO_DEFAULT: Final = ("NO", "DEFAULT") SUPPRESS_HELP: Final = "SUPPRESSHELP" SUPPRESS_USAGE: Final = "SUPPRESSUSAGE" diff --git a/mypy/typeshed/stdlib/os/__init__.pyi b/mypy/typeshed/stdlib/os/__init__.pyi index dd4479f9030a2..4047bb0f1c4dc 100644 --- a/mypy/typeshed/stdlib/os/__init__.pyi +++ b/mypy/typeshed/stdlib/os/__init__.pyi @@ -39,6 +39,7 @@ from typing import ( final, overload, runtime_checkable, + type_check_only, ) from typing_extensions import Self, TypeAlias, Unpack, deprecated @@ -597,12 +598,12 @@ if sys.platform == "darwin" and sys.version_info >= (3, 12): PRIO_DARWIN_PROCESS: int PRIO_DARWIN_THREAD: int -SEEK_SET: int -SEEK_CUR: int -SEEK_END: int +SEEK_SET: Final = 0 +SEEK_CUR: Final = 1 +SEEK_END: Final = 2 if sys.platform != "win32": - SEEK_DATA: int - SEEK_HOLE: int + SEEK_DATA: Final = 3 + SEEK_HOLE: Final = 4 O_RDONLY: int O_WRONLY: int @@ -1241,6 +1242,7 @@ def replace( ) -> None: ... def rmdir(path: StrOrBytesPath, *, dir_fd: int | None = None) -> None: ... @final +@type_check_only class _ScandirIterator(Generic[AnyStr]): def __del__(self) -> None: ... def __iter__(self) -> Self: ... diff --git a/mypy/typeshed/stdlib/pathlib/__init__.pyi b/mypy/typeshed/stdlib/pathlib/__init__.pyi index b84fc69313a15..774478bb2ff42 100644 --- a/mypy/typeshed/stdlib/pathlib/__init__.pyi +++ b/mypy/typeshed/stdlib/pathlib/__init__.pyi @@ -67,7 +67,14 @@ class PurePath(PathLike[str]): def as_posix(self) -> str: ... def as_uri(self) -> str: ... def is_absolute(self) -> bool: ... - def is_reserved(self) -> bool: ... + if sys.version_info >= (3, 13): + @deprecated( + "Deprecated since Python 3.13; will be removed in Python 3.15. " + "Use `os.path.isreserved()` to detect reserved paths on Windows." + ) + def is_reserved(self) -> bool: ... + else: + def is_reserved(self) -> bool: ... if sys.version_info >= (3, 14): def is_relative_to(self, other: StrPath) -> bool: ... elif sys.version_info >= (3, 12): @@ -163,7 +170,6 @@ class Path(PurePath): def mkdir(self, mode: int = 0o777, parents: bool = False, exist_ok: bool = False) -> None: ... if sys.version_info >= (3, 14): - @property def info(self) -> PathInfo: ... @overload diff --git a/mypy/typeshed/stdlib/platform.pyi b/mypy/typeshed/stdlib/platform.pyi index fbc73c6c91775..c6125bd3a56fa 100644 --- a/mypy/typeshed/stdlib/platform.pyi +++ b/mypy/typeshed/stdlib/platform.pyi @@ -1,6 +1,6 @@ import sys from typing import NamedTuple, type_check_only -from typing_extensions import Self +from typing_extensions import Self, deprecated def libc_ver(executable: str | None = None, lib: str = "", version: str = "", chunksize: int = 16384) -> tuple[str, str]: ... def win32_ver(release: str = "", version: str = "", csd: str = "", ptype: str = "") -> tuple[str, str, str, str]: ... @@ -9,9 +9,24 @@ def win32_is_iot() -> bool: ... def mac_ver( release: str = "", versioninfo: tuple[str, str, str] = ("", "", ""), machine: str = "" ) -> tuple[str, tuple[str, str, str], str]: ... -def java_ver( - release: str = "", vendor: str = "", vminfo: tuple[str, str, str] = ("", "", ""), osinfo: tuple[str, str, str] = ("", "", "") -) -> tuple[str, str, tuple[str, str, str], tuple[str, str, str]]: ... + +if sys.version_info >= (3, 13): + @deprecated("Deprecated since Python 3.13; will be removed in Python 3.15.") + def java_ver( + release: str = "", + vendor: str = "", + vminfo: tuple[str, str, str] = ("", "", ""), + osinfo: tuple[str, str, str] = ("", "", ""), + ) -> tuple[str, str, tuple[str, str, str], tuple[str, str, str]]: ... + +else: + def java_ver( + release: str = "", + vendor: str = "", + vminfo: tuple[str, str, str] = ("", "", ""), + osinfo: tuple[str, str, str] = ("", "", ""), + ) -> tuple[str, str, tuple[str, str, str], tuple[str, str, str]]: ... + def system_alias(system: str, release: str, version: str) -> tuple[str, str, str]: ... def architecture(executable: str = sys.executable, bits: str = "", linkage: str = "") -> tuple[str, str]: ... diff --git a/mypy/typeshed/stdlib/pprint.pyi b/mypy/typeshed/stdlib/pprint.pyi index 171878f4165dd..1e80462e25657 100644 --- a/mypy/typeshed/stdlib/pprint.pyi +++ b/mypy/typeshed/stdlib/pprint.pyi @@ -1,4 +1,6 @@ import sys +from _typeshed import SupportsWrite +from collections import deque from typing import IO __all__ = ["pprint", "pformat", "isreadable", "isrecursive", "saferepr", "PrettyPrinter", "pp"] @@ -29,25 +31,25 @@ else: if sys.version_info >= (3, 10): def pp( object: object, - stream: IO[str] | None = ..., - indent: int = ..., - width: int = ..., - depth: int | None = ..., + stream: IO[str] | None = None, + indent: int = 1, + width: int = 80, + depth: int | None = None, *, - compact: bool = ..., + compact: bool = False, sort_dicts: bool = False, - underscore_numbers: bool = ..., + underscore_numbers: bool = False, ) -> None: ... else: def pp( object: object, - stream: IO[str] | None = ..., - indent: int = ..., - width: int = ..., - depth: int | None = ..., + stream: IO[str] | None = None, + indent: int = 1, + width: int = 80, + depth: int | None = None, *, - compact: bool = ..., + compact: bool = False, sort_dicts: bool = False, ) -> None: ... @@ -110,3 +112,48 @@ class PrettyPrinter: def isreadable(self, object: object) -> bool: ... def isrecursive(self, object: object) -> bool: ... def format(self, object: object, context: dict[int, int], maxlevels: int, level: int) -> tuple[str, bool, bool]: ... + def _format( + self, object: object, stream: SupportsWrite[str], indent: int, allowance: int, context: dict[int, int], level: int + ) -> None: ... + def _pprint_dict( + self, + object: dict[object, object], + stream: SupportsWrite[str], + indent: int, + allowance: int, + context: dict[int, int], + level: int, + ) -> None: ... + def _pprint_list( + self, object: list[object], stream: SupportsWrite[str], indent: int, allowance: int, context: dict[int, int], level: int + ) -> None: ... + def _pprint_tuple( + self, + object: tuple[object, ...], + stream: SupportsWrite[str], + indent: int, + allowance: int, + context: dict[int, int], + level: int, + ) -> None: ... + def _pprint_set( + self, object: set[object], stream: SupportsWrite[str], indent: int, allowance: int, context: dict[int, int], level: int + ) -> None: ... + def _pprint_deque( + self, object: deque[object], stream: SupportsWrite[str], indent: int, allowance: int, context: dict[int, int], level: int + ) -> None: ... + def _format_dict_items( + self, + items: list[tuple[object, object]], + stream: SupportsWrite[str], + indent: int, + allowance: int, + context: dict[int, int], + level: int, + ) -> None: ... + def _format_items( + self, items: list[object], stream: SupportsWrite[str], indent: int, allowance: int, context: dict[int, int], level: int + ) -> None: ... + def _repr(self, object: object, context: dict[int, int], level: int) -> str: ... + if sys.version_info >= (3, 10): + def _safe_repr(self, object: object, context: dict[int, int], maxlevels: int, level: int) -> tuple[str, bool, bool]: ... diff --git a/mypy/typeshed/stdlib/pydoc.pyi b/mypy/typeshed/stdlib/pydoc.pyi index f14b9d1bb6998..3c78f9d2de8e9 100644 --- a/mypy/typeshed/stdlib/pydoc.pyi +++ b/mypy/typeshed/stdlib/pydoc.pyi @@ -5,7 +5,7 @@ from builtins import list as _list # "list" conflicts with method name from collections.abc import Callable, Container, Mapping, MutableMapping from reprlib import Repr from types import MethodType, ModuleType, TracebackType -from typing import IO, Any, AnyStr, Final, NoReturn, Protocol, TypeVar +from typing import IO, Any, AnyStr, Final, NoReturn, Protocol, TypeVar, type_check_only from typing_extensions import TypeGuard, deprecated __all__ = ["help"] @@ -17,6 +17,7 @@ __date__: Final[str] __version__: Final[str] __credits__: Final[str] +@type_check_only class _Pager(Protocol): def __call__(self, text: str, title: str = "") -> None: ... diff --git a/mypy/typeshed/stdlib/queue.pyi b/mypy/typeshed/stdlib/queue.pyi index f5d9179e079d4..65e2ac1559adf 100644 --- a/mypy/typeshed/stdlib/queue.pyi +++ b/mypy/typeshed/stdlib/queue.pyi @@ -1,5 +1,6 @@ import sys from _queue import Empty as Empty, SimpleQueue as SimpleQueue +from _typeshed import SupportsRichComparisonT from threading import Condition, Lock from types import GenericAlias from typing import Any, Generic, TypeVar @@ -47,8 +48,8 @@ class Queue(Generic[_T]): def task_done(self) -> None: ... def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... -class PriorityQueue(Queue[_T]): - queue: list[_T] +class PriorityQueue(Queue[SupportsRichComparisonT]): + queue: list[SupportsRichComparisonT] class LifoQueue(Queue[_T]): queue: list[_T] diff --git a/mypy/typeshed/stdlib/quopri.pyi b/mypy/typeshed/stdlib/quopri.pyi index b652e139bd0e2..be6892fcbcd78 100644 --- a/mypy/typeshed/stdlib/quopri.pyi +++ b/mypy/typeshed/stdlib/quopri.pyi @@ -1,8 +1,9 @@ from _typeshed import ReadableBuffer, SupportsNoArgReadline, SupportsRead, SupportsWrite -from typing import Protocol +from typing import Protocol, type_check_only __all__ = ["encode", "decode", "encodestring", "decodestring"] +@type_check_only class _Input(SupportsRead[bytes], SupportsNoArgReadline[bytes], Protocol): ... def encode(input: _Input, output: SupportsWrite[bytes], quotetabs: int, header: bool = False) -> None: ... diff --git a/mypy/typeshed/stdlib/re.pyi b/mypy/typeshed/stdlib/re.pyi index f25a0a376704b..b080626c5802f 100644 --- a/mypy/typeshed/stdlib/re.pyi +++ b/mypy/typeshed/stdlib/re.pyi @@ -239,9 +239,7 @@ if sys.version_info < (3, 13): T: Final = RegexFlag.T TEMPLATE: Final = RegexFlag.TEMPLATE if sys.version_info >= (3, 11): - # pytype chokes on `NOFLAG: Final = RegexFlag.NOFLAG` with `LiteralValueError` - # mypy chokes on `NOFLAG: Final[Literal[RegexFlag.NOFLAG]]` with `Literal[...] is invalid` - NOFLAG = RegexFlag.NOFLAG + NOFLAG: Final = RegexFlag.NOFLAG _FlagsType: TypeAlias = int | RegexFlag # Type-wise the compile() overloads are unnecessary, they could also be modeled using diff --git a/mypy/typeshed/stdlib/shutil.pyi b/mypy/typeshed/stdlib/shutil.pyi index c66d8fa128bec..cc26cfc556a00 100644 --- a/mypy/typeshed/stdlib/shutil.pyi +++ b/mypy/typeshed/stdlib/shutil.pyi @@ -3,7 +3,7 @@ import sys from _typeshed import BytesPath, ExcInfo, FileDescriptorOrPath, MaybeNone, StrOrBytesPath, StrPath, SupportsRead, SupportsWrite from collections.abc import Callable, Iterable, Sequence from tarfile import _TarfileFilter -from typing import Any, AnyStr, NamedTuple, NoReturn, Protocol, TypeVar, overload +from typing import Any, AnyStr, NamedTuple, NoReturn, Protocol, TypeVar, overload, type_check_only from typing_extensions import TypeAlias, deprecated __all__ = [ @@ -79,6 +79,7 @@ def copytree( _OnErrorCallback: TypeAlias = Callable[[Callable[..., Any], str, ExcInfo], object] _OnExcCallback: TypeAlias = Callable[[Callable[..., Any], str, BaseException], object] +@type_check_only class _RmtreeType(Protocol): avoids_symlink_attacks: bool if sys.version_info >= (3, 12): diff --git a/mypy/typeshed/stdlib/signal.pyi b/mypy/typeshed/stdlib/signal.pyi index d50565d1c8ac3..c2668bd8b32d9 100644 --- a/mypy/typeshed/stdlib/signal.pyi +++ b/mypy/typeshed/stdlib/signal.pyi @@ -3,7 +3,7 @@ from _typeshed import structseq from collections.abc import Callable, Iterable from enum import IntEnum from types import FrameType -from typing import Any, Final, Literal, final +from typing import Any, Final, final from typing_extensions import Never, TypeAlias NSIG: int @@ -61,8 +61,8 @@ class Handlers(IntEnum): SIG_DFL = 0 SIG_IGN = 1 -SIG_DFL: Literal[Handlers.SIG_DFL] -SIG_IGN: Literal[Handlers.SIG_IGN] +SIG_DFL: Final = Handlers.SIG_DFL +SIG_IGN: Final = Handlers.SIG_IGN _SIGNUM: TypeAlias = int | Signals _HANDLER: TypeAlias = Callable[[int, FrameType | None], Any] | int | Handlers | None @@ -77,45 +77,45 @@ else: def getsignal(signalnum: _SIGNUM, /) -> _HANDLER: ... def signal(signalnum: _SIGNUM, handler: _HANDLER, /) -> _HANDLER: ... -SIGABRT: Literal[Signals.SIGABRT] -SIGFPE: Literal[Signals.SIGFPE] -SIGILL: Literal[Signals.SIGILL] -SIGINT: Literal[Signals.SIGINT] -SIGSEGV: Literal[Signals.SIGSEGV] -SIGTERM: Literal[Signals.SIGTERM] +SIGABRT: Final = Signals.SIGABRT +SIGFPE: Final = Signals.SIGFPE +SIGILL: Final = Signals.SIGILL +SIGINT: Final = Signals.SIGINT +SIGSEGV: Final = Signals.SIGSEGV +SIGTERM: Final = Signals.SIGTERM if sys.platform == "win32": - SIGBREAK: Literal[Signals.SIGBREAK] - CTRL_C_EVENT: Literal[Signals.CTRL_C_EVENT] - CTRL_BREAK_EVENT: Literal[Signals.CTRL_BREAK_EVENT] + SIGBREAK: Final = Signals.SIGBREAK + CTRL_C_EVENT: Final = Signals.CTRL_C_EVENT + CTRL_BREAK_EVENT: Final = Signals.CTRL_BREAK_EVENT else: if sys.platform != "linux": - SIGINFO: Literal[Signals.SIGINFO] - SIGEMT: Literal[Signals.SIGEMT] - SIGALRM: Literal[Signals.SIGALRM] - SIGBUS: Literal[Signals.SIGBUS] - SIGCHLD: Literal[Signals.SIGCHLD] - SIGCONT: Literal[Signals.SIGCONT] - SIGHUP: Literal[Signals.SIGHUP] - SIGIO: Literal[Signals.SIGIO] - SIGIOT: Literal[Signals.SIGABRT] # alias - SIGKILL: Literal[Signals.SIGKILL] - SIGPIPE: Literal[Signals.SIGPIPE] - SIGPROF: Literal[Signals.SIGPROF] - SIGQUIT: Literal[Signals.SIGQUIT] - SIGSTOP: Literal[Signals.SIGSTOP] - SIGSYS: Literal[Signals.SIGSYS] - SIGTRAP: Literal[Signals.SIGTRAP] - SIGTSTP: Literal[Signals.SIGTSTP] - SIGTTIN: Literal[Signals.SIGTTIN] - SIGTTOU: Literal[Signals.SIGTTOU] - SIGURG: Literal[Signals.SIGURG] - SIGUSR1: Literal[Signals.SIGUSR1] - SIGUSR2: Literal[Signals.SIGUSR2] - SIGVTALRM: Literal[Signals.SIGVTALRM] - SIGWINCH: Literal[Signals.SIGWINCH] - SIGXCPU: Literal[Signals.SIGXCPU] - SIGXFSZ: Literal[Signals.SIGXFSZ] + SIGINFO: Final = Signals.SIGINFO + SIGEMT: Final = Signals.SIGEMT + SIGALRM: Final = Signals.SIGALRM + SIGBUS: Final = Signals.SIGBUS + SIGCHLD: Final = Signals.SIGCHLD + SIGCONT: Final = Signals.SIGCONT + SIGHUP: Final = Signals.SIGHUP + SIGIO: Final = Signals.SIGIO + SIGIOT: Final = Signals.SIGABRT # alias + SIGKILL: Final = Signals.SIGKILL + SIGPIPE: Final = Signals.SIGPIPE + SIGPROF: Final = Signals.SIGPROF + SIGQUIT: Final = Signals.SIGQUIT + SIGSTOP: Final = Signals.SIGSTOP + SIGSYS: Final = Signals.SIGSYS + SIGTRAP: Final = Signals.SIGTRAP + SIGTSTP: Final = Signals.SIGTSTP + SIGTTIN: Final = Signals.SIGTTIN + SIGTTOU: Final = Signals.SIGTTOU + SIGURG: Final = Signals.SIGURG + SIGUSR1: Final = Signals.SIGUSR1 + SIGUSR2: Final = Signals.SIGUSR2 + SIGVTALRM: Final = Signals.SIGVTALRM + SIGWINCH: Final = Signals.SIGWINCH + SIGXCPU: Final = Signals.SIGXCPU + SIGXFSZ: Final = Signals.SIGXFSZ class ItimerError(OSError): ... ITIMER_PROF: int @@ -127,9 +127,9 @@ else: SIG_UNBLOCK = 1 SIG_SETMASK = 2 - SIG_BLOCK: Literal[Sigmasks.SIG_BLOCK] - SIG_UNBLOCK: Literal[Sigmasks.SIG_UNBLOCK] - SIG_SETMASK: Literal[Sigmasks.SIG_SETMASK] + SIG_BLOCK: Final = Sigmasks.SIG_BLOCK + SIG_UNBLOCK: Final = Sigmasks.SIG_UNBLOCK + SIG_SETMASK: Final = Sigmasks.SIG_SETMASK def alarm(seconds: int, /) -> int: ... def getitimer(which: int, /) -> tuple[float, float]: ... def pause() -> None: ... @@ -147,13 +147,13 @@ else: else: def sigwait(sigset: Iterable[int], /) -> _SIGNUM: ... if sys.platform != "darwin": - SIGCLD: Literal[Signals.SIGCHLD] # alias - SIGPOLL: Literal[Signals.SIGIO] # alias - SIGPWR: Literal[Signals.SIGPWR] - SIGRTMAX: Literal[Signals.SIGRTMAX] - SIGRTMIN: Literal[Signals.SIGRTMIN] + SIGCLD: Final = Signals.SIGCHLD # alias + SIGPOLL: Final = Signals.SIGIO # alias + SIGPWR: Final = Signals.SIGPWR + SIGRTMAX: Final = Signals.SIGRTMAX + SIGRTMIN: Final = Signals.SIGRTMIN if sys.version_info >= (3, 11): - SIGSTKFLT: Literal[Signals.SIGSTKFLT] + SIGSTKFLT: Final = Signals.SIGSTKFLT @final class struct_siginfo(structseq[int], tuple[int, int, int, int, int, int, int]): @@ -181,7 +181,7 @@ else: def strsignal(signalnum: _SIGNUM, /) -> str | None: ... def valid_signals() -> set[Signals]: ... def raise_signal(signalnum: _SIGNUM, /) -> None: ... -def set_wakeup_fd(fd: int, /, *, warn_on_full_buffer: bool = ...) -> int: ... +def set_wakeup_fd(fd: int, /, *, warn_on_full_buffer: bool = True) -> int: ... if sys.platform == "linux": - def pidfd_send_signal(pidfd: int, sig: int, siginfo: None = None, flags: int = ..., /) -> None: ... + def pidfd_send_signal(pidfd: int, sig: int, siginfo: None = None, flags: int = 0, /) -> None: ... diff --git a/mypy/typeshed/stdlib/smtplib.pyi b/mypy/typeshed/stdlib/smtplib.pyi index 609b3e6426c4e..3d392c0479935 100644 --- a/mypy/typeshed/stdlib/smtplib.pyi +++ b/mypy/typeshed/stdlib/smtplib.pyi @@ -7,7 +7,7 @@ from re import Pattern from socket import socket from ssl import SSLContext from types import TracebackType -from typing import Any, Protocol, overload +from typing import Any, Protocol, overload, type_check_only from typing_extensions import Self, TypeAlias __all__ = [ @@ -65,7 +65,7 @@ class SMTPAuthenticationError(SMTPResponseException): ... def quoteaddr(addrstring: str) -> str: ... def quotedata(data: str) -> str: ... - +@type_check_only class _AuthObject(Protocol): @overload def __call__(self, challenge: None = None, /) -> str | None: ... diff --git a/mypy/typeshed/stdlib/socket.pyi b/mypy/typeshed/stdlib/socket.pyi index b4fa4381a72ca..491551dd52b15 100644 --- a/mypy/typeshed/stdlib/socket.pyi +++ b/mypy/typeshed/stdlib/socket.pyi @@ -136,7 +136,7 @@ from _typeshed import ReadableBuffer, Unused, WriteableBuffer from collections.abc import Iterable from enum import IntEnum, IntFlag from io import BufferedReader, BufferedRWPair, BufferedWriter, IOBase, RawIOBase, TextIOWrapper -from typing import Any, Literal, Protocol, SupportsIndex, overload +from typing import Any, Literal, Protocol, SupportsIndex, overload, type_check_only from typing_extensions import Self __all__ = [ @@ -1290,6 +1290,7 @@ if sys.platform != "win32" and sys.platform != "linux": if sys.platform == "win32": errorTab: dict[int, str] # undocumented +@type_check_only class _SendableFile(Protocol): def read(self, size: int, /) -> bytes: ... def seek(self, offset: int, /) -> object: ... diff --git a/mypy/typeshed/stdlib/sqlite3/__init__.pyi b/mypy/typeshed/stdlib/sqlite3/__init__.pyi index ab783dbde121c..5a659deaccf68 100644 --- a/mypy/typeshed/stdlib/sqlite3/__init__.pyi +++ b/mypy/typeshed/stdlib/sqlite3/__init__.pyi @@ -220,23 +220,29 @@ _SqliteData: TypeAlias = str | ReadableBuffer | int | float | None _AdaptedInputData: TypeAlias = _SqliteData | Any # The Mapping must really be a dict, but making it invariant is too annoying. _Parameters: TypeAlias = SupportsLenAndGetItem[_AdaptedInputData] | Mapping[str, _AdaptedInputData] +# Controls the legacy transaction handling mode of sqlite3. +_IsolationLevel: TypeAlias = Literal["DEFERRED", "EXCLUSIVE", "IMMEDIATE"] | None +@type_check_only class _AnyParamWindowAggregateClass(Protocol): def step(self, *args: Any) -> object: ... def inverse(self, *args: Any) -> object: ... def value(self) -> _SqliteData: ... def finalize(self) -> _SqliteData: ... +@type_check_only class _WindowAggregateClass(Protocol): step: Callable[..., object] inverse: Callable[..., object] def value(self) -> _SqliteData: ... def finalize(self) -> _SqliteData: ... +@type_check_only class _AggregateProtocol(Protocol): def step(self, value: int, /) -> object: ... def finalize(self) -> int: ... +@type_check_only class _SingleParamWindowAggregateClass(Protocol): def step(self, param: Any, /) -> object: ... def inverse(self, param: Any, /) -> object: ... @@ -285,7 +291,7 @@ class Connection: def Warning(self) -> type[Warning]: ... @property def in_transaction(self) -> bool: ... - isolation_level: str | None # one of '', 'DEFERRED', 'IMMEDIATE' or 'EXCLUSIVE' + isolation_level: _IsolationLevel @property def total_changes(self) -> int: ... if sys.version_info >= (3, 12): @@ -299,26 +305,26 @@ class Connection: def __init__( self, database: StrOrBytesPath, - timeout: float = ..., - detect_types: int = ..., - isolation_level: str | None = ..., - check_same_thread: bool = ..., + timeout: float = 5.0, + detect_types: int = 0, + isolation_level: _IsolationLevel = "DEFERRED", + check_same_thread: bool = True, factory: type[Connection] | None = ..., - cached_statements: int = ..., - uri: bool = ..., + cached_statements: int = 128, + uri: bool = False, autocommit: bool = ..., ) -> None: ... else: def __init__( self, database: StrOrBytesPath, - timeout: float = ..., - detect_types: int = ..., - isolation_level: str | None = ..., - check_same_thread: bool = ..., + timeout: float = 5.0, + detect_types: int = 0, + isolation_level: _IsolationLevel = "DEFERRED", + check_same_thread: bool = True, factory: type[Connection] | None = ..., - cached_statements: int = ..., - uri: bool = ..., + cached_statements: int = 128, + uri: bool = False, ) -> None: ... def close(self) -> None: ... diff --git a/mypy/typeshed/stdlib/sqlite3/dbapi2.pyi b/mypy/typeshed/stdlib/sqlite3/dbapi2.pyi index d3ea3ef0e8963..d37a0d391ec6a 100644 --- a/mypy/typeshed/stdlib/sqlite3/dbapi2.pyi +++ b/mypy/typeshed/stdlib/sqlite3/dbapi2.pyi @@ -66,6 +66,7 @@ from sqlite3 import ( Row as Row, Warning as Warning, ) +from typing import Literal if sys.version_info >= (3, 12): from _sqlite3 import ( @@ -223,7 +224,7 @@ if sys.version_info < (3, 10): from _sqlite3 import OptimizedUnicode as OptimizedUnicode paramstyle: str -threadsafety: int +threadsafety: Literal[0, 1, 3] apilevel: str Date = date Time = time diff --git a/mypy/typeshed/stdlib/ssl.pyi b/mypy/typeshed/stdlib/ssl.pyi index 9fbf5e8dfa847..1b8631d3fb12b 100644 --- a/mypy/typeshed/stdlib/ssl.pyi +++ b/mypy/typeshed/stdlib/ssl.pyi @@ -50,6 +50,7 @@ _SrvnmeCbType: TypeAlias = Callable[[SSLSocket | SSLObject, str | None, SSLSocke socket_error = OSError +@type_check_only class _Cipher(TypedDict): aead: bool alg_bits: int @@ -80,6 +81,7 @@ class SSLCertVerificationError(SSLError, ValueError): CertificateError = SSLCertVerificationError if sys.version_info < (3, 12): + @deprecated("Deprecated since Python 3.7. Removed in Python 3.12. Use `SSLContext.wrap_socket()` instead.") def wrap_socket( sock: socket.socket, keyfile: StrOrBytesPath | None = None, @@ -92,46 +94,7 @@ if sys.version_info < (3, 12): suppress_ragged_eofs: bool = True, ciphers: str | None = None, ) -> SSLSocket: ... - -def create_default_context( - purpose: Purpose = ..., - *, - cafile: StrOrBytesPath | None = None, - capath: StrOrBytesPath | None = None, - cadata: str | ReadableBuffer | None = None, -) -> SSLContext: ... - -if sys.version_info >= (3, 10): - def _create_unverified_context( - protocol: int | None = None, - *, - cert_reqs: int = ..., - check_hostname: bool = False, - purpose: Purpose = ..., - certfile: StrOrBytesPath | None = None, - keyfile: StrOrBytesPath | None = None, - cafile: StrOrBytesPath | None = None, - capath: StrOrBytesPath | None = None, - cadata: str | ReadableBuffer | None = None, - ) -> SSLContext: ... - -else: - def _create_unverified_context( - protocol: int = ..., - *, - cert_reqs: int = ..., - check_hostname: bool = False, - purpose: Purpose = ..., - certfile: StrOrBytesPath | None = None, - keyfile: StrOrBytesPath | None = None, - cafile: StrOrBytesPath | None = None, - capath: StrOrBytesPath | None = None, - cadata: str | ReadableBuffer | None = None, - ) -> SSLContext: ... - -_create_default_https_context: Callable[..., SSLContext] - -if sys.version_info < (3, 12): + @deprecated("Deprecated since Python 3.7. Removed in Python 3.12.") def match_hostname(cert: _PeerCertRetDictType, hostname: str) -> None: ... def cert_time_to_seconds(cert_time: str) -> int: ... @@ -370,7 +333,7 @@ class SSLSocket(socket.socket): def get_channel_binding(self, cb_type: str = "tls-unique") -> bytes | None: ... def selected_alpn_protocol(self) -> str | None: ... if sys.version_info >= (3, 10): - @deprecated("Deprecated in 3.10. Use ALPN instead.") + @deprecated("Deprecated since Python 3.10. Use ALPN instead.") def selected_npn_protocol(self) -> str | None: ... else: def selected_npn_protocol(self) -> str | None: ... @@ -416,13 +379,15 @@ class SSLContext(_SSLContext): if sys.version_info >= (3, 10): security_level: int if sys.version_info >= (3, 10): - # Using the default (None) for the `protocol` parameter is deprecated, - # but there isn't a good way of marking that in the stub unless/until PEP 702 is accepted - def __new__(cls, protocol: int | None = None, *args: Any, **kwargs: Any) -> Self: ... + @overload + def __new__(cls, protocol: int, *args: Any, **kwargs: Any) -> Self: ... + @overload + @deprecated("Deprecated since Python 3.10. Use a specific version of the SSL protocol.") + def __new__(cls, protocol: None = None, *args: Any, **kwargs: Any) -> Self: ... else: def __new__(cls, protocol: int = ..., *args: Any, **kwargs: Any) -> Self: ... - def load_default_certs(self, purpose: Purpose = ...) -> None: ... + def load_default_certs(self, purpose: Purpose = Purpose.SERVER_AUTH) -> None: ... def load_verify_locations( self, cafile: StrOrBytesPath | None = None, @@ -440,7 +405,7 @@ class SSLContext(_SSLContext): def set_ciphers(self, cipherlist: str, /) -> None: ... def set_alpn_protocols(self, alpn_protocols: Iterable[str]) -> None: ... if sys.version_info >= (3, 10): - @deprecated("Deprecated in 3.10. Use ALPN instead.") + @deprecated("Deprecated since Python 3.10. Use ALPN instead.") def set_npn_protocols(self, npn_protocols: Iterable[str]) -> None: ... else: def set_npn_protocols(self, npn_protocols: Iterable[str]) -> None: ... @@ -466,6 +431,44 @@ class SSLContext(_SSLContext): session: SSLSession | None = None, ) -> SSLObject: ... +def create_default_context( + purpose: Purpose = Purpose.SERVER_AUTH, + *, + cafile: StrOrBytesPath | None = None, + capath: StrOrBytesPath | None = None, + cadata: str | ReadableBuffer | None = None, +) -> SSLContext: ... + +if sys.version_info >= (3, 10): + def _create_unverified_context( + protocol: int | None = None, + *, + cert_reqs: int = ..., + check_hostname: bool = False, + purpose: Purpose = Purpose.SERVER_AUTH, + certfile: StrOrBytesPath | None = None, + keyfile: StrOrBytesPath | None = None, + cafile: StrOrBytesPath | None = None, + capath: StrOrBytesPath | None = None, + cadata: str | ReadableBuffer | None = None, + ) -> SSLContext: ... + +else: + def _create_unverified_context( + protocol: int = ..., + *, + cert_reqs: int = ..., + check_hostname: bool = False, + purpose: Purpose = Purpose.SERVER_AUTH, + certfile: StrOrBytesPath | None = None, + keyfile: StrOrBytesPath | None = None, + cafile: StrOrBytesPath | None = None, + capath: StrOrBytesPath | None = None, + cadata: str | ReadableBuffer | None = None, + ) -> SSLContext: ... + +_create_default_https_context = create_default_context + class SSLObject: context: SSLContext @property @@ -486,7 +489,7 @@ class SSLObject: def getpeercert(self, binary_form: bool) -> _PeerCertRetType: ... def selected_alpn_protocol(self) -> str | None: ... if sys.version_info >= (3, 10): - @deprecated("Deprecated in 3.10. Use ALPN instead.") + @deprecated("Deprecated since Python 3.10. Use ALPN instead.") def selected_npn_protocol(self) -> str | None: ... else: def selected_npn_protocol(self) -> str | None: ... diff --git a/mypy/typeshed/stdlib/string/templatelib.pyi b/mypy/typeshed/stdlib/string/templatelib.pyi index 3f460006a7965..9906d31c63915 100644 --- a/mypy/typeshed/stdlib/string/templatelib.pyi +++ b/mypy/typeshed/stdlib/string/templatelib.pyi @@ -1,8 +1,8 @@ from collections.abc import Iterator from types import GenericAlias -from typing import Any, Literal, final +from typing import Any, Literal, TypeVar, final, overload -__all__ = ["Interpolation", "Template"] +_T = TypeVar("_T") @final class Template: # TODO: consider making `Template` generic on `TypeVarTuple` @@ -11,7 +11,7 @@ class Template: # TODO: consider making `Template` generic on `TypeVarTuple` def __new__(cls, *args: str | Interpolation) -> Template: ... def __iter__(self) -> Iterator[str | Interpolation]: ... - def __add__(self, other: Template | str) -> Template: ... + def __add__(self, other: Template, /) -> Template: ... def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... @property def values(self) -> tuple[Any, ...]: ... # Tuple of interpolation values, which can have any type @@ -29,3 +29,8 @@ class Interpolation: cls, value: Any, expression: str = "", conversion: Literal["a", "r", "s"] | None = None, format_spec: str = "" ) -> Interpolation: ... def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +@overload +def convert(obj: _T, /, conversion: None) -> _T: ... +@overload +def convert(obj: object, /, conversion: Literal["r", "s", "a"]) -> str: ... diff --git a/mypy/typeshed/stdlib/sys/__init__.pyi b/mypy/typeshed/stdlib/sys/__init__.pyi index 0ca30396a8785..149f374d6e179 100644 --- a/mypy/typeshed/stdlib/sys/__init__.pyi +++ b/mypy/typeshed/stdlib/sys/__init__.pyi @@ -378,6 +378,7 @@ def settrace(function: TraceFunction | None, /) -> None: ... if sys.platform == "win32": # A tuple of length 5, even though it has more than 5 attributes. @final + @type_check_only class _WinVersion(_UninstantiableStructseq, tuple[int, int, int, int, str]): @property def major(self) -> int: ... @@ -454,7 +455,14 @@ def get_asyncgen_hooks() -> _asyncgen_hooks: ... def set_asyncgen_hooks(firstiter: _AsyncgenHook = ..., finalizer: _AsyncgenHook = ...) -> None: ... if sys.platform == "win32": - def _enablelegacywindowsfsencoding() -> None: ... + if sys.version_info >= (3, 13): + @deprecated( + "Deprecated since Python 3.13; will be removed in Python 3.16. " + "Use the `PYTHONLEGACYWINDOWSFSENCODING` environment variable instead." + ) + def _enablelegacywindowsfsencoding() -> None: ... + else: + def _enablelegacywindowsfsencoding() -> None: ... def get_coroutine_origin_tracking_depth() -> int: ... def set_coroutine_origin_tracking_depth(depth: int) -> None: ... diff --git a/mypy/typeshed/stdlib/sys/_monitoring.pyi b/mypy/typeshed/stdlib/sys/_monitoring.pyi index 0507eeedc26d0..3a8292ea0df4e 100644 --- a/mypy/typeshed/stdlib/sys/_monitoring.pyi +++ b/mypy/typeshed/stdlib/sys/_monitoring.pyi @@ -5,40 +5,52 @@ # of being a `types.ModuleType` instance that cannot be directly imported, # and exists in the `sys`-module namespace despite `sys` not being a package. +import sys from collections.abc import Callable from types import CodeType -from typing import Any +from typing import Any, Final, type_check_only +from typing_extensions import deprecated -DEBUGGER_ID: int -COVERAGE_ID: int -PROFILER_ID: int -OPTIMIZER_ID: int +DEBUGGER_ID: Final[int] +COVERAGE_ID: Final[int] +PROFILER_ID: Final[int] +OPTIMIZER_ID: Final[int] def use_tool_id(tool_id: int, name: str, /) -> None: ... def free_tool_id(tool_id: int, /) -> None: ... def get_tool(tool_id: int, /) -> str | None: ... -events: _events +events: Final[_events] +@type_check_only class _events: - BRANCH: int - CALL: int - C_RAISE: int - C_RETURN: int - EXCEPTION_HANDLED: int - INSTRUCTION: int - JUMP: int - LINE: int - NO_EVENTS: int - PY_RESUME: int - PY_RETURN: int - PY_START: int - PY_THROW: int - PY_UNWIND: int - PY_YIELD: int - RAISE: int - RERAISE: int - STOP_ITERATION: int + CALL: Final[int] + C_RAISE: Final[int] + C_RETURN: Final[int] + EXCEPTION_HANDLED: Final[int] + INSTRUCTION: Final[int] + JUMP: Final[int] + LINE: Final[int] + NO_EVENTS: Final[int] + PY_RESUME: Final[int] + PY_RETURN: Final[int] + PY_START: Final[int] + PY_THROW: Final[int] + PY_UNWIND: Final[int] + PY_YIELD: Final[int] + RAISE: Final[int] + RERAISE: Final[int] + STOP_ITERATION: Final[int] + if sys.version_info >= (3, 14): + BRANCH_LEFT: Final[int] + BRANCH_TAKEN: Final[int] + + @property + @deprecated("BRANCH is deprecated; use BRANCH_LEFT or BRANCH_TAKEN instead") + def BRANCH(self) -> int: ... + + else: + BRANCH: Final[int] def get_events(tool_id: int, /) -> int: ... def set_events(tool_id: int, event_set: int, /) -> None: ... @@ -46,7 +58,7 @@ def get_local_events(tool_id: int, code: CodeType, /) -> int: ... def set_local_events(tool_id: int, code: CodeType, event_set: int, /) -> int: ... def restart_events() -> None: ... -DISABLE: object -MISSING: object +DISABLE: Final[object] +MISSING: Final[object] def register_callback(tool_id: int, event: int, func: Callable[..., Any] | None, /) -> Callable[..., Any] | None: ... diff --git a/mypy/typeshed/stdlib/tarfile.pyi b/mypy/typeshed/stdlib/tarfile.pyi index dba250f2d3533..4e394409bbe0f 100644 --- a/mypy/typeshed/stdlib/tarfile.pyi +++ b/mypy/typeshed/stdlib/tarfile.pyi @@ -6,7 +6,7 @@ from builtins import list as _list # aliases to avoid name clashes with fields from collections.abc import Callable, Iterable, Iterator, Mapping from gzip import _ReadableFileobj as _GzipReadableFileobj, _WritableFileobj as _GzipWritableFileobj from types import TracebackType -from typing import IO, ClassVar, Literal, Protocol, overload +from typing import IO, ClassVar, Final, Literal, Protocol, overload, type_check_only from typing_extensions import Self, TypeAlias, deprecated if sys.version_info >= (3, 14): @@ -47,6 +47,7 @@ if sys.version_info >= (3, 13): _FilterFunction: TypeAlias = Callable[[TarInfo, str], TarInfo | None] _TarfileFilter: TypeAlias = Literal["fully_trusted", "tar", "data"] | _FilterFunction +@type_check_only class _Fileobj(Protocol): def read(self, size: int, /) -> bytes: ... def write(self, b: bytes, /) -> object: ... @@ -57,58 +58,61 @@ class _Fileobj(Protocol): # name: str | bytes # mode: Literal["rb", "r+b", "wb", "xb"] +@type_check_only class _Bz2ReadableFileobj(bz2._ReadableFileobj): def close(self) -> object: ... +@type_check_only class _Bz2WritableFileobj(bz2._WritableFileobj): def close(self) -> object: ... # tar constants -NUL: bytes -BLOCKSIZE: int -RECORDSIZE: int -GNU_MAGIC: bytes -POSIX_MAGIC: bytes - -LENGTH_NAME: int -LENGTH_LINK: int -LENGTH_PREFIX: int - -REGTYPE: bytes -AREGTYPE: bytes -LNKTYPE: bytes -SYMTYPE: bytes -CONTTYPE: bytes -BLKTYPE: bytes -DIRTYPE: bytes -FIFOTYPE: bytes -CHRTYPE: bytes - -GNUTYPE_LONGNAME: bytes -GNUTYPE_LONGLINK: bytes -GNUTYPE_SPARSE: bytes - -XHDTYPE: bytes -XGLTYPE: bytes -SOLARIS_XHDTYPE: bytes - -USTAR_FORMAT: int -GNU_FORMAT: int -PAX_FORMAT: int -DEFAULT_FORMAT: int +NUL: Final = b"\0" +BLOCKSIZE: Final = 512 +RECORDSIZE: Final = 10240 +GNU_MAGIC: Final = b"ustar \0" +POSIX_MAGIC: Final = b"ustar\x0000" + +LENGTH_NAME: Final = 100 +LENGTH_LINK: Final = 100 +LENGTH_PREFIX: Final = 155 + +REGTYPE: Final = b"0" +AREGTYPE: Final = b"\0" +LNKTYPE: Final = b"1" +SYMTYPE: Final = b"2" +CHRTYPE: Final = b"3" +BLKTYPE: Final = b"4" +DIRTYPE: Final = b"5" +FIFOTYPE: Final = b"6" +CONTTYPE: Final = b"7" + +GNUTYPE_LONGNAME: Final = b"L" +GNUTYPE_LONGLINK: Final = b"K" +GNUTYPE_SPARSE: Final = b"S" + +XHDTYPE: Final = b"x" +XGLTYPE: Final = b"g" +SOLARIS_XHDTYPE: Final = b"X" + +_TarFormat: TypeAlias = Literal[0, 1, 2] # does not exist at runtime +USTAR_FORMAT: Final = 0 +GNU_FORMAT: Final = 1 +PAX_FORMAT: Final = 2 +DEFAULT_FORMAT: Final = PAX_FORMAT # tarfile constants -SUPPORTED_TYPES: tuple[bytes, ...] -REGULAR_TYPES: tuple[bytes, ...] -GNU_TYPES: tuple[bytes, ...] -PAX_FIELDS: tuple[str, ...] -PAX_NUMBER_FIELDS: dict[str, type] -PAX_NAME_FIELDS: set[str] +SUPPORTED_TYPES: Final[tuple[bytes, ...]] +REGULAR_TYPES: Final[tuple[bytes, ...]] +GNU_TYPES: Final[tuple[bytes, ...]] +PAX_FIELDS: Final[tuple[str, ...]] +PAX_NUMBER_FIELDS: Final[dict[str, type]] +PAX_NAME_FIELDS: Final[set[str]] -ENCODING: str +ENCODING: Final[str] -class ExFileObject(io.BufferedReader): +class ExFileObject(io.BufferedReader): # undocumented def __init__(self, tarfile: TarFile, tarinfo: TarInfo) -> None: ... class TarFile: @@ -116,13 +120,13 @@ class TarFile: name: StrOrBytesPath | None mode: Literal["r", "a", "w", "x"] fileobj: _Fileobj | None - format: int | None + format: _TarFormat | None tarinfo: type[TarInfo] dereference: bool | None ignore_zeros: bool | None encoding: str | None errors: str - fileobject: type[ExFileObject] + fileobject: type[ExFileObject] # undocumented pax_headers: Mapping[str, str] | None debug: int | None errorlevel: int | None @@ -642,7 +646,7 @@ class TarFile: def getmember(self, name: str) -> TarInfo: ... def getmembers(self) -> _list[TarInfo]: ... def getnames(self) -> _list[str]: ... - def list(self, verbose: bool = True, *, members: _list[TarInfo] | None = None) -> None: ... + def list(self, verbose: bool = True, *, members: Iterable[TarInfo] | None = None) -> None: ... def next(self) -> TarInfo | None: ... # Calling this method without `filter` is deprecated, but it may be set either on the class or in an # individual call, so we can't mark it as @deprecated here. @@ -751,7 +755,7 @@ class TarInfo: offset_data: int sparse: bytes | None mode: int - type: bytes + type: bytes # usually one of the TYPE constants, but could be an arbitrary byte linkname: str uid: int gid: int @@ -791,7 +795,7 @@ class TarInfo: deep: bool = True, ) -> Self: ... def get_info(self) -> Mapping[str, str | int | bytes | Mapping[str, str]]: ... - def tobuf(self, format: int | None = 2, encoding: str | None = "utf-8", errors: str = "surrogateescape") -> bytes: ... + def tobuf(self, format: _TarFormat | None = 2, encoding: str | None = "utf-8", errors: str = "surrogateescape") -> bytes: ... def create_ustar_header( self, info: Mapping[str, str | int | bytes | Mapping[str, str]], encoding: str, errors: str ) -> bytes: ... diff --git a/mypy/typeshed/stdlib/tempfile.pyi b/mypy/typeshed/stdlib/tempfile.pyi index ea6e057e410d4..6b2abe4398d2f 100644 --- a/mypy/typeshed/stdlib/tempfile.pyi +++ b/mypy/typeshed/stdlib/tempfile.pyi @@ -15,7 +15,7 @@ from _typeshed import ( from collections.abc import Iterable, Iterator from types import GenericAlias, TracebackType from typing import IO, Any, AnyStr, Generic, Literal, overload -from typing_extensions import Self +from typing_extensions import Self, deprecated __all__ = [ "NamedTemporaryFile", @@ -471,6 +471,7 @@ def mkstemp( def mkdtemp(suffix: str | None = None, prefix: str | None = None, dir: StrPath | None = None) -> str: ... @overload def mkdtemp(suffix: bytes | None = None, prefix: bytes | None = None, dir: BytesPath | None = None) -> bytes: ... +@deprecated("Deprecated since Python 2.3. Use `mkstemp()` or `NamedTemporaryFile(delete=False)` instead.") def mktemp(suffix: str = "", prefix: str = "tmp", dir: StrPath | None = None) -> str: ... def gettempdirb() -> bytes: ... def gettempprefixb() -> bytes: ... diff --git a/mypy/typeshed/stdlib/termios.pyi b/mypy/typeshed/stdlib/termios.pyi index 5a5a1f53be3c6..a35be5dfe740a 100644 --- a/mypy/typeshed/stdlib/termios.pyi +++ b/mypy/typeshed/stdlib/termios.pyi @@ -1,6 +1,6 @@ import sys from _typeshed import FileDescriptorLike -from typing import Any +from typing import Any, Final from typing_extensions import TypeAlias # Must be a list of length 7, containing 6 ints and a list of NCCS 1-character bytes or ints. @@ -9,286 +9,287 @@ _Attr: TypeAlias = list[int | list[bytes | int]] | list[int | list[bytes]] | lis _AttrReturn: TypeAlias = list[Any] if sys.platform != "win32": - B0: int - B110: int - B115200: int - B1200: int - B134: int - B150: int - B1800: int - B19200: int - B200: int - B230400: int - B2400: int - B300: int - B38400: int - B4800: int - B50: int - B57600: int - B600: int - B75: int - B9600: int - BRKINT: int - BS0: int - BS1: int - BSDLY: int - CDSUSP: int - CEOF: int - CEOL: int - CEOT: int - CERASE: int - CFLUSH: int - CINTR: int - CKILL: int - CLNEXT: int - CLOCAL: int - CQUIT: int - CR0: int - CR1: int - CR2: int - CR3: int - CRDLY: int - CREAD: int - CRPRNT: int - CRTSCTS: int - CS5: int - CS6: int - CS7: int - CS8: int - CSIZE: int - CSTART: int - CSTOP: int - CSTOPB: int - CSUSP: int - CWERASE: int - ECHO: int - ECHOCTL: int - ECHOE: int - ECHOK: int - ECHOKE: int - ECHONL: int - ECHOPRT: int - EXTA: int - EXTB: int - FF0: int - FF1: int - FFDLY: int - FIOASYNC: int - FIOCLEX: int - FIONBIO: int - FIONCLEX: int - FIONREAD: int - FLUSHO: int - HUPCL: int - ICANON: int - ICRNL: int - IEXTEN: int - IGNBRK: int - IGNCR: int - IGNPAR: int - IMAXBEL: int - INLCR: int - INPCK: int - ISIG: int - ISTRIP: int - IXANY: int - IXOFF: int - IXON: int - NCCS: int - NL0: int - NL1: int - NLDLY: int - NOFLSH: int - OCRNL: int - OFDEL: int - OFILL: int - ONLCR: int - ONLRET: int - ONOCR: int - OPOST: int - PARENB: int - PARMRK: int - PARODD: int - PENDIN: int - TAB0: int - TAB1: int - TAB2: int - TAB3: int - TABDLY: int - TCIFLUSH: int - TCIOFF: int - TCIOFLUSH: int - TCION: int - TCOFLUSH: int - TCOOFF: int - TCOON: int - TCSADRAIN: int - TCSAFLUSH: int - TCSANOW: int - TIOCCONS: int - TIOCEXCL: int - TIOCGETD: int - TIOCGPGRP: int - TIOCGWINSZ: int - TIOCM_CAR: int - TIOCM_CD: int - TIOCM_CTS: int - TIOCM_DSR: int - TIOCM_DTR: int - TIOCM_LE: int - TIOCM_RI: int - TIOCM_RNG: int - TIOCM_RTS: int - TIOCM_SR: int - TIOCM_ST: int - TIOCMBIC: int - TIOCMBIS: int - TIOCMGET: int - TIOCMSET: int - TIOCNOTTY: int - TIOCNXCL: int - TIOCOUTQ: int - TIOCPKT_DATA: int - TIOCPKT_DOSTOP: int - TIOCPKT_FLUSHREAD: int - TIOCPKT_FLUSHWRITE: int - TIOCPKT_NOSTOP: int - TIOCPKT_START: int - TIOCPKT_STOP: int - TIOCPKT: int - TIOCSCTTY: int - TIOCSETD: int - TIOCSPGRP: int - TIOCSTI: int - TIOCSWINSZ: int - TOSTOP: int - VDISCARD: int - VEOF: int - VEOL: int - VEOL2: int - VERASE: int - VINTR: int - VKILL: int - VLNEXT: int - VMIN: int - VQUIT: int - VREPRINT: int - VSTART: int - VSTOP: int - VSUSP: int - VT0: int - VT1: int - VTDLY: int - VTIME: int - VWERASE: int + # Values depends on the platform + B0: Final[int] + B110: Final[int] + B115200: Final[int] + B1200: Final[int] + B134: Final[int] + B150: Final[int] + B1800: Final[int] + B19200: Final[int] + B200: Final[int] + B230400: Final[int] + B2400: Final[int] + B300: Final[int] + B38400: Final[int] + B4800: Final[int] + B50: Final[int] + B57600: Final[int] + B600: Final[int] + B75: Final[int] + B9600: Final[int] + BRKINT: Final[int] + BS0: Final[int] + BS1: Final[int] + BSDLY: Final[int] + CDSUSP: Final[int] + CEOF: Final[int] + CEOL: Final[int] + CEOT: Final[int] + CERASE: Final[int] + CFLUSH: Final[int] + CINTR: Final[int] + CKILL: Final[int] + CLNEXT: Final[int] + CLOCAL: Final[int] + CQUIT: Final[int] + CR0: Final[int] + CR1: Final[int] + CR2: Final[int] + CR3: Final[int] + CRDLY: Final[int] + CREAD: Final[int] + CRPRNT: Final[int] + CRTSCTS: Final[int] + CS5: Final[int] + CS6: Final[int] + CS7: Final[int] + CS8: Final[int] + CSIZE: Final[int] + CSTART: Final[int] + CSTOP: Final[int] + CSTOPB: Final[int] + CSUSP: Final[int] + CWERASE: Final[int] + ECHO: Final[int] + ECHOCTL: Final[int] + ECHOE: Final[int] + ECHOK: Final[int] + ECHOKE: Final[int] + ECHONL: Final[int] + ECHOPRT: Final[int] + EXTA: Final[int] + EXTB: Final[int] + FF0: Final[int] + FF1: Final[int] + FFDLY: Final[int] + FIOASYNC: Final[int] + FIOCLEX: Final[int] + FIONBIO: Final[int] + FIONCLEX: Final[int] + FIONREAD: Final[int] + FLUSHO: Final[int] + HUPCL: Final[int] + ICANON: Final[int] + ICRNL: Final[int] + IEXTEN: Final[int] + IGNBRK: Final[int] + IGNCR: Final[int] + IGNPAR: Final[int] + IMAXBEL: Final[int] + INLCR: Final[int] + INPCK: Final[int] + ISIG: Final[int] + ISTRIP: Final[int] + IXANY: Final[int] + IXOFF: Final[int] + IXON: Final[int] + NCCS: Final[int] + NL0: Final[int] + NL1: Final[int] + NLDLY: Final[int] + NOFLSH: Final[int] + OCRNL: Final[int] + OFDEL: Final[int] + OFILL: Final[int] + ONLCR: Final[int] + ONLRET: Final[int] + ONOCR: Final[int] + OPOST: Final[int] + PARENB: Final[int] + PARMRK: Final[int] + PARODD: Final[int] + PENDIN: Final[int] + TAB0: Final[int] + TAB1: Final[int] + TAB2: Final[int] + TAB3: Final[int] + TABDLY: Final[int] + TCIFLUSH: Final[int] + TCIOFF: Final[int] + TCIOFLUSH: Final[int] + TCION: Final[int] + TCOFLUSH: Final[int] + TCOOFF: Final[int] + TCOON: Final[int] + TCSADRAIN: Final[int] + TCSAFLUSH: Final[int] + TCSANOW: Final[int] + TIOCCONS: Final[int] + TIOCEXCL: Final[int] + TIOCGETD: Final[int] + TIOCGPGRP: Final[int] + TIOCGWINSZ: Final[int] + TIOCM_CAR: Final[int] + TIOCM_CD: Final[int] + TIOCM_CTS: Final[int] + TIOCM_DSR: Final[int] + TIOCM_DTR: Final[int] + TIOCM_LE: Final[int] + TIOCM_RI: Final[int] + TIOCM_RNG: Final[int] + TIOCM_RTS: Final[int] + TIOCM_SR: Final[int] + TIOCM_ST: Final[int] + TIOCMBIC: Final[int] + TIOCMBIS: Final[int] + TIOCMGET: Final[int] + TIOCMSET: Final[int] + TIOCNOTTY: Final[int] + TIOCNXCL: Final[int] + TIOCOUTQ: Final[int] + TIOCPKT_DATA: Final[int] + TIOCPKT_DOSTOP: Final[int] + TIOCPKT_FLUSHREAD: Final[int] + TIOCPKT_FLUSHWRITE: Final[int] + TIOCPKT_NOSTOP: Final[int] + TIOCPKT_START: Final[int] + TIOCPKT_STOP: Final[int] + TIOCPKT: Final[int] + TIOCSCTTY: Final[int] + TIOCSETD: Final[int] + TIOCSPGRP: Final[int] + TIOCSTI: Final[int] + TIOCSWINSZ: Final[int] + TOSTOP: Final[int] + VDISCARD: Final[int] + VEOF: Final[int] + VEOL: Final[int] + VEOL2: Final[int] + VERASE: Final[int] + VINTR: Final[int] + VKILL: Final[int] + VLNEXT: Final[int] + VMIN: Final[int] + VQUIT: Final[int] + VREPRINT: Final[int] + VSTART: Final[int] + VSTOP: Final[int] + VSUSP: Final[int] + VT0: Final[int] + VT1: Final[int] + VTDLY: Final[int] + VTIME: Final[int] + VWERASE: Final[int] if sys.version_info >= (3, 13): - EXTPROC: int - IUTF8: int + EXTPROC: Final[int] + IUTF8: Final[int] if sys.platform == "darwin" and sys.version_info >= (3, 13): - ALTWERASE: int - B14400: int - B28800: int - B7200: int - B76800: int - CCAR_OFLOW: int - CCTS_OFLOW: int - CDSR_OFLOW: int - CDTR_IFLOW: int - CIGNORE: int - CRTS_IFLOW: int - MDMBUF: int - NL2: int - NL3: int - NOKERNINFO: int - ONOEOT: int - OXTABS: int - VDSUSP: int - VSTATUS: int + ALTWERASE: Final[int] + B14400: Final[int] + B28800: Final[int] + B7200: Final[int] + B76800: Final[int] + CCAR_OFLOW: Final[int] + CCTS_OFLOW: Final[int] + CDSR_OFLOW: Final[int] + CDTR_IFLOW: Final[int] + CIGNORE: Final[int] + CRTS_IFLOW: Final[int] + MDMBUF: Final[int] + NL2: Final[int] + NL3: Final[int] + NOKERNINFO: Final[int] + ONOEOT: Final[int] + OXTABS: Final[int] + VDSUSP: Final[int] + VSTATUS: Final[int] if sys.platform == "darwin" and sys.version_info >= (3, 11): - TIOCGSIZE: int - TIOCSSIZE: int + TIOCGSIZE: Final[int] + TIOCSSIZE: Final[int] if sys.platform == "linux": - B1152000: int - B576000: int - CBAUD: int - CBAUDEX: int - CIBAUD: int - IOCSIZE_MASK: int - IOCSIZE_SHIFT: int - IUCLC: int - N_MOUSE: int - N_PPP: int - N_SLIP: int - N_STRIP: int - N_TTY: int - NCC: int - OLCUC: int - TCFLSH: int - TCGETA: int - TCGETS: int - TCSBRK: int - TCSBRKP: int - TCSETA: int - TCSETAF: int - TCSETAW: int - TCSETS: int - TCSETSF: int - TCSETSW: int - TCXONC: int - TIOCGICOUNT: int - TIOCGLCKTRMIOS: int - TIOCGSERIAL: int - TIOCGSOFTCAR: int - TIOCINQ: int - TIOCLINUX: int - TIOCMIWAIT: int - TIOCTTYGSTRUCT: int - TIOCSER_TEMT: int - TIOCSERCONFIG: int - TIOCSERGETLSR: int - TIOCSERGETMULTI: int - TIOCSERGSTRUCT: int - TIOCSERGWILD: int - TIOCSERSETMULTI: int - TIOCSERSWILD: int - TIOCSLCKTRMIOS: int - TIOCSSERIAL: int - TIOCSSOFTCAR: int - VSWTC: int - VSWTCH: int - XCASE: int - XTABS: int + B1152000: Final[int] + B576000: Final[int] + CBAUD: Final[int] + CBAUDEX: Final[int] + CIBAUD: Final[int] + IOCSIZE_MASK: Final[int] + IOCSIZE_SHIFT: Final[int] + IUCLC: Final[int] + N_MOUSE: Final[int] + N_PPP: Final[int] + N_SLIP: Final[int] + N_STRIP: Final[int] + N_TTY: Final[int] + NCC: Final[int] + OLCUC: Final[int] + TCFLSH: Final[int] + TCGETA: Final[int] + TCGETS: Final[int] + TCSBRK: Final[int] + TCSBRKP: Final[int] + TCSETA: Final[int] + TCSETAF: Final[int] + TCSETAW: Final[int] + TCSETS: Final[int] + TCSETSF: Final[int] + TCSETSW: Final[int] + TCXONC: Final[int] + TIOCGICOUNT: Final[int] + TIOCGLCKTRMIOS: Final[int] + TIOCGSERIAL: Final[int] + TIOCGSOFTCAR: Final[int] + TIOCINQ: Final[int] + TIOCLINUX: Final[int] + TIOCMIWAIT: Final[int] + TIOCTTYGSTRUCT: Final[int] + TIOCSER_TEMT: Final[int] + TIOCSERCONFIG: Final[int] + TIOCSERGETLSR: Final[int] + TIOCSERGETMULTI: Final[int] + TIOCSERGSTRUCT: Final[int] + TIOCSERGWILD: Final[int] + TIOCSERSETMULTI: Final[int] + TIOCSERSWILD: Final[int] + TIOCSLCKTRMIOS: Final[int] + TIOCSSERIAL: Final[int] + TIOCSSOFTCAR: Final[int] + VSWTC: Final[int] + VSWTCH: Final[int] + XCASE: Final[int] + XTABS: Final[int] if sys.platform != "darwin": - B1000000: int - B1500000: int - B2000000: int - B2500000: int - B3000000: int - B3500000: int - B4000000: int - B460800: int - B500000: int - B921600: int + B1000000: Final[int] + B1500000: Final[int] + B2000000: Final[int] + B2500000: Final[int] + B3000000: Final[int] + B3500000: Final[int] + B4000000: Final[int] + B460800: Final[int] + B500000: Final[int] + B921600: Final[int] if sys.platform != "linux": - TCSASOFT: int + TCSASOFT: Final[int] if sys.platform != "darwin" and sys.platform != "linux": # not available on FreeBSD either. - CDEL: int - CEOL2: int - CESC: int - CNUL: int - COMMON: int - CSWTCH: int - IBSHIFT: int - INIT_C_CC: int - NSWTCH: int + CDEL: Final[int] + CEOL2: Final[int] + CESC: Final[int] + CNUL: Final[int] + COMMON: Final[int] + CSWTCH: Final[int] + IBSHIFT: Final[int] + INIT_C_CC: Final[int] + NSWTCH: Final[int] def tcgetattr(fd: FileDescriptorLike, /) -> _AttrReturn: ... def tcsetattr(fd: FileDescriptorLike, when: int, attributes: _Attr, /) -> None: ... diff --git a/mypy/typeshed/stdlib/threading.pyi b/mypy/typeshed/stdlib/threading.pyi index d31351754d056..033cad3931f5c 100644 --- a/mypy/typeshed/stdlib/threading.pyi +++ b/mypy/typeshed/stdlib/threading.pyi @@ -142,7 +142,7 @@ class Condition: def __exit__( self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None ) -> None: ... - def acquire(self, blocking: bool = ..., timeout: float = ...) -> bool: ... + def acquire(self, blocking: bool = True, timeout: float = -1) -> bool: ... def release(self) -> None: ... def wait(self, timeout: float | None = None) -> bool: ... def wait_for(self, predicate: Callable[[], _T], timeout: float | None = None) -> _T: ... diff --git a/mypy/typeshed/stdlib/time.pyi b/mypy/typeshed/stdlib/time.pyi index 6d2538ea7e3ee..a921722b62c5b 100644 --- a/mypy/typeshed/stdlib/time.pyi +++ b/mypy/typeshed/stdlib/time.pyi @@ -1,6 +1,6 @@ import sys from _typeshed import structseq -from typing import Any, Final, Literal, Protocol, final +from typing import Any, Final, Literal, Protocol, final, type_check_only from typing_extensions import TypeAlias _TimeTuple: TypeAlias = tuple[int, int, int, int, int, int, int, int, int] @@ -80,6 +80,7 @@ def time() -> float: ... if sys.platform != "win32": def tzset() -> None: ... # Unix only +@type_check_only class _ClockInfo(Protocol): adjustable: bool implementation: str diff --git a/mypy/typeshed/stdlib/tkinter/__init__.pyi b/mypy/typeshed/stdlib/tkinter/__init__.pyi index db0e34d737a62..b802d5e97c840 100644 --- a/mypy/typeshed/stdlib/tkinter/__init__.pyi +++ b/mypy/typeshed/stdlib/tkinter/__init__.pyi @@ -366,12 +366,14 @@ def getboolean(s): ... _Ts = TypeVarTuple("_Ts") +@type_check_only class _GridIndexInfo(TypedDict, total=False): minsize: _ScreenUnits pad: _ScreenUnits uniform: str | None weight: int +@type_check_only class _BusyInfo(TypedDict): cursor: _Cursor @@ -1039,6 +1041,7 @@ def Tcl(screenName: str | None = None, baseName: str | None = None, className: s _InMiscTotal = TypedDict("_InMiscTotal", {"in": Misc}) _InMiscNonTotal = TypedDict("_InMiscNonTotal", {"in": Misc}, total=False) +@type_check_only class _PackInfo(_InMiscTotal): # 'before' and 'after' never appear in _PackInfo anchor: _Anchor @@ -1080,6 +1083,7 @@ class Pack: forget = pack_forget propagate = Misc.pack_propagate +@type_check_only class _PlaceInfo(_InMiscNonTotal): # empty dict if widget hasn't been placed anchor: _Anchor bordermode: Literal["inside", "outside", "ignore"] @@ -1116,6 +1120,7 @@ class Place: place = place_configure info = place_info +@type_check_only class _GridInfo(_InMiscNonTotal): # empty dict if widget hasn't been gridded column: int columnspan: int diff --git a/mypy/typeshed/stdlib/tkinter/dnd.pyi b/mypy/typeshed/stdlib/tkinter/dnd.pyi index fe2961701c61d..521f451a9b2c5 100644 --- a/mypy/typeshed/stdlib/tkinter/dnd.pyi +++ b/mypy/typeshed/stdlib/tkinter/dnd.pyi @@ -1,8 +1,9 @@ from tkinter import Event, Misc, Tk, Widget -from typing import ClassVar, Protocol +from typing import ClassVar, Protocol, type_check_only __all__ = ["dnd_start", "DndHandler"] +@type_check_only class _DndSource(Protocol): def dnd_end(self, target: Widget | None, event: Event[Misc] | None, /) -> None: ... diff --git a/mypy/typeshed/stdlib/tkinter/font.pyi b/mypy/typeshed/stdlib/tkinter/font.pyi index cab97490be340..327ba7a2432e0 100644 --- a/mypy/typeshed/stdlib/tkinter/font.pyi +++ b/mypy/typeshed/stdlib/tkinter/font.pyi @@ -2,7 +2,7 @@ import _tkinter import itertools import sys import tkinter -from typing import Any, ClassVar, Final, Literal, TypedDict, overload +from typing import Any, ClassVar, Final, Literal, TypedDict, overload, type_check_only from typing_extensions import TypeAlias, Unpack __all__ = ["NORMAL", "ROMAN", "BOLD", "ITALIC", "nametofont", "Font", "families", "names"] @@ -23,6 +23,7 @@ _FontDescription: TypeAlias = ( | _tkinter.Tcl_Obj # A font object constructed in Tcl ) +@type_check_only class _FontDict(TypedDict): family: str size: int @@ -31,6 +32,7 @@ class _FontDict(TypedDict): underline: bool overstrike: bool +@type_check_only class _MetricsDict(TypedDict): ascent: int descent: int diff --git a/mypy/typeshed/stdlib/tkinter/ttk.pyi b/mypy/typeshed/stdlib/tkinter/ttk.pyi index 50b9cd8f9bcde..c46239df81eb4 100644 --- a/mypy/typeshed/stdlib/tkinter/ttk.pyi +++ b/mypy/typeshed/stdlib/tkinter/ttk.pyi @@ -3,7 +3,7 @@ import tkinter from _typeshed import Incomplete, MaybeNone from collections.abc import Callable from tkinter.font import _FontDescription -from typing import Any, Literal, TypedDict, overload +from typing import Any, Literal, TypedDict, overload, type_check_only from typing_extensions import TypeAlias __all__ = [ @@ -928,6 +928,7 @@ class Spinbox(Entry): config = configure # type: ignore[assignment] def set(self, value: Any) -> None: ... +@type_check_only class _TreeviewItemDict(TypedDict): text: str image: list[str] | Literal[""] # no idea why it's wrapped in list @@ -935,6 +936,7 @@ class _TreeviewItemDict(TypedDict): open: bool # actually 0 or 1 tags: list[str] | Literal[""] +@type_check_only class _TreeviewTagDict(TypedDict): # There is also 'text' and 'anchor', but they don't seem to do anything, using them is likely a bug foreground: str @@ -942,6 +944,7 @@ class _TreeviewTagDict(TypedDict): font: _FontDescription image: str # not wrapped in list :D +@type_check_only class _TreeviewHeaderDict(TypedDict): text: str image: list[str] | Literal[""] @@ -949,6 +952,7 @@ class _TreeviewHeaderDict(TypedDict): command: str state: str # Doesn't seem to appear anywhere else than in these dicts +@type_check_only class _TreeviewColumnDict(TypedDict): width: int minwidth: int diff --git a/mypy/typeshed/stdlib/tty.pyi b/mypy/typeshed/stdlib/tty.pyi index 0611879cf1b29..ca3f0013b20ec 100644 --- a/mypy/typeshed/stdlib/tty.pyi +++ b/mypy/typeshed/stdlib/tty.pyi @@ -15,13 +15,13 @@ if sys.platform != "win32": _FD: TypeAlias = int | IO[str] # XXX: Undocumented integer constants - IFLAG: Final[int] - OFLAG: Final[int] - CFLAG: Final[int] - LFLAG: Final[int] - ISPEED: Final[int] - OSPEED: Final[int] - CC: Final[int] + IFLAG: Final = 0 + OFLAG: Final = 1 + CFLAG: Final = 2 + LFLAG: Final = 3 + ISPEED: Final = 4 + OSPEED: Final = 5 + CC: Final = 6 def setraw(fd: _FD, when: int = 2) -> _ModeSetterReturn: ... def setcbreak(fd: _FD, when: int = 2) -> _ModeSetterReturn: ... diff --git a/mypy/typeshed/stdlib/turtle.pyi b/mypy/typeshed/stdlib/turtle.pyi index 9c62c64e718aa..7d39026b80413 100644 --- a/mypy/typeshed/stdlib/turtle.pyi +++ b/mypy/typeshed/stdlib/turtle.pyi @@ -3,7 +3,7 @@ from _typeshed import StrPath from collections.abc import Callable, Generator, Sequence from contextlib import contextmanager from tkinter import Canvas, Frame, Misc, PhotoImage, Scrollbar -from typing import Any, ClassVar, Literal, TypedDict, overload +from typing import Any, ClassVar, Literal, TypedDict, overload, type_check_only from typing_extensions import Self, TypeAlias __all__ = [ @@ -146,6 +146,7 @@ if sys.version_info < (3, 13): _Color: TypeAlias = str | tuple[float, float, float] _AnyColor: TypeAlias = Any +@type_check_only class _PenState(TypedDict): shown: bool pendown: bool @@ -487,19 +488,8 @@ Pen = Turtle def write_docstringdict(filename: str = "turtle_docstringdict") -> None: ... -# Note: it's somewhat unfortunate that we have to copy the function signatures. -# It would be nice if we could partially reduce the redundancy by doing something -# like the following: -# -# _screen: Screen -# clear = _screen.clear -# -# However, it seems pytype does not support this type of syntax in pyi files. - # Functions copied from TurtleScreenBase: -# Note: mainloop() was always present in the global scope, but was added to -# TurtleScreenBase in Python 3.0 def mainloop() -> None: ... def textinput(title: str, prompt: str) -> str | None: ... def numinput( diff --git a/mypy/typeshed/stdlib/typing.pyi b/mypy/typeshed/stdlib/typing.pyi index d296c8d921498..a85aa2e2dc83a 100644 --- a/mypy/typeshed/stdlib/typing.pyi +++ b/mypy/typeshed/stdlib/typing.pyi @@ -419,6 +419,7 @@ def type_check_only(func_or_cls: _FT) -> _FT: ... # Type aliases and type constructors +@type_check_only class _Alias: # Class for defining generic aliases for library types. def __getitem__(self, typeargs: Any) -> Any: ... @@ -1125,6 +1126,7 @@ if sys.version_info >= (3, 13): def is_protocol(tp: type, /) -> bool: ... def get_protocol_members(tp: type, /) -> frozenset[str]: ... @final + @type_check_only class _NoDefaultType: ... NoDefault: _NoDefaultType diff --git a/mypy/typeshed/stdlib/typing_extensions.pyi b/mypy/typeshed/stdlib/typing_extensions.pyi index 3f7c257120814..22b6ada8ffb7b 100644 --- a/mypy/typeshed/stdlib/typing_extensions.pyi +++ b/mypy/typeshed/stdlib/typing_extensions.pyi @@ -59,6 +59,7 @@ from typing import ( # noqa: Y022,Y037,Y038,Y039,UP035 TypeVar as _TypeVar, Union as Union, _Alias, + _SpecialForm, cast as cast, no_type_check as no_type_check, no_type_check_decorator as no_type_check_decorator, @@ -204,15 +205,6 @@ _TC = _TypeVar("_TC", bound=type[object]) _T_co = _TypeVar("_T_co", covariant=True) # Any type covariant containers. _T_contra = _TypeVar("_T_contra", contravariant=True) -class _Final: ... # This should be imported from typing but that breaks pytype - -# unfortunately we have to duplicate this class definition from typing.pyi or we break pytype -class _SpecialForm(_Final): - def __getitem__(self, parameters: Any) -> object: ... - if sys.version_info >= (3, 10): - def __or__(self, other: Any) -> _SpecialForm: ... - def __ror__(self, other: Any) -> _SpecialForm: ... - # Do not import (and re-export) Protocol or runtime_checkable from # typing module because type checkers need to be able to distinguish # typing.Protocol and typing_extensions.Protocol so they can properly @@ -480,6 +472,7 @@ else: def is_protocol(tp: type, /) -> bool: ... def get_protocol_members(tp: type, /) -> frozenset[str]: ... @final + @type_check_only class _NoDefaultType: ... NoDefault: _NoDefaultType @@ -611,6 +604,7 @@ class Doc: def __eq__(self, other: object) -> bool: ... # PEP 728 +@type_check_only class _NoExtraItemsType: ... NoExtraItems: _NoExtraItemsType diff --git a/mypy/typeshed/stdlib/unittest/case.pyi b/mypy/typeshed/stdlib/unittest/case.pyi index 89bcabf104c25..a602196e73c64 100644 --- a/mypy/typeshed/stdlib/unittest/case.pyi +++ b/mypy/typeshed/stdlib/unittest/case.pyi @@ -2,18 +2,16 @@ import logging import sys import unittest.result from _typeshed import SupportsDunderGE, SupportsDunderGT, SupportsDunderLE, SupportsDunderLT, SupportsRSub, SupportsSub +from builtins import _ClassInfo from collections.abc import Callable, Container, Iterable, Mapping, Sequence, Set as AbstractSet from contextlib import AbstractContextManager from re import Pattern from types import GenericAlias, TracebackType -from typing import Any, AnyStr, Final, Generic, NoReturn, Protocol, SupportsAbs, SupportsRound, TypeVar, overload -from typing_extensions import Never, ParamSpec, Self, TypeAlias +from typing import Any, AnyStr, Final, Generic, NoReturn, Protocol, SupportsAbs, SupportsRound, TypeVar, overload, type_check_only +from typing_extensions import Never, ParamSpec, Self from unittest._log import _AssertLogsContext, _LoggingWatcher from warnings import WarningMessage -if sys.version_info >= (3, 10): - from types import UnionType - _T = TypeVar("_T") _S = TypeVar("_S", bound=SupportsSub[Any, Any]) _E = TypeVar("_E", bound=BaseException) @@ -58,16 +56,9 @@ def skipUnless(condition: object, reason: str) -> Callable[[_FT], _FT]: ... class SkipTest(Exception): def __init__(self, reason: str) -> None: ... +@type_check_only class _SupportsAbsAndDunderGE(SupportsDunderGE[Any], SupportsAbs[Any], Protocol): ... -# Keep this alias in sync with builtins._ClassInfo -# We can't import it from builtins or pytype crashes, -# due to the fact that pytype uses a custom builtins stub rather than typeshed's builtins stub -if sys.version_info >= (3, 10): - _ClassInfo: TypeAlias = type | UnionType | tuple[_ClassInfo, ...] -else: - _ClassInfo: TypeAlias = type | tuple[_ClassInfo, ...] - class TestCase: failureException: type[BaseException] longMessage: bool diff --git a/mypy/typeshed/stdlib/unittest/main.pyi b/mypy/typeshed/stdlib/unittest/main.pyi index 22f2ec10634d6..152e9c33209ca 100644 --- a/mypy/typeshed/stdlib/unittest/main.pyi +++ b/mypy/typeshed/stdlib/unittest/main.pyi @@ -5,12 +5,13 @@ import unittest.result import unittest.suite from collections.abc import Iterable from types import ModuleType -from typing import Any, Final, Protocol +from typing import Any, Final, Protocol, type_check_only from typing_extensions import deprecated MAIN_EXAMPLES: Final[str] MODULE_EXAMPLES: Final[str] +@type_check_only class _TestRunner(Protocol): def run(self, test: unittest.suite.TestSuite | unittest.case.TestCase, /) -> unittest.result.TestResult: ... diff --git a/mypy/typeshed/stdlib/unittest/mock.pyi b/mypy/typeshed/stdlib/unittest/mock.pyi index 9e353900f2d7f..6b0941a917190 100644 --- a/mypy/typeshed/stdlib/unittest/mock.pyi +++ b/mypy/typeshed/stdlib/unittest/mock.pyi @@ -3,7 +3,7 @@ from _typeshed import MaybeNone from collections.abc import Awaitable, Callable, Coroutine, Iterable, Mapping, Sequence from contextlib import _GeneratorContextManager from types import TracebackType -from typing import Any, ClassVar, Final, Generic, Literal, TypeVar, overload +from typing import Any, ClassVar, Final, Generic, Literal, TypeVar, overload, type_check_only from typing_extensions import ParamSpec, Self, TypeAlias _T = TypeVar("_T") @@ -262,7 +262,8 @@ class _patch(Generic[_T]): # This class does not exist at runtime, it's a hack to make this work: # @patch("foo") # def bar(..., mock: MagicMock) -> None: ... -class _patch_default_new(_patch[MagicMock | AsyncMock]): +@type_check_only +class _patch_pass_arg(_patch[_T]): @overload def __call__(self, func: _TT) -> _TT: ... # Can't use the following as ParamSpec is only allowed as last parameter: @@ -288,6 +289,7 @@ class _patch_dict: # This class does not exist at runtime, it's a hack to add methods to the # patch() function. +@type_check_only class _patcher: TEST_PREFIX: str dict: type[_patch_dict] @@ -303,7 +305,7 @@ class _patcher: create: bool = ..., spec_set: Any | None = ..., autospec: Any | None = ..., - new_callable: Any | None = ..., + new_callable: Callable[..., Any] | None = ..., **kwargs: Any, ) -> _patch[_T]: ... @overload @@ -315,9 +317,21 @@ class _patcher: create: bool = ..., spec_set: Any | None = ..., autospec: Any | None = ..., - new_callable: Any | None = ..., + new_callable: Callable[..., _T], **kwargs: Any, - ) -> _patch_default_new: ... + ) -> _patch_pass_arg[_T]: ... + @overload + def __call__( + self, + target: str, + *, + spec: Any | None = ..., + create: bool = ..., + spec_set: Any | None = ..., + autospec: Any | None = ..., + new_callable: None = ..., + **kwargs: Any, + ) -> _patch_pass_arg[MagicMock | AsyncMock]: ... @overload @staticmethod def object( @@ -328,7 +342,7 @@ class _patcher: create: bool = ..., spec_set: Any | None = ..., autospec: Any | None = ..., - new_callable: Any | None = ..., + new_callable: Callable[..., Any] | None = ..., **kwargs: Any, ) -> _patch[_T]: ... @overload @@ -341,9 +355,22 @@ class _patcher: create: bool = ..., spec_set: Any | None = ..., autospec: Any | None = ..., - new_callable: Any | None = ..., + new_callable: Callable[..., _T], + **kwargs: Any, + ) -> _patch_pass_arg[_T]: ... + @overload + @staticmethod + def object( + target: Any, + attribute: str, + *, + spec: Any | None = ..., + create: bool = ..., + spec_set: Any | None = ..., + autospec: Any | None = ..., + new_callable: None = ..., **kwargs: Any, - ) -> _patch[MagicMock | AsyncMock]: ... + ) -> _patch_pass_arg[MagicMock | AsyncMock]: ... @staticmethod def multiple( target: Any, diff --git a/mypy/typeshed/stdlib/unittest/runner.pyi b/mypy/typeshed/stdlib/unittest/runner.pyi index 783764464a53c..f76771f55e131 100644 --- a/mypy/typeshed/stdlib/unittest/runner.pyi +++ b/mypy/typeshed/stdlib/unittest/runner.pyi @@ -4,15 +4,17 @@ import unittest.result import unittest.suite from _typeshed import SupportsFlush, SupportsWrite from collections.abc import Callable, Iterable -from typing import Any, Generic, Protocol, TypeVar +from typing import Any, Generic, Protocol, TypeVar, type_check_only from typing_extensions import Never, TypeAlias from warnings import _ActionKind _ResultClassType: TypeAlias = Callable[[_TextTestStream, bool, int], TextTestResult[Any]] +@type_check_only class _SupportsWriteAndFlush(SupportsWrite[str], SupportsFlush, Protocol): ... # All methods used by unittest.runner.TextTestResult's stream +@type_check_only class _TextTestStream(_SupportsWriteAndFlush, Protocol): def writeln(self, arg: str | None = None, /) -> None: ... diff --git a/mypy/typeshed/stdlib/urllib/request.pyi b/mypy/typeshed/stdlib/urllib/request.pyi index d8fc5e0d8f48d..b99577c1cf71b 100644 --- a/mypy/typeshed/stdlib/urllib/request.pyi +++ b/mypy/typeshed/stdlib/urllib/request.pyi @@ -6,7 +6,7 @@ from email.message import Message from http.client import HTTPConnection, HTTPMessage, HTTPResponse from http.cookiejar import CookieJar from re import Pattern -from typing import IO, Any, ClassVar, NoReturn, Protocol, TypeVar, overload +from typing import IO, Any, ClassVar, NoReturn, Protocol, TypeVar, overload, type_check_only from typing_extensions import TypeAlias, deprecated from urllib.error import HTTPError as HTTPError from urllib.response import addclosehook, addinfourl @@ -237,6 +237,7 @@ class ProxyDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler): auth_header: ClassVar[str] # undocumented def http_error_407(self, req: Request, fp: IO[bytes], code: int, msg: str, headers: HTTPMessage) -> _UrlopenRet | None: ... +@type_check_only class _HTTPConnectionProtocol(Protocol): def __call__( self, diff --git a/mypy/typeshed/stdlib/uuid.pyi b/mypy/typeshed/stdlib/uuid.pyi index 99ac6eb223ef3..0aa2f76d40cc5 100644 --- a/mypy/typeshed/stdlib/uuid.pyi +++ b/mypy/typeshed/stdlib/uuid.pyi @@ -21,7 +21,7 @@ class UUID: int: builtins.int | None = None, version: builtins.int | None = None, *, - is_safe: SafeUUID = ..., + is_safe: SafeUUID = SafeUUID.unknown, ) -> None: ... @property def is_safe(self) -> SafeUUID: ... diff --git a/mypy/typeshed/stdlib/xml/dom/minidom.pyi b/mypy/typeshed/stdlib/xml/dom/minidom.pyi index ab2ef87e38a85..b9da9f3558ff3 100644 --- a/mypy/typeshed/stdlib/xml/dom/minidom.pyi +++ b/mypy/typeshed/stdlib/xml/dom/minidom.pyi @@ -3,7 +3,7 @@ from _collections_abc import dict_keys, dict_values from _typeshed import Incomplete, ReadableBuffer, SupportsRead, SupportsWrite from collections.abc import Iterable, Sequence from types import TracebackType -from typing import Any, ClassVar, Generic, Literal, NoReturn, Protocol, TypeVar, overload +from typing import Any, ClassVar, Generic, Literal, NoReturn, Protocol, TypeVar, overload, type_check_only from typing_extensions import Self, TypeAlias from xml.dom.minicompat import EmptyNodeList, NodeList from xml.dom.xmlbuilder import DocumentLS, DOMImplementationLS @@ -40,9 +40,11 @@ _ImportableNodeVar = TypeVar( | Notation, ) +@type_check_only class _DOMErrorHandler(Protocol): def handleError(self, error: Exception) -> bool: ... +@type_check_only class _UserDataHandler(Protocol): def handle(self, operation: int, key: str, data: Any, src: Node, dst: Node) -> None: ... diff --git a/mypy/typeshed/stdlib/xml/etree/ElementInclude.pyi b/mypy/typeshed/stdlib/xml/etree/ElementInclude.pyi index 8f20ee15a14e5..fd829fdaa5ffc 100644 --- a/mypy/typeshed/stdlib/xml/etree/ElementInclude.pyi +++ b/mypy/typeshed/stdlib/xml/etree/ElementInclude.pyi @@ -1,7 +1,8 @@ from _typeshed import FileDescriptorOrPath -from typing import Final, Literal, Protocol, overload +from typing import Final, Literal, Protocol, overload, type_check_only from xml.etree.ElementTree import Element +@type_check_only class _Loader(Protocol): @overload def __call__(self, href: FileDescriptorOrPath, parse: Literal["xml"], encoding: str | None = None) -> Element: ... diff --git a/mypy/typeshed/stdlib/xml/etree/ElementTree.pyi b/mypy/typeshed/stdlib/xml/etree/ElementTree.pyi index 4c55a1a7452ef..1d7e1725dd8ee 100644 --- a/mypy/typeshed/stdlib/xml/etree/ElementTree.pyi +++ b/mypy/typeshed/stdlib/xml/etree/ElementTree.pyi @@ -335,6 +335,7 @@ class C14NWriterTarget: # The target type is tricky, because the implementation doesn't # require any particular attribute to be present. This documents the attributes # that can be present, but uncommenting any of them would require them. +@type_check_only class _Target(Protocol): # start: Callable[str, dict[str, str], Any] | None # end: Callable[[str], Any] | None diff --git a/mypy/typeshed/stdlib/xml/sax/__init__.pyi b/mypy/typeshed/stdlib/xml/sax/__init__.pyi index ebe92d28c74d9..5a82b48c1e19d 100644 --- a/mypy/typeshed/stdlib/xml/sax/__init__.pyi +++ b/mypy/typeshed/stdlib/xml/sax/__init__.pyi @@ -1,7 +1,7 @@ import sys from _typeshed import ReadableBuffer, StrPath, SupportsRead, _T_co from collections.abc import Iterable -from typing import Protocol +from typing import Protocol, type_check_only from typing_extensions import TypeAlias from xml.sax._exceptions import ( SAXException as SAXException, @@ -13,6 +13,7 @@ from xml.sax._exceptions import ( from xml.sax.handler import ContentHandler as ContentHandler, ErrorHandler as ErrorHandler from xml.sax.xmlreader import InputSource as InputSource, XMLReader +@type_check_only class _SupportsReadClose(SupportsRead[_T_co], Protocol[_T_co]): def close(self) -> None: ... diff --git a/mypy/typeshed/stdlib/xmlrpc/client.pyi b/mypy/typeshed/stdlib/xmlrpc/client.pyi index 6cc4361f4a096..42420ee85848f 100644 --- a/mypy/typeshed/stdlib/xmlrpc/client.pyi +++ b/mypy/typeshed/stdlib/xmlrpc/client.pyi @@ -6,9 +6,10 @@ from collections.abc import Callable, Iterable, Mapping from datetime import datetime from io import BytesIO from types import TracebackType -from typing import Any, ClassVar, Final, Literal, Protocol, overload +from typing import Any, ClassVar, Final, Literal, Protocol, overload, type_check_only from typing_extensions import Self, TypeAlias +@type_check_only class _SupportsTimeTuple(Protocol): def timetuple(self) -> time.struct_time: ... diff --git a/mypy/typeshed/stdlib/xmlrpc/server.pyi b/mypy/typeshed/stdlib/xmlrpc/server.pyi index 5f497aa7190e9..286aaf980fbf5 100644 --- a/mypy/typeshed/stdlib/xmlrpc/server.pyi +++ b/mypy/typeshed/stdlib/xmlrpc/server.pyi @@ -4,28 +4,34 @@ import socketserver from _typeshed import ReadableBuffer from collections.abc import Callable, Iterable, Mapping from re import Pattern -from typing import Any, ClassVar, Protocol +from typing import Any, ClassVar, Protocol, type_check_only from typing_extensions import TypeAlias from xmlrpc.client import Fault, _Marshallable # The dispatch accepts anywhere from 0 to N arguments, no easy way to allow this in mypy +@type_check_only class _DispatchArity0(Protocol): def __call__(self) -> _Marshallable: ... +@type_check_only class _DispatchArity1(Protocol): def __call__(self, arg1: _Marshallable, /) -> _Marshallable: ... +@type_check_only class _DispatchArity2(Protocol): def __call__(self, arg1: _Marshallable, arg2: _Marshallable, /) -> _Marshallable: ... +@type_check_only class _DispatchArity3(Protocol): def __call__(self, arg1: _Marshallable, arg2: _Marshallable, arg3: _Marshallable, /) -> _Marshallable: ... +@type_check_only class _DispatchArity4(Protocol): def __call__( self, arg1: _Marshallable, arg2: _Marshallable, arg3: _Marshallable, arg4: _Marshallable, / ) -> _Marshallable: ... +@type_check_only class _DispatchArityN(Protocol): def __call__(self, *args: _Marshallable) -> _Marshallable: ... diff --git a/mypy/typeshed/stdlib/zipfile/__init__.pyi b/mypy/typeshed/stdlib/zipfile/__init__.pyi index 27c1ef0246c7a..73e3a92fd0e29 100644 --- a/mypy/typeshed/stdlib/zipfile/__init__.pyi +++ b/mypy/typeshed/stdlib/zipfile/__init__.pyi @@ -5,7 +5,7 @@ from collections.abc import Callable, Iterable, Iterator from io import TextIOWrapper from os import PathLike from types import TracebackType -from typing import IO, Final, Literal, Protocol, overload +from typing import IO, Final, Literal, Protocol, overload, type_check_only from typing_extensions import Self, TypeAlias __all__ = [ @@ -41,6 +41,7 @@ error = BadZipfile class LargeZipFile(Exception): ... +@type_check_only class _ZipStream(Protocol): def read(self, n: int, /) -> bytes: ... # The following methods are optional: @@ -49,11 +50,13 @@ class _ZipStream(Protocol): # def seek(self, n: int, /) -> object: ... # Stream shape as required by _EndRecData() and _EndRecData64(). +@type_check_only class _SupportsReadSeekTell(Protocol): def read(self, n: int = ..., /) -> bytes: ... def seek(self, cookie: int, whence: int, /) -> object: ... def tell(self) -> int: ... +@type_check_only class _ClosableZipStream(_ZipStream, Protocol): def close(self) -> object: ... @@ -93,18 +96,23 @@ class ZipExtFile(io.BufferedIOBase): def read1(self, n: int | None) -> bytes: ... # type: ignore[override] def seek(self, offset: int, whence: int = 0) -> int: ... +@type_check_only class _Writer(Protocol): def write(self, s: str, /) -> object: ... +@type_check_only class _ZipReadable(Protocol): def seek(self, offset: int, whence: int = 0, /) -> int: ... def read(self, n: int = -1, /) -> bytes: ... +@type_check_only class _ZipTellable(Protocol): def tell(self) -> int: ... +@type_check_only class _ZipReadableTellable(_ZipReadable, _ZipTellable, Protocol): ... +@type_check_only class _ZipWritable(Protocol): def flush(self) -> None: ... def close(self) -> None: ... @@ -254,9 +262,6 @@ class ZipFile: ) -> None: ... if sys.version_info >= (3, 11): def mkdir(self, zinfo_or_directory_name: str | ZipInfo, mode: int = 0o777) -> None: ... - if sys.version_info >= (3, 14): - @property - def data_offset(self) -> int | None: ... def __del__(self) -> None: ... diff --git a/mypy/typeshed/stdlib/zoneinfo/_common.pyi b/mypy/typeshed/stdlib/zoneinfo/_common.pyi index a2f29f2d14f08..e6d2d83caac18 100644 --- a/mypy/typeshed/stdlib/zoneinfo/_common.pyi +++ b/mypy/typeshed/stdlib/zoneinfo/_common.pyi @@ -1,6 +1,7 @@ import io -from typing import Any, Protocol +from typing import Any, Protocol, type_check_only +@type_check_only class _IOBytes(Protocol): def read(self, size: int, /) -> bytes: ... def seek(self, size: int, whence: int = ..., /) -> Any: ... From 8f48f1baf5dcdd0c08f2a3de2235ad587a677ef5 Mon Sep 17 00:00:00 2001 From: Emma Smith Date: Mon, 4 Aug 2025 08:34:53 -0700 Subject: [PATCH 144/424] Fix `--package-root` tests for Windows and Python 3.13+ (#19583) This PR should fix the rest of the test failures in https://github.com/python/mypy/pull/19545. A change to `os.path.relpath` in 3.13 seems to have broken the handling of Windows paths beginning with `\\`. To resolve this issue, we don't split the drive letter off of the path and instead verify the path is on the current drive. If it isn't it will never resolve to the package root because that must be on the same drive as the CWD: https://github.com/python/mypy/blob/5b03024e829940cf3c3e3d99fc6625f569d02728/mypy/main.py#L1571-L1572 Keeping the drive letter allows relpath to properly generate a relative path and make the tests pass. I need to investigate if the relpath change is a regression in CPython. --- mypy/fscache.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mypy/fscache.py b/mypy/fscache.py index 8251f4bd94880..240370159fff6 100644 --- a/mypy/fscache.py +++ b/mypy/fscache.py @@ -117,7 +117,12 @@ def init_under_package_root(self, path: str) -> bool: if not stat.S_ISDIR(st.st_mode): return False ok = False - drive, path = os.path.splitdrive(path) # Ignore Windows drive name + + # skip if on a different drive + current_drive, _ = os.path.splitdrive(os.getcwd()) + drive, _ = os.path.splitdrive(path) + if drive != current_drive: + return False if os.path.isabs(path): path = os.path.relpath(path) path = os.path.normpath(path) From 64dff4273d7b7140d66fe947b598005be3fd0a9c Mon Sep 17 00:00:00 2001 From: Emma Smith Date: Mon, 4 Aug 2025 08:40:29 -0700 Subject: [PATCH 145/424] [mypyc] Fix async mypyc tests on Windows (#19578) This is part of fixing the failed tests in https://github.com/python/mypy/pull/19545 The async tests previously used many invocations of `asyncio.run`, which likely caused issues with event loop management. The documentation for `asyncio.run` states: > This function cannot be called when another asyncio event loop is running in the same thread. ... > This function should be used as a main entry point for asyncio programs, and should ideally only be called once. Calling `asyncio.run` multiple times could cause the test processes to hang for strange event loop reasons. This commit converts most test cases to be run in a single event loop managed by the default driver, which is now async aware. Not all tests could be converted, e.g. the test that runs an async function in a sync context. However, the test suite does succeed with these changes, and these tests can be further modified if needed. --- mypyc/test-data/driver/driver.py | 9 +- mypyc/test-data/run-async.test | 394 ++++++++++++++----------------- 2 files changed, 185 insertions(+), 218 deletions(-) diff --git a/mypyc/test-data/driver/driver.py b/mypyc/test-data/driver/driver.py index 1ec1c48dfb751..395be6e1630e4 100644 --- a/mypyc/test-data/driver/driver.py +++ b/mypyc/test-data/driver/driver.py @@ -9,6 +9,10 @@ import sys import native +import asyncio +import inspect + +evloop = asyncio.new_event_loop() failures = [] tests_run = 0 @@ -18,7 +22,10 @@ test_func = getattr(native, name) tests_run += 1 try: - test_func() + if inspect.iscoroutinefunction(test_func): + evloop.run_until_complete(test_func) + else: + test_func() except Exception as e: failures.append((name, sys.exc_info())) diff --git a/mypyc/test-data/run-async.test b/mypyc/test-data/run-async.test index f1ec7e8f85e08..a1112e9646712 100644 --- a/mypyc/test-data/run-async.test +++ b/mypyc/test-data/run-async.test @@ -22,12 +22,12 @@ async def f2() -> int: x += i + await f() + await g() return x -def test_simple_call() -> None: - result = asyncio.run(f()) +async def test_simple_call() -> None: + result = await f() assert result == 3 -def test_multiple_awaits_in_expression() -> None: - result = asyncio.run(f2()) +async def test_multiple_awaits_in_expression() -> None: + result = await f2() assert result == 9 class MyError(Exception): @@ -61,17 +61,17 @@ async def exc6() -> int: return 3 return 4 -def test_exception() -> None: +async def test_exception() -> None: with assertRaises(MyError): - asyncio.run(exc1()) + await exc1() with assertRaises(MyError): - asyncio.run(exc2()) + await exc2() with assertRaises(MyError): - asyncio.run(exc3()) + await exc3() with assertRaises(MyError): - asyncio.run(exc4()) - assert asyncio.run(exc5()) == 3 - assert asyncio.run(exc6()) == 3 + await exc4() + assert await exc5() == 3 + assert await exc6() == 3 async def indirect_call(x: int, c: Callable[[int], Awaitable[int]]) -> int: return await c(x) @@ -92,20 +92,20 @@ async def ident(x: float, err: bool = False) -> float: raise MyError() return x + float("0.0") -def test_indirect_call() -> None: - assert asyncio.run(indirect_call(3, inc)) == 4 +async def test_indirect_call() -> None: + assert await indirect_call(3, inc) == 4 with assertRaises(MyError): - asyncio.run(indirect_call_2(exc1())) + await indirect_call_2(exc1()) - assert asyncio.run(indirect_call_3(ident(2.0))) == 3.0 - assert asyncio.run(indirect_call_3(ident(-113.0))) == -112.0 - assert asyncio.run(indirect_call_3(ident(-114.0))) == -113.0 + assert await indirect_call_3(ident(2.0)) == 3.0 + assert await indirect_call_3(ident(-113.0)) == -112.0 + assert await indirect_call_3(ident(-114.0)) == -113.0 with assertRaises(MyError): - asyncio.run(indirect_call_3(ident(1.0, True))) + await indirect_call_3(ident(1.0, True)) with assertRaises(MyError): - asyncio.run(indirect_call_3(ident(-113.0, True))) + await indirect_call_3(ident(-113.0, True)) class C: def __init__(self, n: int) -> None: @@ -125,15 +125,13 @@ async def method_call_exception() -> int: c = C(5) return await c.add(3, err=True) -def test_async_method_call() -> None: - assert asyncio.run(method_call(3)) == 8 +async def test_async_method_call() -> None: + assert await method_call(3) == 8 with assertRaises(MyError): - asyncio.run(method_call_exception()) + await method_call_exception() [file asyncio/__init__.pyi] async def sleep(t: float) -> None: ... -# eh, we could use the real type but it doesn't seem important -def run(x: object) -> object: ... [typing fixtures/typing-full.pyi] @@ -159,16 +157,16 @@ async def branch_await_not() -> int: return 3 return 2 -def test_branch() -> None: - assert asyncio.run(branch_await()) == 3 - assert asyncio.run(branch_await_not()) == 2 +async def test_branch() -> None: + assert await branch_await() == 3 + assert await branch_await_not() == 2 async def assign_multi() -> int: _, x = int(), await one() return x + 1 -def test_assign_multi() -> None: - assert asyncio.run(assign_multi()) == 2 +async def test_assign_multi() -> None: + assert await assign_multi() == 2 class C: def __init__(self, s: str) -> None: @@ -188,8 +186,8 @@ async def concat(s: str, t: str) -> str: async def set_attr(s: str) -> None: (await make_c("xyz")).s = await concat(s, "!") -def test_set_attr() -> None: - asyncio.run(set_attr("foo")) # Just check that it compiles and runs +async def test_set_attr() -> None: + await set_attr("foo") # Just check that it compiles and runs def concat2(x: str, y: str) -> str: return x + y @@ -200,15 +198,15 @@ async def call1(s: str) -> str: async def call2(s: str) -> str: return await concat(str(int()), await concat(s, "b")) -def test_call() -> None: - assert asyncio.run(call1("foo")) == "0fooa" - assert asyncio.run(call2("foo")) == "0foob" +async def test_call() -> None: + assert await call1("foo") == "0fooa" + assert await call2("foo") == "0foob" async def method_call(s: str) -> str: return C("<").concat(await concat(s, ">")) -def test_method_call() -> None: - assert asyncio.run(method_call("foo")) == "" +async def test_method_call() -> None: + assert await method_call("foo") == "" class D: def __init__(self, a: str, b: str) -> None: @@ -219,13 +217,11 @@ async def construct(s: str) -> str: c = D(await concat(s, "!"), await concat(s, "?")) return c.a + c.b -def test_construct() -> None: - assert asyncio.run(construct("foo")) == "foo!foo?" +async def test_construct() -> None: + assert await construct("foo") == "foo!foo?" [file asyncio/__init__.pyi] async def sleep(t: float) -> None: ... -# eh, we could use the real type but it doesn't seem important -def run(x: object) -> object: ... [typing fixtures/typing-full.pyi] @@ -361,7 +357,7 @@ class ConManB: async def __aexit__(self, *exc: object): pass -async def x() -> None: +async def test_x() -> None: value = 2 async with ConMan() as f: value += f @@ -370,12 +366,6 @@ async def x() -> None: value += f assert value == 5, value -[typing fixtures/typing-full.pyi] -[file driver.py] -import asyncio -import native -asyncio.run(native.x()) - [case testRunAsyncSpecialCases] import asyncio @@ -385,8 +375,8 @@ async def t() -> tuple[int, str, str]: async def f() -> tuple[int, str, str]: return await t() -def test_tuple_return() -> None: - result = asyncio.run(f()) +async def test_tuple_return() -> None: + result = await f() assert result == (1, "x", "y") async def e() -> ValueError: @@ -395,14 +385,12 @@ async def e() -> ValueError: async def g() -> ValueError: return await e() -def test_exception_return() -> None: - result = asyncio.run(g()) +async def test_exception_return() -> None: + result = await g() assert isinstance(result, ValueError) [file asyncio/__init__.pyi] async def sleep(t: float) -> None: ... -# eh, we could use the real type but it doesn't seem important -def run(x: object) -> object: ... [typing fixtures/typing-full.pyi] @@ -410,15 +398,15 @@ def run(x: object) -> object: ... import asyncio import gc -def assert_no_leaks(fn, max_new): +async def assert_no_leaks(fn, max_new): # Warm-up, in case asyncio allocates something on first use - asyncio.run(fn()) + await fn() gc.collect() old_objs = gc.get_objects() for i in range(10): - asyncio.run(fn()) + await fn() gc.collect() new_objs = gc.get_objects() @@ -438,8 +426,8 @@ async def foo(n: int) -> str: s = await concat_one(s) return s -def test_trivial() -> None: - assert_no_leaks(lambda: foo(1000), 5) +async def test_trivial() -> None: + await assert_no_leaks(lambda: foo(1000), 5) async def make_list(a: list[int]) -> list[int]: await concat_one("foobar") @@ -456,8 +444,8 @@ async def bar(n: int) -> None: for i in range(n): await spill() -def test_spilled() -> None: - assert_no_leaks(lambda: bar(40), 2) +async def test_spilled() -> None: + await assert_no_leaks(lambda: bar(80), 2) async def raise_deep(n: int) -> str: if n == 0: @@ -484,8 +472,8 @@ async def exc(n: int) -> list[str]: a.append(str(int() + 5)) return a -def test_exception() -> None: - assert_no_leaks(lambda: exc(50), 2) +async def test_exception() -> None: + await assert_no_leaks(lambda: exc(50), 2) class C: def __init__(self, s: str) -> None: @@ -507,11 +495,10 @@ async def stolen(n: int) -> int: assert s == str(i + 2) + "1" return n -def test_stolen() -> None: - assert_no_leaks(lambda: stolen(100), 2) +async def test_stolen() -> None: + await assert_no_leaks(lambda: stolen(200), 2) [file asyncio/__init__.pyi] -def run(x: object) -> object: ... async def sleep(t: float) -> None: ... [case testRunAsyncMiscTypesInEnvironment] @@ -559,8 +546,8 @@ async def float_ops(x: float) -> float: n = float("0.5") + await inc_float(n) return n -def test_float() -> None: - assert asyncio.run(float_ops(2.5)) == 5.0 +async def test_float() -> None: + assert await float_ops(2.5) == 5.0 async def i64_ops(x: i64) -> i64: n = x @@ -568,8 +555,8 @@ async def i64_ops(x: i64) -> i64: n = i64("1") + await inc_i64(n) return n -def test_i64() -> None: - assert asyncio.run(i64_ops(2)) == 5 +async def test_i64() -> None: + assert await i64_ops(2) == 5 async def i32_ops(x: i32) -> i32: n = x @@ -577,8 +564,8 @@ async def i32_ops(x: i32) -> i32: n = i32("1") + await inc_i32(n) return n -def test_i32() -> None: - assert asyncio.run(i32_ops(3)) == 6 +async def test_i32() -> None: + assert await i32_ops(3) == 6 async def i16_ops(x: i16) -> i16: n = x @@ -586,8 +573,8 @@ async def i16_ops(x: i16) -> i16: n = i16("1") + await inc_i16(n) return n -def test_i16() -> None: - assert asyncio.run(i16_ops(4)) == 7 +async def test_i16() -> None: + assert await i16_ops(4) == 7 async def u8_ops(x: u8) -> u8: n = x @@ -595,8 +582,8 @@ async def u8_ops(x: u8) -> u8: n = u8("1") + await inc_u8(n) return n -def test_u8() -> None: - assert asyncio.run(u8_ops(5)) == 8 +async def test_u8() -> None: + assert await u8_ops(5) == 8 async def tuple_ops(x: tuple[i64, float]) -> tuple[i64, float]: n = x @@ -604,8 +591,8 @@ async def tuple_ops(x: tuple[i64, float]) -> tuple[i64, float]: m = ((i64("1"), float("0.5")), await inc_tuple(n)) return m[1] -def test_tuple() -> None: - assert asyncio.run(tuple_ops((1, 2.5))) == (3, 5.5) +async def test_tuple() -> None: + assert await tuple_ops((1, 2.5)) == (3, 5.5) async def bool_ops(x: bool) -> bool: n = x @@ -613,9 +600,9 @@ async def bool_ops(x: bool) -> bool: m = (bool("1"), await neg_bool(n)) return m[0] and m[1] -def test_bool() -> None: - assert asyncio.run(bool_ops(True)) is True - assert asyncio.run(bool_ops(False)) is False +async def test_bool() -> None: + assert await bool_ops(True) is True + assert await bool_ops(False) is False [file asyncio/__init__.pyi] def run(x: object) -> object: ... @@ -648,8 +635,8 @@ async def async_def_contains_normal(x: int) -> int: a += nested((await inc(3)), (await inc(4))) return a -def test_async_def_contains_normal() -> None: - assert normal_contains_async_def(2) == (2 + 2 + 4 + 5) +async def test_async_def_contains_normal() -> None: + assert await async_def_contains_normal(2) == (2 + 2 + 4 + 5) async def async_def_contains_async_def(x: int) -> int: async def f(y: int) -> int: @@ -657,8 +644,8 @@ async def async_def_contains_async_def(x: int) -> int: return (await f(1)) + (await f(2)) -def test_async_def_contains_async_def() -> None: - assert asyncio.run(async_def_contains_async_def(3)) == (3 + 1 + 1 + 1) + (3 + 1 + 2 + 1) +async def test_async_def_contains_async_def() -> None: + assert await async_def_contains_async_def(3) == (3 + 1 + 1 + 1) + (3 + 1 + 2 + 1) async def async_def_contains_generator(x: int) -> tuple[int, int, int]: def gen(y: int) -> Iterator[int]: @@ -673,8 +660,8 @@ async def async_def_contains_generator(x: int) -> tuple[int, int, int]: return res -def test_async_def_contains_generator() -> None: - assert asyncio.run(async_def_contains_generator(3)) == (13, 4, 7) +async def test_async_def_contains_generator() -> None: + assert await async_def_contains_generator(3) == (13, 4, 7) def generator_contains_async_def(x: int) -> Iterator[int]: async def f(y: int) -> int: @@ -696,8 +683,8 @@ async def async_def_contains_two_nested_functions(x: int, y: int) -> tuple[int, return (await inc(f(3))), (await inc(g(4, 10))) -def test_async_def_contains_two_nested_functions() -> None: - assert asyncio.run(async_def_contains_two_nested_functions(5, 7)) == ( +async def test_async_def_contains_two_nested_functions() -> None: + assert await async_def_contains_two_nested_functions(5, 7) == ( (5 + 3 + 1), (7 + 4 + 10 + 1) ) @@ -714,8 +701,8 @@ async def async_def_contains_overloaded_async_def(n: int) -> int: return (await f(n)) + 1 -def test_async_def_contains_overloaded_async_def() -> None: - assert asyncio.run(async_def_contains_overloaded_async_def(5)) == 6 +async def test_async_def_contains_overloaded_async_def() -> None: + assert await async_def_contains_overloaded_async_def(5) == 6 T = TypeVar("T") @@ -730,8 +717,9 @@ async def async_def_contains_decorated_async_def(n: int) -> int: return (await f(n)) + 1 -def test_async_def_contains_decorated_async_def() -> None: - assert asyncio.run(async_def_contains_decorated_async_def(7)) == 10 +async def test_async_def_contains_decorated_async_def() -> None: + assert await async_def_contains_decorated_async_def(7) == 10 + [file asyncio/__init__.pyi] def run(x: object) -> object: ... @@ -742,10 +730,7 @@ def run(x: object) -> object: ... # - at least one of those does not explicitly return # - the non-returning path is taken at runtime -import asyncio - - -async def test_mixed_return(b: bool) -> bool: +async def mixed_return(b: bool) -> bool: try: if b: return b @@ -754,33 +739,21 @@ async def test_mixed_return(b: bool) -> bool: return b -async def test_run() -> None: +async def test_async_try_finally_mixed_return() -> None: # Test return path - result1 = await test_mixed_return(True) + result1 = await mixed_return(True) assert result1 == True # Test non-return path - result2 = await test_mixed_return(False) + result2 = await mixed_return(False) assert result2 == False - -def test_async_try_finally_mixed_return() -> None: - asyncio.run(test_run()) - -[file driver.py] -from native import test_async_try_finally_mixed_return -test_async_try_finally_mixed_return() - -[file asyncio/__init__.pyi] -def run(x: object) -> object: ... - [case testAsyncWithMixedReturn] # This used to raise an AttributeError, related to # testAsyncTryFinallyMixedReturn, this is essentially # a far more extensive version of that test surfacing # more edge cases -import asyncio from typing import Optional, Type, Literal @@ -798,14 +771,14 @@ class AsyncContextManager: # Simple async functions (generator class) -async def test_gen_1(b: bool) -> bool: +async def gen_1(b: bool) -> bool: async with AsyncContextManager(): if b: return b return b -async def test_gen_2(b: bool) -> bool: +async def gen_2(b: bool) -> bool: async with AsyncContextManager(): if b: return b @@ -813,7 +786,7 @@ async def test_gen_2(b: bool) -> bool: return b -async def test_gen_3(b: bool) -> bool: +async def gen_3(b: bool) -> bool: async with AsyncContextManager(): if b: return b @@ -822,7 +795,7 @@ async def test_gen_3(b: bool) -> bool: return b -async def test_gen_4(b: bool) -> bool: +async def gen_4(b: bool) -> bool: ret: bool async with AsyncContextManager(): if b: @@ -832,7 +805,7 @@ async def test_gen_4(b: bool) -> bool: return ret -async def test_gen_5(i: int) -> int: +async def gen_5(i: int) -> int: async with AsyncContextManager(): if i == 1: return i @@ -843,7 +816,7 @@ async def test_gen_5(i: int) -> int: return i -async def test_gen_6(i: int) -> int: +async def gen_6(i: int) -> int: async with AsyncContextManager(): if i == 1: return i @@ -854,7 +827,7 @@ async def test_gen_6(i: int) -> int: return i -async def test_gen_7(i: int) -> int: +async def gen_7(i: int) -> int: async with AsyncContextManager(): if i == 1: return i @@ -867,7 +840,7 @@ async def test_gen_7(i: int) -> int: # Async functions with nested functions (environment class) -async def test_env_1(b: bool) -> bool: +async def env_1(b: bool) -> bool: def helper() -> bool: return True @@ -877,7 +850,7 @@ async def test_env_1(b: bool) -> bool: return b -async def test_env_2(b: bool) -> bool: +async def env_2(b: bool) -> bool: def helper() -> bool: return True @@ -888,7 +861,7 @@ async def test_env_2(b: bool) -> bool: return b -async def test_env_3(b: bool) -> bool: +async def env_3(b: bool) -> bool: def helper() -> bool: return True @@ -900,7 +873,7 @@ async def test_env_3(b: bool) -> bool: return b -async def test_env_4(b: bool) -> bool: +async def env_4(b: bool) -> bool: def helper() -> bool: return True @@ -913,7 +886,7 @@ async def test_env_4(b: bool) -> bool: return ret -async def test_env_5(i: int) -> int: +async def env_5(i: int) -> int: def helper() -> int: return 1 @@ -927,7 +900,7 @@ async def test_env_5(i: int) -> int: return i -async def test_env_6(i: int) -> int: +async def env_6(i: int) -> int: def helper() -> int: return 1 @@ -941,7 +914,7 @@ async def test_env_6(i: int) -> int: return i -async def test_env_7(i: int) -> int: +async def env_7(i: int) -> int: def helper() -> int: return 1 @@ -956,87 +929,76 @@ async def test_env_7(i: int) -> int: return i -async def run_all_tests() -> None: +async def test_async_with_mixed_return() -> None: # Test simple async functions (generator class) - # test_env_1: mixed return/no-return - assert await test_gen_1(True) is True - assert await test_gen_1(False) is False - - # test_gen_2: all branches return - assert await test_gen_2(True) is True - assert await test_gen_2(False) is False - - # test_gen_3: mixed return/pass - assert await test_gen_3(True) is True - assert await test_gen_3(False) is False - - # test_gen_4: no returns in async with - assert await test_gen_4(True) is True - assert await test_gen_4(False) is False - - # test_gen_5: multiple branches, some return - assert await test_gen_5(0) == 0 - assert await test_gen_5(1) == 1 - assert await test_gen_5(2) == 2 - assert await test_gen_5(3) == 3 - - # test_gen_6: all explicit branches return, implicit fallthrough - assert await test_gen_6(0) == 0 - assert await test_gen_6(1) == 1 - assert await test_gen_6(2) == 2 - assert await test_gen_6(3) == 3 - - # test_gen_7: all branches return including else - assert await test_gen_7(0) == 0 - assert await test_gen_7(1) == 1 - assert await test_gen_7(2) == 2 - assert await test_gen_7(3) == 3 + # env_1: mixed return/no-return + assert await gen_1(True) is True + assert await gen_1(False) is False + + # gen_2: all branches return + assert await gen_2(True) is True + assert await gen_2(False) is False + + # gen_3: mixed return/pass + assert await gen_3(True) is True + assert await gen_3(False) is False + + # gen_4: no returns in async with + assert await gen_4(True) is True + assert await gen_4(False) is False + + # gen_5: multiple branches, some return + assert await gen_5(0) == 0 + assert await gen_5(1) == 1 + assert await gen_5(2) == 2 + assert await gen_5(3) == 3 + + # gen_6: all explicit branches return, implicit fallthrough + assert await gen_6(0) == 0 + assert await gen_6(1) == 1 + assert await gen_6(2) == 2 + assert await gen_6(3) == 3 + + # gen_7: all branches return including else + assert await gen_7(0) == 0 + assert await gen_7(1) == 1 + assert await gen_7(2) == 2 + assert await gen_7(3) == 3 # Test async functions with nested functions (environment class) - # test_env_1: mixed return/no-return - assert await test_env_1(True) is True - assert await test_env_1(False) is False - - # test_env_2: all branches return - assert await test_env_2(True) is True - assert await test_env_2(False) is False - - # test_env_3: mixed return/pass - assert await test_env_3(True) is True - assert await test_env_3(False) is False - - # test_env_4: no returns in async with - assert await test_env_4(True) is True - assert await test_env_4(False) is False - - # test_env_5: multiple branches, some return - assert await test_env_5(0) == 0 - assert await test_env_5(1) == 1 - assert await test_env_5(2) == 2 - assert await test_env_5(3) == 3 - - # test_env_6: all explicit branches return, implicit fallthrough - assert await test_env_6(0) == 0 - assert await test_env_6(1) == 1 - assert await test_env_6(2) == 2 - assert await test_env_6(3) == 3 - - # test_env_7: all branches return including else - assert await test_env_7(0) == 0 - assert await test_env_7(1) == 1 - assert await test_env_7(2) == 2 - assert await test_env_7(3) == 3 - - -def test_async_with_mixed_return() -> None: - asyncio.run(run_all_tests()) - -[file driver.py] -from native import test_async_with_mixed_return -test_async_with_mixed_return() - -[file asyncio/__init__.pyi] -def run(x: object) -> object: ... + # env_1: mixed return/no-return + assert await env_1(True) is True + assert await env_1(False) is False + + # env_2: all branches return + assert await env_2(True) is True + assert await env_2(False) is False + + # env_3: mixed return/pass + assert await env_3(True) is True + assert await env_3(False) is False + + # env_4: no returns in async with + assert await env_4(True) is True + assert await env_4(False) is False + + # env_5: multiple branches, some return + assert await env_5(0) == 0 + assert await env_5(1) == 1 + assert await env_5(2) == 2 + assert await env_5(3) == 3 + + # env_6: all explicit branches return, implicit fallthrough + assert await env_6(0) == 0 + assert await env_6(1) == 1 + assert await env_6(2) == 2 + assert await env_6(3) == 3 + + # env_7: all branches return including else + assert await env_7(0) == 0 + assert await env_7(1) == 1 + assert await env_7(2) == 2 + assert await env_7(3) == 3 [case testAsyncTryExceptFinallyAwait] import asyncio @@ -1127,49 +1089,48 @@ async def async_no_exception_with_await_in_finally() -> int: await asyncio.sleep(0) return 2 # Should not reach this -def test_async_try_except_finally_await() -> None: +async def test_async_try_except_finally_await() -> None: # Test 0: Simplest case - just try/finally with exception # Expected: ValueError propagates with assertRaises(ValueError): - asyncio.run(simple_try_finally_await()) + await simple_try_finally_await() # Test 1: Exception caught, not re-raised # Expected: return 2 (from except block) - result = asyncio.run(async_try_except_no_reraise()) + result = await async_try_except_no_reraise() assert result == 2, f"Expected 2, got {result}" # Test 2: Exception caught and re-raised # Expected: ValueError propagates with assertRaises(ValueError): - asyncio.run(async_try_except_reraise()) + await async_try_except_reraise() # Test 3: Exception caught, different exception raised # Expected: RuntimeError propagates with assertRaises(RuntimeError): - asyncio.run(async_try_except_raise_different()) + await async_try_except_raise_different() # Test 4: Try/except inside finally # Expected: ValueError propagates (outer exception) with assertRaises(ValueError): - asyncio.run(async_try_except_inside_finally()) + await async_try_except_inside_finally() # Test 5: Try/finally inside finally # Expected: RuntimeError propagates (inner error) with assertRaises(RuntimeError): - asyncio.run(async_try_finally_inside_finally()) + await async_try_finally_inside_finally() # Control case: No await in finally (should work correctly) with assertRaises(TestError): - asyncio.run(async_exception_no_await_in_finally()) + await async_exception_no_await_in_finally() # Test normal flow (no exception) # Expected: return 1 - result = asyncio.run(async_no_exception_with_await_in_finally()) + result = await async_no_exception_with_await_in_finally() assert result == 1, f"Expected 1, got {result}" [file asyncio/__init__.pyi] async def sleep(t: float) -> None: ... -def run(x: object) -> object: ... [case testAsyncContextManagerExceptionHandling] import asyncio @@ -1233,18 +1194,17 @@ async def test_exception_in_aexit() -> str: except Exception as e: return f"caught different exception: {type(e).__name__}" -def test_async_context_manager_exception_handling() -> None: +async def test_async_context_manager_exception_handling() -> None: # Test 1: Basic exception propagation - result = asyncio.run(test_basic_exception()) + result = await test_basic_exception() # Expected: "caught ValueError - correct!" assert result == "caught ValueError - correct!", f"Expected exception to propagate, got: {result}" # Test 2: Exception raised in __aexit__ replaces original exception - result = asyncio.run(test_exception_in_aexit()) + result = await test_exception_in_aexit() # Expected: "caught RuntimeError - correct!" # (The RuntimeError from __aexit__ should replace the ValueError) assert result == "caught RuntimeError - correct!", f"Expected RuntimeError from __aexit__, got: {result}" [file asyncio/__init__.pyi] async def sleep(t: float) -> None: ... -def run(x: object) -> object: ... From 3387d6fd9761cb39c952abe5251ee538bcdea598 Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Tue, 5 Aug 2025 02:23:45 +0200 Subject: [PATCH 146/424] Fix dict assignment to a wider context containing an incompatible typeddict of the same shape (#19592) Fixes #19590. Fixes #14991 (oops, I forgot I have already reported this...). When a typeddict context does not cover all available options, proceed with checking as usual against the whole context if none of the items matches in full despite being structurally compatible. --- mypy/checkexpr.py | 24 ++++++++++++++++-------- test-data/unit/check-typeddict.test | 18 ++++++++++++++++++ 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 1b10370b08cb4..6e0915179f90a 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -5350,9 +5350,9 @@ def visit_dict_expr(self, e: DictExpr) -> Type: # an error, but returns the TypedDict type that matches the literal it found # that would cause a second error when that TypedDict type is returned upstream # to avoid the second error, we always return TypedDict type that was requested - typeddict_contexts = self.find_typeddict_context(self.type_context[-1], e) + typeddict_contexts, exhaustive = self.find_typeddict_context(self.type_context[-1], e) if typeddict_contexts: - if len(typeddict_contexts) == 1: + if len(typeddict_contexts) == 1 and exhaustive: return self.check_typeddict_literal_in_context(e, typeddict_contexts[0]) # Multiple items union, check if at least one of them matches cleanly. for typeddict_context in typeddict_contexts: @@ -5363,7 +5363,8 @@ def visit_dict_expr(self, e: DictExpr) -> Type: self.chk.store_types(tmap) return ret_type # No item matched without an error, so we can't unambiguously choose the item. - self.msg.typeddict_context_ambiguous(typeddict_contexts, e) + if exhaustive: + self.msg.typeddict_context_ambiguous(typeddict_contexts, e) # fast path attempt dt = self.fast_dict_type(e) @@ -5425,22 +5426,29 @@ def visit_dict_expr(self, e: DictExpr) -> Type: def find_typeddict_context( self, context: Type | None, dict_expr: DictExpr - ) -> list[TypedDictType]: + ) -> tuple[list[TypedDictType], bool]: + """Extract `TypedDict` members of the enclosing context. + + Returns: + a 2-tuple, (found_candidates, is_exhaustive) + """ context = get_proper_type(context) if isinstance(context, TypedDictType): - return [context] + return [context], True elif isinstance(context, UnionType): items = [] + exhaustive = True for item in context.items: - item_contexts = self.find_typeddict_context(item, dict_expr) + item_contexts, item_exhaustive = self.find_typeddict_context(item, dict_expr) for item_context in item_contexts: if self.match_typeddict_call_with_dict( item_context, dict_expr.items, dict_expr ): items.append(item_context) - return items + exhaustive = exhaustive and item_exhaustive + return items, exhaustive # No TypedDict type in context. - return [] + return [], False def visit_lambda_expr(self, e: LambdaExpr) -> Type: """Type check lambda expression.""" diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index be5a6c655d8ca..34cae74d795b0 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -4289,3 +4289,21 @@ inputs: Sequence[Component] = [{ }] [builtins fixtures/dict.pyi] [typing fixtures/typing-typeddict.pyi] + +[case testTypedDictAssignableToWiderContext] +from typing import TypedDict, Union + +class TD(TypedDict): + x: int + +x: Union[TD, dict[str, str]] = {"x": "foo"} +y: Union[TD, dict[str, int]] = {"x": "foo"} # E: Dict entry 0 has incompatible type "str": "str"; expected "str": "int" + +def ok(d: Union[TD, dict[str, str]]) -> None: ... +ok({"x": "foo"}) + +def bad(d: Union[TD, dict[str, int]]) -> None: ... +bad({"x": "foo"}) # E: Dict entry 0 has incompatible type "str": "str"; expected "str": "int" + +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] From 83fed5ab1f307016e1fbb00527e2b6f88e9522b5 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 5 Aug 2025 14:26:17 +0100 Subject: [PATCH 147/424] Skip more method bodies in third-party libraries (#19586) A while ago we started stripping function bodies when checking third-party libraries. This PR pushes this idea further: * Tighten the check in `fastparse.py` to only consider `foo.bar` as possible self attribute definition. * Do not type-check bodies where we didn't find any `self` attribute _definitions_ during semantic analysis. * Skip method override checks in third-party libraries. In total this makes e.g. `mypy -c 'import torch'` ~10% faster. Surprisingly, this also has some visible impact on self-check. --- mypy/checker.py | 23 ++++++++++++++++++++--- mypy/fastparse.py | 2 +- mypy/nodes.py | 3 +++ mypy/semanal.py | 3 +++ 4 files changed, 27 insertions(+), 4 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 4d82214157549..68f9bd4c1383c 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -832,8 +832,10 @@ def check_overlapping_overloads(self, defn: OverloadedFuncDef) -> None: # At this point we should have set the impl already, and all remaining # items are decorators - if self.msg.errors.file in self.msg.errors.ignored_files or ( - self.is_typeshed_stub and self.options.test_env + if ( + self.options.ignore_errors + or self.msg.errors.file in self.msg.errors.ignored_files + or (self.is_typeshed_stub and self.options.test_env) ): # This is a little hacky, however, the quadratic check here is really expensive, this # method has no side effects, so we should skip it if we aren't going to report @@ -1444,7 +1446,19 @@ def check_func_def( # TODO: Find a way of working around this limitation if _is_empty_generator_function(item) or len(expanded) >= 2: self.binder.suppress_unreachable_warnings() - self.accept(item.body) + # When checking a third-party library, we can skip function body, + # if during semantic analysis we found that there are no attributes + # defined via self here. + if ( + not ( + self.options.ignore_errors + or self.msg.errors.file in self.msg.errors.ignored_files + ) + or self.options.preserve_asts + or not isinstance(defn, FuncDef) + or defn.has_self_attr_def + ): + self.accept(item.body) unreachable = self.binder.is_unreachable() if new_frame is not None: self.binder.pop_frame(True, 0) @@ -2127,6 +2141,9 @@ def check_method_override( Return a list of base classes which contain an attribute with the method name. """ + if self.options.ignore_errors or self.msg.errors.file in self.msg.errors.ignored_files: + # Method override checks may be expensive, so skip them in third-party libraries. + return None # Check against definitions in base classes. check_override_compatibility = ( defn.name not in ("__init__", "__new__", "__init_subclass__", "__post_init__") diff --git a/mypy/fastparse.py b/mypy/fastparse.py index bb71242182f19..0e1b66f0db59d 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -2234,7 +2234,7 @@ def visit_index_expr(self, e: IndexExpr) -> None: pass def visit_member_expr(self, e: MemberExpr) -> None: - if self.lvalue: + if self.lvalue and isinstance(e.expr, NameExpr): self.found = True diff --git a/mypy/nodes.py b/mypy/nodes.py index 6ffe579efe71f..d5fa4a79699e5 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -818,6 +818,7 @@ class FuncDef(FuncItem, SymbolNode, Statement): "original_def", "is_trivial_body", "is_trivial_self", + "has_self_attr_def", "is_mypy_only", # Present only when a function is decorated with @typing.dataclass_transform or similar "dataclass_transform_spec", @@ -856,6 +857,8 @@ def __init__( # the majority). In cases where self is not annotated and there are no Self # in the signature we can simply drop the first argument. self.is_trivial_self = False + # Keep track of functions where self attributes are defined. + self.has_self_attr_def = False # This is needed because for positional-only arguments the name is set to None, # but we sometimes still want to show it in error messages. if arguments: diff --git a/mypy/semanal.py b/mypy/semanal.py index ab9075cd06ce1..0aace6b1165aa 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -4570,6 +4570,9 @@ def analyze_member_lvalue( lval.node = v # TODO: should we also set lval.kind = MDEF? self.type.names[lval.name] = SymbolTableNode(MDEF, v, implicit=True) + for func in self.scope.functions: + if isinstance(func, FuncDef): + func.has_self_attr_def = True self.check_lvalue_validity(lval.node, lval) def is_self_member_ref(self, memberexpr: MemberExpr) -> bool: From d7753ef05393eaa9f7902ac5cb53ced30a0b7b2e Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 5 Aug 2025 14:27:43 +0100 Subject: [PATCH 148/424] Move Windows tests from oldest to newest version (#19545) This is mostly to speed-up CI, but also as we discussed with Jukka this may make more sense as we want to test new features on more platforms. --- .github/workflows/test.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 97fb7755563be..47f725170bd8b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,11 +37,6 @@ jobs: toxenv: py tox_extra_args: "-n 4" test_mypyc: true - - name: Test suite with py39-windows-64 - python: '3.9' - os: windows-latest - toxenv: py39 - tox_extra_args: "-n 4" - name: Test suite with py310-ubuntu python: '3.10' os: ubuntu-24.04-arm @@ -64,6 +59,11 @@ jobs: toxenv: py tox_extra_args: "-n 4" test_mypyc: true + - name: Test suite with py313-windows-64 + python: '3.13' + os: windows-latest + toxenv: py + tox_extra_args: "-n 4" - name: Test suite with py314-dev-ubuntu python: '3.14-dev' From db67fac952a390cf2cb533beb9bcce1cf15ce918 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 5 Aug 2025 17:05:54 +0100 Subject: [PATCH 149/424] Assorted niche optimizations (#19587) These are few random micro-optimizations (plus few correctness fixes I noticed in the process): * De-serializing large callables/overloads is very slaw because of `Enum.__call__()` * Large unions _without_ literals were forced through `make_simplified_union()` in code path that supposed to handle literals * Now that `SomeType.__eq__()` is called more often and we preserve more original types in `expand_type()` I added couple fast paths after gathering call stats there. In total this gives ~0.5% on self-check, but makes loading some numeric libraries from cache up to 10% faster. --- mypy/nodes.py | 8 +++++++- mypy/semanal.py | 2 ++ mypy/typeops.py | 14 ++++++++------ mypy/types.py | 25 +++++++++++++------------ 4 files changed, 30 insertions(+), 19 deletions(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index d5fa4a79699e5..9d5867c5371dd 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -918,7 +918,7 @@ def deserialize(cls, data: JsonDict) -> FuncDef: # NOTE: ret.info is set in the fixup phase. ret.arg_names = data["arg_names"] ret.original_first_arg = data.get("original_first_arg") - ret.arg_kinds = [ArgKind(x) for x in data["arg_kinds"]] + ret.arg_kinds = [ARG_KINDS[x] for x in data["arg_kinds"]] ret.abstract_status = data["abstract_status"] ret.dataclass_transform_spec = ( DataclassTransformSpec.deserialize(data["dataclass_transform_spec"]) @@ -2016,6 +2016,8 @@ def is_star(self) -> bool: ARG_STAR2: Final = ArgKind.ARG_STAR2 ARG_NAMED_OPT: Final = ArgKind.ARG_NAMED_OPT +ARG_KINDS: Final = (ARG_POS, ARG_OPT, ARG_STAR, ARG_NAMED, ARG_STAR2, ARG_NAMED_OPT) + class CallExpr(Expression): """Call expression. @@ -3491,6 +3493,8 @@ def update_tuple_type(self, typ: mypy.types.TupleType) -> None: self.special_alias = alias else: self.special_alias.target = alias.target + # Invalidate recursive status cache in case it was previously set. + self.special_alias._is_recursive = None def update_typeddict_type(self, typ: mypy.types.TypedDictType) -> None: """Update typeddict_type and special_alias as needed.""" @@ -3500,6 +3504,8 @@ def update_typeddict_type(self, typ: mypy.types.TypedDictType) -> None: self.special_alias = alias else: self.special_alias.target = alias.target + # Invalidate recursive status cache in case it was previously set. + self.special_alias._is_recursive = None def __str__(self) -> str: """Return a string representation of the type. diff --git a/mypy/semanal.py b/mypy/semanal.py index 0aace6b1165aa..99d1eb36e7887 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -5636,6 +5636,8 @@ def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None: existing.node.target = res existing.node.alias_tvars = alias_tvars updated = True + # Invalidate recursive status cache in case it was previously set. + existing.node._is_recursive = None else: # Otherwise just replace existing placeholder with type alias. existing.node = alias_node diff --git a/mypy/typeops.py b/mypy/typeops.py index 75213bd936744..13082225e5fda 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -63,6 +63,7 @@ flatten_nested_unions, get_proper_type, get_proper_types, + remove_dups, ) from mypy.typetraverser import TypeTraverserVisitor from mypy.typevars import fill_typevars @@ -995,7 +996,7 @@ def is_singleton_type(typ: Type) -> bool: return typ.is_singleton_type() -def try_expanding_sum_type_to_union(typ: Type, target_fullname: str) -> ProperType: +def try_expanding_sum_type_to_union(typ: Type, target_fullname: str) -> Type: """Attempts to recursively expand any enum Instances with the given target_fullname into a Union of all of its component LiteralTypes. @@ -1017,21 +1018,22 @@ class Status(Enum): typ = get_proper_type(typ) if isinstance(typ, UnionType): + # Non-empty enums cannot subclass each other so simply removing duplicates is enough. items = [ - try_expanding_sum_type_to_union(item, target_fullname) for item in typ.relevant_items() + try_expanding_sum_type_to_union(item, target_fullname) + for item in remove_dups(flatten_nested_unions(typ.relevant_items())) ] - return make_simplified_union(items, contract_literals=False) + return UnionType.make_union(items) if isinstance(typ, Instance) and typ.type.fullname == target_fullname: if typ.type.fullname == "builtins.bool": - items = [LiteralType(True, typ), LiteralType(False, typ)] - return make_simplified_union(items, contract_literals=False) + return UnionType([LiteralType(True, typ), LiteralType(False, typ)]) if typ.type.is_enum: items = [LiteralType(name, typ) for name in typ.type.enum_members] if not items: return typ - return make_simplified_union(items, contract_literals=False) + return UnionType.make_union(items) return typ diff --git a/mypy/types.py b/mypy/types.py index 029477c1d5c47..b4771b15f77ad 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -21,7 +21,7 @@ import mypy.nodes from mypy.bogus_type import Bogus -from mypy.nodes import ARG_POS, ARG_STAR, ARG_STAR2, INVARIANT, ArgKind, FakeInfo, SymbolNode +from mypy.nodes import ARG_KINDS, ARG_POS, ARG_STAR, ARG_STAR2, INVARIANT, ArgKind, SymbolNode from mypy.options import Options from mypy.state import state from mypy.util import IdMapper @@ -538,6 +538,10 @@ def __repr__(self) -> str: return self.raw_id.__repr__() def __eq__(self, other: object) -> bool: + # Although this call is not expensive (like UnionType or TypedDictType), + # most of the time we get the same object here, so add a fast path. + if self is other: + return True return ( isinstance(other, TypeVarId) and self.raw_id == other.raw_id @@ -1780,7 +1784,9 @@ def deserialize(cls, data: JsonDict) -> Parameters: assert data[".class"] == "Parameters" return Parameters( [deserialize_type(t) for t in data["arg_types"]], - [ArgKind(x) for x in data["arg_kinds"]], + # This is a micro-optimization until mypyc gets dedicated enum support. Otherwise, + # we would spend ~20% of types deserialization time in Enum.__call__(). + [ARG_KINDS[x] for x in data["arg_kinds"]], data["arg_names"], variables=[cast(TypeVarLikeType, deserialize_type(v)) for v in data["variables"]], imprecise_arg_kinds=data["imprecise_arg_kinds"], @@ -1797,7 +1803,7 @@ def __hash__(self) -> int: ) def __eq__(self, other: object) -> bool: - if isinstance(other, (Parameters, CallableType)): + if isinstance(other, Parameters): return ( self.arg_types == other.arg_types and self.arg_names == other.arg_names @@ -2210,15 +2216,9 @@ def with_normalized_var_args(self) -> Self: ) def __hash__(self) -> int: - # self.is_type_obj() will fail if self.fallback.type is a FakeInfo - if isinstance(self.fallback.type, FakeInfo): - is_type_obj = 2 - else: - is_type_obj = self.is_type_obj() return hash( ( self.ret_type, - is_type_obj, self.is_ellipsis_args, self.name, tuple(self.arg_types), @@ -2236,7 +2236,6 @@ def __eq__(self, other: object) -> bool: and self.arg_names == other.arg_names and self.arg_kinds == other.arg_kinds and self.name == other.name - and self.is_type_obj() == other.is_type_obj() and self.is_ellipsis_args == other.is_ellipsis_args and self.type_guard == other.type_guard and self.type_is == other.type_is @@ -2271,10 +2270,10 @@ def serialize(self) -> JsonDict: @classmethod def deserialize(cls, data: JsonDict) -> CallableType: assert data[".class"] == "CallableType" - # TODO: Set definition to the containing SymbolNode? + # The .definition link is set in fixup.py. return CallableType( [deserialize_type(t) for t in data["arg_types"]], - [ArgKind(x) for x in data["arg_kinds"]], + [ARG_KINDS[x] for x in data["arg_kinds"]], data["arg_names"], deserialize_type(data["ret_type"]), Instance.deserialize(data["fallback"]), @@ -2931,6 +2930,8 @@ def __hash__(self) -> int: def __eq__(self, other: object) -> bool: if not isinstance(other, UnionType): return NotImplemented + if self is other: + return True return frozenset(self.items) == frozenset(other.items) @overload From a6bfb2eed74e156f48daa8266945708c0a50366d Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Thu, 7 Aug 2025 22:05:57 -0700 Subject: [PATCH 150/424] Fix crash when using enable_error_code value of wrong type in pyproject.toml (#19494) Fixes #19491 If you give a string in toml, you get Invalid error code(s): whatever. However, if you give a value that doesn't mean a string in toml, you get a crash like TypeError: 'int' object is not iterable. I suspect this would also apply to many other pyproject.toml values if you set them wrong, because we pass many of them immediately into try_split, which tries to iterate them. I have added a (fairly minimal) test for this behavior. --------- Co-authored-by: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> --- mypy/config_parser.py | 29 +++++++++++++++++++++------ test-data/unit/cmdline.pyproject.test | 11 ++++++++++ 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/mypy/config_parser.py b/mypy/config_parser.py index 208c12adafbe7..5f08f342241ee 100644 --- a/mypy/config_parser.py +++ b/mypy/config_parser.py @@ -17,15 +17,15 @@ from collections.abc import Mapping, MutableMapping, Sequence from typing import Any, Callable, Final, TextIO, Union -from typing_extensions import TypeAlias as _TypeAlias +from typing_extensions import Never, TypeAlias from mypy import defaults from mypy.options import PER_MODULE_OPTIONS, Options -_CONFIG_VALUE_TYPES: _TypeAlias = Union[ +_CONFIG_VALUE_TYPES: TypeAlias = Union[ str, bool, int, float, dict[str, str], list[str], tuple[int, int] ] -_INI_PARSER_CALLABLE: _TypeAlias = Callable[[Any], _CONFIG_VALUE_TYPES] +_INI_PARSER_CALLABLE: TypeAlias = Callable[[Any], _CONFIG_VALUE_TYPES] class VersionTypeError(argparse.ArgumentTypeError): @@ -60,14 +60,31 @@ def parse_version(v: str | float) -> tuple[int, int]: return major, minor -def try_split(v: str | Sequence[str], split_regex: str = "[,]") -> list[str]: - """Split and trim a str or list of str into a list of str""" +def try_split(v: str | Sequence[str] | object, split_regex: str = ",") -> list[str]: + """Split and trim a str or sequence (eg: list) of str into a list of str. + If an element of the input is not str, a type error will be raised.""" + + def complain(x: object, additional_info: str = "") -> Never: + raise argparse.ArgumentTypeError( + f"Expected a list or a stringified version thereof, but got: '{x}', of type {type(x).__name__}.{additional_info}" + ) + if isinstance(v, str): items = [p.strip() for p in re.split(split_regex, v)] if items and items[-1] == "": items.pop(-1) return items - return [p.strip() for p in v] + elif isinstance(v, Sequence): + return [ + ( + p.strip() + if isinstance(p, str) + else complain(p, additional_info=" (As an element of the list.)") + ) + for p in v + ] + else: + complain(v) def validate_codes(codes: list[str]) -> list[str]: diff --git a/test-data/unit/cmdline.pyproject.test b/test-data/unit/cmdline.pyproject.test index f9691ba245f9c..68dfacb372fba 100644 --- a/test-data/unit/cmdline.pyproject.test +++ b/test-data/unit/cmdline.pyproject.test @@ -226,3 +226,14 @@ y: int = 'y' # E: Incompatible types in assignment (expression has type "str", # This should not trigger any errors, because it is not included: z: int = 'z' [out] + +[case testPyprojectTOMLSettingOfWrongType] +# cmd: mypy a.py +[file pyproject.toml] +\[tool.mypy] +enable_error_code = true +[file a.py] +x: int = 1 +[out] +pyproject.toml: [mypy]: enable_error_code: Expected a list or a stringified version thereof, but got: 'True', of type bool. +== Return code: 0 From de6e7426ab3e7fac87ca5afc575846dc05763ea7 Mon Sep 17 00:00:00 2001 From: Saul Shanabrook Date: Fri, 8 Aug 2025 01:06:52 -0400 Subject: [PATCH 151/424] Fix TypeGuard with call on temporary object (#19577) Fixes #19575 by adding support for TypeGaurd/TypeIs when they are used on methods off of classes which were not saved to a variable. Solution adapted from copilot answer here and then refined: https://github.com/saulshanabrook/mypy/pull/1 --- mypy/checker.py | 43 ++++++++++++++------------ test-data/unit/check-typeguard.test | 47 +++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 20 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 68f9bd4c1383c..3b94b84bb975f 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -6218,21 +6218,26 @@ def find_isinstance_check_helper( attr = try_getting_str_literals(node.args[1], self.lookup_type(node.args[1])) if literal(expr) == LITERAL_TYPE and attr and len(attr) == 1: return self.hasattr_type_maps(expr, self.lookup_type(expr), attr[0]) - elif isinstance(node.callee, RefExpr): - if node.callee.type_guard is not None or node.callee.type_is is not None: + else: + type_is, type_guard = None, None + called_type = self.lookup_type_or_none(node.callee) + if called_type is not None: + called_type = get_proper_type(called_type) + # TODO: there are some more cases in check_call() to handle. + # If the callee is an instance, try to extract TypeGuard/TypeIs from its __call__ method. + if isinstance(called_type, Instance): + call = find_member("__call__", called_type, called_type, is_operator=True) + if call is not None: + called_type = get_proper_type(call) + if isinstance(called_type, CallableType): + type_is, type_guard = called_type.type_is, called_type.type_guard + + # If the callee is a RefExpr, extract TypeGuard/TypeIs directly. + if isinstance(node.callee, RefExpr): + type_is, type_guard = node.callee.type_is, node.callee.type_guard + if type_guard is not None or type_is is not None: # TODO: Follow *args, **kwargs if node.arg_kinds[0] != nodes.ARG_POS: - # the first argument might be used as a kwarg - called_type = get_proper_type(self.lookup_type(node.callee)) - - # TODO: there are some more cases in check_call() to handle. - if isinstance(called_type, Instance): - call = find_member( - "__call__", called_type, called_type, is_operator=True - ) - if call is not None: - called_type = get_proper_type(call) - # *assuming* the overloaded function is correct, there's a couple cases: # 1) The first argument has different names, but is pos-only. We don't # care about this case, the argument must be passed positionally. @@ -6245,9 +6250,7 @@ def find_isinstance_check_helper( # we want the idx-th variable to be narrowed expr = collapse_walrus(node.args[idx]) else: - kind = ( - "guard" if node.callee.type_guard is not None else "narrower" - ) + kind = "guard" if type_guard is not None else "narrower" self.fail( message_registry.TYPE_GUARD_POS_ARG_REQUIRED.format(kind), node ) @@ -6258,15 +6261,15 @@ def find_isinstance_check_helper( # considered "always right" (i.e. even if the types are not overlapping). # Also note that a care must be taken to unwrap this back at read places # where we use this to narrow down declared type. - if node.callee.type_guard is not None: - return {expr: TypeGuardedType(node.callee.type_guard)}, {} + if type_guard is not None: + return {expr: TypeGuardedType(type_guard)}, {} else: - assert node.callee.type_is is not None + assert type_is is not None return conditional_types_to_typemaps( expr, *self.conditional_types_with_intersection( self.lookup_type(expr), - [TypeRange(node.callee.type_is, is_upper_bound=False)], + [TypeRange(type_is, is_upper_bound=False)], expr, consider_runtime_isinstance=False, ), diff --git a/test-data/unit/check-typeguard.test b/test-data/unit/check-typeguard.test index fdcfcc969adc1..93e665e4548c3 100644 --- a/test-data/unit/check-typeguard.test +++ b/test-data/unit/check-typeguard.test @@ -731,6 +731,53 @@ assert a(x=x) reveal_type(x) # N: Revealed type is "builtins.int" [builtins fixtures/tuple.pyi] +# https://github.com/python/mypy/issues/19575 +[case testNoCrashOnDunderCallTypeGuardTemporaryObject] +from typing_extensions import TypeGuard +class E: + def __init__(self) -> None: ... + def __call__(self, o: object) -> TypeGuard[int]: + return True +x = object() +if E()(x): + reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testNoCrashOnDunderCallTypeIsTemporaryObject] +from typing_extensions import TypeIs +class E: + def __init__(self) -> None: ... + def __call__(self, o: object) -> TypeIs[int]: + return True +x = object() +if E()(x): + reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testNoCrashOnDunderCallTypeIsTemporaryObjectGeneric] +from typing import Generic, TypeVar +from typing_extensions import TypeIs +T = TypeVar("T") +class E(Generic[T]): + def __init__(self) -> None: ... + def __call__(self, o: object) -> TypeIs[T]: + return True +x = object() +if E[int]()(x): + reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testTypeGuardTemporaryObjectWithKeywordArg] +from typing_extensions import TypeGuard +class E: + def __init__(self) -> None: ... + def __call__(self, o: object) -> TypeGuard[int]: + return True +x = object() +if E()(o=x): + reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + [case testTypeGuardRestrictAwaySingleInvariant] from typing import List from typing_extensions import TypeGuard From a3eb219fb9a1c4d700b897ebd5e0ba95d2846c35 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Fri, 8 Aug 2025 06:38:45 -0700 Subject: [PATCH 152/424] Fix overload diagnostic when vararg and varkwarg can match (#19614) Fixes #19612 --- mypy/typeops.py | 9 ++++++--- test-data/unit/check-overloading.test | 14 +++++++++++++- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/mypy/typeops.py b/mypy/typeops.py index 13082225e5fda..866eb72ede3ec 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -521,15 +521,18 @@ def callable_corresponding_argument( # def right(a: int = ...) -> None: ... # def left(__a: int = ..., *, a: int = ...) -> None: ... - from mypy.subtypes import is_equivalent + from mypy.subtypes import is_subtype if ( not (by_name.required or by_pos.required) and by_pos.name is None and by_name.pos is None - and is_equivalent(by_name.typ, by_pos.typ) ): - return FormalArgument(by_name.name, by_pos.pos, by_name.typ, False) + # We actually want the intersection of by_name.typ and by_pos.typ + if is_subtype(by_name.typ, by_pos.typ): + return FormalArgument(by_name.name, by_pos.pos, by_name.typ, False) + if is_subtype(by_pos.typ, by_name.typ): + return FormalArgument(by_name.name, by_pos.pos, by_pos.typ, False) return by_name if by_name is not None else by_pos diff --git a/test-data/unit/check-overloading.test b/test-data/unit/check-overloading.test index e7f6ff04c13ee..560d4a5c12fc2 100644 --- a/test-data/unit/check-overloading.test +++ b/test-data/unit/check-overloading.test @@ -231,9 +231,21 @@ def f(x: 'A') -> Any: # E: Overloaded function implementation does not accept al reveal_type(f(A())) # N: Revealed type is "__main__.B" reveal_type(f(B())) # N: Revealed type is "__main__.A" - [builtins fixtures/isinstance.pyi] +[case testTypeCheckOverloadImplOverlapVarArgsAndKwargs] +from __future__ import annotations +from typing import overload + +@overload +def foo(x: int) -> None: ... +@overload +def foo(a: str, /) -> None: ... + +def foo(*args: int | str, **kw: int) -> None: + pass +[builtins fixtures/tuple.pyi] + [case testTypeCheckOverloadWithImplTooSpecificRetType] from typing import overload, Any From 0d791b29b7ba4e5a9b04c0b6bdba11faf4a186a2 Mon Sep 17 00:00:00 2001 From: Christoph Tyralla Date: Fri, 8 Aug 2025 15:48:09 +0200 Subject: [PATCH 153/424] PEP 702 (@deprecated): consider overloads in snapshot descriptions (#19613) This change is taken from #18682. The tests are unmodified. The code is simplified [as suggested by Ivan](https://github.com/python/mypy/pull/18682#issuecomment-3158248336). --- mypy/server/astdiff.py | 11 +- test-data/unit/fine-grained.test | 308 +++++++++++++++++++++++++++++++ 2 files changed, 318 insertions(+), 1 deletion(-) diff --git a/mypy/server/astdiff.py b/mypy/server/astdiff.py index 16a0d882a8aa6..1df85a163e0fd 100644 --- a/mypy/server/astdiff.py +++ b/mypy/server/astdiff.py @@ -252,6 +252,15 @@ def snapshot_definition(node: SymbolNode | None, common: SymbolSnapshot) -> Symb setter_type = snapshot_optional_type(first_item.var.setter_type) is_trivial_body = impl.is_trivial_body if impl else False dataclass_transform_spec = find_dataclass_transform_spec(node) + + deprecated: str | list[str | None] | None = None + if isinstance(node, FuncDef): + deprecated = node.deprecated + elif isinstance(node, OverloadedFuncDef): + deprecated = [node.deprecated] + [ + i.func.deprecated for i in node.items if isinstance(i, Decorator) + ] + return ( "Func", common, @@ -262,7 +271,7 @@ def snapshot_definition(node: SymbolNode | None, common: SymbolSnapshot) -> Symb signature, is_trivial_body, dataclass_transform_spec.serialize() if dataclass_transform_spec is not None else None, - node.deprecated if isinstance(node, FuncDef) else None, + deprecated, setter_type, # multi-part properties are stored as OverloadedFuncDef ) elif isinstance(node, Var): diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index c25ed79e73562..0d10559d0692c 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -11033,6 +11033,314 @@ b.py:1: error: class a.C is deprecated: use C2 instead b.py:2: error: class a.D is deprecated: use D2 instead +[case testDeprecatedAddKeepChangeAndRemoveOverloadedFunctionDeprecation] +# flags: --enable-error-code=deprecated + +from a import f +f(1) +f("y") +import a +a.f(1) +a.f("y") + +[file a.py] +from typing import overload, Union +@overload +def f(x: int) -> int: ... +@overload +def f(x: str) -> str: ... +def f(x: Union[int, str]) -> Union[int, str]: ... + +[file a.py.2] +from typing import overload, Union +from typing_extensions import deprecated +@overload +def f(x: int) -> int: ... +@overload +@deprecated("pass int") +def f(x: str) -> str: ... +def f(x: Union[int, str]) -> Union[int, str]: ... + +[file a.py.3] +from typing import overload, Union +from typing_extensions import deprecated +@overload +def f(x: int) -> int: ... +@overload +@deprecated("pass int") +def f(x: str) -> str: ... +def f(x: Union[int, str]) -> Union[int, str]: ... + +[file a.py.4] +from typing import overload, Union +from typing_extensions import deprecated +@overload +def f(x: int) -> int: ... +@overload +@deprecated("pass int, please") +def f(x: str) -> str: ... +def f(x: Union[int, str]) -> Union[int, str]: ... + +[file a.py.5] +from typing import overload, Union +@overload +def f(x: int) -> int: ... +@overload +def f(x: str) -> str: ... +def f(x: Union[int, str]) -> Union[int, str]: ... + +[builtins fixtures/tuple.pyi] +[out] +== +main:5: error: overload def (x: builtins.str) -> builtins.str of function a.f is deprecated: pass int +main:8: error: overload def (x: builtins.str) -> builtins.str of function a.f is deprecated: pass int +== +main:5: error: overload def (x: builtins.str) -> builtins.str of function a.f is deprecated: pass int +main:8: error: overload def (x: builtins.str) -> builtins.str of function a.f is deprecated: pass int +== +main:5: error: overload def (x: builtins.str) -> builtins.str of function a.f is deprecated: pass int, please +main:8: error: overload def (x: builtins.str) -> builtins.str of function a.f is deprecated: pass int, please +== + + +[case testDeprecatedRemoveOverloadedFunctionDeprecation] +# flags: --enable-error-code=deprecated + +from a import f +f(1) +f("y") +import a +a.f(1) +a.f("y") + +[file a.py] +from typing import overload, Union +from typing_extensions import deprecated +@overload +def f(x: int) -> int: ... +@overload +@deprecated("pass int") +def f(x: str) -> str: ... +def f(x: Union[int, str]) -> Union[int, str]: ... + +[file a.py.2] +from typing import overload, Union +@overload +def f(x: int) -> int: ... +@overload +def f(x: str) -> str: ... +def f(x: Union[int, str]) -> Union[int, str]: ... + +[builtins fixtures/tuple.pyi] +[out] +main:5: error: overload def (x: builtins.str) -> builtins.str of function a.f is deprecated: pass int +main:8: error: overload def (x: builtins.str) -> builtins.str of function a.f is deprecated: pass int +== + + +[case testDeprecatedKeepOverloadedFunctionDeprecation] +# flags: --enable-error-code=deprecated + +from a import f +f(1) +f("y") +import a +a.f(1) +a.f("y") + +[file a.py] +from typing import overload, Union +from typing_extensions import deprecated +@overload +def f(x: int) -> int: ... +@overload +@deprecated("pass int") +def f(x: str) -> str: ... +def f(x: Union[int, str]) -> Union[int, str]: ... + +[file a.py.2] +from typing import overload, Union +from typing_extensions import deprecated +@overload +def f(x: int) -> int: ... +@overload +@deprecated("pass int") +def f(x: str) -> str: ... +def f(x: Union[int, str]) -> Union[int, str]: ... + +[builtins fixtures/tuple.pyi] +[out] +main:5: error: overload def (x: builtins.str) -> builtins.str of function a.f is deprecated: pass int +main:8: error: overload def (x: builtins.str) -> builtins.str of function a.f is deprecated: pass int +== +main:5: error: overload def (x: builtins.str) -> builtins.str of function a.f is deprecated: pass int +main:8: error: overload def (x: builtins.str) -> builtins.str of function a.f is deprecated: pass int + + +[case testDeprecatedAddOverloadedFunctionDeprecationIndirectImport] +# flags: --enable-error-code=deprecated + +from b import f +f(1) +f("y") +import b +b.f(1) +b.f("y") + +[file b.py] +from a import f + +[file a.py] +from typing import overload, Union +@overload +def f(x: int) -> int: ... +@overload +def f(x: str) -> str: ... +def f(x: Union[int, str]) -> Union[int, str]: ... + +[file a.py.2] +from typing import overload, Union +from typing_extensions import deprecated +@overload +def f(x: int) -> int: ... +@overload +@deprecated("pass int") +def f(x: str) -> str: ... +def f(x: Union[int, str]) -> Union[int, str]: ... + +[builtins fixtures/tuple.pyi] +[out] +== +main:5: error: overload def (x: builtins.str) -> builtins.str of function a.f is deprecated: pass int +main:8: error: overload def (x: builtins.str) -> builtins.str of function a.f is deprecated: pass int + + +[case testDeprecatedChangeOverloadedFunctionDeprecationIndirectImport] +# flags: --enable-error-code=deprecated + +from b import f +f(1) +f("y") +import b +b.f(1) +b.f("y") + +[file b.py] +from a import f + +[file a.py] +from typing import overload, Union +from typing_extensions import deprecated +@overload +def f(x: int) -> int: ... +@overload +@deprecated("pass int") +def f(x: str) -> str: ... +def f(x: Union[int, str]) -> Union[int, str]: ... + +[file a.py.2] +from typing import overload, Union +from typing_extensions import deprecated +@overload +def f(x: int) -> int: ... +@overload +@deprecated("pass int, please") +def f(x: str) -> str: ... +def f(x: Union[int, str]) -> Union[int, str]: ... + +[builtins fixtures/tuple.pyi] +[out] +main:5: error: overload def (x: builtins.str) -> builtins.str of function a.f is deprecated: pass int +main:8: error: overload def (x: builtins.str) -> builtins.str of function a.f is deprecated: pass int +== +main:5: error: overload def (x: builtins.str) -> builtins.str of function a.f is deprecated: pass int, please +main:8: error: overload def (x: builtins.str) -> builtins.str of function a.f is deprecated: pass int, please + + +[case testDeprecatedRemoveOverloadedFunctionDeprecationIndirectImport] +# flags: --enable-error-code=deprecated + +from b import f +f(1) +f("y") +import b +b.f(1) +b.f("y") + +[file b.py] +from a import f + +[file a.py] +from typing import overload, Union +from typing_extensions import deprecated +@overload +def f(x: int) -> int: ... +@overload +@deprecated("pass int") +def f(x: str) -> str: ... +def f(x: Union[int, str]) -> Union[int, str]: ... + +[file a.py.2] +from typing import overload, Union +@overload +def f(x: int) -> int: ... +@overload +def f(x: str) -> str: ... +def f(x: Union[int, str]) -> Union[int, str]: ... + +[builtins fixtures/tuple.pyi] +[out] +main:5: error: overload def (x: builtins.str) -> builtins.str of function a.f is deprecated: pass int +main:8: error: overload def (x: builtins.str) -> builtins.str of function a.f is deprecated: pass int +== + + +[case testDeprecatedOverloadedFunctionAlreadyDecorated] +# flags: --enable-error-code=deprecated + +from b import f +f(1) +f("y") +import b +b.f(1) +b.f("y") + +[file b.py] +from a import f + +[file a.py] +from typing import Callable, overload, Union + +def d(t: Callable[[str], str]) -> Callable[[str], str]: ... + +@overload +def f(x: int) -> int: ... +@overload +@d +def f(x: str) -> str: ... +def f(x: Union[int, str]) -> Union[int, str]: ... + +[file a.py.2] +from typing import Callable, overload, Union +from typing_extensions import deprecated + +def d(t: Callable[[str], str]) -> Callable[[str], str]: ... + +@overload +def f(x: int) -> int: ... +@overload +@deprecated("deprecated decorated overload") +@d +def f(x: str) -> str: ... +def f(x: Union[int, str]) -> Union[int, str]: ... + +[builtins fixtures/tuple.pyi] +[out] +== +main:5: error: overload def (x: builtins.str) -> builtins.str of function a.f is deprecated: deprecated decorated overload +main:8: error: overload def (x: builtins.str) -> builtins.str of function a.f is deprecated: deprecated decorated overload + + [case testDeprecatedChangeClassDeprecationIndirectImport] # flags: --enable-error-code=deprecated from b import C From 04f38f5862547a3f695da0dc6a8843de5370574e Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 8 Aug 2025 20:01:59 +0100 Subject: [PATCH 154/424] Fix crash on settable property alias (#19615) Fixes https://github.com/python/mypy/issues/19572 Surprisingly, when working on this I found another inconsistency in how `.definiton` is set. So after all I decided to do some cleanup, now `.definiton` should always point do the `Decortor` if the definition is a decorated function (no matter whether it is a trivial decorator like `@abstractmethod` or `@overload` or a "real" one). --------- Co-authored-by: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> --- mypy/checker.py | 27 ++++++-- mypy/checkmember.py | 11 ++- mypy/fixup.py | 8 +-- mypy/messages.py | 22 +++--- mypy/nodes.py | 7 ++ mypy/semanal.py | 2 + mypy/types.py | 3 + test-data/unit/check-classes.test | 47 +++++++++++++ test-data/unit/check-dataclasses.test | 2 +- test-data/unit/fine-grained.test | 98 +-------------------------- test-data/unit/pythoneval.test | 1 + 11 files changed, 111 insertions(+), 117 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 3b94b84bb975f..6176df84c225b 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -130,6 +130,7 @@ WhileStmt, WithStmt, YieldExpr, + get_func_def, is_final_node, ) from mypy.operators import flip_ops, int_op_to_method, neg_ops @@ -703,6 +704,12 @@ def _visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: # TODO: keep precise type for callables with tricky but valid signatures. setter_type = fallback_setter_type defn.items[0].var.setter_type = setter_type + if isinstance(defn.type, Overloaded): + # Update legacy property type for decorated properties. + getter_type = self.extract_callable_type(defn.items[0].var.type, defn) + if getter_type is not None: + getter_type.definition = defn.items[0] + defn.type.items[0] = getter_type for i, fdef in enumerate(defn.items): assert isinstance(fdef, Decorator) if defn.is_property: @@ -730,7 +737,7 @@ def _visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: assert isinstance(item, Decorator) item_type = self.extract_callable_type(item.var.type, item) if item_type is not None: - item_type.definition = item.func + item_type.definition = item item_types.append(item_type) if item_types: defn.type = Overloaded(item_types) @@ -2501,8 +2508,9 @@ def check_override( override_ids = override.type_var_ids() type_name = None - if isinstance(override.definition, FuncDef): - type_name = override.definition.info.name + definition = get_func_def(override) + if isinstance(definition, FuncDef): + type_name = definition.info.name def erase_override(t: Type) -> Type: return erase_typevars(t, ids_to_erase=override_ids) @@ -3509,6 +3517,7 @@ def check_compatibility_all_supers(self, lvalue: RefExpr, rvalue: Expression) -> continue base_type, base_node = self.node_type_from_base(lvalue_node.name, base, lvalue) + # TODO: if the r.h.s. is a descriptor, we should check setter override as well. custom_setter = is_custom_settable_property(base_node) if isinstance(base_type, PartialType): base_type = None @@ -4494,6 +4503,8 @@ def set_inferred_type(self, var: Var, lvalue: Lvalue, type: Type) -> None: if isinstance(p_type, Overloaded): # TODO: in theory we can have a property with a deleter only. var.is_settable_property = True + assert isinstance(definition, Decorator), definition + var.setter_type = definition.var.setter_type def set_inference_error_fallback_type(self, var: Var, lvalue: Lvalue, type: Type) -> None: """Store best known type for variable if type inference failed. @@ -5356,6 +5367,8 @@ def visit_decorator_inner( self.check_untyped_after_decorator(sig, e.func) self.require_correct_self_argument(sig, e.func) sig = set_callable_name(sig, e.func) + if isinstance(sig, CallableType): + sig.definition = e e.var.type = sig e.var.is_ready = True if e.func.is_property: @@ -8654,8 +8667,10 @@ def visit_type_alias_type(self, t: TypeAliasType) -> Type: return t.copy_modified(args=[a.accept(self) for a in t.args]) -def is_classmethod_node(node: Node | None) -> bool | None: +def is_classmethod_node(node: SymbolNode | None) -> bool | None: """Find out if a node describes a classmethod.""" + if isinstance(node, Decorator): + node = node.func if isinstance(node, FuncDef): return node.is_class if isinstance(node, Var): @@ -8663,8 +8678,10 @@ def is_classmethod_node(node: Node | None) -> bool | None: return None -def is_node_static(node: Node | None) -> bool | None: +def is_node_static(node: SymbolNode | None) -> bool | None: """Find out if a node describes a static function method.""" + if isinstance(node, Decorator): + node = node.func if isinstance(node, FuncDef): return node.is_static if isinstance(node, Var): diff --git a/mypy/checkmember.py b/mypy/checkmember.py index da67591a4553f..2c41f2e273cc0 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -976,8 +976,15 @@ def expand_and_bind_callable( freeze_all_type_vars(expanded) if not var.is_property: return expanded - # TODO: a decorated property can result in Overloaded here. - assert isinstance(expanded, CallableType) + if isinstance(expanded, Overloaded): + # Legacy way to store settable properties is with overloads. Also in case it is + # an actual overloaded property, selecting first item that passed check_self_arg() + # is a good approximation, long-term we should use check_call() inference below. + if not expanded.items: + # A broken overload, error should be already reported. + return AnyType(TypeOfAny.from_error) + expanded = expanded.items[0] + assert isinstance(expanded, CallableType), expanded if var.is_settable_property and mx.is_lvalue and var.setter_type is not None: if expanded.variables: type_ctx = mx.rvalue or TempNode(AnyType(TypeOfAny.special_form), context=mx.context) diff --git a/mypy/fixup.py b/mypy/fixup.py index 0007fe8faabf1..18bdc1c6f497f 100644 --- a/mypy/fixup.py +++ b/mypy/fixup.py @@ -181,8 +181,7 @@ def visit_overloaded_func_def(self, o: OverloadedFuncDef) -> None: if isinstance(o.type, Overloaded): # For error messages we link the original definition for each item. for typ, item in zip(o.type.items, o.items): - if isinstance(item, Decorator): - typ.definition = item.func + typ.definition = item def visit_decorator(self, d: Decorator) -> None: if self.current_info is not None: @@ -193,8 +192,9 @@ def visit_decorator(self, d: Decorator) -> None: d.var.accept(self) for node in d.decorators: node.accept(self) - if isinstance(d.var.type, ProperType) and isinstance(d.var.type, CallableType): - d.var.type.definition = d.func + typ = d.var.type + if isinstance(typ, ProperType) and isinstance(typ, CallableType): + typ.definition = d.func def visit_class_def(self, c: ClassDef) -> None: for v in c.type_vars: diff --git a/mypy/messages.py b/mypy/messages.py index 6b55da59d183a..f626d4c71916f 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -55,6 +55,7 @@ SymbolTable, TypeInfo, Var, + get_func_def, reverse_builtin_aliases, ) from mypy.operators import op_methods, op_methods_to_symbols @@ -2938,10 +2939,11 @@ def format_single(arg: Type) -> str: def pretty_class_or_static_decorator(tp: CallableType) -> str | None: """Return @classmethod or @staticmethod, if any, for the given callable type.""" - if tp.definition is not None and isinstance(tp.definition, SYMBOL_FUNCBASE_TYPES): - if tp.definition.is_class: + definition = get_func_def(tp) + if definition is not None and isinstance(definition, SYMBOL_FUNCBASE_TYPES): + if definition.is_class: return "@classmethod" - if tp.definition.is_static: + if definition.is_static: return "@staticmethod" return None @@ -2991,12 +2993,13 @@ def [T <: int] f(self, x: int, y: T) -> None slash = True # If we got a "special arg" (i.e: self, cls, etc...), prepend it to the arg list + definition = get_func_def(tp) if ( - isinstance(tp.definition, FuncDef) - and hasattr(tp.definition, "arguments") + isinstance(definition, FuncDef) + and hasattr(definition, "arguments") and not tp.from_concatenate ): - definition_arg_names = [arg.variable.name for arg in tp.definition.arguments] + definition_arg_names = [arg.variable.name for arg in definition.arguments] if ( len(definition_arg_names) > len(tp.arg_names) and definition_arg_names[0] @@ -3005,7 +3008,7 @@ def [T <: int] f(self, x: int, y: T) -> None if s: s = ", " + s s = definition_arg_names[0] + s - s = f"{tp.definition.name}({s})" + s = f"{definition.name}({s})" elif tp.name: first_arg = get_first_arg(tp) if first_arg: @@ -3051,9 +3054,10 @@ def [T <: int] f(self, x: int, y: T) -> None def get_first_arg(tp: CallableType) -> str | None: - if not isinstance(tp.definition, FuncDef) or not tp.definition.info or tp.definition.is_static: + definition = get_func_def(tp) + if not isinstance(definition, FuncDef) or not definition.info or definition.is_static: return None - return tp.definition.original_first_arg + return definition.original_first_arg def variance_string(variance: int) -> str: diff --git a/mypy/nodes.py b/mypy/nodes.py index 9d5867c5371dd..99b9bf72c9484 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -4380,6 +4380,13 @@ def is_final_node(node: SymbolNode | None) -> bool: return isinstance(node, (Var, FuncDef, OverloadedFuncDef, Decorator)) and node.is_final +def get_func_def(typ: mypy.types.CallableType) -> SymbolNode | None: + definition = typ.definition + if isinstance(definition, Decorator): + definition = definition.func + return definition + + def local_definitions( names: SymbolTable, name_prefix: str, info: TypeInfo | None = None ) -> Iterator[Definition]: diff --git a/mypy/semanal.py b/mypy/semanal.py index 99d1eb36e7887..fb66fb5158db4 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1243,6 +1243,7 @@ def analyze_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: bare_setter_type = self.analyze_property_with_multi_part_definition(defn) typ = function_type(first_item.func, self.named_type("builtins.function")) assert isinstance(typ, CallableType) + typ.definition = first_item types = [typ] else: # This is a normal overload. Find the item signatures, the @@ -1374,6 +1375,7 @@ def analyze_overload_sigs_and_impl( if isinstance(item, Decorator): callable = function_type(item.func, self.named_type("builtins.function")) assert isinstance(callable, CallableType) + callable.definition = item if not any(refers_to_fullname(dec, OVERLOAD_NAMES) for dec in item.decorators): if i == len(defn.items) - 1 and not self.is_stub_file: # Last item outside a stub is impl diff --git a/mypy/types.py b/mypy/types.py index b4771b15f77ad..a73ac3c3524ab 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1887,6 +1887,9 @@ def __init__( self.fallback = fallback assert not name or " __main__.CGI" reveal_type(CGT) # N: Revealed type is "def [T] () -> __main__.CGT[T`1]" + +[case testSettablePropertyAlias] +from typing import Any, TypeVar + +class A: + @property + def prop(self: Any) -> str: ... + @prop.setter + def prop(self, val: str) -> None: ... + +T = TypeVar("T") +class AT: + @property + def prop(self: T) -> T: ... + @prop.setter + def prop(self: T, val: list[T]) -> None: ... + +class B: + prop: str + prop_t: str + +class C(B): + prop = A.prop + prop_t = AT.prop # E: Incompatible types in assignment (expression has type "C", base class "B" defined the type as "str") + +reveal_type(C().prop) # N: Revealed type is "builtins.str" +C().prop = "no" # E: Invalid self argument "C" to attribute function "prop" with type "Callable[[A, str], None]" +reveal_type(C().prop_t) # N: Revealed type is "__main__.C" +C().prop_t = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "list[C]") +[builtins fixtures/property.pyi] + +[case testClassEqDecoratedAbstractNote] +from abc import abstractmethod + +class C: + @abstractmethod + def __eq__(self, other: C) -> bool: ... +[builtins fixtures/plugin_attrs.pyi] +[out] +main:5: error: Argument 1 of "__eq__" is incompatible with supertype "builtins.object"; supertype defines the argument type as "object" +main:5: note: This violates the Liskov substitution principle +main:5: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides +main:5: note: It is recommended for "__eq__" to work with arbitrary objects, for example: +main:5: note: def __eq__(self, other: object) -> bool: +main:5: note: if not isinstance(other, C): +main:5: note: return NotImplemented +main:5: note: return diff --git a/test-data/unit/check-dataclasses.test b/test-data/unit/check-dataclasses.test index a6ac30e20c36c..f43c49c200c86 100644 --- a/test-data/unit/check-dataclasses.test +++ b/test-data/unit/check-dataclasses.test @@ -2055,7 +2055,7 @@ from dataclasses import dataclass, replace, InitVar from typing import ClassVar @dataclass -class A: +class A: # N: "replace" of "A" defined here x: int q: InitVar[int] q2: InitVar[int] = 0 diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index 0d10559d0692c..888b7bc7e97f1 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -7312,9 +7312,7 @@ class C: == mod.py:9: error: Incompatible types in assignment (expression has type "int", variable has type "str") -[case testOverloadedMethodSupertype-only_when_cache] --- Different cache/no-cache tests because --- CallableType.def_extras.first_arg differs ("self"/None) +[case testOverloadedMethodSupertype] from typing import overload, Any import b class Child(b.Parent): @@ -7355,49 +7353,6 @@ main:4: note: def f(self, arg: int) -> int main:4: note: @overload main:4: note: def f(self, arg: str) -> str -[case testOverloadedMethodSupertype2-only_when_nocache] --- Different cache/no-cache tests because --- CallableType.def_extras.first_arg differs ("self"/None) -from typing import overload, Any -import b -class Child(b.Parent): - @overload # Fail - def f(self, arg: int) -> int: ... - @overload - def f(self, arg: str) -> str: ... - def f(self, arg: Any) -> Any: ... -[file b.py] -from typing import overload, Any -class C: pass -class Parent: - @overload - def f(self, arg: int) -> int: ... - @overload - def f(self, arg: str) -> str: ... - def f(self, arg: Any) -> Any: ... -[file b.py.2] -from typing import overload, Any -class C: pass -class Parent: - @overload - def f(self, arg: int) -> int: ... - @overload - def f(self, arg: str) -> C: ... - def f(self, arg: Any) -> Any: ... -[out] -== -main:4: error: Signature of "f" incompatible with supertype "b.Parent" -main:4: note: Superclass: -main:4: note: @overload -main:4: note: def f(self, arg: int) -> int -main:4: note: @overload -main:4: note: def f(self, arg: str) -> C -main:4: note: Subclass: -main:4: note: @overload -main:4: note: def f(arg: int) -> int -main:4: note: @overload -main:4: note: def f(arg: str) -> str - [case testOverloadedInitSupertype] import a [file a.py] @@ -8486,9 +8441,7 @@ class D: == a.py:3: error: Cannot override final attribute "meth" (previously declared in base class "C") -[case testFinalBodyReprocessedAndStillFinalOverloaded-only_when_cache] --- Different cache/no-cache tests because --- CallableType.def_extras.first_arg differs ("self"/None) +[case testFinalBodyReprocessedAndStillFinalOverloaded] import a [file a.py] from c import C @@ -8533,53 +8486,6 @@ a.py:3: note: def meth(self, x: str) -> str a.py:3: note: Subclass: a.py:3: note: def meth(self) -> None -[case testFinalBodyReprocessedAndStillFinalOverloaded2-only_when_nocache] --- Different cache/no-cache tests because --- CallableType.def_extras.first_arg differs ("self"/None) -import a -[file a.py] -from c import C -class A: - def meth(self) -> None: ... - -[file a.py.3] -from c import C -class A(C): - def meth(self) -> None: ... - -[file c.py] -from typing import final, overload, Union -from d import D - -class C: - @overload - def meth(self, x: int) -> int: ... - @overload - def meth(self, x: str) -> str: ... - @final - def meth(self, x: Union[int, str]) -> Union[int, str]: - D(int()) - return x -[file d.py] -class D: - def __init__(self, x: int) -> None: ... -[file d.py.2] -from typing import Optional -class D: - def __init__(self, x: Optional[int]) -> None: ... -[out] -== -== -a.py:3: error: Cannot override final attribute "meth" (previously declared in base class "C") -a.py:3: error: Signature of "meth" incompatible with supertype "c.C" -a.py:3: note: Superclass: -a.py:3: note: @overload -a.py:3: note: def meth(x: int) -> int -a.py:3: note: @overload -a.py:3: note: def meth(x: str) -> str -a.py:3: note: Subclass: -a.py:3: note: def meth(self) -> None - [case testIfMypyUnreachableClass] from a import x diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index 4bd94dfce03e5..9b5d8a1ac54c0 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -1970,6 +1970,7 @@ a2 = replace() a2 = replace(a, x='spam') a2 = replace(a, x=42, q=42) [out] +_testDataclassReplace.py:4: note: "replace" of "A" defined here _testDataclassReplace.py:9: note: Revealed type is "_testDataclassReplace.A" _testDataclassReplace.py:10: error: Too few arguments for "replace" _testDataclassReplace.py:11: error: Argument "x" to "replace" of "A" has incompatible type "str"; expected "int" From adacbbf563e2768cac40ed2f962caca5619c5518 Mon Sep 17 00:00:00 2001 From: Emma Smith Date: Fri, 8 Aug 2025 17:27:32 -0700 Subject: [PATCH 155/424] [mypyc] Provide instructions for resolving missing test module on Windows (#19579) Python doesn't come with the test module by default on Windows if installed through pymanager. Since `test.support.EqualToForwardRef` is used as part of the mypyc run tests, we should provide a helpful error message when the test fails due to the missing `test` module. Co-authored-by: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> --- mypyc/test-data/run-tuples.test | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/mypyc/test-data/run-tuples.test b/mypyc/test-data/run-tuples.test index ea0a1cb8d8529..5d9485288cfb0 100644 --- a/mypyc/test-data/run-tuples.test +++ b/mypyc/test-data/run-tuples.test @@ -131,10 +131,23 @@ import sys from typing import Optional from native import ClassIR, FuncIR, Record +HAVE_TEST = False if sys.version_info >= (3, 14): - from test.support import EqualToForwardRef - type_forward_ref = EqualToForwardRef -else: + try: + from test.support import EqualToForwardRef + type_forward_ref = EqualToForwardRef + HAVE_TEST = True + except ImportError as e: + # catch the case of a pymanager installed Python + # without the test module. It is excluded by default + # on Windows. + msg = 'Missing "test" module.' + if sys.platform == "win32": + msg += (' Please install a version of Python with the test module.' + ' If you are using pymanager, try running pymanager install --force PythonTest\\') + raise ImportError(msg) from e + +if not HAVE_TEST: from typing import ForwardRef type_forward_ref = ForwardRef From 186515f66fc47d8afc0c9d43482265f295c83a08 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Fri, 8 Aug 2025 21:16:01 -0700 Subject: [PATCH 156/424] Further fix overload diagnostic for vararg and varkwarg (#19619) #19614 got merged faster than I expected :-) --- mypy/typeops.py | 10 ++++------ test-data/unit/check-overloading.test | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/mypy/typeops.py b/mypy/typeops.py index 866eb72ede3ec..88b3c5da48ce0 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -521,18 +521,16 @@ def callable_corresponding_argument( # def right(a: int = ...) -> None: ... # def left(__a: int = ..., *, a: int = ...) -> None: ... - from mypy.subtypes import is_subtype + from mypy.meet import meet_types if ( not (by_name.required or by_pos.required) and by_pos.name is None and by_name.pos is None ): - # We actually want the intersection of by_name.typ and by_pos.typ - if is_subtype(by_name.typ, by_pos.typ): - return FormalArgument(by_name.name, by_pos.pos, by_name.typ, False) - if is_subtype(by_pos.typ, by_name.typ): - return FormalArgument(by_name.name, by_pos.pos, by_pos.typ, False) + return FormalArgument( + by_name.name, by_pos.pos, meet_types(by_name.typ, by_pos.typ), False + ) return by_name if by_name is not None else by_pos diff --git a/test-data/unit/check-overloading.test b/test-data/unit/check-overloading.test index 560d4a5c12fc2..be55a182b87bb 100644 --- a/test-data/unit/check-overloading.test +++ b/test-data/unit/check-overloading.test @@ -246,6 +246,23 @@ def foo(*args: int | str, **kw: int) -> None: pass [builtins fixtures/tuple.pyi] +[case testTypeCheckOverloadImplOverlapVarArgsAndKwargsUnion] +from __future__ import annotations +from typing import overload + +class Foo: ... + +@overload +def foo(x: int) -> None: ... +@overload +def foo(*, x: Foo) -> None: ... +@overload +def foo(a: str, /) -> None: ... + +def foo(*args: int | str, **kw: int | Foo) -> None: + pass +[builtins fixtures/tuple.pyi] + [case testTypeCheckOverloadWithImplTooSpecificRetType] from typing import overload, Any From cc5f1e1b9763d712d5db7b89c40e20d881407361 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 9 Aug 2025 09:09:21 +0100 Subject: [PATCH 157/424] Cache common instances (#19621) These few types account for a significant proportion of all types created: * `str` is just everywhere * `object` and `function` are used as fallbacks in many places * `int` and `bool` are coming from various literals This gives around 1.5% performance improvement on my desktop. This is a bit ugly, but also looks like an easy win. Note that during semantic analysis I am caching types more conservatively, just in case some plugins modify them in place (`named_type()` is a part of semantic analyzer plugin interface). --- mypy/checker.py | 30 ++++++++++++++++++++++++++++++ mypy/checkexpr.py | 21 ++++++++++++++++----- mypy/semanal.py | 25 ++++++++++++++++++------- 3 files changed, 64 insertions(+), 12 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 6176df84c225b..32ef3701df9eb 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -431,6 +431,13 @@ def __init__( self._expr_checker = mypy.checkexpr.ExpressionChecker( self, self.msg, self.plugin, per_line_checking_time_ns ) + + self._str_type: Instance | None = None + self._function_type: Instance | None = None + self._int_type: Instance | None = None + self._bool_type: Instance | None = None + self._object_type: Instance | None = None + self.pattern_checker = PatternChecker(self, self.msg, self.plugin, options) self._unique_id = 0 @@ -7369,6 +7376,29 @@ def named_type(self, name: str) -> Instance: For example, named_type('builtins.object') produces the 'object' type. """ + if name == "builtins.str": + if self._str_type is None: + self._str_type = self._named_type(name) + return self._str_type + if name == "builtins.function": + if self._function_type is None: + self._function_type = self._named_type(name) + return self._function_type + if name == "builtins.int": + if self._int_type is None: + self._int_type = self._named_type(name) + return self._int_type + if name == "builtins.bool": + if self._bool_type is None: + self._bool_type = self._named_type(name) + return self._bool_type + if name == "builtins.object": + if self._object_type is None: + self._object_type = self._named_type(name) + return self._object_type + return self._named_type(name) + + def _named_type(self, name: str) -> Instance: # Assume that the name refers to a type. sym = self.lookup_qualified(name) node = sym.node diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 6e0915179f90a..04ea678a2736a 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -360,6 +360,9 @@ def __init__( ] = {} self.in_lambda_expr = False + self._literal_true: Instance | None = None + self._literal_false: Instance | None = None + def reset(self) -> None: self.resolved_type = {} self.expr_cache.clear() @@ -3428,11 +3431,19 @@ def infer_literal_expr_type(self, value: LiteralValue, fallback_name: str) -> Ty if self.is_literal_context(): return LiteralType(value=value, fallback=typ) else: - return typ.copy_modified( - last_known_value=LiteralType( - value=value, fallback=typ, line=typ.line, column=typ.column - ) - ) + if value is True: + if self._literal_true is None: + self._literal_true = typ.copy_modified( + last_known_value=LiteralType(value=value, fallback=typ) + ) + return self._literal_true + if value is False: + if self._literal_false is None: + self._literal_false = typ.copy_modified( + last_known_value=LiteralType(value=value, fallback=typ) + ) + return self._literal_false + return typ.copy_modified(last_known_value=LiteralType(value=value, fallback=typ)) def concat_tuples(self, left: TupleType, right: TupleType) -> TupleType: """Concatenate two fixed length tuples.""" diff --git a/mypy/semanal.py b/mypy/semanal.py index fb66fb5158db4..dfa2102346069 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -497,6 +497,10 @@ def __init__( # Used to track edge case when return is still inside except* if it enters a loop self.return_stmt_inside_except_star_block: bool = False + self._str_type: Instance | None = None + self._function_type: Instance | None = None + self._object_type: Instance | None = None + # mypyc doesn't properly handle implementing an abstractproperty # with a regular attribute so we make them properties @property @@ -1241,7 +1245,7 @@ def analyze_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: # This is a property. first_item.func.is_overload = True bare_setter_type = self.analyze_property_with_multi_part_definition(defn) - typ = function_type(first_item.func, self.named_type("builtins.function")) + typ = function_type(first_item.func, self.function_type()) assert isinstance(typ, CallableType) typ.definition = first_item types = [typ] @@ -1373,7 +1377,7 @@ def analyze_overload_sigs_and_impl( item.accept(self) # TODO: support decorated overloaded functions properly if isinstance(item, Decorator): - callable = function_type(item.func, self.named_type("builtins.function")) + callable = function_type(item.func, self.function_type()) assert isinstance(callable, CallableType) callable.definition = item if not any(refers_to_fullname(dec, OVERLOAD_NAMES) for dec in item.decorators): @@ -1536,9 +1540,7 @@ def analyze_property_with_multi_part_definition( if first_node.name == "setter": # The first item represents the entire property. first_item.var.is_settable_property = True - setter_func_type = function_type( - item.func, self.named_type("builtins.function") - ) + setter_func_type = function_type(item.func, self.function_type()) assert isinstance(setter_func_type, CallableType) bare_setter_type = setter_func_type defn.setter_index = i + 1 @@ -6630,10 +6632,19 @@ def lookup_fully_qualified_or_none(self, fullname: str) -> SymbolTableNode | Non return result def object_type(self) -> Instance: - return self.named_type("builtins.object") + if self._object_type is None: + self._object_type = self.named_type("builtins.object") + return self._object_type def str_type(self) -> Instance: - return self.named_type("builtins.str") + if self._str_type is None: + self._str_type = self.named_type("builtins.str") + return self._str_type + + def function_type(self) -> Instance: + if self._function_type is None: + self._function_type = self.named_type("builtins.function") + return self._function_type def named_type(self, fullname: str, args: list[Type] | None = None) -> Instance: sym = self.lookup_fully_qualified(fullname) From 660d911223da3516c496e3a0cfdefa3eaf982290 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 9 Aug 2025 19:47:25 +0100 Subject: [PATCH 158/424] Two more micro-optimizations (#19627) This has two things (totalling 1.5% locally, but see caveat below): * Do not use `@contextmanger` (that is relatively slow) for `local_type_map`, since it appears in multiple hot paths. * Do not show name suggestions for import errors in third party packages (since those errors are ignored anyway). It calls `difflib` that can be extremely slow with large modules. Btw the second will probably not affect self-check, although it did affect _my_ self-check, since apparently `pytest` depends on `numpy`. Well, they don't specify it as a package dependency, but https://github.com/pytest-dev/pytest/blob/main/src/_pytest/python_api.py#L17-L18 ```python if TYPE_CHECKING: from numpy import ndarray ``` (and I have numpy installed in all my environments, LOL) --- mypy/checker.py | 48 +++++++++++++++++++++++++++++++++-------------- mypy/checkexpr.py | 15 ++++++--------- mypy/semanal.py | 4 +++- 3 files changed, 43 insertions(+), 24 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 32ef3701df9eb..206abae6adec4 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -6,7 +6,18 @@ from collections import defaultdict from collections.abc import Iterable, Iterator, Mapping, Sequence, Set as AbstractSet from contextlib import ExitStack, contextmanager -from typing import Callable, Final, Generic, NamedTuple, Optional, TypeVar, Union, cast, overload +from typing import ( + Callable, + Final, + Generic, + Literal, + NamedTuple, + Optional, + TypeVar, + Union, + cast, + overload, +) from typing_extensions import TypeAlias as _TypeAlias, TypeGuard import mypy.checkexpr @@ -277,6 +288,26 @@ class PartialTypeScope(NamedTuple): is_local: bool +class LocalTypeMap: + """Store inferred types into a temporary type map (returned). + + This can be used to perform type checking "experiments" without + affecting exported types (which are used by mypyc). + """ + + def __init__(self, chk: TypeChecker) -> None: + self.chk = chk + + def __enter__(self) -> dict[Expression, Type]: + temp_type_map: dict[Expression, Type] = {} + self.chk._type_maps.append(temp_type_map) + return temp_type_map + + def __exit__(self, exc_type: object, exc_val: object, exc_tb: object) -> Literal[False]: + self.chk._type_maps.pop() + return False + + class TypeChecker(NodeVisitor[None], TypeCheckerSharedApi): """Mypy type checker. @@ -402,6 +433,7 @@ def __init__( self.is_typeshed_stub = tree.is_typeshed_file(options) self.inferred_attribute_types = None self.allow_constructor_cache = True + self.local_type_map = LocalTypeMap(self) # If True, process function definitions. If False, don't. This is used # for processing module top levels in fine-grained incremental mode. @@ -4631,7 +4663,7 @@ def check_simple_assignment( # may cause some perf impact, plus we want to partially preserve # the old behavior. This helps with various practical examples, see # e.g. testOptionalTypeNarrowedByGenericCall. - with self.msg.filter_errors() as local_errors, self.local_type_map() as type_map: + with self.msg.filter_errors() as local_errors, self.local_type_map as type_map: alt_rvalue_type = self.expr_checker.accept( rvalue, None, always_allow_any=always_allow_any ) @@ -7458,18 +7490,6 @@ def lookup_type(self, node: Expression) -> Type: def store_types(self, d: dict[Expression, Type]) -> None: self._type_maps[-1].update(d) - @contextmanager - def local_type_map(self) -> Iterator[dict[Expression, Type]]: - """Store inferred types into a temporary type map (returned). - - This can be used to perform type checking "experiments" without - affecting exported types (which are used by mypyc). - """ - temp_type_map: dict[Expression, Type] = {} - self._type_maps.append(temp_type_map) - yield temp_type_map - self._type_maps.pop() - def in_checked_function(self) -> bool: """Should we type-check the current function? diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 04ea678a2736a..63f39b6416028 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1075,7 +1075,7 @@ def check_typeddict_call_with_kwargs( # We don't show any errors, just infer types in a generic TypedDict type, # a custom error message will be given below, if there are errors. - with self.msg.filter_errors(), self.chk.local_type_map(): + with self.msg.filter_errors(), self.chk.local_type_map: orig_ret_type, _ = self.check_callable_call( infer_callee, # We use first expression for each key to infer type variables of a generic @@ -1440,7 +1440,7 @@ def is_generic_decorator_overload_call( return None if not isinstance(get_proper_type(callee_type.ret_type), CallableType): return None - with self.chk.local_type_map(): + with self.chk.local_type_map: with self.msg.filter_errors(): arg_type = get_proper_type(self.accept(args[0], type_context=None)) if isinstance(arg_type, Overloaded): @@ -2920,7 +2920,7 @@ def infer_overload_return_type( for typ in plausible_targets: assert self.msg is self.chk.msg with self.msg.filter_errors() as w: - with self.chk.local_type_map() as m: + with self.chk.local_type_map as m: ret_type, infer_type = self.check_call( callee=typ, args=args, @@ -5367,7 +5367,7 @@ def visit_dict_expr(self, e: DictExpr) -> Type: return self.check_typeddict_literal_in_context(e, typeddict_contexts[0]) # Multiple items union, check if at least one of them matches cleanly. for typeddict_context in typeddict_contexts: - with self.msg.filter_errors() as err, self.chk.local_type_map() as tmap: + with self.msg.filter_errors() as err, self.chk.local_type_map as tmap: ret_type = self.check_typeddict_literal_in_context(e, typeddict_context) if err.has_new_errors(): continue @@ -6095,15 +6095,12 @@ def accept( def accept_maybe_cache(self, node: Expression, type_context: Type | None = None) -> Type: binder_version = self.chk.binder.version - # Micro-optimization: inline local_type_map() as it is somewhat slow in mypyc. - type_map: dict[Expression, Type] = {} - self.chk._type_maps.append(type_map) with self.msg.filter_errors(filter_errors=True, save_filtered_errors=True) as msg: - typ = node.accept(self) + with self.chk.local_type_map as type_map: + typ = node.accept(self) messages = msg.filtered_errors() if binder_version == self.chk.binder.version and not self.chk.current_node_deferred: self.expr_cache[(node, type_context)] = (binder_version, typ, messages, type_map) - self.chk._type_maps.pop() self.chk.store_types(type_map) self.msg.add_errors(messages) return typ diff --git a/mypy/semanal.py b/mypy/semanal.py index dfa2102346069..bebabfd3233c8 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -3048,7 +3048,9 @@ def report_missing_module_attribute( message = ( f'Module "{import_id}" does not explicitly export attribute "{source_id}"' ) - else: + elif not ( + self.options.ignore_errors or self.cur_mod_node.path in self.errors.ignored_files + ): alternatives = set(module.names.keys()).difference({source_id}) matches = best_matches(source_id, alternatives, n=3) if matches: From 94eb6b75388e48f3beb0691f27d3d16cee149a93 Mon Sep 17 00:00:00 2001 From: Brian Schubert Date: Sun, 10 Aug 2025 11:18:08 -0400 Subject: [PATCH 159/424] [mypyc] Fix seg fault due to heap type objects with static tp_doc (#19636) See https://github.com/python/mypy/pull/19634#issuecomment-3172291620 Also took the opportunity to add `PyDoc_STR` to the static docstrings. AFAIK this isn't strictly necessary, but it's better style and in theory makes it possible to compile without docstrings if someone wanted to do that. --- mypyc/codegen/emitclass.py | 4 ++-- mypyc/codegen/emitmodule.py | 2 +- mypyc/lib-rt/misc_ops.c | 15 +++++++++++++++ mypyc/test-data/run-signatures.test | 16 ++++++++++++++++ 4 files changed, 34 insertions(+), 3 deletions(-) diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 0c2d470104d03..ecf8c37f83c9f 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -375,7 +375,7 @@ def emit_line() -> None: flags.append("Py_TPFLAGS_MANAGED_DICT") fields["tp_flags"] = " | ".join(flags) - fields["tp_doc"] = native_class_doc_initializer(cl) + fields["tp_doc"] = f"PyDoc_STR({native_class_doc_initializer(cl)})" emitter.emit_line(f"static PyTypeObject {emitter.type_struct_name(cl)}_template_ = {{") emitter.emit_line("PyVarObject_HEAD_INIT(NULL, 0)") @@ -925,7 +925,7 @@ def generate_methods_table(cl: ClassIR, name: str, emitter: Emitter) -> None: flags.append("METH_CLASS") doc = native_function_doc_initializer(fn) - emitter.emit_line(" {}, {}}},".format(" | ".join(flags), doc)) + emitter.emit_line(" {}, PyDoc_STR({})}},".format(" | ".join(flags), doc)) # Provide a default __getstate__ and __setstate__ if not cl.has_method("__setstate__") and not cl.has_method("__getstate__"): diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index de34ed9fc7da4..1e49b1320b26d 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -983,7 +983,7 @@ def emit_module_methods( emitter.emit_line( ( '{{"{name}", (PyCFunction){prefix}{cname}, {flag} | METH_KEYWORDS, ' - "{doc} /* docstring */}}," + "PyDoc_STR({doc}) /* docstring */}}," ).format( name=name, cname=fn.cname(emitter.names), prefix=PREFIX, flag=flag, doc=doc ) diff --git a/mypyc/lib-rt/misc_ops.c b/mypyc/lib-rt/misc_ops.c index 3787ea553037b..0c9d7812ac6c7 100644 --- a/mypyc/lib-rt/misc_ops.c +++ b/mypyc/lib-rt/misc_ops.c @@ -300,6 +300,21 @@ PyObject *CPyType_FromTemplate(PyObject *template, Py_XDECREF(dummy_class); + // Unlike the tp_doc slots of most other object, a heap type's tp_doc + // must be heap allocated. + if (template_->tp_doc) { + // Silently truncate the docstring if it contains a null byte + Py_ssize_t size = strlen(template_->tp_doc) + 1; + char *tp_doc = (char *)PyMem_Malloc(size); + if (tp_doc == NULL) { + PyErr_NoMemory(); + goto error; + } + + memcpy(tp_doc, template_->tp_doc, size); + t->ht_type.tp_doc = tp_doc; + } + #if PY_MINOR_VERSION == 11 // This is a hack. Python 3.11 doesn't include good public APIs to work with managed // dicts, which are the default for heap types. So we try to opt-out until Python 3.12. diff --git a/mypyc/test-data/run-signatures.test b/mypyc/test-data/run-signatures.test index a2de7076f5ef4..0a9ea32f5357d 100644 --- a/mypyc/test-data/run-signatures.test +++ b/mypyc/test-data/run-signatures.test @@ -184,6 +184,22 @@ for cls in [Empty, HasInit, InheritedInit]: assert getattr(cls, "__doc__") == "" assert getattr(HasInitBad, "__doc__") is None +[case testSignaturesConstructorsNonExt] +from mypy_extensions import mypyc_attr + +@mypyc_attr(native_class=False) +class NonExt: + def __init__(self, x) -> None: pass + +[file driver.py] +import inspect +from testutil import assertRaises +from native import * + +# TODO: support constructor signatures for non-extension classes +with assertRaises(ValueError, "no signature found for builtin"): + inspect.signature(NonExt) + [case testSignaturesHistoricalPositionalOnly] import inspect From a07abb64c46b43700c0f9a58cfeaa5c16cff5d93 Mon Sep 17 00:00:00 2001 From: Christoph Tyralla Date: Sun, 10 Aug 2025 17:19:07 +0200 Subject: [PATCH 160/424] PEP 702 (@deprecated): handle "combined" overloads (#19626) This change is taken from #18682. The new code and the tests are unmodified. I only had to remove two now unnecessary calls of `warn_deprecated` which were introduced after opening #18682. --- mypy/checkexpr.py | 30 ++++++----- test-data/unit/check-deprecated.test | 76 ++++++++++++++++++++++++++-- 2 files changed, 89 insertions(+), 17 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 63f39b6416028..9752a5e68638f 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -2722,6 +2722,7 @@ def check_overload_call( # for example, when we have a fallback alternative that accepts an unrestricted # typevar. See https://github.com/python/mypy/issues/4063 for related discussion. erased_targets: list[CallableType] | None = None + inferred_types: list[Type] | None = None unioned_result: tuple[Type, Type] | None = None # Determine whether we need to encourage union math. This should be generally safe, @@ -2749,13 +2750,14 @@ def check_overload_call( # Record if we succeeded. Next we need to see if maybe normal procedure # gives a narrower type. if unioned_return: - returns, inferred_types = zip(*unioned_return) + returns = [u[0] for u in unioned_return] + inferred_types = [u[1] for u in unioned_return] # Note that we use `combine_function_signatures` instead of just returning # a union of inferred callables because for example a call # Union[int -> int, str -> str](Union[int, str]) is invalid and # we don't want to introduce internal inconsistencies. unioned_result = ( - make_simplified_union(list(returns), context.line, context.column), + make_simplified_union(returns, context.line, context.column), self.combine_function_signatures(get_proper_types(inferred_types)), ) @@ -2770,7 +2772,7 @@ def check_overload_call( object_type, context, ) - # If any of checks succeed, stop early. + # If any of checks succeed, perform deprecation tests and stop early. if inferred_result is not None and unioned_result is not None: # Both unioned and direct checks succeeded, choose the more precise type. if ( @@ -2778,11 +2780,18 @@ def check_overload_call( and not isinstance(get_proper_type(inferred_result[0]), AnyType) and not none_type_var_overlap ): - return inferred_result - return unioned_result - elif unioned_result is not None: + unioned_result = None + else: + inferred_result = None + if unioned_result is not None: + if inferred_types is not None: + for inferred_type in inferred_types: + if isinstance(c := get_proper_type(inferred_type), CallableType): + self.chk.warn_deprecated(c.definition, context) return unioned_result - elif inferred_result is not None: + if inferred_result is not None: + if isinstance(c := get_proper_type(inferred_result[1]), CallableType): + self.chk.warn_deprecated(c.definition, context) return inferred_result # Step 4: Failure. At this point, we know there is no match. We fall back to trying @@ -2936,8 +2945,6 @@ def infer_overload_return_type( # check for ambiguity due to 'Any' below. if not args_contain_any: self.chk.store_types(m) - if isinstance(infer_type, ProperType) and isinstance(infer_type, CallableType): - self.chk.warn_deprecated(infer_type.definition, context) return ret_type, infer_type p_infer_type = get_proper_type(infer_type) if isinstance(p_infer_type, CallableType): @@ -2974,11 +2981,6 @@ def infer_overload_return_type( else: # Success! No ambiguity; return the first match. self.chk.store_types(type_maps[0]) - inferred_callable = inferred_types[0] - if isinstance(inferred_callable, ProperType) and isinstance( - inferred_callable, CallableType - ): - self.chk.warn_deprecated(inferred_callable.definition, context) return return_types[0], inferred_types[0] def overload_erased_call_targets( diff --git a/test-data/unit/check-deprecated.test b/test-data/unit/check-deprecated.test index e1173ac425bad..607e9d7679561 100644 --- a/test-data/unit/check-deprecated.test +++ b/test-data/unit/check-deprecated.test @@ -671,9 +671,11 @@ C().g = "x" # E: function __main__.C.g is deprecated: use g2 instead \ [case testDeprecatedDescriptor] # flags: --enable-error-code=deprecated -from typing import Any, Optional, Union, overload +from typing import Any, Generic, Optional, overload, TypeVar, Union from typing_extensions import deprecated +T = TypeVar("T") + @deprecated("use E1 instead") class D1: def __get__(self, obj: Optional[C], objtype: Any) -> Union[D1, int]: ... @@ -701,10 +703,19 @@ class D3: def __set__(self, obj: C, value: str) -> None: ... def __set__(self, obj: C, value: Union[int, str]) -> None: ... +class D4(Generic[T]): + @overload + def __get__(self, obj: None, objtype: Any) -> T: ... + @overload + @deprecated("deprecated instance access") + def __get__(self, obj: C, objtype: Any) -> T: ... + def __get__(self, obj: Optional[C], objtype: Any) -> T: ... + class C: d1 = D1() # E: class __main__.D1 is deprecated: use E1 instead d2 = D2() d3 = D3() + d4 = D4[int]() c: C C.d1 @@ -719,15 +730,21 @@ C.d3 # E: overload def (self: __main__.D3, obj: None, objtype: Any) -> __main__ c.d3 # E: overload def (self: __main__.D3, obj: __main__.C, objtype: Any) -> builtins.int of function __main__.D3.__get__ is deprecated: use E3.__get__ instead c.d3 = 1 c.d3 = "x" # E: overload def (self: __main__.D3, obj: __main__.C, value: builtins.str) of function __main__.D3.__set__ is deprecated: use E3.__set__ instead + +C.d4 +c.d4 # E: overload def (self: __main__.D4[T`1], obj: __main__.C, objtype: Any) -> T`1 of function __main__.D4.__get__ is deprecated: deprecated instance access [builtins fixtures/property.pyi] [case testDeprecatedOverloadedFunction] # flags: --enable-error-code=deprecated -from typing import Union, overload +from typing import Any, overload, Union from typing_extensions import deprecated +int_or_str: Union[int, str] +any: Any + @overload def f(x: int) -> int: ... @overload @@ -738,6 +755,8 @@ def f(x: Union[int, str]) -> Union[int, str]: ... f # E: function __main__.f is deprecated: use f2 instead f(1) # E: function __main__.f is deprecated: use f2 instead f("x") # E: function __main__.f is deprecated: use f2 instead +f(int_or_str) # E: function __main__.f is deprecated: use f2 instead +f(any) # E: function __main__.f is deprecated: use f2 instead f(1.0) # E: function __main__.f is deprecated: use f2 instead \ # E: No overload variant of "f" matches argument type "float" \ # N: Possible overload variants: \ @@ -754,6 +773,8 @@ def g(x: Union[int, str]) -> Union[int, str]: ... g g(1) # E: overload def (x: builtins.int) -> builtins.int of function __main__.g is deprecated: work with str instead g("x") +g(int_or_str) # E: overload def (x: builtins.int) -> builtins.int of function __main__.g is deprecated: work with str instead +g(any) g(1.0) # E: No overload variant of "g" matches argument type "float" \ # N: Possible overload variants: \ # N: def g(x: int) -> int \ @@ -769,13 +790,62 @@ def h(x: Union[int, str]) -> Union[int, str]: ... h h(1) h("x") # E: overload def (x: builtins.str) -> builtins.str of function __main__.h is deprecated: work with int instead +h(int_or_str) # E: overload def (x: builtins.str) -> builtins.str of function __main__.h is deprecated: work with int instead +h(any) h(1.0) # E: No overload variant of "h" matches argument type "float" \ # N: Possible overload variants: \ # N: def h(x: int) -> int \ # N: def h(x: str) -> str -[builtins fixtures/tuple.pyi] +@overload +def i(x: int) -> int: ... +@overload +@deprecated("work with int instead") +def i(x: str) -> str: ... +@overload +def i(x: Any) -> Any: ... +def i(x: Union[int, str]) -> Union[int, str]: ... +i +i(1) +i("x") # E: overload def (x: builtins.str) -> builtins.str of function __main__.i is deprecated: work with int instead +i(int_or_str) # E: overload def (x: builtins.str) -> builtins.str of function __main__.i is deprecated: work with int instead +i(any) +i(1.0) + +@overload +def j(x: int) -> int: ... +@overload +def j(x: str) -> str: ... +@overload +@deprecated("work with int or str instead") +def j(x: Any) -> Any: ... +def j(x: Union[int, str]) -> Union[int, str]: ... + +j +j(1) +j("x") +j(int_or_str) +j(any) +j(1.0) # E: overload def (x: Any) -> Any of function __main__.j is deprecated: work with int or str instead + +@overload +@deprecated("work with str instead") +def k(x: int) -> int: ... +@overload +def k(x: str) -> str: ... +@overload +@deprecated("work with str instead") +def k(x: object) -> Any: ... +def k(x: object) -> Union[int, str]: ... + +k +k(1) # E: overload def (x: builtins.int) -> builtins.int of function __main__.k is deprecated: work with str instead +k("x") +k(int_or_str) # E: overload def (x: builtins.int) -> builtins.int of function __main__.k is deprecated: work with str instead +k(any) +k(1.0) # E: overload def (x: builtins.object) -> Any of function __main__.k is deprecated: work with str instead +[builtins fixtures/tuple.pyi] [case testDeprecatedImportedOverloadedFunction] # flags: --enable-error-code=deprecated From 6d0ce5eb7ffa3322bd9a63708682a9805c7996e1 Mon Sep 17 00:00:00 2001 From: Piotr Sawicki Date: Mon, 11 Aug 2025 14:32:22 +0200 Subject: [PATCH 161/424] [mypyc] Add prefix to attributes of generator classes (#19535) Fixes https://github.com/mypyc/mypyc/issues/1120 `async` functions were recently changed to be represented as generator classes in the IR. Their parameters are represented as attributes in those classes. There are several methods added to every generator class by the compiler, one of those is called `send`. If there is also a parameter called `send`, mypyc assumes that the method is a property getter/setter and inserts method calls in the generated code for `GetAttr` and `SetAttr` nodes in the IR. This is incorrect and led to the compilation error in the linked issue because the `send` method of generator classes takes one more argument than a property getter would. The name clash is fixed by adding a prefix `__mypyc_generator_attribute__` to attribute names derived from parameters, which should ensure that argument references are not converted into method calls. --- mypyc/codegen/emitfunc.py | 7 ++-- mypyc/common.py | 1 + mypyc/irbuild/builder.py | 11 +++++-- mypyc/irbuild/env_class.py | 51 ++++++++++++++++++++--------- mypyc/irbuild/generator.py | 18 ++++++---- mypyc/test-data/run-async.test | 26 +++++++++++++++ mypyc/test-data/run-generators.test | 36 ++++++++++++++++++++ 7 files changed, 122 insertions(+), 28 deletions(-) diff --git a/mypyc/codegen/emitfunc.py b/mypyc/codegen/emitfunc.py index 086be293d5b36..f00f2e7002171 100644 --- a/mypyc/codegen/emitfunc.py +++ b/mypyc/codegen/emitfunc.py @@ -8,6 +8,7 @@ from mypyc.codegen.cstring import c_string_initializer from mypyc.codegen.emit import DEBUG_ERRORS, Emitter, TracebackAndGotoHandler, c_array_initializer from mypyc.common import ( + GENERATOR_ATTRIBUTE_PREFIX, HAVE_IMMORTAL, MODULE_PREFIX, NATIVE_PREFIX, @@ -436,7 +437,9 @@ def visit_get_attr(self, op: GetAttr) -> None: exc_class = "PyExc_AttributeError" self.emitter.emit_line( 'PyErr_SetString({}, "attribute {} of {} undefined");'.format( - exc_class, repr(op.attr), repr(cl.name) + exc_class, + repr(op.attr.removeprefix(GENERATOR_ATTRIBUTE_PREFIX)), + repr(cl.name), ) ) @@ -938,7 +941,7 @@ def emit_attribute_error(self, op: Branch, class_name: str, attr: str) -> None: self.source_path.replace("\\", "\\\\"), op.traceback_entry[0], class_name, - attr, + attr.removeprefix(GENERATOR_ATTRIBUTE_PREFIX), op.traceback_entry[1], globals_static, ) diff --git a/mypyc/common.py b/mypyc/common.py index b5506eed89c22..3a77e9e60c355 100644 --- a/mypyc/common.py +++ b/mypyc/common.py @@ -22,6 +22,7 @@ LAMBDA_NAME: Final = "__mypyc_lambda__" PROPSET_PREFIX: Final = "__mypyc_setter__" SELF_NAME: Final = "__mypyc_self__" +GENERATOR_ATTRIBUTE_PREFIX: Final = "__mypyc_generator_attribute__" # Max short int we accept as a literal is based on 32-bit platforms, # so that we can just always emit the same code. diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index ec3c1b1b1f3cf..608c524b5d4fb 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -59,7 +59,7 @@ ) from mypy.util import module_prefix, split_target from mypy.visitor import ExpressionVisitor, StatementVisitor -from mypyc.common import BITMAP_BITS, SELF_NAME, TEMP_ATTR_NAME +from mypyc.common import BITMAP_BITS, GENERATOR_ATTRIBUTE_PREFIX, SELF_NAME, TEMP_ATTR_NAME from mypyc.crash import catch_errors from mypyc.errors import Errors from mypyc.ir.class_ir import ClassIR, NonExtClassInfo @@ -651,7 +651,11 @@ def get_assignment_target( # current environment. if self.fn_info.is_generator: return self.add_var_to_env_class( - symbol, reg_type, self.fn_info.generator_class, reassign=False + symbol, + reg_type, + self.fn_info.generator_class, + reassign=False, + prefix=GENERATOR_ATTRIBUTE_PREFIX, ) # Otherwise define a new local variable. @@ -1333,10 +1337,11 @@ def add_var_to_env_class( base: FuncInfo | ImplicitClass, reassign: bool = False, always_defined: bool = False, + prefix: str = "", ) -> AssignmentTarget: # First, define the variable name as an attribute of the environment class, and then # construct a target for that attribute. - name = remangle_redefinition_name(var.name) + name = prefix + remangle_redefinition_name(var.name) self.fn_info.env_class.attributes[name] = rtype if always_defined: self.fn_info.env_class.attrs_with_defaults.add(name) diff --git a/mypyc/irbuild/env_class.py b/mypyc/irbuild/env_class.py index 51c854a4a2b2d..2334b4370103f 100644 --- a/mypyc/irbuild/env_class.py +++ b/mypyc/irbuild/env_class.py @@ -18,7 +18,13 @@ def g() -> int: from __future__ import annotations from mypy.nodes import Argument, FuncDef, SymbolNode, Var -from mypyc.common import BITMAP_BITS, ENV_ATTR_NAME, SELF_NAME, bitmap_name +from mypyc.common import ( + BITMAP_BITS, + ENV_ATTR_NAME, + GENERATOR_ATTRIBUTE_PREFIX, + SELF_NAME, + bitmap_name, +) from mypyc.ir.class_ir import ClassIR from mypyc.ir.ops import Call, GetAttr, SetAttr, Value from mypyc.ir.rtypes import RInstance, bitmap_rprimitive, object_rprimitive @@ -60,7 +66,7 @@ class is generated, the function environment has not yet been return env_class -def finalize_env_class(builder: IRBuilder) -> None: +def finalize_env_class(builder: IRBuilder, prefix: str = "") -> None: """Generate, instantiate, and set up the environment of an environment class.""" if not builder.fn_info.can_merge_generator_and_env_classes(): instantiate_env_class(builder) @@ -69,9 +75,9 @@ def finalize_env_class(builder: IRBuilder) -> None: # that were previously added to the environment with references to the function's # environment class. if builder.fn_info.is_nested: - add_args_to_env(builder, local=False, base=builder.fn_info.callable_class) + add_args_to_env(builder, local=False, base=builder.fn_info.callable_class, prefix=prefix) else: - add_args_to_env(builder, local=False, base=builder.fn_info) + add_args_to_env(builder, local=False, base=builder.fn_info, prefix=prefix) def instantiate_env_class(builder: IRBuilder) -> Value: @@ -96,7 +102,7 @@ def instantiate_env_class(builder: IRBuilder) -> Value: return curr_env_reg -def load_env_registers(builder: IRBuilder) -> None: +def load_env_registers(builder: IRBuilder, prefix: str = "") -> None: """Load the registers for the current FuncItem being visited. Adds the arguments of the FuncItem to the environment. If the @@ -104,7 +110,7 @@ def load_env_registers(builder: IRBuilder) -> None: loads all of the outer environments of the FuncItem into registers so that they can be used when accessing free variables. """ - add_args_to_env(builder, local=True) + add_args_to_env(builder, local=True, prefix=prefix) fn_info = builder.fn_info fitem = fn_info.fitem @@ -113,7 +119,7 @@ def load_env_registers(builder: IRBuilder) -> None: # If this is a FuncDef, then make sure to load the FuncDef into its own environment # class so that the function can be called recursively. if isinstance(fitem, FuncDef) and fn_info.add_nested_funcs_to_env: - setup_func_for_recursive_call(builder, fitem, fn_info.callable_class) + setup_func_for_recursive_call(builder, fitem, fn_info.callable_class, prefix=prefix) def load_outer_env( @@ -134,8 +140,11 @@ def load_outer_env( assert isinstance(env.type, RInstance), f"{env} must be of type RInstance" for symbol, target in outer_env.items(): - env.type.class_ir.attributes[symbol.name] = target.type - symbol_target = AssignmentTargetAttr(env, symbol.name) + attr_name = symbol.name + if isinstance(target, AssignmentTargetAttr): + attr_name = target.attr + env.type.class_ir.attributes[attr_name] = target.type + symbol_target = AssignmentTargetAttr(env, attr_name) builder.add_target(symbol, symbol_target) return env @@ -178,6 +187,7 @@ def add_args_to_env( local: bool = True, base: FuncInfo | ImplicitClass | None = None, reassign: bool = True, + prefix: str = "", ) -> None: fn_info = builder.fn_info args = fn_info.fitem.arguments @@ -193,10 +203,12 @@ def add_args_to_env( if is_free_variable(builder, arg.variable) or fn_info.is_generator: rtype = builder.type_to_rtype(arg.variable.type) assert base is not None, "base cannot be None for adding nonlocal args" - builder.add_var_to_env_class(arg.variable, rtype, base, reassign=reassign) + builder.add_var_to_env_class( + arg.variable, rtype, base, reassign=reassign, prefix=prefix + ) -def add_vars_to_env(builder: IRBuilder) -> None: +def add_vars_to_env(builder: IRBuilder, prefix: str = "") -> None: """Add relevant local variables and nested functions to the environment class. Add all variables and functions that are declared/defined within current @@ -216,7 +228,9 @@ def add_vars_to_env(builder: IRBuilder) -> None: for var in sorted(builder.free_variables[builder.fn_info.fitem], key=lambda x: x.name): if isinstance(var, Var): rtype = builder.type_to_rtype(var.type) - builder.add_var_to_env_class(var, rtype, env_for_func, reassign=False) + builder.add_var_to_env_class( + var, rtype, env_for_func, reassign=False, prefix=prefix + ) if builder.fn_info.fitem in builder.encapsulating_funcs: for nested_fn in builder.encapsulating_funcs[builder.fn_info.fitem]: @@ -226,12 +240,16 @@ def add_vars_to_env(builder: IRBuilder) -> None: # the same name and signature across conditional blocks # will generate different callable classes, so the callable # class that gets instantiated must be generic. + if nested_fn.is_generator: + prefix = GENERATOR_ATTRIBUTE_PREFIX builder.add_var_to_env_class( - nested_fn, object_rprimitive, env_for_func, reassign=False + nested_fn, object_rprimitive, env_for_func, reassign=False, prefix=prefix ) -def setup_func_for_recursive_call(builder: IRBuilder, fdef: FuncDef, base: ImplicitClass) -> None: +def setup_func_for_recursive_call( + builder: IRBuilder, fdef: FuncDef, base: ImplicitClass, prefix: str = "" +) -> None: """Enable calling a nested function (with a callable class) recursively. Adds the instance of the callable class representing the given @@ -241,7 +259,8 @@ def setup_func_for_recursive_call(builder: IRBuilder, fdef: FuncDef, base: Impli """ # First, set the attribute of the environment class so that GetAttr can be called on it. prev_env = builder.fn_infos[-2].env_class - prev_env.attributes[fdef.name] = builder.type_to_rtype(fdef.type) + attr_name = prefix + fdef.name + prev_env.attributes[attr_name] = builder.type_to_rtype(fdef.type) if isinstance(base, GeneratorClass): # If we are dealing with a generator class, then we need to first get the register @@ -253,7 +272,7 @@ def setup_func_for_recursive_call(builder: IRBuilder, fdef: FuncDef, base: Impli # Obtain the instance of the callable class representing the FuncDef, and add it to the # current environment. - val = builder.add(GetAttr(prev_env_reg, fdef.name, -1)) + val = builder.add(GetAttr(prev_env_reg, attr_name, -1)) target = builder.add_local_reg(fdef, object_rprimitive) builder.assign(target, val, -1) diff --git a/mypyc/irbuild/generator.py b/mypyc/irbuild/generator.py index c858946f33c40..b3a417ed6a3ef 100644 --- a/mypyc/irbuild/generator.py +++ b/mypyc/irbuild/generator.py @@ -13,7 +13,7 @@ from typing import Callable from mypy.nodes import ARG_OPT, FuncDef, Var -from mypyc.common import ENV_ATTR_NAME, NEXT_LABEL_ATTR_NAME +from mypyc.common import ENV_ATTR_NAME, GENERATOR_ATTRIBUTE_PREFIX, NEXT_LABEL_ATTR_NAME from mypyc.ir.class_ir import ClassIR from mypyc.ir.func_ir import FuncDecl, FuncIR from mypyc.ir.ops import ( @@ -68,14 +68,14 @@ def gen_generator_func( ) -> tuple[FuncIR, Value | None]: """Generate IR for generator function that returns generator object.""" setup_generator_class(builder) - load_env_registers(builder) + load_env_registers(builder, prefix=GENERATOR_ATTRIBUTE_PREFIX) gen_arg_defaults(builder) if builder.fn_info.can_merge_generator_and_env_classes(): gen = instantiate_generator_class(builder) builder.fn_info._curr_env_reg = gen - finalize_env_class(builder) + finalize_env_class(builder, prefix=GENERATOR_ATTRIBUTE_PREFIX) else: - finalize_env_class(builder) + finalize_env_class(builder, prefix=GENERATOR_ATTRIBUTE_PREFIX) gen = instantiate_generator_class(builder) builder.add(Return(gen)) @@ -104,11 +104,13 @@ class that implements the function (each function gets a separate class). and top_level and top_level.add_nested_funcs_to_env ): - setup_func_for_recursive_call(builder, fitem, builder.fn_info.generator_class) + setup_func_for_recursive_call( + builder, fitem, builder.fn_info.generator_class, prefix=GENERATOR_ATTRIBUTE_PREFIX + ) create_switch_for_generator_class(builder) add_raise_exception_blocks_to_generator_class(builder, fitem.line) - add_vars_to_env(builder) + add_vars_to_env(builder, prefix=GENERATOR_ATTRIBUTE_PREFIX) builder.accept(fitem.body) builder.maybe_add_implicit_return() @@ -429,7 +431,9 @@ def setup_env_for_generator_class(builder: IRBuilder) -> None: # Add arguments from the original generator function to the # environment of the generator class. - add_args_to_env(builder, local=False, base=cls, reassign=False) + add_args_to_env( + builder, local=False, base=cls, reassign=False, prefix=GENERATOR_ATTRIBUTE_PREFIX + ) # Set the next label register for the generator class. cls.next_label_reg = builder.read(cls.next_label_target, fitem.line) diff --git a/mypyc/test-data/run-async.test b/mypyc/test-data/run-async.test index a1112e9646712..55cde4ab44f1f 100644 --- a/mypyc/test-data/run-async.test +++ b/mypyc/test-data/run-async.test @@ -1208,3 +1208,29 @@ async def test_async_context_manager_exception_handling() -> None: [file asyncio/__init__.pyi] async def sleep(t: float) -> None: ... + +[case testCallableArgWithSameNameAsHelperMethod] +import asyncio +from typing import Awaitable, Callable + + +MyCallable = Callable[[int, int], Awaitable[int]] + +async def add(a: int, b: int) -> int: + return a + b + +async def await_send(send: MyCallable) -> int: + return await send(1, 2) + +async def await_throw(throw: MyCallable) -> int: + return await throw(3, 4) + +async def tests() -> None: + assert await await_send(add) == 3 + assert await await_throw(add) == 7 + +def test_callable_arg_same_name_as_helper() -> None: + asyncio.run(tests()) + +[file asyncio/__init__.pyi] +def run(x: object) -> object: ... diff --git a/mypyc/test-data/run-generators.test b/mypyc/test-data/run-generators.test index 3b4581f849e9f..bfbd5b83696b1 100644 --- a/mypyc/test-data/run-generators.test +++ b/mypyc/test-data/run-generators.test @@ -272,8 +272,17 @@ def call_nested_decorated(x: int) -> list[int]: a.append(x) return a +def call_nested_recursive(x: int) -> Iterator: + def recursive(x: int) -> Iterator: + if x > 0: + yield from recursive(x - 1) + yield x + + yield from recursive(x) + def test_call_nested_generator_in_function() -> None: assert call_nested_decorated(5) == [5, 15] + assert list(call_nested_recursive(5)) == [0, 1, 2, 3, 4, 5] [case testYieldThrow] from typing import Generator, Iterable, Any, Union @@ -871,3 +880,30 @@ def test_undefined_int_in_environment() -> None: with assertRaises(AttributeError): # TODO: Should be UnboundLocalError list(gen2(False)) + +[case testVariableWithSameNameAsHelperMethod] +from testutil import assertRaises +from typing import Iterator + +def gen_send() -> Iterator[int]: + send = 1 + yield send + 1 + +def gen_throw() -> Iterator[int]: + throw = 42 + yield throw * 2 + +def undefined() -> Iterator[int]: + if int(): + send = 1 + yield send + 1 + +def test_same_names() -> None: + assert list(gen_send()) == [2] + assert list(gen_throw()) == [84] + + with assertRaises(AttributeError, "attribute 'send' of 'undefined_gen' undefined"): + # TODO: Should be UnboundLocalError, this test verifies that the attribute name + # matches the variable name in the input code, since internally it's generated + # with a prefix. + list(undefined()) From 5a786075d8c366ee753c62fa36857589023ed561 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 11 Aug 2025 16:24:15 +0100 Subject: [PATCH 162/424] Another two micro-optimizations (#19633) Here are two things: * Make `FormalArgument` a native class. We create huge amount of these (as callable subtyping is one of the most common subtype checks), and named tuples creation is significantly slower than native classes. * Do not call `re.match()` in a code path of `format_type()`. This is relatively slow (as it is a `py_call()`) and it is called in almost every error message. This creates problems for code with many third-party dependencies where these errors are ignored anyway. FWIW in total these give ~0.5% together (I didn't measure individually, but I guess the most benefit for self-check is from the first one). --------- Co-authored-by: Ali Hamdan --- mypy/messages.py | 9 +++----- mypy/types.py | 37 +++++++++++++++++--------------- test-data/unit/check-tuples.test | 2 +- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/mypy/messages.py b/mypy/messages.py index f626d4c71916f..571cebb1b174c 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -2432,13 +2432,13 @@ def format_long_tuple_type(self, typ: TupleType) -> str: """Format very long tuple type using an ellipsis notation""" item_cnt = len(typ.items) if item_cnt > MAX_TUPLE_ITEMS: - return "tuple[{}, {}, ... <{} more items>]".format( + return '"tuple[{}, {}, ... <{} more items>]"'.format( format_type_bare(typ.items[0], self.options), format_type_bare(typ.items[1], self.options), str(item_cnt - 2), ) else: - return format_type_bare(typ, self.options) + return format_type(typ, self.options) def generate_incompatible_tuple_error( self, @@ -2517,15 +2517,12 @@ def iteration_dependent_errors(self, iter_errors: IterationDependentErrors) -> N def quote_type_string(type_string: str) -> str: """Quotes a type representation for use in messages.""" - no_quote_regex = r"^<(tuple|union): \d+ items>$" if ( type_string in ["Module", "overloaded function", ""] or type_string.startswith("Module ") - or re.match(no_quote_regex, type_string) is not None or type_string.endswith("?") ): - # Messages are easier to read if these aren't quoted. We use a - # regex to match strings with variable contents. + # These messages are easier to read if these aren't quoted. return type_string return f'"{type_string}"' diff --git a/mypy/types.py b/mypy/types.py index a73ac3c3524ab..d7dd3e1f2dce8 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -5,18 +5,7 @@ import sys from abc import abstractmethod from collections.abc import Iterable, Sequence -from typing import ( - TYPE_CHECKING, - Any, - ClassVar, - Final, - NamedTuple, - NewType, - TypeVar, - Union, - cast, - overload, -) +from typing import TYPE_CHECKING, Any, ClassVar, Final, NewType, TypeVar, Union, cast, overload from typing_extensions import Self, TypeAlias as _TypeAlias, TypeGuard import mypy.nodes @@ -1607,11 +1596,25 @@ def bound(self) -> bool: return bool(self.items) and self.items[0].is_bound -class FormalArgument(NamedTuple): - name: str | None - pos: int | None - typ: Type - required: bool +class FormalArgument: + def __init__(self, name: str | None, pos: int | None, typ: Type, required: bool) -> None: + self.name = name + self.pos = pos + self.typ = typ + self.required = required + + def __eq__(self, other: object) -> bool: + if not isinstance(other, FormalArgument): + return NotImplemented + return ( + self.name == other.name + and self.pos == other.pos + and self.typ == other.typ + and self.required == other.required + ) + + def __hash__(self) -> int: + return hash((self.name, self.pos, self.typ, self.required)) class Parameters(ProperType): diff --git a/test-data/unit/check-tuples.test b/test-data/unit/check-tuples.test index 615ba129dad5d..cfdd2aacc4d25 100644 --- a/test-data/unit/check-tuples.test +++ b/test-data/unit/check-tuples.test @@ -1612,7 +1612,7 @@ t4: Tuple[int, int, int, int, int, int, int, int, int, int, int, int] = (1, 2, 3 t5: Tuple[int, int] = (1, 2, "s", 4) # E: Incompatible types in assignment (expression has type "tuple[int, int, str, int]", variable has type "tuple[int, int]") # long initializer assignment with mismatched pairs -t6: Tuple[int, int, int, int, int, int, int, int, int, int, int, int] = (1, 2, 3, 4, 5, 6, 7, 8, "str", "str", "str", "str", 1, 1, 1, 1, 1) # E: Incompatible types in assignment (expression has type tuple[int, int, ... <15 more items>], variable has type tuple[int, int, ... <10 more items>]) +t6: Tuple[int, int, int, int, int, int, int, int, int, int, int, int] = (1, 2, 3, 4, 5, 6, 7, 8, "str", "str", "str", "str", 1, 1, 1, 1, 1) # E: Incompatible types in assignment (expression has type "tuple[int, int, ... <15 more items>]", variable has type "tuple[int, int, ... <10 more items>]") [builtins fixtures/tuple.pyi] From 982853dc318c9aaf84001a98f7bd0aa4fc9cdbe1 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 12 Aug 2025 23:47:24 +0100 Subject: [PATCH 163/424] Special-case enum method calls (#19634) Improves https://github.com/mypyc/mypyc/issues/1121, this gives a bit above 1% on mypy self-check. This only adds support for regular and overloaded methods without decorators (class/static methods and properties stay slow). When working on this I considered (and actually tried) four options: * Make enums extension classes, then many methods will use fast calls ~automatically (we will just need to set a final flag). This just didn't work, in the sense no segfaults, but it looks like we don't call `__prepare__()`, or don't call it at the right moment. Or maybe I just didn't try hard enough. In general, for some reason this feels risky. * Use existing `CPyDef`s for (non-extension) enum methods, but since they have an extra argument, `__mypyc_self__`, we can supply `NULL` there, since we know it is unused. This is actually easy and it works, but IMO it is ultra-ugly, so I decided to not do it. * Write a separate `CPyDef` without `__mypy_self__`, use it for direct calls, and make existing callable classes `CPyDef`s one-line functions that simply call the first one. This is possible, but quite complicated, and I am not sure it is easy to generalize (e.g. on classmethods). * Finally, the way I do this is to simply generate a second method, that is almost a copy of the original one. This involves a bit of code duplication (in C), but the benefit is that it is conceptually simple, and easily extendable. We can cover more special cases on as-needed basis. --- mypyc/common.py | 1 + mypyc/ir/class_ir.py | 5 +++ mypyc/irbuild/function.py | 17 +++++++- mypyc/irbuild/ll_builder.py | 5 ++- mypyc/irbuild/prepare.py | 35 +++++++++++++++- mypyc/test-data/irbuild-classes.test | 60 ++++++++++++++++++++++++++++ mypyc/test-data/run-classes.test | 50 +++++++++++++++++++++++ 7 files changed, 169 insertions(+), 4 deletions(-) diff --git a/mypyc/common.py b/mypyc/common.py index 3a77e9e60c355..2de63c09bb2ce 100644 --- a/mypyc/common.py +++ b/mypyc/common.py @@ -15,6 +15,7 @@ MODULE_PREFIX: Final = "CPyModule_" # Cached modules TYPE_VAR_PREFIX: Final = "CPyTypeVar_" # Type variables when using new-style Python 3.12 syntax ATTR_PREFIX: Final = "_" # Attributes +FAST_PREFIX: Final = "__mypyc_fast_" # Optimized methods in non-extension classes ENV_ATTR_NAME: Final = "__mypyc_env__" NEXT_LABEL_ATTR_NAME: Final = "__mypyc_next_label__" diff --git a/mypyc/ir/class_ir.py b/mypyc/ir/class_ir.py index 561dc9d438c43..f6015b64dcdd2 100644 --- a/mypyc/ir/class_ir.py +++ b/mypyc/ir/class_ir.py @@ -210,6 +210,9 @@ def __init__( # per-type free "list" of up to length 1. self.reuse_freed_instance = False + # Is this a class inheriting from enum.Enum? Such classes can be special-cased. + self.is_enum = False + def __repr__(self) -> str: return ( "ClassIR(" @@ -410,6 +413,7 @@ def serialize(self) -> JsonDict: "init_self_leak": self.init_self_leak, "env_user_function": self.env_user_function.id if self.env_user_function else None, "reuse_freed_instance": self.reuse_freed_instance, + "is_enum": self.is_enum, } @classmethod @@ -466,6 +470,7 @@ def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> ClassIR: ctx.functions[data["env_user_function"]] if data["env_user_function"] else None ) ir.reuse_freed_instance = data["reuse_freed_instance"] + ir.is_enum = data["is_enum"] return ir diff --git a/mypyc/irbuild/function.py b/mypyc/irbuild/function.py index 90506adde672c..d70b164755037 100644 --- a/mypyc/irbuild/function.py +++ b/mypyc/irbuild/function.py @@ -29,7 +29,7 @@ Var, ) from mypy.types import CallableType, Type, UnboundType, get_proper_type -from mypyc.common import LAMBDA_NAME, PROPSET_PREFIX, SELF_NAME +from mypyc.common import FAST_PREFIX, LAMBDA_NAME, PROPSET_PREFIX, SELF_NAME from mypyc.ir.class_ir import ClassIR, NonExtClassInfo from mypyc.ir.func_ir import ( FUNC_CLASSMETHOD, @@ -166,6 +166,7 @@ def gen_func_item( name: str, sig: FuncSignature, cdef: ClassDef | None = None, + make_ext_method: bool = False, ) -> tuple[FuncIR, Value | None]: """Generate and return the FuncIR for a given FuncDef. @@ -217,7 +218,7 @@ def c() -> None: class_name = None if cdef: ir = builder.mapper.type_to_ir[cdef.info] - in_non_ext = not ir.is_ext_class + in_non_ext = not ir.is_ext_class and not make_ext_method class_name = cdef.name if is_singledispatch: @@ -339,6 +340,9 @@ def gen_func_ir( fitem = fn_info.fitem assert isinstance(fitem, FuncDef), fitem func_decl = builder.mapper.func_to_decl[fitem] + if cdef and fn_info.name == FAST_PREFIX + func_decl.name: + # Special-cased version of a method has a separate FuncDecl, use that one. + func_decl = builder.mapper.type_to_ir[cdef.info].method_decls[fn_info.name] if fn_info.is_decorated or is_singledispatch_main_func: class_name = None if cdef is None else cdef.name func_decl = FuncDecl( @@ -453,6 +457,15 @@ def handle_non_ext_method( builder.add_to_non_ext_dict(non_ext, name, func_reg, fdef.line) + # If we identified that this non-extension class method can be special-cased for + # direct access during prepare phase, generate a "static" version of it. + class_ir = builder.mapper.type_to_ir[cdef.info] + name = FAST_PREFIX + fdef.name + if name in class_ir.method_decls: + func_ir, func_reg = gen_func_item(builder, fdef, name, sig, cdef, make_ext_method=True) + class_ir.methods[name] = func_ir + builder.functions.append(func_ir) + def gen_func_ns(builder: IRBuilder) -> str: """Generate a namespace for a nested function using its outer function names.""" diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index a5e28268efeda..05d558e0822ac 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -17,6 +17,7 @@ from mypyc.common import ( BITMAP_BITS, FAST_ISINSTANCE_MAX_SUBCLASSES, + FAST_PREFIX, IS_FREE_THREADED, MAX_LITERAL_SHORT_INT, MAX_SHORT_INT, @@ -1171,11 +1172,13 @@ def gen_method_call( return self.py_method_call(base, name, arg_values, line, arg_kinds, arg_names) # If the base type is one of ours, do a MethodCall + fast_name = FAST_PREFIX + name if ( isinstance(base.type, RInstance) - and base.type.class_ir.is_ext_class + and (base.type.class_ir.is_ext_class or base.type.class_ir.has_method(fast_name)) and not base.type.class_ir.builtin_base ): + name = name if base.type.class_ir.is_ext_class else fast_name if base.type.class_ir.has_method(name): decl = base.type.class_ir.method_decl(name) if arg_kinds is None: diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index 1d6117ab7b1ed..83ec3f7c1d382 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -38,7 +38,7 @@ from mypy.semanal import refers_to_fullname from mypy.traverser import TraverserVisitor from mypy.types import Instance, Type, get_proper_type -from mypyc.common import PROPSET_PREFIX, SELF_NAME, get_id_from_name +from mypyc.common import FAST_PREFIX, PROPSET_PREFIX, SELF_NAME, get_id_from_name from mypyc.crash import catch_errors from mypyc.errors import Errors from mypyc.ir.class_ir import ClassIR @@ -106,6 +106,7 @@ def build_type_map( class_ir.children = None mapper.type_to_ir[cdef.info] = class_ir mapper.symbol_fullnames.add(class_ir.fullname) + class_ir.is_enum = cdef.info.is_enum and len(cdef.info.enum_members) > 0 # Populate structural information in class IR for extension classes. for module, cdef in classes: @@ -270,6 +271,36 @@ def prepare_method_def( ir.property_types[node.name] = decl.sig.ret_type +def prepare_fast_path( + ir: ClassIR, + module_name: str, + cdef: ClassDef, + mapper: Mapper, + node: SymbolNode | None, + options: CompilerOptions, +) -> None: + """Add fast (direct) variants of methods in non-extension classes.""" + if ir.is_enum: + # We check that non-empty enums are implicitly final in mypy, so we + # can generate direct calls to enum methods. + if isinstance(node, OverloadedFuncDef): + if node.is_property: + return + node = node.impl + if not isinstance(node, FuncDef): + # TODO: support decorated methods (at least @classmethod and @staticmethod). + return + # The simplest case is a regular or overloaded method without decorators. In this + # case we can generate practically identical IR method body, but with a signature + # suitable for direct calls (usual non-extension class methods are converted to + # callable classes, and thus have an extra __mypyc_self__ argument). + name = FAST_PREFIX + node.name + sig = mapper.fdef_to_sig(node, options.strict_dunders_typing) + decl = FuncDecl(name, cdef.name, module_name, sig, FUNC_NORMAL) + ir.method_decls[name] = decl + return + + def is_valid_multipart_property_def(prop: OverloadedFuncDef) -> bool: # Checks to ensure supported property decorator semantics if len(prop.items) != 2: @@ -579,6 +610,8 @@ def prepare_non_ext_class_def( else: prepare_method_def(ir, module_name, cdef, mapper, get_func_def(node.node), options) + prepare_fast_path(ir, module_name, cdef, mapper, node.node, options) + if any(cls in mapper.type_to_ir and mapper.type_to_ir[cls].is_ext_class for cls in info.mro): errors.error( "Non-extension classes may not inherit from extension classes", path, cdef.line diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index 1a2c237cc3c9b..f8ea26cd41e89 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -1408,3 +1408,63 @@ class TestOverload: def __mypyc_generator_helper__(self, x: Any) -> Any: return x + +[case testEnumFastPath] +from enum import Enum + +def test(e: E) -> bool: + return e.is_one() + +class E(Enum): + ONE = 1 + TWO = 2 + + def is_one(self) -> bool: + return self == E.ONE +[out] +def test(e): + e :: __main__.E + r0 :: bool +L0: + r0 = e.__mypyc_fast_is_one() + return r0 +def is_one_E_obj.__get__(__mypyc_self__, instance, owner): + __mypyc_self__, instance, owner, r0 :: object + r1 :: bit + r2 :: object +L0: + r0 = load_address _Py_NoneStruct + r1 = instance == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return __mypyc_self__ +L2: + r2 = PyMethod_New(__mypyc_self__, instance) + return r2 +def is_one_E_obj.__call__(__mypyc_self__, self): + __mypyc_self__ :: __main__.is_one_E_obj + self, r0 :: __main__.E + r1 :: bool + r2 :: bit +L0: + r0 = __main__.E.ONE :: static + if is_error(r0) goto L1 else goto L2 +L1: + r1 = raise NameError('value for final name "ONE" was not set') + unreachable +L2: + r2 = self == r0 + return r2 +def E.__mypyc_fast_is_one(self): + self, r0 :: __main__.E + r1 :: bool + r2 :: bit +L0: + r0 = __main__.E.ONE :: static + if is_error(r0) goto L1 else goto L2 +L1: + r1 = raise NameError('value for final name "ONE" was not set') + unreachable +L2: + r2 = self == r0 + return r2 diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 54f5343bc7bb8..1481f3e068715 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -2710,6 +2710,56 @@ from native import Player [out] Player.MIN = +[case testEnumMethodCalls] +from enum import Enum +from typing import overload, Optional, Union + +class C: + def foo(self, x: Test) -> bool: + assert Test.ONE.is_one() + assert x.next(2) == Test.THREE + assert x.prev(2) == Test.ONE + assert x.enigma(22) + assert x.enigma("22") == 22 + return x.is_one(inverse=True) + +class Test(Enum): + ONE = 1 + TWO = 2 + THREE = 3 + + def is_one(self, *, inverse: bool = False) -> bool: + if inverse: + return self != Test.ONE + return self == Test.ONE + + @classmethod + def next(cls, val: int) -> Test: + return cls(val + 1) + + @staticmethod + def prev(val: int) -> Test: + return Test(val - 1) + + @overload + def enigma(self, val: int) -> bool: ... + @overload + def enigma(self, val: Optional[str] = None) -> int: ... + def enigma(self, val: Union[int, str, None] = None) -> Union[int, bool]: + if isinstance(val, int): + return self.is_one() + return 22 +[file driver.py] +from native import Test, C + +assert Test.ONE.is_one() +assert Test.TWO.is_one(inverse=True) +assert not C().foo(Test.ONE) +assert Test.next(2) == Test.THREE +assert Test.prev(2) == Test.ONE +assert Test.ONE.enigma(22) +assert Test.ONE.enigma("22") == 22 + [case testStaticCallsWithUnpackingArgs] from typing import Tuple From a0c5238cb881f93fdfabb375c2b9a6921fefea63 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 13 Aug 2025 11:32:13 +0100 Subject: [PATCH 164/424] [mypyc] Fix remaining failing test on free-threaded builds (#19646) Just skip an in irbuild test if running on a free-threaded build, since the IR looks different if free threading is enabled. Only this one test was failing for me on Python 3.14.0rc1. --- mypyc/test-data/irbuild-classes.test | 2 +- mypyc/test/test_irbuild.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index f8ea26cd41e89..8a2cc42fbb0f5 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -180,7 +180,7 @@ L0: o.x = r1; r2 = is_error return o -[case testSubclass_toplevel] +[case testSubclass_withgil_toplevel] from typing import TypeVar, Generic from mypy_extensions import trait T = TypeVar('T') diff --git a/mypyc/test/test_irbuild.py b/mypyc/test/test_irbuild.py index d8f974ef201b6..e79cbec392f45 100644 --- a/mypyc/test/test_irbuild.py +++ b/mypyc/test/test_irbuild.py @@ -8,7 +8,7 @@ from mypy.errors import CompileError from mypy.test.config import test_temp_dir from mypy.test.data import DataDrivenTestCase -from mypyc.common import TOP_LEVEL_NAME +from mypyc.common import IS_FREE_THREADED, TOP_LEVEL_NAME from mypyc.ir.pprint import format_func from mypyc.test.testutil import ( ICODE_GEN_BUILTINS, @@ -71,6 +71,9 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: if options is None: # Skipped test case return + if "_withgil" in testcase.name and IS_FREE_THREADED: + # Test case should only run on a non-free-threaded build. + return with use_custom_builtins(os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase): expected_output = remove_comment_lines(testcase.output) expected_output = replace_word_size(expected_output) From b3d5021503624cd55b50801091166add22e7d81d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 13 Aug 2025 11:32:52 +0100 Subject: [PATCH 165/424] [mypyc] Include more operations in the trace log (#19647) Add these operations to the trace log (note that trace logging is disabled by default): * Native attribute get/set * Boxing and unboxing * Casts * Incref/decref (including some implicit increfs) All of these are common operations and can be performance bottlenecks. There are ways to avoid or speed up most of these (possibly through new mypyc features). For example, incref/decref can sometimes be avoided by using borrowing. --- mypyc/transform/log_trace.py | 89 +++++++++++++++++++++++++++++++++--- 1 file changed, 82 insertions(+), 7 deletions(-) diff --git a/mypyc/transform/log_trace.py b/mypyc/transform/log_trace.py index 5b20940c66bb6..cec76b9b4f888 100644 --- a/mypyc/transform/log_trace.py +++ b/mypyc/transform/log_trace.py @@ -9,8 +9,27 @@ from __future__ import annotations +from typing import Final + from mypyc.ir.func_ir import FuncIR -from mypyc.ir.ops import Call, CallC, CString, LoadLiteral, LoadStatic, Op, PrimitiveOp, Value +from mypyc.ir.ops import ( + Box, + Call, + CallC, + Cast, + CString, + DecRef, + GetAttr, + IncRef, + LoadLiteral, + LoadStatic, + Op, + PrimitiveOp, + SetAttr, + Unbox, + Value, +) +from mypyc.ir.rtypes import none_rprimitive from mypyc.irbuild.ll_builder import LowLevelIRBuilder from mypyc.options import CompilerOptions from mypyc.primitives.misc_ops import log_trace_event @@ -38,6 +57,18 @@ def get_load_global_name(op: CallC) -> str | None: return None +# These primitives perform an implicit IncRef for the return value. Only some of the most common ones +# are included, and mostly ops that could be switched to use borrowing in some contexts. +primitives_that_inc_ref: Final = { + "list_get_item_unsafe", + "CPyList_GetItemShort", + "CPyDict_GetWithNone", + "CPyList_GetItem", + "CPyDict_GetItem", + "CPyList_PopLast", +} + + class LogTraceEventTransform(IRTransform): def __init__(self, builder: LowLevelIRBuilder, fullname: str) -> None: super().__init__(builder) @@ -48,7 +79,10 @@ def visit_call(self, op: Call) -> Value: return self.log(op, "call", op.fn.fullname) def visit_primitive_op(self, op: PrimitiveOp) -> Value: - return self.log(op, "primitive_op", op.desc.name) + value = self.log(op, "primitive_op", op.desc.name) + if op.desc.name in primitives_that_inc_ref: + self.log_inc_ref(value) + return value def visit_call_c(self, op: CallC) -> Value: if global_name := get_load_global_name(op): @@ -63,11 +97,53 @@ def visit_call_c(self, op: CallC) -> Value: elif func_name == "PyObject_VectorcallMethod" and isinstance(op.args[0], LoadLiteral): return self.log(op, "python_call_method", str(op.args[0].value)) - return self.log(op, "call_c", func_name) + value = self.log(op, "call_c", func_name) + if func_name in primitives_that_inc_ref: + self.log_inc_ref(value) + return value + + def visit_get_attr(self, op: GetAttr) -> Value: + value = self.log(op, "get_attr", f"{op.class_type.name}.{op.attr}") + if not op.is_borrowed and op.type.is_refcounted: + self.log_inc_ref(op) + return value + + def visit_set_attr(self, op: SetAttr) -> Value: + name = "set_attr" if not op.is_init else "set_attr_init" + return self.log(op, name, f"{op.class_type.name}.{op.attr}") + + def visit_box(self, op: Box) -> Value: + if op.src.type is none_rprimitive: + # Boxing 'None' is a very quick operation, so we don't log it. + return self.add(op) + else: + return self.log(op, "box", str(op.src.type)) + + def visit_unbox(self, op: Unbox) -> Value: + return self.log(op, "unbox", str(op.type)) + + def visit_cast(self, op: Cast) -> Value | None: + value = self.log(op, "cast", str(op.type)) + if not op.is_borrowed: + self.log_inc_ref(value) + return value + + def visit_inc_ref(self, op: IncRef) -> Value: + return self.log(op, "inc_ref", str(op.src.type)) + + def visit_dec_ref(self, op: DecRef) -> Value: + return self.log(op, "dec_ref", str(op.src.type)) + + def log_inc_ref(self, value: Value) -> None: + self.log_event("inc_ref", str(value.type), value.line) def log(self, op: Op, name: str, details: str) -> Value: - if op.line >= 0: - line_str = str(op.line) + self.log_event(name, details, op.line) + return self.add(op) + + def log_event(self, name: str, details: str, line: int) -> None: + if line >= 0: + line_str = str(line) else: line_str = "" self.builder.primitive_op( @@ -78,6 +154,5 @@ def log(self, op: Op, name: str, details: str) -> Value: CString(name.encode("utf-8")), CString(details.encode("utf-8")), ], - op.line, + line, ) - return self.add(op) From fb41108b945c562bb02a9ff30a3530f6a2cc9c70 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Wed, 13 Aug 2025 11:52:02 -0400 Subject: [PATCH 166/424] [mypyc] feat: stararg fastpath when calling fn(*args) with tuple (#19623) There are 3 safe cases where we can reuse a tuple when calling a python function: fn(*args) fn(*args, **kwargs) fn(*args, k=1, k2=2, **kwargs) This PR covers the first two cases. The IR diff will probably demonstrate this change better than I can explain it. --- mypyc/irbuild/ll_builder.py | 16 +++++++- mypyc/test-data/irbuild-generics.test | 57 +++++++++++---------------- 2 files changed, 37 insertions(+), 36 deletions(-) diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index 05d558e0822ac..c5f9503b8c663 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -789,6 +789,18 @@ def _construct_varargs( for value, kind, name in args: if kind == ARG_STAR: if star_result is None: + # fast path if star expr is a tuple: + # we can pass the immutable tuple straight into the function call. + if is_tuple_rprimitive(value.type): + if len(args) == 1: + # fn(*args) + return value, self._create_dict([], [], line) + elif len(args) == 2 and args[1][1] == ARG_STAR2: + # fn(*args, **kwargs) + star_result = value + continue + # elif ...: TODO extend this to optimize fn(*args, k=1, **kwargs) case + # TODO optimize this case using the length utils - currently in review star_result = self.new_list_op(star_values, line) self.primitive_op(list_extend_op, [star_result, value], line) elif kind == ARG_STAR2: @@ -886,9 +898,11 @@ def _construct_varargs( # tuple. Otherwise create the tuple from the list. if star_result is None: star_result = self.new_tuple(star_values, line) - else: + elif not is_tuple_rprimitive(star_result.type): + # if star_result is a tuple we took the fast path star_result = self.primitive_op(list_tuple_op, [star_result], line) if has_star2 and star2_result is None: + # TODO: use dict_copy_op for simple cases of **kwargs star2_result = self._create_dict(star2_keys, star2_values, line) return star_result, star2_result diff --git a/mypyc/test-data/irbuild-generics.test b/mypyc/test-data/irbuild-generics.test index d39d47e397a1f..03032a7746c0e 100644 --- a/mypyc/test-data/irbuild-generics.test +++ b/mypyc/test-data/irbuild-generics.test @@ -166,25 +166,18 @@ execute(f, 1) def execute(func, args, kwargs): func :: object args :: tuple - kwargs :: dict - r0 :: list - r1 :: object - r2 :: dict - r3 :: i32 - r4 :: bit - r5 :: tuple - r6 :: object - r7 :: int + kwargs, r0 :: dict + r1 :: i32 + r2 :: bit + r3 :: object + r4 :: int L0: - r0 = PyList_New(0) - r1 = CPyList_Extend(r0, args) - r2 = PyDict_New() - r3 = CPyDict_UpdateInDisplay(r2, kwargs) - r4 = r3 >= 0 :: signed - r5 = PyList_AsTuple(r0) - r6 = PyObject_Call(func, r5, r2) - r7 = unbox(int, r6) - return r7 + r0 = PyDict_New() + r1 = CPyDict_UpdateInDisplay(r0, kwargs) + r2 = r1 >= 0 :: signed + r3 = PyObject_Call(func, args, r0) + r4 = unbox(int, r3) + return r4 def f(x): x :: int L0: @@ -709,14 +702,11 @@ def inner_deco_obj.__call__(__mypyc_self__, args, kwargs): can_dictcomp :: dict r22, can_iter, r23, can_use_keys, r24, can_use_values :: list r25 :: object - r26 :: list - r27 :: object - r28 :: dict - r29 :: i32 - r30 :: bit - r31 :: tuple - r32 :: object - r33 :: int + r26 :: dict + r27 :: i32 + r28 :: bit + r29 :: object + r30 :: int L0: r0 = __mypyc_self__.__mypyc_env__ r1 = var_object_size args @@ -768,15 +758,12 @@ L9: r24 = CPyDict_Values(kwargs) can_use_values = r24 r25 = r0.func - r26 = PyList_New(0) - r27 = CPyList_Extend(r26, args) - r28 = PyDict_New() - r29 = CPyDict_UpdateInDisplay(r28, kwargs) - r30 = r29 >= 0 :: signed - r31 = PyList_AsTuple(r26) - r32 = PyObject_Call(r25, r31, r28) - r33 = unbox(int, r32) - return r33 + r26 = PyDict_New() + r27 = CPyDict_UpdateInDisplay(r26, kwargs) + r28 = r27 >= 0 :: signed + r29 = PyObject_Call(r25, args, r26) + r30 = unbox(int, r29) + return r30 def deco(func): func :: object r0 :: __main__.deco_env From c95d8ab3d48dbb2192e2179278a60a951b98bcba Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Thu, 14 Aug 2025 11:15:20 -0400 Subject: [PATCH 167/424] [mypyc] feat: cache len for iterating over immutable types (#19656) Currently, if a user uses an immutable type as the sequence input for a for loop, the length is checked once at each iteration which, while necessary for some container types such as list and dictionaries, is not necessary for iterating over immutable types tuple, str, and bytes. This PR modifies the codebase such that the length is only checked at the first iteration, and reused from there. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- mypyc/ir/rtypes.py | 14 +- mypyc/irbuild/builder.py | 7 +- mypyc/irbuild/for_helpers.py | 35 +- mypyc/irbuild/specialize.py | 2 +- mypyc/test-data/fixtures/ir.py | 1 + mypyc/test-data/irbuild-generics.test | 14 +- mypyc/test-data/irbuild-lists.test | 354 ++++++++++++++++++++ mypyc/test-data/irbuild-tuple.test | 460 ++++++++++++++++++++++++-- 8 files changed, 846 insertions(+), 41 deletions(-) diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index c0871bba258c4..7a82a884256d6 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -628,7 +628,19 @@ def is_range_rprimitive(rtype: RType) -> bool: def is_sequence_rprimitive(rtype: RType) -> bool: return isinstance(rtype, RPrimitive) and ( - is_list_rprimitive(rtype) or is_tuple_rprimitive(rtype) or is_str_rprimitive(rtype) + is_list_rprimitive(rtype) + or is_tuple_rprimitive(rtype) + or is_str_rprimitive(rtype) + or is_bytes_rprimitive(rtype) + ) + + +def is_immutable_rprimitive(rtype: RType) -> TypeGuard[RPrimitive]: + return ( + is_str_rprimitive(rtype) + or is_bytes_rprimitive(rtype) + or is_tuple_rprimitive(rtype) + or is_frozenset_rprimitive(rtype) ) diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index 608c524b5d4fb..4f2f539118d74 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -91,6 +91,7 @@ RType, RUnion, bitmap_rprimitive, + bytes_rprimitive, c_pyssize_t_rprimitive, dict_rprimitive, int_rprimitive, @@ -962,8 +963,12 @@ def get_sequence_type_from_type(self, target_type: Type) -> RType: elif isinstance(target_type, Instance): if target_type.type.fullname == "builtins.str": return str_rprimitive - else: + elif target_type.type.fullname == "builtins.bytes": + return bytes_rprimitive + try: return self.type_to_rtype(target_type.args[0]) + except IndexError: + raise ValueError(f"{target_type!r} is not a valid sequence.") from None # This elif-blocks are needed for iterating over classes derived from NamedTuple. elif isinstance(target_type, TypeVarLikeType): return self.get_sequence_type_from_type(target_type.upper_bound) diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index 5cf89f579ec48..762b41866a057 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -48,6 +48,7 @@ int_rprimitive, is_dict_rprimitive, is_fixed_width_rtype, + is_immutable_rprimitive, is_list_rprimitive, is_sequence_rprimitive, is_short_int_rprimitive, @@ -205,9 +206,9 @@ def sequence_from_generator_preallocate_helper( there is no condition list in the generator and only one original sequence with one index is allowed. - e.g. (1) tuple(f(x) for x in a_list/a_tuple) - (2) list(f(x) for x in a_list/a_tuple) - (3) [f(x) for x in a_list/a_tuple] + e.g. (1) tuple(f(x) for x in a_list/a_tuple/a_str/a_bytes) + (2) list(f(x) for x in a_list/a_tuple/a_str/a_bytes) + (3) [f(x) for x in a_list/a_tuple/a_str/a_bytes] RTuple as an original sequence is not supported yet. Args: @@ -224,7 +225,7 @@ def sequence_from_generator_preallocate_helper( """ if len(gen.sequences) == 1 and len(gen.indices) == 1 and len(gen.condlists[0]) == 0: rtype = builder.node_type(gen.sequences[0]) - if is_list_rprimitive(rtype) or is_tuple_rprimitive(rtype) or is_str_rprimitive(rtype): + if is_sequence_rprimitive(rtype): sequence = builder.accept(gen.sequences[0]) length = builder.builder.builtin_len(sequence, gen.line, use_pyssize_t=True) target_op = empty_op_llbuilder(length, gen.line) @@ -785,17 +786,31 @@ class ForSequence(ForGenerator): Supports iterating in both forward and reverse. """ + length_reg: Value | AssignmentTarget | None + def init(self, expr_reg: Value, target_type: RType, reverse: bool) -> None: + assert is_sequence_rprimitive(expr_reg.type), expr_reg builder = self.builder self.reverse = reverse # Define target to contain the expression, along with the index that will be used # for the for-loop. If we are inside of a generator function, spill these into the # environment class. self.expr_target = builder.maybe_spill(expr_reg) + if is_immutable_rprimitive(expr_reg.type): + # If the expression is an immutable type, we can load the length just once. + self.length_reg = builder.maybe_spill(self.load_len(self.expr_target)) + else: + # Otherwise, even if the length is known, we must recalculate the length + # at every iteration for compatibility with python semantics. + self.length_reg = None if not reverse: index_reg: Value = Integer(0, c_pyssize_t_rprimitive) else: - index_reg = builder.builder.int_sub(self.load_len(self.expr_target), 1) + if self.length_reg is not None: + len_val = builder.read(self.length_reg) + else: + len_val = self.load_len(self.expr_target) + index_reg = builder.builder.int_sub(len_val, 1) self.index_target = builder.maybe_spill_assignable(index_reg) self.target_type = target_type @@ -814,9 +829,13 @@ def gen_condition(self) -> None: second_check = BasicBlock() builder.add_bool_branch(comparison, second_check, self.loop_exit) builder.activate_block(second_check) - # For compatibility with python semantics we recalculate the length - # at every iteration. - len_reg = self.load_len(self.expr_target) + if self.length_reg is None: + # For compatibility with python semantics we recalculate the length + # at every iteration. + len_reg = self.load_len(self.expr_target) + else: + # (unless input is immutable type). + len_reg = builder.read(self.length_reg, line) comparison = builder.binary_op(builder.read(self.index_target, line), len_reg, "<", line) builder.add_bool_branch(comparison, self.body_block, self.loop_exit) diff --git a/mypyc/irbuild/specialize.py b/mypyc/irbuild/specialize.py index 3015640fb3fd5..748cda1256a74 100644 --- a/mypyc/irbuild/specialize.py +++ b/mypyc/irbuild/specialize.py @@ -288,7 +288,7 @@ def translate_tuple_from_generator_call( """Special case for simplest tuple creation from a generator. For example: - tuple(f(x) for x in some_list/some_tuple/some_str) + tuple(f(x) for x in some_list/some_tuple/some_str/some_bytes) 'translate_safe_generator_call()' would take care of other cases if this fails. """ diff --git a/mypyc/test-data/fixtures/ir.py b/mypyc/test-data/fixtures/ir.py index 76afc1ea58ccb..661ae50fd5f3c 100644 --- a/mypyc/test-data/fixtures/ir.py +++ b/mypyc/test-data/fixtures/ir.py @@ -172,6 +172,7 @@ def __getitem__(self, i: int) -> int: ... def __getitem__(self, i: slice) -> bytes: ... def join(self, x: Iterable[object]) -> bytes: ... def decode(self, x: str=..., y: str=...) -> str: ... + def __iter__(self) -> Iterator[int]: ... class bytearray: @overload diff --git a/mypyc/test-data/irbuild-generics.test b/mypyc/test-data/irbuild-generics.test index 03032a7746c0e..4e9391e0d59e9 100644 --- a/mypyc/test-data/irbuild-generics.test +++ b/mypyc/test-data/irbuild-generics.test @@ -711,18 +711,18 @@ L0: r0 = __mypyc_self__.__mypyc_env__ r1 = var_object_size args r2 = PyList_New(r1) - r3 = 0 + r3 = var_object_size args + r4 = 0 L1: - r4 = var_object_size args - r5 = r3 < r4 :: signed + r5 = r4 < r3 :: signed if r5 goto L2 else goto L4 :: bool L2: - r6 = CPySequenceTuple_GetItemUnsafe(args, r3) + r6 = CPySequenceTuple_GetItemUnsafe(args, r4) x = r6 - CPyList_SetItemUnsafe(r2, r3, x) + CPyList_SetItemUnsafe(r2, r4, x) L3: - r7 = r3 + 1 - r3 = r7 + r7 = r4 + 1 + r4 = r7 goto L1 L4: can_listcomp = r2 diff --git a/mypyc/test-data/irbuild-lists.test b/mypyc/test-data/irbuild-lists.test index 06120e077af95..d83fb88390db4 100644 --- a/mypyc/test-data/irbuild-lists.test +++ b/mypyc/test-data/irbuild-lists.test @@ -572,3 +572,357 @@ def sort_iterable(a): L0: r0 = CPySequence_Sort(a) return 1 + +[case testListBuiltFromStr] +def f2(val: str) -> str: + return val + "f2" + +def test() -> None: + source = "abc" + a = [f2(x) for x in source] +[out] +def f2(val): + val, r0, r1 :: str +L0: + r0 = 'f2' + r1 = PyUnicode_Concat(val, r0) + return r1 +def test(): + r0, source :: str + r1 :: native_int + r2 :: bit + r3 :: list + r4 :: native_int + r5 :: bit + r6 :: native_int + r7 :: bit + r8, x, r9 :: str + r10 :: native_int + a :: list +L0: + r0 = 'abc' + source = r0 + r1 = CPyStr_Size_size_t(source) + r2 = r1 >= 0 :: signed + r3 = PyList_New(r1) + r4 = CPyStr_Size_size_t(source) + r5 = r4 >= 0 :: signed + r6 = 0 +L1: + r7 = r6 < r4 :: signed + if r7 goto L2 else goto L4 :: bool +L2: + r8 = CPyStr_GetItemUnsafe(source, r6) + x = r8 + r9 = f2(x) + CPyList_SetItemUnsafe(r3, r6, r9) +L3: + r10 = r6 + 1 + r6 = r10 + goto L1 +L4: + a = r3 + return 1 + +[case testListBuiltFromStrExpr] +def f2(val: str) -> str: + return val + "f2" + +def test() -> None: + a = [f2(x) for x in "abc"] +[out] +def f2(val): + val, r0, r1 :: str +L0: + r0 = 'f2' + r1 = PyUnicode_Concat(val, r0) + return r1 +def test(): + r0 :: str + r1 :: native_int + r2 :: bit + r3 :: list + r4 :: native_int + r5 :: bit + r6 :: native_int + r7 :: bit + r8, x, r9 :: str + r10 :: native_int + a :: list +L0: + r0 = 'abc' + r1 = CPyStr_Size_size_t(r0) + r2 = r1 >= 0 :: signed + r3 = PyList_New(r1) + r4 = CPyStr_Size_size_t(r0) + r5 = r4 >= 0 :: signed + r6 = 0 +L1: + r7 = r6 < r4 :: signed + if r7 goto L2 else goto L4 :: bool +L2: + r8 = CPyStr_GetItemUnsafe(r0, r6) + x = r8 + r9 = f2(x) + CPyList_SetItemUnsafe(r3, r6, r9) +L3: + r10 = r6 + 1 + r6 = r10 + goto L1 +L4: + a = r3 + return 1 + +[case testListBuiltFromFinalStr] +from typing import Final + +source: Final = "abc" + +def f2(val: str) -> str: + return val + "f2" + +def test() -> None: + a = [f2(x) for x in source] +[out] +def f2(val): + val, r0, r1 :: str +L0: + r0 = 'f2' + r1 = PyUnicode_Concat(val, r0) + return r1 +def test(): + r0 :: str + r1 :: native_int + r2 :: bit + r3 :: list + r4 :: native_int + r5 :: bit + r6 :: native_int + r7 :: bit + r8, x, r9 :: str + r10 :: native_int + a :: list +L0: + r0 = 'abc' + r1 = CPyStr_Size_size_t(r0) + r2 = r1 >= 0 :: signed + r3 = PyList_New(r1) + r4 = CPyStr_Size_size_t(r0) + r5 = r4 >= 0 :: signed + r6 = 0 +L1: + r7 = r6 < r4 :: signed + if r7 goto L2 else goto L4 :: bool +L2: + r8 = CPyStr_GetItemUnsafe(r0, r6) + x = r8 + r9 = f2(x) + CPyList_SetItemUnsafe(r3, r6, r9) +L3: + r10 = r6 + 1 + r6 = r10 + goto L1 +L4: + a = r3 + return 1 + +[case testListBuiltFromBytes_64bit] +def f2(val: int) -> int: + return val + 2 + +def test() -> None: + source = b"abc" + a = [f2(x) for x in source] + +[out] +def f2(val): + val, r0 :: int +L0: + r0 = CPyTagged_Add(val, 4) + return r0 +def test(): + r0, source :: bytes + r1 :: native_int + r2 :: list + r3, r4 :: native_int + r5, r6, r7 :: bit + r8, r9, r10, r11 :: int + r12 :: object + r13, x, r14 :: int + r15 :: object + r16 :: native_int + a :: list +L0: + r0 = b'abc' + source = r0 + r1 = var_object_size source + r2 = PyList_New(r1) + r3 = var_object_size source + r4 = 0 +L1: + r5 = r4 < r3 :: signed + if r5 goto L2 else goto L8 :: bool +L2: + r6 = r4 <= 4611686018427387903 :: signed + if r6 goto L3 else goto L4 :: bool +L3: + r7 = r4 >= -4611686018427387904 :: signed + if r7 goto L5 else goto L4 :: bool +L4: + r8 = CPyTagged_FromInt64(r4) + r9 = r8 + goto L6 +L5: + r10 = r4 << 1 + r9 = r10 +L6: + r11 = CPyBytes_GetItem(source, r9) + r12 = box(int, r11) + r13 = unbox(int, r12) + x = r13 + r14 = f2(x) + r15 = box(int, r14) + CPyList_SetItemUnsafe(r2, r4, r15) +L7: + r16 = r4 + 1 + r4 = r16 + goto L1 +L8: + a = r2 + return 1 + +[case testListBuiltFromBytesExpr_64bit] +def f2(val: int) -> int: + return val + 2 + +def test() -> None: + a = [f2(x) for x in b"abc"] + +[out] +def f2(val): + val, r0 :: int +L0: + r0 = CPyTagged_Add(val, 4) + return r0 +def test(): + r0 :: bytes + r1 :: native_int + r2 :: list + r3, r4 :: native_int + r5, r6, r7 :: bit + r8, r9, r10, r11 :: int + r12 :: object + r13, x, r14 :: int + r15 :: object + r16 :: native_int + a :: list +L0: + r0 = b'abc' + r1 = var_object_size r0 + r2 = PyList_New(r1) + r3 = var_object_size r0 + r4 = 0 +L1: + r5 = r4 < r3 :: signed + if r5 goto L2 else goto L8 :: bool +L2: + r6 = r4 <= 4611686018427387903 :: signed + if r6 goto L3 else goto L4 :: bool +L3: + r7 = r4 >= -4611686018427387904 :: signed + if r7 goto L5 else goto L4 :: bool +L4: + r8 = CPyTagged_FromInt64(r4) + r9 = r8 + goto L6 +L5: + r10 = r4 << 1 + r9 = r10 +L6: + r11 = CPyBytes_GetItem(r0, r9) + r12 = box(int, r11) + r13 = unbox(int, r12) + x = r13 + r14 = f2(x) + r15 = box(int, r14) + CPyList_SetItemUnsafe(r2, r4, r15) +L7: + r16 = r4 + 1 + r4 = r16 + goto L1 +L8: + a = r2 + return 1 + +[case testListBuiltFromFinalBytes_64bit] +from typing import Final + +source: Final = b"abc" + +def f2(val: int) -> int: + return val + 2 + +def test() -> None: + a = [f2(x) for x in source] + +[out] +def f2(val): + val, r0 :: int +L0: + r0 = CPyTagged_Add(val, 4) + return r0 +def test(): + r0 :: bytes + r1 :: bool + r2 :: native_int + r3 :: list + r4, r5 :: native_int + r6, r7, r8 :: bit + r9, r10, r11, r12 :: int + r13 :: object + r14, x, r15 :: int + r16 :: object + r17 :: native_int + a :: list +L0: + r0 = __main__.source :: static + if is_error(r0) goto L1 else goto L2 +L1: + r1 = raise NameError('value for final name "source" was not set') + unreachable +L2: + r2 = var_object_size r0 + r3 = PyList_New(r2) + r4 = var_object_size r0 + r5 = 0 +L3: + r6 = r5 < r4 :: signed + if r6 goto L4 else goto L10 :: bool +L4: + r7 = r5 <= 4611686018427387903 :: signed + if r7 goto L5 else goto L6 :: bool +L5: + r8 = r5 >= -4611686018427387904 :: signed + if r8 goto L7 else goto L6 :: bool +L6: + r9 = CPyTagged_FromInt64(r5) + r10 = r9 + goto L8 +L7: + r11 = r5 << 1 + r10 = r11 +L8: + r12 = CPyBytes_GetItem(r0, r10) + r13 = box(int, r12) + r14 = unbox(int, r13) + x = r14 + r15 = f2(x) + r16 = box(int, r15) + CPyList_SetItemUnsafe(r3, r5, r16) +L9: + r17 = r5 + 1 + r5 = r17 + goto L3 +L10: + a = r3 + return 1 diff --git a/mypyc/test-data/irbuild-tuple.test b/mypyc/test-data/irbuild-tuple.test index 5c5ec27b18829..0342ec304c25c 100644 --- a/mypyc/test-data/irbuild-tuple.test +++ b/mypyc/test-data/irbuild-tuple.test @@ -133,18 +133,18 @@ def f(xs): r4, x :: str r5 :: native_int L0: - r0 = 0 + r0 = var_object_size xs + r1 = 0 L1: - r1 = var_object_size xs - r2 = r0 < r1 :: signed + r2 = r1 < r0 :: signed if r2 goto L2 else goto L4 :: bool L2: - r3 = CPySequenceTuple_GetItemUnsafe(xs, r0) + r3 = CPySequenceTuple_GetItemUnsafe(xs, r1) r4 = cast(str, r3) x = r4 L3: - r5 = r0 + 1 - r0 = r5 + r5 = r1 + 1 + r1 = r5 goto L1 L4: return 1 @@ -291,8 +291,10 @@ def test(): r1 :: native_int r2 :: bit r3 :: tuple - r4, r5 :: native_int - r6, r7 :: bit + r4 :: native_int + r5 :: bit + r6 :: native_int + r7 :: bit r8, x, r9 :: str r10 :: native_int a :: tuple @@ -302,25 +304,437 @@ L0: r1 = CPyStr_Size_size_t(source) r2 = r1 >= 0 :: signed r3 = PyTuple_New(r1) - r4 = 0 + r4 = CPyStr_Size_size_t(source) + r5 = r4 >= 0 :: signed + r6 = 0 L1: - r5 = CPyStr_Size_size_t(source) - r6 = r5 >= 0 :: signed - r7 = r4 < r5 :: signed + r7 = r6 < r4 :: signed if r7 goto L2 else goto L4 :: bool L2: - r8 = CPyStr_GetItemUnsafe(source, r4) + r8 = CPyStr_GetItemUnsafe(source, r6) x = r8 r9 = f2(x) - CPySequenceTuple_SetItemUnsafe(r3, r4, r9) + CPySequenceTuple_SetItemUnsafe(r3, r6, r9) L3: - r10 = r4 + 1 - r4 = r10 + r10 = r6 + 1 + r6 = r10 goto L1 L4: a = r3 return 1 +[case testTupleBuiltFromStrExpr] +def f2(val: str) -> str: + return val + "f2" + +def test() -> None: + a = tuple(f2(x) for x in "abc") + +[out] +def f2(val): + val, r0, r1 :: str +L0: + r0 = 'f2' + r1 = PyUnicode_Concat(val, r0) + return r1 +def test(): + r0 :: str + r1 :: native_int + r2 :: bit + r3 :: tuple + r4 :: native_int + r5 :: bit + r6 :: native_int + r7 :: bit + r8, x, r9 :: str + r10 :: native_int + a :: tuple +L0: + r0 = 'abc' + r1 = CPyStr_Size_size_t(r0) + r2 = r1 >= 0 :: signed + r3 = PyTuple_New(r1) + r4 = CPyStr_Size_size_t(r0) + r5 = r4 >= 0 :: signed + r6 = 0 +L1: + r7 = r6 < r4 :: signed + if r7 goto L2 else goto L4 :: bool +L2: + r8 = CPyStr_GetItemUnsafe(r0, r6) + x = r8 + r9 = f2(x) + CPySequenceTuple_SetItemUnsafe(r3, r6, r9) +L3: + r10 = r6 + 1 + r6 = r10 + goto L1 +L4: + a = r3 + return 1 + +[case testTupleBuiltFromFinalStr] +from typing import Final + +source: Final = "abc" + +def f2(val: str) -> str: + return val + "f2" + +def test() -> None: + a = tuple(f2(x) for x in source) +[out] +def f2(val): + val, r0, r1 :: str +L0: + r0 = 'f2' + r1 = PyUnicode_Concat(val, r0) + return r1 +def test(): + r0 :: str + r1 :: native_int + r2 :: bit + r3 :: tuple + r4 :: native_int + r5 :: bit + r6 :: native_int + r7 :: bit + r8, x, r9 :: str + r10 :: native_int + a :: tuple +L0: + r0 = 'abc' + r1 = CPyStr_Size_size_t(r0) + r2 = r1 >= 0 :: signed + r3 = PyTuple_New(r1) + r4 = CPyStr_Size_size_t(r0) + r5 = r4 >= 0 :: signed + r6 = 0 +L1: + r7 = r6 < r4 :: signed + if r7 goto L2 else goto L4 :: bool +L2: + r8 = CPyStr_GetItemUnsafe(r0, r6) + x = r8 + r9 = f2(x) + CPySequenceTuple_SetItemUnsafe(r3, r6, r9) +L3: + r10 = r6 + 1 + r6 = r10 + goto L1 +L4: + a = r3 + return 1 + +[case testTupleBuiltFromBytes_64bit] +def f2(val: int) -> int: + return val + 2 + +def test() -> None: + source = b"abc" + a = tuple(f2(x) for x in source) + +[out] +def f2(val): + val, r0 :: int +L0: + r0 = CPyTagged_Add(val, 4) + return r0 +def test(): + r0, source :: bytes + r1 :: native_int + r2 :: tuple + r3, r4 :: native_int + r5, r6, r7 :: bit + r8, r9, r10, r11 :: int + r12 :: object + r13, x, r14 :: int + r15 :: object + r16 :: native_int + a :: tuple +L0: + r0 = b'abc' + source = r0 + r1 = var_object_size source + r2 = PyTuple_New(r1) + r3 = var_object_size source + r4 = 0 +L1: + r5 = r4 < r3 :: signed + if r5 goto L2 else goto L8 :: bool +L2: + r6 = r4 <= 4611686018427387903 :: signed + if r6 goto L3 else goto L4 :: bool +L3: + r7 = r4 >= -4611686018427387904 :: signed + if r7 goto L5 else goto L4 :: bool +L4: + r8 = CPyTagged_FromInt64(r4) + r9 = r8 + goto L6 +L5: + r10 = r4 << 1 + r9 = r10 +L6: + r11 = CPyBytes_GetItem(source, r9) + r12 = box(int, r11) + r13 = unbox(int, r12) + x = r13 + r14 = f2(x) + r15 = box(int, r14) + CPySequenceTuple_SetItemUnsafe(r2, r4, r15) +L7: + r16 = r4 + 1 + r4 = r16 + goto L1 +L8: + a = r2 + return 1 + +[case testTupleBuiltFromBytesExpr_64bit] +def f2(val: int) -> int: + return val + 2 + +def test() -> None: + a = tuple(f2(x) for x in b"abc") + +[out] +def f2(val): + val, r0 :: int +L0: + r0 = CPyTagged_Add(val, 4) + return r0 +def test(): + r0 :: bytes + r1 :: native_int + r2 :: tuple + r3, r4 :: native_int + r5, r6, r7 :: bit + r8, r9, r10, r11 :: int + r12 :: object + r13, x, r14 :: int + r15 :: object + r16 :: native_int + a :: tuple +L0: + r0 = b'abc' + r1 = var_object_size r0 + r2 = PyTuple_New(r1) + r3 = var_object_size r0 + r4 = 0 +L1: + r5 = r4 < r3 :: signed + if r5 goto L2 else goto L8 :: bool +L2: + r6 = r4 <= 4611686018427387903 :: signed + if r6 goto L3 else goto L4 :: bool +L3: + r7 = r4 >= -4611686018427387904 :: signed + if r7 goto L5 else goto L4 :: bool +L4: + r8 = CPyTagged_FromInt64(r4) + r9 = r8 + goto L6 +L5: + r10 = r4 << 1 + r9 = r10 +L6: + r11 = CPyBytes_GetItem(r0, r9) + r12 = box(int, r11) + r13 = unbox(int, r12) + x = r13 + r14 = f2(x) + r15 = box(int, r14) + CPySequenceTuple_SetItemUnsafe(r2, r4, r15) +L7: + r16 = r4 + 1 + r4 = r16 + goto L1 +L8: + a = r2 + return 1 + +[case testTupleBuiltFromFinalBytes_64bit] +from typing import Final + +source: Final = b"abc" + +def f2(val: int) -> int: + return val + 2 + +def test() -> None: + a = tuple(f2(x) for x in source) + +[out] +def f2(val): + val, r0 :: int +L0: + r0 = CPyTagged_Add(val, 4) + return r0 +def test(): + r0 :: bytes + r1 :: bool + r2 :: native_int + r3 :: tuple + r4, r5 :: native_int + r6, r7, r8 :: bit + r9, r10, r11, r12 :: int + r13 :: object + r14, x, r15 :: int + r16 :: object + r17 :: native_int + a :: tuple +L0: + r0 = __main__.source :: static + if is_error(r0) goto L1 else goto L2 +L1: + r1 = raise NameError('value for final name "source" was not set') + unreachable +L2: + r2 = var_object_size r0 + r3 = PyTuple_New(r2) + r4 = var_object_size r0 + r5 = 0 +L3: + r6 = r5 < r4 :: signed + if r6 goto L4 else goto L10 :: bool +L4: + r7 = r5 <= 4611686018427387903 :: signed + if r7 goto L5 else goto L6 :: bool +L5: + r8 = r5 >= -4611686018427387904 :: signed + if r8 goto L7 else goto L6 :: bool +L6: + r9 = CPyTagged_FromInt64(r5) + r10 = r9 + goto L8 +L7: + r11 = r5 << 1 + r10 = r11 +L8: + r12 = CPyBytes_GetItem(r0, r10) + r13 = box(int, r12) + r14 = unbox(int, r13) + x = r14 + r15 = f2(x) + r16 = box(int, r15) + CPySequenceTuple_SetItemUnsafe(r3, r5, r16) +L9: + r17 = r5 + 1 + r5 = r17 + goto L3 +L10: + a = r3 + return 1 + +[case testTupleBuiltFromFixedLengthTuple] +def f(val: int) -> bool: + return val % 2 == 0 + +def test() -> None: + source = (1, 2, 3) + a = tuple(f(x) for x in source) +[out] +def f(val): + val, r0 :: int + r1 :: bit +L0: + r0 = CPyTagged_Remainder(val, 4) + r1 = int_eq r0, 0 + return r1 +def test(): + r0, source :: tuple[int, int, int] + r1 :: list + r2, r3, r4 :: object + r5, x :: int + r6 :: bool + r7 :: object + r8 :: i32 + r9, r10 :: bit + r11, a :: tuple +L0: + r0 = (2, 4, 6) + source = r0 + r1 = PyList_New(0) + r2 = box(tuple[int, int, int], source) + r3 = PyObject_GetIter(r2) +L1: + r4 = PyIter_Next(r3) + if is_error(r4) goto L4 else goto L2 +L2: + r5 = unbox(int, r4) + x = r5 + r6 = f(x) + r7 = box(bool, r6) + r8 = PyList_Append(r1, r7) + r9 = r8 >= 0 :: signed +L3: + goto L1 +L4: + r10 = CPy_NoErrOccurred() +L5: + r11 = PyList_AsTuple(r1) + a = r11 + return 1 + +[case testTupleBuiltFromFinalFixedLengthTuple] +from typing import Final + +source: Final = (1, 2, 3) + +def f(val: int) -> bool: + return val % 2 == 0 + +def test() -> None: + a = tuple(f(x) for x in source) +[out] +def f(val): + val, r0 :: int + r1 :: bit +L0: + r0 = CPyTagged_Remainder(val, 4) + r1 = int_eq r0, 0 + return r1 +def test(): + r0 :: list + r1 :: tuple[int, int, int] + r2 :: bool + r3, r4, r5 :: object + r6, x :: int + r7 :: bool + r8 :: object + r9 :: i32 + r10, r11 :: bit + r12, a :: tuple +L0: + r0 = PyList_New(0) + r1 = __main__.source :: static + if is_error(r1) goto L1 else goto L2 +L1: + r2 = raise NameError('value for final name "source" was not set') + unreachable +L2: + r3 = box(tuple[int, int, int], r1) + r4 = PyObject_GetIter(r3) +L3: + r5 = PyIter_Next(r4) + if is_error(r5) goto L6 else goto L4 +L4: + r6 = unbox(int, r5) + x = r6 + r7 = f(x) + r8 = box(bool, r7) + r9 = PyList_Append(r0, r8) + r10 = r9 >= 0 :: signed +L5: + goto L3 +L6: + r11 = CPy_NoErrOccurred() +L7: + r12 = PyList_AsTuple(r0) + a = r12 + return 1 + [case testTupleBuiltFromVariableLengthTuple] from typing import Tuple @@ -349,21 +763,21 @@ def test(source): L0: r0 = var_object_size source r1 = PyTuple_New(r0) - r2 = 0 + r2 = var_object_size source + r3 = 0 L1: - r3 = var_object_size source - r4 = r2 < r3 :: signed + r4 = r3 < r2 :: signed if r4 goto L2 else goto L4 :: bool L2: - r5 = CPySequenceTuple_GetItemUnsafe(source, r2) + r5 = CPySequenceTuple_GetItemUnsafe(source, r3) r6 = unbox(bool, r5) x = r6 r7 = f(x) r8 = box(bool, r7) - CPySequenceTuple_SetItemUnsafe(r1, r2, r8) + CPySequenceTuple_SetItemUnsafe(r1, r3, r8) L3: - r9 = r2 + 1 - r2 = r9 + r9 = r3 + 1 + r3 = r9 goto L1 L4: a = r1 From 27b9ba0031a2ffd1661a1c93c606a1013e04f95f Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Thu, 14 Aug 2025 11:17:07 -0400 Subject: [PATCH 168/424] chore: homogenize TypeGuard usage in rtypes.py (#19655) --- mypyc/ir/rtypes.py | 62 +++++++++++++++++++++++----------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 7a82a884256d6..3c2fbfec10356 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -192,7 +192,7 @@ def may_be_immortal(self) -> bool: def serialize(self) -> str: return "void" - def __eq__(self, other: object) -> bool: + def __eq__(self, other: object) -> TypeGuard[RVoid]: return isinstance(other, RVoid) def __hash__(self) -> int: @@ -279,7 +279,7 @@ def serialize(self) -> str: def __repr__(self) -> str: return "" % self.name - def __eq__(self, other: object) -> bool: + def __eq__(self, other: object) -> TypeGuard[RPrimitive]: return isinstance(other, RPrimitive) and other.name == self.name def __hash__(self) -> int: @@ -513,15 +513,15 @@ def __hash__(self) -> int: range_rprimitive: Final = RPrimitive("builtins.range", is_unboxed=False, is_refcounted=True) -def is_tagged(rtype: RType) -> bool: +def is_tagged(rtype: RType) -> TypeGuard[RPrimitive]: return rtype is int_rprimitive or rtype is short_int_rprimitive -def is_int_rprimitive(rtype: RType) -> bool: +def is_int_rprimitive(rtype: RType) -> TypeGuard[RPrimitive]: return rtype is int_rprimitive -def is_short_int_rprimitive(rtype: RType) -> bool: +def is_short_int_rprimitive(rtype: RType) -> TypeGuard[RPrimitive]: return rtype is short_int_rprimitive @@ -535,7 +535,7 @@ def is_int32_rprimitive(rtype: RType) -> TypeGuard[RPrimitive]: ) -def is_int64_rprimitive(rtype: RType) -> bool: +def is_int64_rprimitive(rtype: RType) -> TypeGuard[RPrimitive]: return rtype is int64_rprimitive or ( rtype is c_pyssize_t_rprimitive and rtype._ctype == "int64_t" ) @@ -554,79 +554,79 @@ def is_uint8_rprimitive(rtype: RType) -> TypeGuard[RPrimitive]: return rtype is uint8_rprimitive -def is_uint32_rprimitive(rtype: RType) -> bool: +def is_uint32_rprimitive(rtype: RType) -> TypeGuard[RPrimitive]: return rtype is uint32_rprimitive -def is_uint64_rprimitive(rtype: RType) -> bool: +def is_uint64_rprimitive(rtype: RType) -> TypeGuard[RPrimitive]: return rtype is uint64_rprimitive -def is_c_py_ssize_t_rprimitive(rtype: RType) -> bool: +def is_c_py_ssize_t_rprimitive(rtype: RType) -> TypeGuard[RPrimitive]: return rtype is c_pyssize_t_rprimitive -def is_pointer_rprimitive(rtype: RType) -> bool: +def is_pointer_rprimitive(rtype: RType) -> TypeGuard[RPrimitive]: return rtype is pointer_rprimitive -def is_float_rprimitive(rtype: RType) -> bool: +def is_float_rprimitive(rtype: RType) -> TypeGuard[RPrimitive]: return isinstance(rtype, RPrimitive) and rtype.name == "builtins.float" -def is_bool_rprimitive(rtype: RType) -> bool: +def is_bool_rprimitive(rtype: RType) -> TypeGuard[RPrimitive]: return isinstance(rtype, RPrimitive) and rtype.name == "builtins.bool" -def is_bit_rprimitive(rtype: RType) -> bool: +def is_bit_rprimitive(rtype: RType) -> TypeGuard[RPrimitive]: return isinstance(rtype, RPrimitive) and rtype.name == "bit" -def is_bool_or_bit_rprimitive(rtype: RType) -> bool: +def is_bool_or_bit_rprimitive(rtype: RType) -> TypeGuard[RPrimitive]: return is_bool_rprimitive(rtype) or is_bit_rprimitive(rtype) -def is_object_rprimitive(rtype: RType) -> bool: +def is_object_rprimitive(rtype: RType) -> TypeGuard[RPrimitive]: return isinstance(rtype, RPrimitive) and rtype.name == "builtins.object" -def is_none_rprimitive(rtype: RType) -> bool: +def is_none_rprimitive(rtype: RType) -> TypeGuard[RPrimitive]: return isinstance(rtype, RPrimitive) and rtype.name == "builtins.None" -def is_list_rprimitive(rtype: RType) -> bool: +def is_list_rprimitive(rtype: RType) -> TypeGuard[RPrimitive]: return isinstance(rtype, RPrimitive) and rtype.name == "builtins.list" -def is_dict_rprimitive(rtype: RType) -> bool: +def is_dict_rprimitive(rtype: RType) -> TypeGuard[RPrimitive]: return isinstance(rtype, RPrimitive) and rtype.name == "builtins.dict" -def is_set_rprimitive(rtype: RType) -> bool: +def is_set_rprimitive(rtype: RType) -> TypeGuard[RPrimitive]: return isinstance(rtype, RPrimitive) and rtype.name == "builtins.set" -def is_frozenset_rprimitive(rtype: RType) -> bool: +def is_frozenset_rprimitive(rtype: RType) -> TypeGuard[RPrimitive]: return isinstance(rtype, RPrimitive) and rtype.name == "builtins.frozenset" -def is_str_rprimitive(rtype: RType) -> bool: +def is_str_rprimitive(rtype: RType) -> TypeGuard[RPrimitive]: return isinstance(rtype, RPrimitive) and rtype.name == "builtins.str" -def is_bytes_rprimitive(rtype: RType) -> bool: +def is_bytes_rprimitive(rtype: RType) -> TypeGuard[RPrimitive]: return isinstance(rtype, RPrimitive) and rtype.name == "builtins.bytes" -def is_tuple_rprimitive(rtype: RType) -> bool: +def is_tuple_rprimitive(rtype: RType) -> TypeGuard[RPrimitive]: return isinstance(rtype, RPrimitive) and rtype.name == "builtins.tuple" -def is_range_rprimitive(rtype: RType) -> bool: +def is_range_rprimitive(rtype: RType) -> TypeGuard[RPrimitive]: return isinstance(rtype, RPrimitive) and rtype.name == "builtins.range" -def is_sequence_rprimitive(rtype: RType) -> bool: +def is_sequence_rprimitive(rtype: RType) -> TypeGuard[RPrimitive]: return isinstance(rtype, RPrimitive) and ( is_list_rprimitive(rtype) or is_tuple_rprimitive(rtype) @@ -729,7 +729,7 @@ def __str__(self) -> str: def __repr__(self) -> str: return "" % ", ".join(repr(typ) for typ in self.types) - def __eq__(self, other: object) -> bool: + def __eq__(self, other: object) -> TypeGuard[RTuple]: return isinstance(other, RTuple) and self.types == other.types def __hash__(self) -> int: @@ -862,7 +862,7 @@ def __repr__(self) -> str: ", ".join(name + ":" + repr(typ) for name, typ in zip(self.names, self.types)), ) - def __eq__(self, other: object) -> bool: + def __eq__(self, other: object) -> TypeGuard[RStruct]: return ( isinstance(other, RStruct) and self.name == other.name @@ -932,7 +932,7 @@ def attr_type(self, name: str) -> RType: def __repr__(self) -> str: return "" % self.name - def __eq__(self, other: object) -> bool: + def __eq__(self, other: object) -> TypeGuard[RInstance]: return isinstance(other, RInstance) and other.name == self.name def __hash__(self) -> int: @@ -986,7 +986,7 @@ def __str__(self) -> str: return "union[%s]" % ", ".join(str(item) for item in self.items) # We compare based on the set because order in a union doesn't matter - def __eq__(self, other: object) -> bool: + def __eq__(self, other: object) -> TypeGuard[RUnion]: return isinstance(other, RUnion) and self.items_set == other.items_set def __hash__(self) -> int: @@ -1028,7 +1028,7 @@ def optional_value_type(rtype: RType) -> RType | None: return None -def is_optional_type(rtype: RType) -> bool: +def is_optional_type(rtype: RType) -> TypeGuard[RUnion]: """Is rtype an optional type with exactly two union items?""" return optional_value_type(rtype) is not None @@ -1060,7 +1060,7 @@ def __str__(self) -> str: def __repr__(self) -> str: return f"" - def __eq__(self, other: object) -> bool: + def __eq__(self, other: object) -> TypeGuard[RArray]: return ( isinstance(other, RArray) and self.item_type == other.item_type From 766c43c6d66b64a3e8e61cc537a26a86286de968 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Thu, 14 Aug 2025 13:08:23 -0400 Subject: [PATCH 169/424] [mypyc] feat: exact_dict_set_item_op (#19657) This PR implements a `exact_dict_set_item_op` custom_op in preparation for the addition of exact_dict_rprimitive. We don't actually need to implement exact_dict_rprimitive before implementing this op as long as we accept that it will not be used for CallExpr specialization. This is okay for us now, as it is already addressed in the larger PR. --- mypyc/irbuild/classdef.py | 14 ++++++++------ mypyc/irbuild/expression.py | 4 ++-- mypyc/irbuild/function.py | 14 +++++++++----- mypyc/primitives/dict_ops.py | 9 +++++++++ mypyc/test-data/irbuild-basic.test | 6 +++--- mypyc/test-data/irbuild-classes.test | 6 +++--- mypyc/test-data/irbuild-generics.test | 8 ++++---- mypyc/test-data/irbuild-singledispatch.test | 4 ++-- 8 files changed, 40 insertions(+), 25 deletions(-) diff --git a/mypyc/irbuild/classdef.py b/mypyc/irbuild/classdef.py index 3282e836ac9e1..72482710208af 100644 --- a/mypyc/irbuild/classdef.py +++ b/mypyc/irbuild/classdef.py @@ -66,7 +66,7 @@ ) from mypyc.irbuild.prepare import GENERATOR_HELPER_NAME from mypyc.irbuild.util import dataclass_type, get_func_def, is_constant, is_dataclass_decorator -from mypyc.primitives.dict_ops import dict_new_op, dict_set_item_op +from mypyc.primitives.dict_ops import dict_new_op, exact_dict_set_item_op from mypyc.primitives.generic_ops import ( iter_op, next_op, @@ -271,8 +271,8 @@ def finalize(self, ir: ClassIR) -> None: ) # Add the non-extension class to the dict - self.builder.primitive_op( - dict_set_item_op, + self.builder.call_c( + exact_dict_set_item_op, [ self.builder.load_globals_dict(), self.builder.load_str(self.cdef.name), @@ -487,8 +487,10 @@ def allocate_class(builder: IRBuilder, cdef: ClassDef) -> Value: builder.add(InitStatic(tp, cdef.name, builder.module_name, NAMESPACE_TYPE)) # Add it to the dict - builder.primitive_op( - dict_set_item_op, [builder.load_globals_dict(), builder.load_str(cdef.name), tp], cdef.line + builder.call_c( + exact_dict_set_item_op, + [builder.load_globals_dict(), builder.load_str(cdef.name), tp], + cdef.line, ) return tp @@ -672,7 +674,7 @@ def add_non_ext_class_attr_ann( typ = builder.add(LoadAddress(type_object_op.type, type_object_op.src, stmt.line)) key = builder.load_str(lvalue.name) - builder.primitive_op(dict_set_item_op, [non_ext.anns, key, typ], stmt.line) + builder.call_c(exact_dict_set_item_op, [non_ext.anns, key, typ], stmt.line) def add_non_ext_class_attr( diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index 990c904dc4473..c3d863fa96dee 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -97,7 +97,7 @@ ) from mypyc.irbuild.specialize import apply_function_specialization, apply_method_specialization from mypyc.primitives.bytes_ops import bytes_slice_op -from mypyc.primitives.dict_ops import dict_get_item_op, dict_new_op, dict_set_item_op +from mypyc.primitives.dict_ops import dict_get_item_op, dict_new_op, exact_dict_set_item_op from mypyc.primitives.generic_ops import iter_op from mypyc.primitives.list_ops import list_append_op, list_extend_op, list_slice_op from mypyc.primitives.misc_ops import ellipsis_op, get_module_dict_op, new_slice_op, type_op @@ -1030,7 +1030,7 @@ def transform_dictionary_comprehension(builder: IRBuilder, o: DictionaryComprehe def gen_inner_stmts() -> None: k = builder.accept(o.key) v = builder.accept(o.value) - builder.primitive_op(dict_set_item_op, [builder.read(d), k, v], o.line) + builder.call_c(exact_dict_set_item_op, [builder.read(d), k, v], o.line) comprehension_helper(builder, loop_params, gen_inner_stmts, o.line) return builder.read(d) diff --git a/mypyc/irbuild/function.py b/mypyc/irbuild/function.py index d70b164755037..f0fc424aea540 100644 --- a/mypyc/irbuild/function.py +++ b/mypyc/irbuild/function.py @@ -76,7 +76,11 @@ ) from mypyc.irbuild.generator import gen_generator_func, gen_generator_func_body from mypyc.irbuild.targets import AssignmentTarget -from mypyc.primitives.dict_ops import dict_get_method_with_none, dict_new_op, dict_set_item_op +from mypyc.primitives.dict_ops import ( + dict_get_method_with_none, + dict_new_op, + exact_dict_set_item_op, +) from mypyc.primitives.generic_ops import py_setattr_op from mypyc.primitives.misc_ops import register_function from mypyc.primitives.registry import builtin_names @@ -123,8 +127,8 @@ def transform_decorator(builder: IRBuilder, dec: Decorator) -> None: if decorated_func is not None: # Set the callable object representing the decorated function as a global. - builder.primitive_op( - dict_set_item_op, + builder.call_c( + exact_dict_set_item_op, [builder.load_globals_dict(), builder.load_str(dec.func.name), decorated_func], decorated_func.line, ) @@ -826,7 +830,7 @@ def generate_singledispatch_dispatch_function( find_impl = builder.load_module_attr_by_fullname("functools._find_impl", line) registry = load_singledispatch_registry(builder, dispatch_func_obj, line) uncached_impl = builder.py_call(find_impl, [arg_type, registry], line) - builder.primitive_op(dict_set_item_op, [dispatch_cache, arg_type, uncached_impl], line) + builder.call_c(exact_dict_set_item_op, [dispatch_cache, arg_type, uncached_impl], line) builder.assign(impl_to_use, uncached_impl, line) builder.goto(call_func) @@ -1003,7 +1007,7 @@ def maybe_insert_into_registry_dict(builder: IRBuilder, fitem: FuncDef) -> None: registry = load_singledispatch_registry(builder, dispatch_func_obj, line) for typ in types: loaded_type = load_type(builder, typ, None, line) - builder.primitive_op(dict_set_item_op, [registry, loaded_type, to_insert], line) + builder.call_c(exact_dict_set_item_op, [registry, loaded_type, to_insert], line) dispatch_cache = builder.builder.get_attr( dispatch_func_obj, "dispatch_cache", dict_rprimitive, line ) diff --git a/mypyc/primitives/dict_ops.py b/mypyc/primitives/dict_ops.py index ac928bb0eb504..21f8a4badca33 100644 --- a/mypyc/primitives/dict_ops.py +++ b/mypyc/primitives/dict_ops.py @@ -98,6 +98,15 @@ error_kind=ERR_NEG_INT, ) +# dict[key] = value (exact dict only, no subclasses) +# NOTE: this is currently for internal use only, and not used for CallExpr specialization +exact_dict_set_item_op = custom_op( + arg_types=[dict_rprimitive, object_rprimitive, object_rprimitive], + return_type=c_int_rprimitive, + c_function_name="PyDict_SetItem", + error_kind=ERR_NEG_INT, +) + # key in dict binary_op( name="in", diff --git a/mypyc/test-data/irbuild-basic.test b/mypyc/test-data/irbuild-basic.test index 4a7d315ec8367..8d981db2b3915 100644 --- a/mypyc/test-data/irbuild-basic.test +++ b/mypyc/test-data/irbuild-basic.test @@ -1993,7 +1993,7 @@ L6: r13 = CPyTagged_Multiply(x, x) r14 = box(int, x) r15 = box(int, r13) - r16 = CPyDict_SetItem(r0, r14, r15) + r16 = PyDict_SetItem(r0, r14, r15) r17 = r16 >= 0 :: signed L7: r18 = r6 + 1 @@ -2620,7 +2620,7 @@ L0: d = r14 r15 = __main__.globals :: static r16 = 'd' - r17 = CPyDict_SetItem(r15, r16, r14) + r17 = PyDict_SetItem(r15, r16, r14) r18 = r17 >= 0 :: signed r19 = 'c' r20 = builtins :: module @@ -2693,7 +2693,7 @@ L2: keep_alive r17 r24 = __main__.globals :: static r25 = 'c' - r26 = CPyDict_SetItem(r24, r25, r23) + r26 = PyDict_SetItem(r24, r25, r23) r27 = r26 >= 0 :: signed return 1 diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index 8a2cc42fbb0f5..c7bf5de852a85 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -303,7 +303,7 @@ L2: __main__.C = r27 :: type r33 = __main__.globals :: static r34 = 'C' - r35 = CPyDict_SetItem(r33, r34, r27) + r35 = PyDict_SetItem(r33, r34, r27) r36 = r35 >= 0 :: signed r37 = :: object r38 = '__main__' @@ -316,7 +316,7 @@ L2: __main__.S = r40 :: type r45 = __main__.globals :: static r46 = 'S' - r47 = CPyDict_SetItem(r45, r46, r40) + r47 = PyDict_SetItem(r45, r46, r40) r48 = r47 >= 0 :: signed r49 = __main__.C :: type r50 = __main__.S :: type @@ -340,7 +340,7 @@ L2: __main__.D = r61 :: type r68 = __main__.globals :: static r69 = 'D' - r70 = CPyDict_SetItem(r68, r69, r61) + r70 = PyDict_SetItem(r68, r69, r61) r71 = r70 >= 0 :: signed return 1 diff --git a/mypyc/test-data/irbuild-generics.test b/mypyc/test-data/irbuild-generics.test index 4e9391e0d59e9..783492e63e472 100644 --- a/mypyc/test-data/irbuild-generics.test +++ b/mypyc/test-data/irbuild-generics.test @@ -351,7 +351,7 @@ L17: k = r49 v = r50 r51 = box(int, v) - r52 = CPyDict_SetItem(r40, k, r51) + r52 = PyDict_SetItem(r40, k, r51) r53 = r52 >= 0 :: signed L18: r54 = CPyDict_CheckSize(m, r42) @@ -493,7 +493,7 @@ L17: r49 = cast(union[int, str], r47) k = r48 v = r49 - r50 = CPyDict_SetItem(r39, k, v) + r50 = PyDict_SetItem(r39, k, v) r51 = r50 >= 0 :: signed L18: r52 = CPyDict_CheckSize(m, r41) @@ -630,7 +630,7 @@ L17: r47 = cast(str, r45) k = r47 v = r46 - r48 = CPyDict_SetItem(r38, k, v) + r48 = PyDict_SetItem(r38, k, v) r49 = r48 >= 0 :: signed L18: r50 = CPyDict_CheckSize(t, r40) @@ -742,7 +742,7 @@ L6: r17 = cast(str, r15) k = r17 v = r16 - r18 = CPyDict_SetItem(r8, k, v) + r18 = PyDict_SetItem(r8, k, v) r19 = r18 >= 0 :: signed L7: r20 = CPyDict_CheckSize(kwargs, r10) diff --git a/mypyc/test-data/irbuild-singledispatch.test b/mypyc/test-data/irbuild-singledispatch.test index ef11ae04dc648..1060ee63c57df 100644 --- a/mypyc/test-data/irbuild-singledispatch.test +++ b/mypyc/test-data/irbuild-singledispatch.test @@ -76,7 +76,7 @@ L2: r12 = load_address r11 r13 = PyObject_Vectorcall(r9, r12, 2, 0) keep_alive r1, r10 - r14 = CPyDict_SetItem(r2, r1, r13) + r14 = PyDict_SetItem(r2, r1, r13) r15 = r14 >= 0 :: signed r6 = r13 L3: @@ -214,7 +214,7 @@ L2: r12 = load_address r11 r13 = PyObject_Vectorcall(r9, r12, 2, 0) keep_alive r1, r10 - r14 = CPyDict_SetItem(r2, r1, r13) + r14 = PyDict_SetItem(r2, r1, r13) r15 = r14 >= 0 :: signed r6 = r13 L3: From 74927d5cbbcdeaefb2ef3c36c8b37a331050f6e5 Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Fri, 15 Aug 2025 00:27:09 +0200 Subject: [PATCH 170/424] Include `ambiguous` into `UninhabitedType` identity (#19648) Fixes #19641, but also reveals a test that was only passing by coincidence. This inference has never worked correctly: ```python from typing import Mapping, Never d: dict[Never, Never] def run() -> Mapping[str, int]: return d ``` As discussed, I updated the test to expect failure in that case. We should only special-case inline collection literals for that, everything else (including locals) can cause undesired false negatives. --- mypy/test/testsolve.py | 12 +++++++----- mypy/test/typefixture.py | 2 ++ mypy/types.py | 4 ++-- test-data/unit/check-generic-subtyping.test | 14 ++++++++++++++ test-data/unit/check-inference.test | 10 +++++++++- 5 files changed, 34 insertions(+), 8 deletions(-) diff --git a/mypy/test/testsolve.py b/mypy/test/testsolve.py index 6566b03ef5e99..d60b2cb3fcc55 100644 --- a/mypy/test/testsolve.py +++ b/mypy/test/testsolve.py @@ -64,12 +64,14 @@ def test_multiple_variables(self) -> None: ) def test_no_constraints_for_var(self) -> None: - self.assert_solve([self.fx.t], [], [self.fx.uninhabited]) - self.assert_solve([self.fx.t, self.fx.s], [], [self.fx.uninhabited, self.fx.uninhabited]) + self.assert_solve([self.fx.t], [], [self.fx.a_uninhabited]) + self.assert_solve( + [self.fx.t, self.fx.s], [], [self.fx.a_uninhabited, self.fx.a_uninhabited] + ) self.assert_solve( [self.fx.t, self.fx.s], [self.supc(self.fx.s, self.fx.a)], - [self.fx.uninhabited, self.fx.a], + [self.fx.a_uninhabited, self.fx.a], ) def test_simple_constraints_with_dynamic_type(self) -> None: @@ -116,7 +118,7 @@ def test_poly_no_constraints(self) -> None: self.assert_solve( [self.fx.t, self.fx.u], [], - [self.fx.uninhabited, self.fx.uninhabited], + [self.fx.a_uninhabited, self.fx.a_uninhabited], allow_polymorphic=True, ) @@ -152,7 +154,7 @@ def test_poly_free_pair_with_bounds_uninhabited(self) -> None: self.assert_solve( [self.fx.ub, self.fx.uc], [self.subc(self.fx.ub, self.fx.uc)], - [self.fx.uninhabited, self.fx.uninhabited], + [self.fx.a_uninhabited, self.fx.a_uninhabited], [], allow_polymorphic=True, ) diff --git a/mypy/test/typefixture.py b/mypy/test/typefixture.py index d6c904732b179..0defcdaebc990 100644 --- a/mypy/test/typefixture.py +++ b/mypy/test/typefixture.py @@ -78,6 +78,8 @@ def make_type_var( self.anyt = AnyType(TypeOfAny.special_form) self.nonet = NoneType() self.uninhabited = UninhabitedType() + self.a_uninhabited = UninhabitedType() + self.a_uninhabited.ambiguous = True # Abstract class TypeInfos diff --git a/mypy/types.py b/mypy/types.py index d7dd3e1f2dce8..26c5b474ba6cf 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1236,10 +1236,10 @@ def accept(self, visitor: TypeVisitor[T]) -> T: return visitor.visit_uninhabited_type(self) def __hash__(self) -> int: - return hash(UninhabitedType) + return hash((UninhabitedType, self.ambiguous)) def __eq__(self, other: object) -> bool: - return isinstance(other, UninhabitedType) + return isinstance(other, UninhabitedType) and other.ambiguous == self.ambiguous def serialize(self) -> JsonDict: return {".class": "UninhabitedType"} diff --git a/test-data/unit/check-generic-subtyping.test b/test-data/unit/check-generic-subtyping.test index f65ef3975852d..ee5bcccf8aceb 100644 --- a/test-data/unit/check-generic-subtyping.test +++ b/test-data/unit/check-generic-subtyping.test @@ -753,6 +753,20 @@ s, s = Nums() # E: Incompatible types in assignment (expression has type "int", [builtins fixtures/for.pyi] [out] +[case testUninhabitedCacheChecksAmbiguous] +# https://github.com/python/mypy/issues/19641 +from typing import Mapping, Never, TypeVar + +M = TypeVar("M", bound=Mapping[str,object]) + +def get(arg: M, /) -> M: + return arg + +get({}) + +def upcast(d: dict[Never, Never]) -> Mapping[str, object]: + return d # E: Incompatible return value type (got "dict[Never, Never]", expected "Mapping[str, object]") +[builtins fixtures/dict.pyi] -- Variance -- -------- diff --git a/test-data/unit/check-inference.test b/test-data/unit/check-inference.test index 53efcc0d22e39..63278d6c4547a 100644 --- a/test-data/unit/check-inference.test +++ b/test-data/unit/check-inference.test @@ -3781,10 +3781,18 @@ from typing import Any, Dict, NoReturn, NoReturn, Union def foo() -> Union[Dict[str, Any], Dict[int, Any]]: return {} +[builtins fixtures/dict.pyi] + +[case testExistingEmptyCollectionDoesNotUpcast] +from typing import Any, Dict, NoReturn, NoReturn, Union empty: Dict[NoReturn, NoReturn] + +def foo() -> Dict[str, Any]: + return empty # E: Incompatible return value type (got "dict[Never, Never]", expected "dict[str, Any]") + def bar() -> Union[Dict[str, Any], Dict[int, Any]]: - return empty + return empty # E: Incompatible return value type (got "dict[Never, Never]", expected "Union[dict[str, Any], dict[int, Any]]") [builtins fixtures/dict.pyi] [case testUpperBoundInferenceFallbackNotOverused] From cb77a9b36775742b726e856af628e2dcc8e714c2 Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Fri, 15 Aug 2025 00:29:13 +0200 Subject: [PATCH 171/424] Reset to previous statement when leaving `return` in semanal (#19642) Unlike other statements, return can be found in lambdas, and so is not immediately followed by another statement to check or EOF. We need to restore the previous value to continue checking it. Fixes #19632 --- mypy/semanal.py | 2 ++ test-data/unit/check-classes.test | 15 +++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/mypy/semanal.py b/mypy/semanal.py index bebabfd3233c8..ebd9515d832df 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -5332,6 +5332,7 @@ def visit_expression_stmt(self, s: ExpressionStmt) -> None: s.expr.accept(self) def visit_return_stmt(self, s: ReturnStmt) -> None: + old = self.statement self.statement = s if not self.is_func_scope(): self.fail('"return" outside function', s) @@ -5339,6 +5340,7 @@ def visit_return_stmt(self, s: ReturnStmt) -> None: self.fail('"return" not allowed in except* block', s, serious=True) if s.expr: s.expr.accept(self) + self.statement = old def visit_raise_stmt(self, s: RaiseStmt) -> None: self.statement = s diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index 62f538260fffd..23dbe2bc07afb 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -9258,3 +9258,18 @@ main:5: note: def __eq__(self, other: object) -> bool: main:5: note: if not isinstance(other, C): main:5: note: return NotImplemented main:5: note: return + +[case testLambdaInAttributeCallValue] +# https://github.com/python/mypy/issues/19632 +import foo + +def nop(fn: object) -> foo.Bar: + return foo.Bar() + +class Bar: + foo: foo.Bar = nop( + lambda: 0 + ) +[file foo.py] +class Bar: + ... From 8e66cf2f56ea3892e8d56fc6f7f8f97d81734c8e Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Fri, 15 Aug 2025 12:16:01 +0200 Subject: [PATCH 172/424] fix: prevent false positive "untyped after decorator transformation" after deferral (#19591) Fixes #19589 --- mypy/checker.py | 2 +- test-data/unit/check-flags.test | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/mypy/checker.py b/mypy/checker.py index 206abae6adec4..47c72924bf3c6 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -5502,7 +5502,7 @@ def visit_with_stmt(self, s: WithStmt) -> None: self.accept(s.body) def check_untyped_after_decorator(self, typ: Type, func: FuncDef) -> None: - if not self.options.disallow_any_decorated or self.is_stub: + if not self.options.disallow_any_decorated or self.is_stub or self.current_node_deferred: return if mypy.checkexpr.has_any_type(typ): diff --git a/test-data/unit/check-flags.test b/test-data/unit/check-flags.test index bb64bb44d2828..8eec979029d00 100644 --- a/test-data/unit/check-flags.test +++ b/test-data/unit/check-flags.test @@ -1116,6 +1116,39 @@ def f(x: Any) -> Any: # E: Function is untyped after decorator transformation def h(x): # E: Function is untyped after decorator transformation pass [builtins fixtures/list.pyi] + +[case testDisallowAnyDecoratedUnannotatedDecoratorDeferred1] +# flags: --disallow-any-decorated +from typing import Callable + +def d(f: Callable[[int], None]) -> Callable[[int], None]: + return f + +def wrapper() -> None: + if c: + @d + def h(x): + pass + +c = [1] +[builtins fixtures/list.pyi] + +[case testDisallowAnyDecoratedUnannotatedDecoratorDeferred2] +# flags: --disallow-any-decorated +from typing import Callable + +def d(f: Callable[[int], None]) -> Callable[[int], None]: + return f + +c = 1 # no deferral - check that the previous testcase is valid + +def wrapper() -> None: + if c: + @d + def h(x): + pass +[builtins fixtures/list.pyi] + [case testDisallowAnyDecoratedErrorIsReportedOnlyOnce] # flags: --disallow-any-decorated From ac646c0c55e790388609065400084fe8e189f86c Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Fri, 15 Aug 2025 08:07:51 -0400 Subject: [PATCH 173/424] [mypyc] feat: optimize f-string building from Final values (#19611) We can do some extra constant folding in cases like this: ```python from typing import Final BASE_URL: Final = "https://example.com" PORT: Final = 1234 def get_url(endpoint: str) -> str: return f"{BASE_URL}:{PORT}/{endpoint}" ``` which should generate the same C code as ```python def get_url(endpoint: str) -> str: return f"https://example.com:1234/{endpoint}" ``` This PR makes it so. --- mypyc/irbuild/format_str_tokenizer.py | 6 ++-- mypyc/irbuild/specialize.py | 17 ++++++++++++ mypyc/test-data/irbuild-str.test | 40 +++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 2 deletions(-) diff --git a/mypyc/irbuild/format_str_tokenizer.py b/mypyc/irbuild/format_str_tokenizer.py index eaa4027ed7684..5a35900006d24 100644 --- a/mypyc/irbuild/format_str_tokenizer.py +++ b/mypyc/irbuild/format_str_tokenizer.py @@ -12,7 +12,7 @@ ) from mypy.errors import Errors from mypy.messages import MessageBuilder -from mypy.nodes import Context, Expression +from mypy.nodes import Context, Expression, StrExpr from mypy.options import Options from mypyc.ir.ops import Integer, Value from mypyc.ir.rtypes import ( @@ -143,7 +143,9 @@ def convert_format_expr_to_str( for x, format_op in zip(exprs, format_ops): node_type = builder.node_type(x) if format_op == FormatOp.STR: - if is_str_rprimitive(node_type): + if is_str_rprimitive(node_type) or isinstance( + x, StrExpr + ): # NOTE: why does mypyc think our fake StrExprs are not str rprimitives? var_str = builder.accept(x) elif is_int_rprimitive(node_type) or is_short_int_rprimitive(node_type): var_str = builder.primitive_op(int_to_str_op, [builder.accept(x)], line) diff --git a/mypyc/irbuild/specialize.py b/mypyc/irbuild/specialize.py index 748cda1256a74..8459682222343 100644 --- a/mypyc/irbuild/specialize.py +++ b/mypyc/irbuild/specialize.py @@ -31,6 +31,7 @@ RefExpr, StrExpr, TupleExpr, + Var, ) from mypy.types import AnyType, TypeOfAny from mypyc.ir.ops import ( @@ -710,6 +711,22 @@ def translate_fstring(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Va format_ops.append(FormatOp.STR) exprs.append(item.args[0]) + def get_literal_str(expr: Expression) -> str | None: + if isinstance(expr, StrExpr): + return expr.value + elif isinstance(expr, RefExpr) and isinstance(expr.node, Var) and expr.node.is_final: + return str(expr.node.final_value) + return None + + for i in range(len(exprs) - 1): + while ( + len(exprs) >= i + 2 + and (first := get_literal_str(exprs[i])) is not None + and (second := get_literal_str(exprs[i + 1])) is not None + ): + exprs = [*exprs[:i], StrExpr(first + second), *exprs[i + 2 :]] + format_ops = [*format_ops[:i], FormatOp.STR, *format_ops[i + 2 :]] + substitutions = convert_format_expr_to_str(builder, format_ops, exprs, expr.line) if substitutions is None: return None diff --git a/mypyc/test-data/irbuild-str.test b/mypyc/test-data/irbuild-str.test index 3e69325a454bf..24807510193d7 100644 --- a/mypyc/test-data/irbuild-str.test +++ b/mypyc/test-data/irbuild-str.test @@ -605,3 +605,43 @@ L3: r6 = r7 L4: return r6 + +[case testFStringFromConstants] +from typing import Final +string: Final = "abc" +integer: Final = 123 +floating: Final = 3.14 +boolean: Final = True + +def test(x: str) -> str: + return f"{string}{integer}{floating}{boolean}{x}{boolean}{floating}{integer}{string}{x}{string}{integer}{floating}{boolean}" +def test2(x: str) -> str: + return f"{string}{integer}{floating}{boolean}{x}{boolean}{floating}{integer}{string}{x}{string}{integer}{floating}{boolean}{x}" +def test3(x: str) -> str: + return f"{x}{string}{integer}{floating}{boolean}{x}{boolean}{floating}{integer}{string}{x}{string}{integer}{floating}{boolean}{x}" + +[out] +def test(x): + x, r0, r1, r2, r3 :: str +L0: + r0 = 'abc1233.14True' + r1 = 'True3.14123abc' + r2 = 'abc1233.14True' + r3 = CPyStr_Build(5, r0, x, r1, x, r2) + return r3 +def test2(x): + x, r0, r1, r2, r3 :: str +L0: + r0 = 'abc1233.14True' + r1 = 'True3.14123abc' + r2 = 'abc1233.14True' + r3 = CPyStr_Build(6, r0, x, r1, x, r2, x) + return r3 +def test3(x): + x, r0, r1, r2, r3 :: str +L0: + r0 = 'abc1233.14True' + r1 = 'True3.14123abc' + r2 = 'abc1233.14True' + r3 = CPyStr_Build(7, x, r0, x, r1, x, r2, x) + return r3 From 206b739489e7448a7a330b385c21053e0ddd7eab Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 15 Aug 2025 15:46:53 +0100 Subject: [PATCH 174/424] [mypyc] Remove unreachable code (#19667) See #19050 for context. --- mypyc/test/test_cheader.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/mypyc/test/test_cheader.py b/mypyc/test/test_cheader.py index 7ab055c735ad5..ec9e2c4cf4504 100644 --- a/mypyc/test/test_cheader.py +++ b/mypyc/test/test_cheader.py @@ -7,7 +7,6 @@ import re import unittest -from mypyc.ir.ops import PrimitiveDescription from mypyc.primitives import registry @@ -32,8 +31,6 @@ def check_name(name: str) -> None: registry.function_ops.values(), ]: for ops in values: - if isinstance(ops, PrimitiveDescription): - ops = [ops] for op in ops: if op.c_function_name is not None: check_name(op.c_function_name) From e30128b1b649bba3c4aab198fe9d35c138c5eeb1 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Fri, 15 Aug 2025 17:51:45 +0200 Subject: [PATCH 175/424] Sort arguments in TypedDict overlap error message (#19666) Discovered in https://github.com/python/mypy/pull/19664#issuecomment-3191182315. Sets are unsorted which could make the primer output differ unnecessarily. --- mypy/semanal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index ebd9515d832df..eef658d9300b2 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1059,7 +1059,7 @@ def remove_unpack_kwargs(self, defn: FuncDef, typ: CallableType) -> CallableType # It is OK for TypedDict to have a key named 'kwargs'. overlap.discard(typ.arg_names[-1]) if overlap: - overlapped = ", ".join([f'"{name}"' for name in overlap]) + overlapped = ", ".join([f'"{name}"' for name in sorted(filter(None, overlap))]) self.fail(f"Overlap between argument names and ** TypedDict items: {overlapped}", defn) new_arg_types = typ.arg_types[:-1] + [AnyType(TypeOfAny.from_error)] return typ.copy_modified(arg_types=new_arg_types) From 3fcfcb8d9e73facab074b6f7dbba76c2fe1212fc Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Sat, 16 Aug 2025 11:09:20 +0200 Subject: [PATCH 176/424] Sync typeshed (#19664) Source commit: https://github.com/python/typeshed/commit/554701e9b6107be8e0bda4955a309d94fefc22ae --- ...e-of-LiteralString-in-builtins-13743.patch | 14 +- mypy/typeshed/stdlib/_contextvars.pyi | 2 +- mypy/typeshed/stdlib/_frozen_importlib.pyi | 11 + .../stdlib/_frozen_importlib_external.pyi | 17 +- mypy/typeshed/stdlib/_interpchannels.pyi | 12 +- mypy/typeshed/stdlib/_interpreters.pyi | 4 +- mypy/typeshed/stdlib/_socket.pyi | 1053 ++++++++--------- mypy/typeshed/stdlib/_ssl.pyi | 212 ++-- mypy/typeshed/stdlib/_tkinter.pyi | 3 +- mypy/typeshed/stdlib/array.pyi | 29 +- mypy/typeshed/stdlib/ast.pyi | 6 +- mypy/typeshed/stdlib/asyncio/coroutines.pyi | 3 +- mypy/typeshed/stdlib/asyncio/trsock.pyi | 31 +- mypy/typeshed/stdlib/binascii.pyi | 6 +- mypy/typeshed/stdlib/builtins.pyi | 4 + .../stdlib/concurrent/futures/interpreter.pyi | 59 +- mypy/typeshed/stdlib/configparser.pyi | 23 +- mypy/typeshed/stdlib/fcntl.pyi | 166 +-- mypy/typeshed/stdlib/gettext.pyi | 19 +- mypy/typeshed/stdlib/glob.pyi | 12 +- mypy/typeshed/stdlib/gzip.pyi | 3 +- mypy/typeshed/stdlib/html/parser.pyi | 6 +- mypy/typeshed/stdlib/importlib/__init__.pyi | 2 + mypy/typeshed/stdlib/importlib/_abc.pyi | 5 + mypy/typeshed/stdlib/importlib/abc.pyi | 4 + .../stdlib/importlib/metadata/__init__.pyi | 5 +- mypy/typeshed/stdlib/importlib/util.pyi | 14 +- mypy/typeshed/stdlib/inspect.pyi | 7 +- mypy/typeshed/stdlib/locale.pyi | 8 +- mypy/typeshed/stdlib/pathlib/__init__.pyi | 10 +- mypy/typeshed/stdlib/pkgutil.pyi | 14 +- mypy/typeshed/stdlib/pty.pyi | 12 +- mypy/typeshed/stdlib/re.pyi | 8 +- mypy/typeshed/stdlib/smtpd.pyi | 3 +- mypy/typeshed/stdlib/socket.pyi | 4 +- mypy/typeshed/stdlib/ssl.pyi | 160 +-- mypy/typeshed/stdlib/sys/__init__.pyi | 8 + mypy/typeshed/stdlib/tkinter/__init__.pyi | 1 + mypy/typeshed/stdlib/turtle.pyi | 4 +- mypy/typeshed/stdlib/typing.pyi | 32 +- mypy/typeshed/stdlib/typing_extensions.pyi | 4 +- mypy/typeshed/stdlib/urllib/request.pyi | 4 +- mypy/typeshed/stdlib/winreg.pyi | 2 +- mypy/typeshed/stdlib/zipimport.pyi | 16 +- mypy/typeshed/stdlib/zlib.pyi | 12 +- 45 files changed, 1104 insertions(+), 930 deletions(-) diff --git a/misc/typeshed_patches/0001-Remove-use-of-LiteralString-in-builtins-13743.patch b/misc/typeshed_patches/0001-Remove-use-of-LiteralString-in-builtins-13743.patch index 9d0cb5271e7d7..a47d5db3cd222 100644 --- a/misc/typeshed_patches/0001-Remove-use-of-LiteralString-in-builtins-13743.patch +++ b/misc/typeshed_patches/0001-Remove-use-of-LiteralString-in-builtins-13743.patch @@ -1,4 +1,4 @@ -From e6995c91231e1915eba43a29a22dd4cbfaf9e08e Mon Sep 17 00:00:00 2001 +From 805d7fc06a8bee350959512e0908a18a87b7f8c2 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 26 Sep 2022 12:55:07 -0700 Subject: [PATCH] Remove use of LiteralString in builtins (#13743) @@ -8,7 +8,7 @@ Subject: [PATCH] Remove use of LiteralString in builtins (#13743) 1 file changed, 1 insertion(+), 99 deletions(-) diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi -index 00728f42d..ea77a730f 100644 +index c7ab95482..3e93da36e 100644 --- a/mypy/typeshed/stdlib/builtins.pyi +++ b/mypy/typeshed/stdlib/builtins.pyi @@ -63,7 +63,6 @@ from typing import ( # noqa: Y022,UP035 @@ -19,7 +19,7 @@ index 00728f42d..ea77a730f 100644 ParamSpec, Self, TypeAlias, -@@ -453,31 +452,16 @@ class str(Sequence[str]): +@@ -468,31 +467,16 @@ class str(Sequence[str]): def __new__(cls, object: object = ...) -> Self: ... @overload def __new__(cls, object: ReadableBuffer, encoding: str = ..., errors: str = ...) -> Self: ... @@ -51,7 +51,7 @@ index 00728f42d..ea77a730f 100644 def format(self, *args: object, **kwargs: object) -> str: ... def format_map(self, mapping: _FormatMapMapping, /) -> str: ... def index(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... -@@ -493,98 +477,34 @@ class str(Sequence[str]): +@@ -508,98 +492,34 @@ class str(Sequence[str]): def isspace(self) -> bool: ... def istitle(self) -> bool: ... def isupper(self) -> bool: ... @@ -150,7 +150,7 @@ index 00728f42d..ea77a730f 100644 def zfill(self, width: SupportsIndex, /) -> str: ... # type: ignore[misc] @staticmethod @overload -@@ -595,39 +515,21 @@ class str(Sequence[str]): +@@ -610,39 +530,21 @@ class str(Sequence[str]): @staticmethod @overload def maketrans(x: str, y: str, z: str, /) -> dict[int, int | None]: ... @@ -190,7 +190,7 @@ index 00728f42d..ea77a730f 100644 - @overload def __rmul__(self, value: SupportsIndex, /) -> str: ... # type: ignore[misc] def __getnewargs__(self) -> tuple[str]: ... - + def __format__(self, format_spec: str, /) -> str: ... -- -2.49.0 +2.50.1 diff --git a/mypy/typeshed/stdlib/_contextvars.pyi b/mypy/typeshed/stdlib/_contextvars.pyi index e2e2e4df9d086..0ddeca7882cd1 100644 --- a/mypy/typeshed/stdlib/_contextvars.pyi +++ b/mypy/typeshed/stdlib/_contextvars.pyi @@ -39,7 +39,7 @@ class Token(Generic[_T]): if sys.version_info >= (3, 14): def __enter__(self) -> Self: ... def __exit__( - self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: TracebackType | None + self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: TracebackType | None, / ) -> None: ... def copy_context() -> Context: ... diff --git a/mypy/typeshed/stdlib/_frozen_importlib.pyi b/mypy/typeshed/stdlib/_frozen_importlib.pyi index 3dbc8c6b52f0d..93aaed82e2e1c 100644 --- a/mypy/typeshed/stdlib/_frozen_importlib.pyi +++ b/mypy/typeshed/stdlib/_frozen_importlib.pyi @@ -6,6 +6,7 @@ from _typeshed.importlib import LoaderProtocol from collections.abc import Mapping, Sequence from types import ModuleType from typing import Any, ClassVar +from typing_extensions import deprecated # Signature of `builtins.__import__` should be kept identical to `importlib.__import__` def __import__( @@ -49,6 +50,7 @@ class BuiltinImporter(importlib.abc.MetaPathFinder, importlib.abc.InspectLoader) # MetaPathFinder if sys.version_info < (3, 12): @classmethod + @deprecated("Deprecated since Python 3.4; removed in Python 3.12. Use `find_spec()` instead.") def find_module(cls, fullname: str, path: Sequence[str] | None = None) -> importlib.abc.Loader | None: ... @classmethod @@ -67,6 +69,10 @@ class BuiltinImporter(importlib.abc.MetaPathFinder, importlib.abc.InspectLoader) # Loader if sys.version_info < (3, 12): @staticmethod + @deprecated( + "Deprecated since Python 3.4; removed in Python 3.12. " + "The module spec is now used by the import machinery to generate a module repr." + ) def module_repr(module: types.ModuleType) -> str: ... if sys.version_info >= (3, 10): @staticmethod @@ -83,6 +89,7 @@ class FrozenImporter(importlib.abc.MetaPathFinder, importlib.abc.InspectLoader): # MetaPathFinder if sys.version_info < (3, 12): @classmethod + @deprecated("Deprecated since Python 3.4; removed in Python 3.12. Use `find_spec()` instead.") def find_module(cls, fullname: str, path: Sequence[str] | None = None) -> importlib.abc.Loader | None: ... @classmethod @@ -101,6 +108,10 @@ class FrozenImporter(importlib.abc.MetaPathFinder, importlib.abc.InspectLoader): # Loader if sys.version_info < (3, 12): @staticmethod + @deprecated( + "Deprecated since Python 3.4; removed in Python 3.12. " + "The module spec is now used by the import machinery to generate a module repr." + ) def module_repr(m: types.ModuleType) -> str: ... if sys.version_info >= (3, 10): @staticmethod diff --git a/mypy/typeshed/stdlib/_frozen_importlib_external.pyi b/mypy/typeshed/stdlib/_frozen_importlib_external.pyi index edad50a8d8583..80eebe45a7d44 100644 --- a/mypy/typeshed/stdlib/_frozen_importlib_external.pyi +++ b/mypy/typeshed/stdlib/_frozen_importlib_external.pyi @@ -43,6 +43,7 @@ def spec_from_file_location( class WindowsRegistryFinder(importlib.abc.MetaPathFinder): if sys.version_info < (3, 12): @classmethod + @deprecated("Deprecated since Python 3.4; removed in Python 3.12. Use `find_spec()` instead.") def find_module(cls, fullname: str, path: Sequence[str] | None = None) -> importlib.abc.Loader | None: ... @classmethod @@ -70,6 +71,7 @@ class PathFinder(importlib.abc.MetaPathFinder): ) -> ModuleSpec | None: ... if sys.version_info < (3, 12): @classmethod + @deprecated("Deprecated since Python 3.4; removed in Python 3.12. Use `find_spec()` instead.") def find_module(cls, fullname: str, path: Sequence[str] | None = None) -> importlib.abc.Loader | None: ... SOURCE_SUFFIXES: list[str] @@ -158,7 +160,10 @@ if sys.version_info >= (3, 11): def get_resource_reader(self, module: types.ModuleType) -> importlib.readers.NamespaceReader: ... if sys.version_info < (3, 12): @staticmethod - @deprecated("module_repr() is deprecated, and has been removed in Python 3.12") + @deprecated( + "Deprecated since Python 3.4; removed in Python 3.12. " + "The module spec is now used by the import machinery to generate a module repr." + ) def module_repr(module: types.ModuleType) -> str: ... _NamespaceLoader = NamespaceLoader @@ -176,12 +181,18 @@ else: def load_module(self, fullname: str) -> types.ModuleType: ... if sys.version_info >= (3, 10): @staticmethod - @deprecated("module_repr() is deprecated, and has been removed in Python 3.12") + @deprecated( + "Deprecated since Python 3.4; removed in Python 3.12. " + "The module spec is now used by the import machinery to generate a module repr." + ) def module_repr(module: types.ModuleType) -> str: ... def get_resource_reader(self, module: types.ModuleType) -> importlib.readers.NamespaceReader: ... else: @classmethod - @deprecated("module_repr() is deprecated, and has been removed in Python 3.12") + @deprecated( + "Deprecated since Python 3.4; removed in Python 3.12. " + "The module spec is now used by the import machinery to generate a module repr." + ) def module_repr(cls, module: types.ModuleType) -> str: ... if sys.version_info >= (3, 13): diff --git a/mypy/typeshed/stdlib/_interpchannels.pyi b/mypy/typeshed/stdlib/_interpchannels.pyi index c03496044df06..a631a6f16616b 100644 --- a/mypy/typeshed/stdlib/_interpchannels.pyi +++ b/mypy/typeshed/stdlib/_interpchannels.pyi @@ -17,15 +17,15 @@ class ChannelID: def send(self) -> Self: ... @property def recv(self) -> Self: ... - def __eq__(self, other: object) -> bool: ... - def __ge__(self, other: ChannelID) -> bool: ... - def __gt__(self, other: ChannelID) -> bool: ... + def __eq__(self, other: object, /) -> bool: ... + def __ge__(self, other: ChannelID, /) -> bool: ... + def __gt__(self, other: ChannelID, /) -> bool: ... def __hash__(self) -> int: ... def __index__(self) -> int: ... def __int__(self) -> int: ... - def __le__(self, other: ChannelID) -> bool: ... - def __lt__(self, other: ChannelID) -> bool: ... - def __ne__(self, other: object) -> bool: ... + def __le__(self, other: ChannelID, /) -> bool: ... + def __lt__(self, other: ChannelID, /) -> bool: ... + def __ne__(self, other: object, /) -> bool: ... @final class ChannelInfo(structseq[int], tuple[bool, bool, bool, int, int, int, int, int]): diff --git a/mypy/typeshed/stdlib/_interpreters.pyi b/mypy/typeshed/stdlib/_interpreters.pyi index 54fc0e39d239d..f89a24e7d85c6 100644 --- a/mypy/typeshed/stdlib/_interpreters.pyi +++ b/mypy/typeshed/stdlib/_interpreters.pyi @@ -34,8 +34,8 @@ def exec( def call( id: SupportsIndex, callable: Callable[..., _R], - args: tuple[object, ...] | None = None, - kwargs: dict[str, object] | None = None, + args: tuple[Any, ...] | None = None, + kwargs: dict[str, Any] | None = None, *, restrict: bool = False, ) -> tuple[_R, types.SimpleNamespace]: ... diff --git a/mypy/typeshed/stdlib/_socket.pyi b/mypy/typeshed/stdlib/_socket.pyi index 41fdce87ec14d..9c153a3a6ba0b 100644 --- a/mypy/typeshed/stdlib/_socket.pyi +++ b/mypy/typeshed/stdlib/_socket.pyi @@ -2,7 +2,7 @@ import sys from _typeshed import ReadableBuffer, WriteableBuffer from collections.abc import Iterable from socket import error as error, gaierror as gaierror, herror as herror, timeout as timeout -from typing import Any, SupportsIndex, overload +from typing import Any, Final, SupportsIndex, overload from typing_extensions import CapsuleType, TypeAlias _CMSG: TypeAlias = tuple[int, int, bytes] @@ -19,23 +19,23 @@ _RetAddress: TypeAlias = Any # https://docs.python.org/3/library/socket.html#constants if sys.platform != "win32": - AF_UNIX: int + AF_UNIX: Final[int] -AF_INET: int -AF_INET6: int +AF_INET: Final[int] +AF_INET6: Final[int] -AF_UNSPEC: int +AF_UNSPEC: Final[int] -SOCK_STREAM: int -SOCK_DGRAM: int -SOCK_RAW: int -SOCK_RDM: int -SOCK_SEQPACKET: int +SOCK_STREAM: Final[int] +SOCK_DGRAM: Final[int] +SOCK_RAW: Final[int] +SOCK_RDM: Final[int] +SOCK_SEQPACKET: Final[int] if sys.platform == "linux": # Availability: Linux >= 2.6.27 - SOCK_CLOEXEC: int - SOCK_NONBLOCK: int + SOCK_CLOEXEC: Final[int] + SOCK_NONBLOCK: Final[int] # -------------------- # Many constants of these forms, documented in the Unix documentation on @@ -56,329 +56,329 @@ if sys.platform == "linux": # TCP_* # -------------------- -SO_ACCEPTCONN: int -SO_BROADCAST: int -SO_DEBUG: int -SO_DONTROUTE: int -SO_ERROR: int -SO_KEEPALIVE: int -SO_LINGER: int -SO_OOBINLINE: int -SO_RCVBUF: int -SO_RCVLOWAT: int -SO_RCVTIMEO: int -SO_REUSEADDR: int -SO_SNDBUF: int -SO_SNDLOWAT: int -SO_SNDTIMEO: int -SO_TYPE: int +SO_ACCEPTCONN: Final[int] +SO_BROADCAST: Final[int] +SO_DEBUG: Final[int] +SO_DONTROUTE: Final[int] +SO_ERROR: Final[int] +SO_KEEPALIVE: Final[int] +SO_LINGER: Final[int] +SO_OOBINLINE: Final[int] +SO_RCVBUF: Final[int] +SO_RCVLOWAT: Final[int] +SO_RCVTIMEO: Final[int] +SO_REUSEADDR: Final[int] +SO_SNDBUF: Final[int] +SO_SNDLOWAT: Final[int] +SO_SNDTIMEO: Final[int] +SO_TYPE: Final[int] if sys.platform != "linux": - SO_USELOOPBACK: int + SO_USELOOPBACK: Final[int] if sys.platform == "win32": - SO_EXCLUSIVEADDRUSE: int + SO_EXCLUSIVEADDRUSE: Final[int] if sys.platform != "win32": - SO_REUSEPORT: int + SO_REUSEPORT: Final[int] if sys.platform != "darwin" or sys.version_info >= (3, 13): - SO_BINDTODEVICE: int + SO_BINDTODEVICE: Final[int] if sys.platform != "win32" and sys.platform != "darwin": - SO_DOMAIN: int - SO_MARK: int - SO_PASSCRED: int - SO_PASSSEC: int - SO_PEERCRED: int - SO_PEERSEC: int - SO_PRIORITY: int - SO_PROTOCOL: int + SO_DOMAIN: Final[int] + SO_MARK: Final[int] + SO_PASSCRED: Final[int] + SO_PASSSEC: Final[int] + SO_PEERCRED: Final[int] + SO_PEERSEC: Final[int] + SO_PRIORITY: Final[int] + SO_PROTOCOL: Final[int] if sys.platform != "win32" and sys.platform != "darwin" and sys.platform != "linux": - SO_SETFIB: int + SO_SETFIB: Final[int] if sys.platform == "linux" and sys.version_info >= (3, 13): - SO_BINDTOIFINDEX: int + SO_BINDTOIFINDEX: Final[int] -SOMAXCONN: int +SOMAXCONN: Final[int] -MSG_CTRUNC: int -MSG_DONTROUTE: int -MSG_OOB: int -MSG_PEEK: int -MSG_TRUNC: int -MSG_WAITALL: int +MSG_CTRUNC: Final[int] +MSG_DONTROUTE: Final[int] +MSG_OOB: Final[int] +MSG_PEEK: Final[int] +MSG_TRUNC: Final[int] +MSG_WAITALL: Final[int] if sys.platform != "win32": - MSG_DONTWAIT: int - MSG_EOR: int - MSG_NOSIGNAL: int # Sometimes this exists on darwin, sometimes not + MSG_DONTWAIT: Final[int] + MSG_EOR: Final[int] + MSG_NOSIGNAL: Final[int] # Sometimes this exists on darwin, sometimes not if sys.platform != "darwin": - MSG_ERRQUEUE: int + MSG_ERRQUEUE: Final[int] if sys.platform == "win32": - MSG_BCAST: int - MSG_MCAST: int + MSG_BCAST: Final[int] + MSG_MCAST: Final[int] if sys.platform != "win32" and sys.platform != "darwin": - MSG_CMSG_CLOEXEC: int - MSG_CONFIRM: int - MSG_FASTOPEN: int - MSG_MORE: int + MSG_CMSG_CLOEXEC: Final[int] + MSG_CONFIRM: Final[int] + MSG_FASTOPEN: Final[int] + MSG_MORE: Final[int] if sys.platform != "win32" and sys.platform != "linux": - MSG_EOF: int + MSG_EOF: Final[int] if sys.platform != "win32" and sys.platform != "linux" and sys.platform != "darwin": - MSG_NOTIFICATION: int - MSG_BTAG: int # Not FreeBSD either - MSG_ETAG: int # Not FreeBSD either - -SOL_IP: int -SOL_SOCKET: int -SOL_TCP: int -SOL_UDP: int + MSG_NOTIFICATION: Final[int] + MSG_BTAG: Final[int] # Not FreeBSD either + MSG_ETAG: Final[int] # Not FreeBSD either + +SOL_IP: Final[int] +SOL_SOCKET: Final[int] +SOL_TCP: Final[int] +SOL_UDP: Final[int] if sys.platform != "win32" and sys.platform != "darwin": # Defined in socket.h for Linux, but these aren't always present for # some reason. - SOL_ATALK: int - SOL_AX25: int - SOL_HCI: int - SOL_IPX: int - SOL_NETROM: int - SOL_ROSE: int + SOL_ATALK: Final[int] + SOL_AX25: Final[int] + SOL_HCI: Final[int] + SOL_IPX: Final[int] + SOL_NETROM: Final[int] + SOL_ROSE: Final[int] if sys.platform != "win32": - SCM_RIGHTS: int + SCM_RIGHTS: Final[int] if sys.platform != "win32" and sys.platform != "darwin": - SCM_CREDENTIALS: int + SCM_CREDENTIALS: Final[int] if sys.platform != "win32" and sys.platform != "linux": - SCM_CREDS: int - -IPPROTO_ICMP: int -IPPROTO_IP: int -IPPROTO_RAW: int -IPPROTO_TCP: int -IPPROTO_UDP: int -IPPROTO_AH: int -IPPROTO_DSTOPTS: int -IPPROTO_EGP: int -IPPROTO_ESP: int -IPPROTO_FRAGMENT: int -IPPROTO_HOPOPTS: int -IPPROTO_ICMPV6: int -IPPROTO_IDP: int -IPPROTO_IGMP: int -IPPROTO_IPV6: int -IPPROTO_NONE: int -IPPROTO_PIM: int -IPPROTO_PUP: int -IPPROTO_ROUTING: int -IPPROTO_SCTP: int + SCM_CREDS: Final[int] + +IPPROTO_ICMP: Final[int] +IPPROTO_IP: Final[int] +IPPROTO_RAW: Final[int] +IPPROTO_TCP: Final[int] +IPPROTO_UDP: Final[int] +IPPROTO_AH: Final[int] +IPPROTO_DSTOPTS: Final[int] +IPPROTO_EGP: Final[int] +IPPROTO_ESP: Final[int] +IPPROTO_FRAGMENT: Final[int] +IPPROTO_HOPOPTS: Final[int] +IPPROTO_ICMPV6: Final[int] +IPPROTO_IDP: Final[int] +IPPROTO_IGMP: Final[int] +IPPROTO_IPV6: Final[int] +IPPROTO_NONE: Final[int] +IPPROTO_PIM: Final[int] +IPPROTO_PUP: Final[int] +IPPROTO_ROUTING: Final[int] +IPPROTO_SCTP: Final[int] if sys.platform != "linux": - IPPROTO_GGP: int - IPPROTO_IPV4: int - IPPROTO_MAX: int - IPPROTO_ND: int + IPPROTO_GGP: Final[int] + IPPROTO_IPV4: Final[int] + IPPROTO_MAX: Final[int] + IPPROTO_ND: Final[int] if sys.platform == "win32": - IPPROTO_CBT: int - IPPROTO_ICLFXBM: int - IPPROTO_IGP: int - IPPROTO_L2TP: int - IPPROTO_PGM: int - IPPROTO_RDP: int - IPPROTO_ST: int + IPPROTO_CBT: Final[int] + IPPROTO_ICLFXBM: Final[int] + IPPROTO_IGP: Final[int] + IPPROTO_L2TP: Final[int] + IPPROTO_PGM: Final[int] + IPPROTO_RDP: Final[int] + IPPROTO_ST: Final[int] if sys.platform != "win32": - IPPROTO_GRE: int - IPPROTO_IPIP: int - IPPROTO_RSVP: int - IPPROTO_TP: int + IPPROTO_GRE: Final[int] + IPPROTO_IPIP: Final[int] + IPPROTO_RSVP: Final[int] + IPPROTO_TP: Final[int] if sys.platform != "win32" and sys.platform != "linux": - IPPROTO_EON: int - IPPROTO_HELLO: int - IPPROTO_IPCOMP: int - IPPROTO_XTP: int + IPPROTO_EON: Final[int] + IPPROTO_HELLO: Final[int] + IPPROTO_IPCOMP: Final[int] + IPPROTO_XTP: Final[int] if sys.platform != "win32" and sys.platform != "darwin" and sys.platform != "linux": - IPPROTO_BIP: int # Not FreeBSD either - IPPROTO_MOBILE: int # Not FreeBSD either - IPPROTO_VRRP: int # Not FreeBSD either + IPPROTO_BIP: Final[int] # Not FreeBSD either + IPPROTO_MOBILE: Final[int] # Not FreeBSD either + IPPROTO_VRRP: Final[int] # Not FreeBSD either if sys.platform == "linux": # Availability: Linux >= 2.6.20, FreeBSD >= 10.1 - IPPROTO_UDPLITE: int + IPPROTO_UDPLITE: Final[int] if sys.version_info >= (3, 10) and sys.platform == "linux": - IPPROTO_MPTCP: int - -IPPORT_RESERVED: int -IPPORT_USERRESERVED: int - -INADDR_ALLHOSTS_GROUP: int -INADDR_ANY: int -INADDR_BROADCAST: int -INADDR_LOOPBACK: int -INADDR_MAX_LOCAL_GROUP: int -INADDR_NONE: int -INADDR_UNSPEC_GROUP: int - -IP_ADD_MEMBERSHIP: int -IP_DROP_MEMBERSHIP: int -IP_HDRINCL: int -IP_MULTICAST_IF: int -IP_MULTICAST_LOOP: int -IP_MULTICAST_TTL: int -IP_OPTIONS: int + IPPROTO_MPTCP: Final[int] + +IPPORT_RESERVED: Final[int] +IPPORT_USERRESERVED: Final[int] + +INADDR_ALLHOSTS_GROUP: Final[int] +INADDR_ANY: Final[int] +INADDR_BROADCAST: Final[int] +INADDR_LOOPBACK: Final[int] +INADDR_MAX_LOCAL_GROUP: Final[int] +INADDR_NONE: Final[int] +INADDR_UNSPEC_GROUP: Final[int] + +IP_ADD_MEMBERSHIP: Final[int] +IP_DROP_MEMBERSHIP: Final[int] +IP_HDRINCL: Final[int] +IP_MULTICAST_IF: Final[int] +IP_MULTICAST_LOOP: Final[int] +IP_MULTICAST_TTL: Final[int] +IP_OPTIONS: Final[int] if sys.platform != "linux": - IP_RECVDSTADDR: int + IP_RECVDSTADDR: Final[int] if sys.version_info >= (3, 10): - IP_RECVTOS: int -IP_TOS: int -IP_TTL: int + IP_RECVTOS: Final[int] +IP_TOS: Final[int] +IP_TTL: Final[int] if sys.platform != "win32": - IP_DEFAULT_MULTICAST_LOOP: int - IP_DEFAULT_MULTICAST_TTL: int - IP_MAX_MEMBERSHIPS: int - IP_RECVOPTS: int - IP_RECVRETOPTS: int - IP_RETOPTS: int + IP_DEFAULT_MULTICAST_LOOP: Final[int] + IP_DEFAULT_MULTICAST_TTL: Final[int] + IP_MAX_MEMBERSHIPS: Final[int] + IP_RECVOPTS: Final[int] + IP_RECVRETOPTS: Final[int] + IP_RETOPTS: Final[int] if sys.version_info >= (3, 13) and sys.platform == "linux": - CAN_RAW_ERR_FILTER: int + CAN_RAW_ERR_FILTER: Final[int] if sys.version_info >= (3, 14): - IP_RECVTTL: int + IP_RECVTTL: Final[int] if sys.platform == "win32" or sys.platform == "linux": - IPV6_RECVERR: int - IP_RECVERR: int - SO_ORIGINAL_DST: int + IPV6_RECVERR: Final[int] + IP_RECVERR: Final[int] + SO_ORIGINAL_DST: Final[int] if sys.platform == "win32": - SOL_RFCOMM: int - SO_BTH_ENCRYPT: int - SO_BTH_MTU: int - SO_BTH_MTU_MAX: int - SO_BTH_MTU_MIN: int - TCP_QUICKACK: int + SOL_RFCOMM: Final[int] + SO_BTH_ENCRYPT: Final[int] + SO_BTH_MTU: Final[int] + SO_BTH_MTU_MAX: Final[int] + SO_BTH_MTU_MIN: Final[int] + TCP_QUICKACK: Final[int] if sys.platform == "linux": - IP_FREEBIND: int - IP_RECVORIGDSTADDR: int - VMADDR_CID_LOCAL: int + IP_FREEBIND: Final[int] + IP_RECVORIGDSTADDR: Final[int] + VMADDR_CID_LOCAL: Final[int] if sys.platform != "win32" and sys.platform != "darwin": - IP_TRANSPARENT: int + IP_TRANSPARENT: Final[int] if sys.platform != "win32" and sys.platform != "darwin" and sys.version_info >= (3, 11): - IP_BIND_ADDRESS_NO_PORT: int + IP_BIND_ADDRESS_NO_PORT: Final[int] if sys.version_info >= (3, 12): - IP_ADD_SOURCE_MEMBERSHIP: int - IP_BLOCK_SOURCE: int - IP_DROP_SOURCE_MEMBERSHIP: int - IP_PKTINFO: int - IP_UNBLOCK_SOURCE: int - -IPV6_CHECKSUM: int -IPV6_JOIN_GROUP: int -IPV6_LEAVE_GROUP: int -IPV6_MULTICAST_HOPS: int -IPV6_MULTICAST_IF: int -IPV6_MULTICAST_LOOP: int -IPV6_RECVTCLASS: int -IPV6_TCLASS: int -IPV6_UNICAST_HOPS: int -IPV6_V6ONLY: int -IPV6_DONTFRAG: int -IPV6_HOPLIMIT: int -IPV6_HOPOPTS: int -IPV6_PKTINFO: int -IPV6_RECVRTHDR: int -IPV6_RTHDR: int + IP_ADD_SOURCE_MEMBERSHIP: Final[int] + IP_BLOCK_SOURCE: Final[int] + IP_DROP_SOURCE_MEMBERSHIP: Final[int] + IP_PKTINFO: Final[int] + IP_UNBLOCK_SOURCE: Final[int] + +IPV6_CHECKSUM: Final[int] +IPV6_JOIN_GROUP: Final[int] +IPV6_LEAVE_GROUP: Final[int] +IPV6_MULTICAST_HOPS: Final[int] +IPV6_MULTICAST_IF: Final[int] +IPV6_MULTICAST_LOOP: Final[int] +IPV6_RECVTCLASS: Final[int] +IPV6_TCLASS: Final[int] +IPV6_UNICAST_HOPS: Final[int] +IPV6_V6ONLY: Final[int] +IPV6_DONTFRAG: Final[int] +IPV6_HOPLIMIT: Final[int] +IPV6_HOPOPTS: Final[int] +IPV6_PKTINFO: Final[int] +IPV6_RECVRTHDR: Final[int] +IPV6_RTHDR: Final[int] if sys.platform != "win32": - IPV6_RTHDR_TYPE_0: int - IPV6_DSTOPTS: int - IPV6_NEXTHOP: int - IPV6_PATHMTU: int - IPV6_RECVDSTOPTS: int - IPV6_RECVHOPLIMIT: int - IPV6_RECVHOPOPTS: int - IPV6_RECVPATHMTU: int - IPV6_RECVPKTINFO: int - IPV6_RTHDRDSTOPTS: int + IPV6_RTHDR_TYPE_0: Final[int] + IPV6_DSTOPTS: Final[int] + IPV6_NEXTHOP: Final[int] + IPV6_PATHMTU: Final[int] + IPV6_RECVDSTOPTS: Final[int] + IPV6_RECVHOPLIMIT: Final[int] + IPV6_RECVHOPOPTS: Final[int] + IPV6_RECVPATHMTU: Final[int] + IPV6_RECVPKTINFO: Final[int] + IPV6_RTHDRDSTOPTS: Final[int] if sys.platform != "win32" and sys.platform != "linux": - IPV6_USE_MIN_MTU: int - -EAI_AGAIN: int -EAI_BADFLAGS: int -EAI_FAIL: int -EAI_FAMILY: int -EAI_MEMORY: int -EAI_NODATA: int -EAI_NONAME: int -EAI_SERVICE: int -EAI_SOCKTYPE: int + IPV6_USE_MIN_MTU: Final[int] + +EAI_AGAIN: Final[int] +EAI_BADFLAGS: Final[int] +EAI_FAIL: Final[int] +EAI_FAMILY: Final[int] +EAI_MEMORY: Final[int] +EAI_NODATA: Final[int] +EAI_NONAME: Final[int] +EAI_SERVICE: Final[int] +EAI_SOCKTYPE: Final[int] if sys.platform != "win32": - EAI_ADDRFAMILY: int - EAI_OVERFLOW: int - EAI_SYSTEM: int + EAI_ADDRFAMILY: Final[int] + EAI_OVERFLOW: Final[int] + EAI_SYSTEM: Final[int] if sys.platform != "win32" and sys.platform != "linux": - EAI_BADHINTS: int - EAI_MAX: int - EAI_PROTOCOL: int - -AI_ADDRCONFIG: int -AI_ALL: int -AI_CANONNAME: int -AI_NUMERICHOST: int -AI_NUMERICSERV: int -AI_PASSIVE: int -AI_V4MAPPED: int + EAI_BADHINTS: Final[int] + EAI_MAX: Final[int] + EAI_PROTOCOL: Final[int] + +AI_ADDRCONFIG: Final[int] +AI_ALL: Final[int] +AI_CANONNAME: Final[int] +AI_NUMERICHOST: Final[int] +AI_NUMERICSERV: Final[int] +AI_PASSIVE: Final[int] +AI_V4MAPPED: Final[int] if sys.platform != "win32" and sys.platform != "linux": - AI_DEFAULT: int - AI_MASK: int - AI_V4MAPPED_CFG: int - -NI_DGRAM: int -NI_MAXHOST: int -NI_MAXSERV: int -NI_NAMEREQD: int -NI_NOFQDN: int -NI_NUMERICHOST: int -NI_NUMERICSERV: int + AI_DEFAULT: Final[int] + AI_MASK: Final[int] + AI_V4MAPPED_CFG: Final[int] + +NI_DGRAM: Final[int] +NI_MAXHOST: Final[int] +NI_MAXSERV: Final[int] +NI_NAMEREQD: Final[int] +NI_NOFQDN: Final[int] +NI_NUMERICHOST: Final[int] +NI_NUMERICSERV: Final[int] if sys.platform == "linux" and sys.version_info >= (3, 13): - NI_IDN: int + NI_IDN: Final[int] -TCP_FASTOPEN: int -TCP_KEEPCNT: int -TCP_KEEPINTVL: int -TCP_MAXSEG: int -TCP_NODELAY: int +TCP_FASTOPEN: Final[int] +TCP_KEEPCNT: Final[int] +TCP_KEEPINTVL: Final[int] +TCP_MAXSEG: Final[int] +TCP_NODELAY: Final[int] if sys.platform != "win32": - TCP_NOTSENT_LOWAT: int + TCP_NOTSENT_LOWAT: Final[int] if sys.platform != "darwin": - TCP_KEEPIDLE: int + TCP_KEEPIDLE: Final[int] if sys.version_info >= (3, 10) and sys.platform == "darwin": - TCP_KEEPALIVE: int + TCP_KEEPALIVE: Final[int] if sys.version_info >= (3, 11) and sys.platform == "darwin": - TCP_CONNECTION_INFO: int + TCP_CONNECTION_INFO: Final[int] if sys.platform != "win32" and sys.platform != "darwin": - TCP_CONGESTION: int - TCP_CORK: int - TCP_DEFER_ACCEPT: int - TCP_INFO: int - TCP_LINGER2: int - TCP_QUICKACK: int - TCP_SYNCNT: int - TCP_USER_TIMEOUT: int - TCP_WINDOW_CLAMP: int + TCP_CONGESTION: Final[int] + TCP_CORK: Final[int] + TCP_DEFER_ACCEPT: Final[int] + TCP_INFO: Final[int] + TCP_LINGER2: Final[int] + TCP_QUICKACK: Final[int] + TCP_SYNCNT: Final[int] + TCP_USER_TIMEOUT: Final[int] + TCP_WINDOW_CLAMP: Final[int] if sys.platform == "linux" and sys.version_info >= (3, 12): - TCP_CC_INFO: int - TCP_FASTOPEN_CONNECT: int - TCP_FASTOPEN_KEY: int - TCP_FASTOPEN_NO_COOKIE: int - TCP_INQ: int - TCP_MD5SIG: int - TCP_MD5SIG_EXT: int - TCP_QUEUE_SEQ: int - TCP_REPAIR: int - TCP_REPAIR_OPTIONS: int - TCP_REPAIR_QUEUE: int - TCP_REPAIR_WINDOW: int - TCP_SAVED_SYN: int - TCP_SAVE_SYN: int - TCP_THIN_DUPACK: int - TCP_THIN_LINEAR_TIMEOUTS: int - TCP_TIMESTAMP: int - TCP_TX_DELAY: int - TCP_ULP: int - TCP_ZEROCOPY_RECEIVE: int + TCP_CC_INFO: Final[int] + TCP_FASTOPEN_CONNECT: Final[int] + TCP_FASTOPEN_KEY: Final[int] + TCP_FASTOPEN_NO_COOKIE: Final[int] + TCP_INQ: Final[int] + TCP_MD5SIG: Final[int] + TCP_MD5SIG_EXT: Final[int] + TCP_QUEUE_SEQ: Final[int] + TCP_REPAIR: Final[int] + TCP_REPAIR_OPTIONS: Final[int] + TCP_REPAIR_QUEUE: Final[int] + TCP_REPAIR_WINDOW: Final[int] + TCP_SAVED_SYN: Final[int] + TCP_SAVE_SYN: Final[int] + TCP_THIN_DUPACK: Final[int] + TCP_THIN_LINEAR_TIMEOUTS: Final[int] + TCP_TIMESTAMP: Final[int] + TCP_TX_DELAY: Final[int] + TCP_ULP: Final[int] + TCP_ZEROCOPY_RECEIVE: Final[int] # -------------------- # Specifically documented constants @@ -386,250 +386,250 @@ if sys.platform == "linux" and sys.version_info >= (3, 12): if sys.platform == "linux": # Availability: Linux >= 2.6.25, NetBSD >= 8 - AF_CAN: int - PF_CAN: int - SOL_CAN_BASE: int - SOL_CAN_RAW: int - CAN_EFF_FLAG: int - CAN_EFF_MASK: int - CAN_ERR_FLAG: int - CAN_ERR_MASK: int - CAN_RAW: int - CAN_RAW_FILTER: int - CAN_RAW_LOOPBACK: int - CAN_RAW_RECV_OWN_MSGS: int - CAN_RTR_FLAG: int - CAN_SFF_MASK: int + AF_CAN: Final[int] + PF_CAN: Final[int] + SOL_CAN_BASE: Final[int] + SOL_CAN_RAW: Final[int] + CAN_EFF_FLAG: Final[int] + CAN_EFF_MASK: Final[int] + CAN_ERR_FLAG: Final[int] + CAN_ERR_MASK: Final[int] + CAN_RAW: Final[int] + CAN_RAW_FILTER: Final[int] + CAN_RAW_LOOPBACK: Final[int] + CAN_RAW_RECV_OWN_MSGS: Final[int] + CAN_RTR_FLAG: Final[int] + CAN_SFF_MASK: Final[int] if sys.version_info < (3, 11): - CAN_RAW_ERR_FILTER: int + CAN_RAW_ERR_FILTER: Final[int] if sys.platform == "linux": # Availability: Linux >= 2.6.25 - CAN_BCM: int - CAN_BCM_TX_SETUP: int - CAN_BCM_TX_DELETE: int - CAN_BCM_TX_READ: int - CAN_BCM_TX_SEND: int - CAN_BCM_RX_SETUP: int - CAN_BCM_RX_DELETE: int - CAN_BCM_RX_READ: int - CAN_BCM_TX_STATUS: int - CAN_BCM_TX_EXPIRED: int - CAN_BCM_RX_STATUS: int - CAN_BCM_RX_TIMEOUT: int - CAN_BCM_RX_CHANGED: int - CAN_BCM_SETTIMER: int - CAN_BCM_STARTTIMER: int - CAN_BCM_TX_COUNTEVT: int - CAN_BCM_TX_ANNOUNCE: int - CAN_BCM_TX_CP_CAN_ID: int - CAN_BCM_RX_FILTER_ID: int - CAN_BCM_RX_CHECK_DLC: int - CAN_BCM_RX_NO_AUTOTIMER: int - CAN_BCM_RX_ANNOUNCE_RESUME: int - CAN_BCM_TX_RESET_MULTI_IDX: int - CAN_BCM_RX_RTR_FRAME: int - CAN_BCM_CAN_FD_FRAME: int + CAN_BCM: Final[int] + CAN_BCM_TX_SETUP: Final[int] + CAN_BCM_TX_DELETE: Final[int] + CAN_BCM_TX_READ: Final[int] + CAN_BCM_TX_SEND: Final[int] + CAN_BCM_RX_SETUP: Final[int] + CAN_BCM_RX_DELETE: Final[int] + CAN_BCM_RX_READ: Final[int] + CAN_BCM_TX_STATUS: Final[int] + CAN_BCM_TX_EXPIRED: Final[int] + CAN_BCM_RX_STATUS: Final[int] + CAN_BCM_RX_TIMEOUT: Final[int] + CAN_BCM_RX_CHANGED: Final[int] + CAN_BCM_SETTIMER: Final[int] + CAN_BCM_STARTTIMER: Final[int] + CAN_BCM_TX_COUNTEVT: Final[int] + CAN_BCM_TX_ANNOUNCE: Final[int] + CAN_BCM_TX_CP_CAN_ID: Final[int] + CAN_BCM_RX_FILTER_ID: Final[int] + CAN_BCM_RX_CHECK_DLC: Final[int] + CAN_BCM_RX_NO_AUTOTIMER: Final[int] + CAN_BCM_RX_ANNOUNCE_RESUME: Final[int] + CAN_BCM_TX_RESET_MULTI_IDX: Final[int] + CAN_BCM_RX_RTR_FRAME: Final[int] + CAN_BCM_CAN_FD_FRAME: Final[int] if sys.platform == "linux": # Availability: Linux >= 3.6 - CAN_RAW_FD_FRAMES: int + CAN_RAW_FD_FRAMES: Final[int] # Availability: Linux >= 4.1 - CAN_RAW_JOIN_FILTERS: int + CAN_RAW_JOIN_FILTERS: Final[int] # Availability: Linux >= 2.6.25 - CAN_ISOTP: int + CAN_ISOTP: Final[int] # Availability: Linux >= 5.4 - CAN_J1939: int - - J1939_MAX_UNICAST_ADDR: int - J1939_IDLE_ADDR: int - J1939_NO_ADDR: int - J1939_NO_NAME: int - J1939_PGN_REQUEST: int - J1939_PGN_ADDRESS_CLAIMED: int - J1939_PGN_ADDRESS_COMMANDED: int - J1939_PGN_PDU1_MAX: int - J1939_PGN_MAX: int - J1939_NO_PGN: int - - SO_J1939_FILTER: int - SO_J1939_PROMISC: int - SO_J1939_SEND_PRIO: int - SO_J1939_ERRQUEUE: int - - SCM_J1939_DEST_ADDR: int - SCM_J1939_DEST_NAME: int - SCM_J1939_PRIO: int - SCM_J1939_ERRQUEUE: int - - J1939_NLA_PAD: int - J1939_NLA_BYTES_ACKED: int - J1939_EE_INFO_NONE: int - J1939_EE_INFO_TX_ABORT: int - J1939_FILTER_MAX: int + CAN_J1939: Final[int] + + J1939_MAX_UNICAST_ADDR: Final[int] + J1939_IDLE_ADDR: Final[int] + J1939_NO_ADDR: Final[int] + J1939_NO_NAME: Final[int] + J1939_PGN_REQUEST: Final[int] + J1939_PGN_ADDRESS_CLAIMED: Final[int] + J1939_PGN_ADDRESS_COMMANDED: Final[int] + J1939_PGN_PDU1_MAX: Final[int] + J1939_PGN_MAX: Final[int] + J1939_NO_PGN: Final[int] + + SO_J1939_FILTER: Final[int] + SO_J1939_PROMISC: Final[int] + SO_J1939_SEND_PRIO: Final[int] + SO_J1939_ERRQUEUE: Final[int] + + SCM_J1939_DEST_ADDR: Final[int] + SCM_J1939_DEST_NAME: Final[int] + SCM_J1939_PRIO: Final[int] + SCM_J1939_ERRQUEUE: Final[int] + + J1939_NLA_PAD: Final[int] + J1939_NLA_BYTES_ACKED: Final[int] + J1939_EE_INFO_NONE: Final[int] + J1939_EE_INFO_TX_ABORT: Final[int] + J1939_FILTER_MAX: Final[int] if sys.version_info >= (3, 12) and sys.platform != "linux" and sys.platform != "win32" and sys.platform != "darwin": # Availability: FreeBSD >= 14.0 - AF_DIVERT: int - PF_DIVERT: int + AF_DIVERT: Final[int] + PF_DIVERT: Final[int] if sys.platform == "linux": # Availability: Linux >= 2.2 - AF_PACKET: int - PF_PACKET: int - PACKET_BROADCAST: int - PACKET_FASTROUTE: int - PACKET_HOST: int - PACKET_LOOPBACK: int - PACKET_MULTICAST: int - PACKET_OTHERHOST: int - PACKET_OUTGOING: int + AF_PACKET: Final[int] + PF_PACKET: Final[int] + PACKET_BROADCAST: Final[int] + PACKET_FASTROUTE: Final[int] + PACKET_HOST: Final[int] + PACKET_LOOPBACK: Final[int] + PACKET_MULTICAST: Final[int] + PACKET_OTHERHOST: Final[int] + PACKET_OUTGOING: Final[int] if sys.version_info >= (3, 12) and sys.platform == "linux": - ETH_P_ALL: int + ETH_P_ALL: Final[int] if sys.platform == "linux": # Availability: Linux >= 2.6.30 - AF_RDS: int - PF_RDS: int - SOL_RDS: int + AF_RDS: Final[int] + PF_RDS: Final[int] + SOL_RDS: Final[int] # These are present in include/linux/rds.h but don't always show up # here. - RDS_CANCEL_SENT_TO: int - RDS_CMSG_RDMA_ARGS: int - RDS_CMSG_RDMA_DEST: int - RDS_CMSG_RDMA_MAP: int - RDS_CMSG_RDMA_STATUS: int - RDS_CONG_MONITOR: int - RDS_FREE_MR: int - RDS_GET_MR: int - RDS_GET_MR_FOR_DEST: int - RDS_RDMA_DONTWAIT: int - RDS_RDMA_FENCE: int - RDS_RDMA_INVALIDATE: int - RDS_RDMA_NOTIFY_ME: int - RDS_RDMA_READWRITE: int - RDS_RDMA_SILENT: int - RDS_RDMA_USE_ONCE: int - RDS_RECVERR: int + RDS_CANCEL_SENT_TO: Final[int] + RDS_CMSG_RDMA_ARGS: Final[int] + RDS_CMSG_RDMA_DEST: Final[int] + RDS_CMSG_RDMA_MAP: Final[int] + RDS_CMSG_RDMA_STATUS: Final[int] + RDS_CONG_MONITOR: Final[int] + RDS_FREE_MR: Final[int] + RDS_GET_MR: Final[int] + RDS_GET_MR_FOR_DEST: Final[int] + RDS_RDMA_DONTWAIT: Final[int] + RDS_RDMA_FENCE: Final[int] + RDS_RDMA_INVALIDATE: Final[int] + RDS_RDMA_NOTIFY_ME: Final[int] + RDS_RDMA_READWRITE: Final[int] + RDS_RDMA_SILENT: Final[int] + RDS_RDMA_USE_ONCE: Final[int] + RDS_RECVERR: Final[int] # This is supported by CPython but doesn't seem to be a real thing. # The closest existing constant in rds.h is RDS_CMSG_CONG_UPDATE - # RDS_CMSG_RDMA_UPDATE: int + # RDS_CMSG_RDMA_UPDATE: Final[int] if sys.platform == "win32": - SIO_RCVALL: int - SIO_KEEPALIVE_VALS: int - SIO_LOOPBACK_FAST_PATH: int - RCVALL_MAX: int - RCVALL_OFF: int - RCVALL_ON: int - RCVALL_SOCKETLEVELONLY: int + SIO_RCVALL: Final[int] + SIO_KEEPALIVE_VALS: Final[int] + SIO_LOOPBACK_FAST_PATH: Final[int] + RCVALL_MAX: Final[int] + RCVALL_OFF: Final[int] + RCVALL_ON: Final[int] + RCVALL_SOCKETLEVELONLY: Final[int] if sys.platform == "linux": - AF_TIPC: int - SOL_TIPC: int - TIPC_ADDR_ID: int - TIPC_ADDR_NAME: int - TIPC_ADDR_NAMESEQ: int - TIPC_CFG_SRV: int - TIPC_CLUSTER_SCOPE: int - TIPC_CONN_TIMEOUT: int - TIPC_CRITICAL_IMPORTANCE: int - TIPC_DEST_DROPPABLE: int - TIPC_HIGH_IMPORTANCE: int - TIPC_IMPORTANCE: int - TIPC_LOW_IMPORTANCE: int - TIPC_MEDIUM_IMPORTANCE: int - TIPC_NODE_SCOPE: int - TIPC_PUBLISHED: int - TIPC_SRC_DROPPABLE: int - TIPC_SUBSCR_TIMEOUT: int - TIPC_SUB_CANCEL: int - TIPC_SUB_PORTS: int - TIPC_SUB_SERVICE: int - TIPC_TOP_SRV: int - TIPC_WAIT_FOREVER: int - TIPC_WITHDRAWN: int - TIPC_ZONE_SCOPE: int + AF_TIPC: Final[int] + SOL_TIPC: Final[int] + TIPC_ADDR_ID: Final[int] + TIPC_ADDR_NAME: Final[int] + TIPC_ADDR_NAMESEQ: Final[int] + TIPC_CFG_SRV: Final[int] + TIPC_CLUSTER_SCOPE: Final[int] + TIPC_CONN_TIMEOUT: Final[int] + TIPC_CRITICAL_IMPORTANCE: Final[int] + TIPC_DEST_DROPPABLE: Final[int] + TIPC_HIGH_IMPORTANCE: Final[int] + TIPC_IMPORTANCE: Final[int] + TIPC_LOW_IMPORTANCE: Final[int] + TIPC_MEDIUM_IMPORTANCE: Final[int] + TIPC_NODE_SCOPE: Final[int] + TIPC_PUBLISHED: Final[int] + TIPC_SRC_DROPPABLE: Final[int] + TIPC_SUBSCR_TIMEOUT: Final[int] + TIPC_SUB_CANCEL: Final[int] + TIPC_SUB_PORTS: Final[int] + TIPC_SUB_SERVICE: Final[int] + TIPC_TOP_SRV: Final[int] + TIPC_WAIT_FOREVER: Final[int] + TIPC_WITHDRAWN: Final[int] + TIPC_ZONE_SCOPE: Final[int] if sys.platform == "linux": # Availability: Linux >= 2.6.38 - AF_ALG: int - SOL_ALG: int - ALG_OP_DECRYPT: int - ALG_OP_ENCRYPT: int - ALG_OP_SIGN: int - ALG_OP_VERIFY: int - ALG_SET_AEAD_ASSOCLEN: int - ALG_SET_AEAD_AUTHSIZE: int - ALG_SET_IV: int - ALG_SET_KEY: int - ALG_SET_OP: int - ALG_SET_PUBKEY: int + AF_ALG: Final[int] + SOL_ALG: Final[int] + ALG_OP_DECRYPT: Final[int] + ALG_OP_ENCRYPT: Final[int] + ALG_OP_SIGN: Final[int] + ALG_OP_VERIFY: Final[int] + ALG_SET_AEAD_ASSOCLEN: Final[int] + ALG_SET_AEAD_AUTHSIZE: Final[int] + ALG_SET_IV: Final[int] + ALG_SET_KEY: Final[int] + ALG_SET_OP: Final[int] + ALG_SET_PUBKEY: Final[int] if sys.platform == "linux": # Availability: Linux >= 4.8 (or maybe 3.9, CPython docs are confusing) - AF_VSOCK: int - IOCTL_VM_SOCKETS_GET_LOCAL_CID: int - VMADDR_CID_ANY: int - VMADDR_CID_HOST: int - VMADDR_PORT_ANY: int - SO_VM_SOCKETS_BUFFER_MAX_SIZE: int - SO_VM_SOCKETS_BUFFER_SIZE: int - SO_VM_SOCKETS_BUFFER_MIN_SIZE: int - VM_SOCKETS_INVALID_VERSION: int # undocumented + AF_VSOCK: Final[int] + IOCTL_VM_SOCKETS_GET_LOCAL_CID: Final = 0x7B9 + VMADDR_CID_ANY: Final = 0xFFFFFFFF + VMADDR_CID_HOST: Final = 2 + VMADDR_PORT_ANY: Final = 0xFFFFFFFF + SO_VM_SOCKETS_BUFFER_MAX_SIZE: Final = 2 + SO_VM_SOCKETS_BUFFER_SIZE: Final = 0 + SO_VM_SOCKETS_BUFFER_MIN_SIZE: Final = 1 + VM_SOCKETS_INVALID_VERSION: Final = 0xFFFFFFFF # undocumented # Documented as only available on BSD, macOS, but empirically sometimes # available on Windows if sys.platform != "linux": - AF_LINK: int + AF_LINK: Final[int] has_ipv6: bool if sys.platform != "darwin" and sys.platform != "linux": - BDADDR_ANY: str - BDADDR_LOCAL: str + BDADDR_ANY: Final = "00:00:00:00:00:00" + BDADDR_LOCAL: Final = "00:00:00:FF:FF:FF" if sys.platform != "win32" and sys.platform != "darwin" and sys.platform != "linux": - HCI_FILTER: int # not in NetBSD or DragonFlyBSD - HCI_TIME_STAMP: int # not in FreeBSD, NetBSD, or DragonFlyBSD - HCI_DATA_DIR: int # not in FreeBSD, NetBSD, or DragonFlyBSD + HCI_FILTER: Final[int] # not in NetBSD or DragonFlyBSD + HCI_TIME_STAMP: Final[int] # not in FreeBSD, NetBSD, or DragonFlyBSD + HCI_DATA_DIR: Final[int] # not in FreeBSD, NetBSD, or DragonFlyBSD if sys.platform == "linux": - AF_QIPCRTR: int # Availability: Linux >= 4.7 + AF_QIPCRTR: Final[int] # Availability: Linux >= 4.7 if sys.version_info >= (3, 11) and sys.platform != "linux" and sys.platform != "win32" and sys.platform != "darwin": # FreeBSD - SCM_CREDS2: int - LOCAL_CREDS: int - LOCAL_CREDS_PERSISTENT: int + SCM_CREDS2: Final[int] + LOCAL_CREDS: Final[int] + LOCAL_CREDS_PERSISTENT: Final[int] if sys.version_info >= (3, 11) and sys.platform == "linux": - SO_INCOMING_CPU: int # Availability: Linux >= 3.9 + SO_INCOMING_CPU: Final[int] # Availability: Linux >= 3.9 if sys.version_info >= (3, 12) and sys.platform == "win32": # Availability: Windows - AF_HYPERV: int - HV_PROTOCOL_RAW: int - HVSOCKET_CONNECT_TIMEOUT: int - HVSOCKET_CONNECT_TIMEOUT_MAX: int - HVSOCKET_CONNECTED_SUSPEND: int - HVSOCKET_ADDRESS_FLAG_PASSTHRU: int - HV_GUID_ZERO: str - HV_GUID_WILDCARD: str - HV_GUID_BROADCAST: str - HV_GUID_CHILDREN: str - HV_GUID_LOOPBACK: str - HV_GUID_PARENT: str + AF_HYPERV: Final[int] + HV_PROTOCOL_RAW: Final[int] + HVSOCKET_CONNECT_TIMEOUT: Final[int] + HVSOCKET_CONNECT_TIMEOUT_MAX: Final[int] + HVSOCKET_CONNECTED_SUSPEND: Final[int] + HVSOCKET_ADDRESS_FLAG_PASSTHRU: Final[int] + HV_GUID_ZERO: Final = "00000000-0000-0000-0000-000000000000" + HV_GUID_WILDCARD: Final = "00000000-0000-0000-0000-000000000000" + HV_GUID_BROADCAST: Final = "FFFFFFFF-FFFF-FFFF-FFFF-FFFFFFFFFFFF" + HV_GUID_CHILDREN: Final = "90DB8B89-0D35-4F79-8CE9-49EA0AC8B7CD" + HV_GUID_LOOPBACK: Final = "E0E16197-DD56-4A10-9195-5EE7A155A838" + HV_GUID_PARENT: Final = "A42E7CDA-D03F-480C-9CC2-A4DE20ABB878" if sys.version_info >= (3, 12): if sys.platform != "win32": # Availability: Linux, FreeBSD, macOS - ETHERTYPE_ARP: int - ETHERTYPE_IP: int - ETHERTYPE_IPV6: int - ETHERTYPE_VLAN: int + ETHERTYPE_ARP: Final[int] + ETHERTYPE_IP: Final[int] + ETHERTYPE_IPV6: Final[int] + ETHERTYPE_VLAN: Final[int] # -------------------- # Semi-documented constants @@ -639,95 +639,95 @@ if sys.version_info >= (3, 12): if sys.platform == "linux": # Netlink is defined by Linux - AF_NETLINK: int - NETLINK_CRYPTO: int - NETLINK_DNRTMSG: int - NETLINK_FIREWALL: int - NETLINK_IP6_FW: int - NETLINK_NFLOG: int - NETLINK_ROUTE: int - NETLINK_USERSOCK: int - NETLINK_XFRM: int + AF_NETLINK: Final[int] + NETLINK_CRYPTO: Final[int] + NETLINK_DNRTMSG: Final[int] + NETLINK_FIREWALL: Final[int] + NETLINK_IP6_FW: Final[int] + NETLINK_NFLOG: Final[int] + NETLINK_ROUTE: Final[int] + NETLINK_USERSOCK: Final[int] + NETLINK_XFRM: Final[int] # Technically still supported by CPython - # NETLINK_ARPD: int # linux 2.0 to 2.6.12 (EOL August 2005) - # NETLINK_ROUTE6: int # linux 2.2 to 2.6.12 (EOL August 2005) - # NETLINK_SKIP: int # linux 2.0 to 2.6.12 (EOL August 2005) - # NETLINK_TAPBASE: int # linux 2.2 to 2.6.12 (EOL August 2005) - # NETLINK_TCPDIAG: int # linux 2.6.0 to 2.6.13 (EOL December 2005) - # NETLINK_W1: int # linux 2.6.13 to 2.6.17 (EOL October 2006) + # NETLINK_ARPD: Final[int] # linux 2.0 to 2.6.12 (EOL August 2005) + # NETLINK_ROUTE6: Final[int] # linux 2.2 to 2.6.12 (EOL August 2005) + # NETLINK_SKIP: Final[int] # linux 2.0 to 2.6.12 (EOL August 2005) + # NETLINK_TAPBASE: Final[int] # linux 2.2 to 2.6.12 (EOL August 2005) + # NETLINK_TCPDIAG: Final[int] # linux 2.6.0 to 2.6.13 (EOL December 2005) + # NETLINK_W1: Final[int] # linux 2.6.13 to 2.6.17 (EOL October 2006) if sys.platform == "darwin": - PF_SYSTEM: int - SYSPROTO_CONTROL: int + PF_SYSTEM: Final[int] + SYSPROTO_CONTROL: Final[int] if sys.platform != "darwin" and sys.platform != "linux": - AF_BLUETOOTH: int + AF_BLUETOOTH: Final[int] if sys.platform != "win32" and sys.platform != "darwin" and sys.platform != "linux": # Linux and some BSD support is explicit in the docs # Windows and macOS do not support in practice - BTPROTO_HCI: int - BTPROTO_L2CAP: int - BTPROTO_SCO: int # not in FreeBSD + BTPROTO_HCI: Final[int] + BTPROTO_L2CAP: Final[int] + BTPROTO_SCO: Final[int] # not in FreeBSD if sys.platform != "darwin" and sys.platform != "linux": - BTPROTO_RFCOMM: int + BTPROTO_RFCOMM: Final[int] if sys.platform == "linux": - UDPLITE_RECV_CSCOV: int - UDPLITE_SEND_CSCOV: int + UDPLITE_RECV_CSCOV: Final[int] + UDPLITE_SEND_CSCOV: Final[int] # -------------------- # Documented under socket.shutdown # -------------------- -SHUT_RD: int -SHUT_RDWR: int -SHUT_WR: int +SHUT_RD: Final[int] +SHUT_RDWR: Final[int] +SHUT_WR: Final[int] # -------------------- # Undocumented constants # -------------------- # Undocumented address families -AF_APPLETALK: int -AF_DECnet: int -AF_IPX: int -AF_SNA: int +AF_APPLETALK: Final[int] +AF_DECnet: Final[int] +AF_IPX: Final[int] +AF_SNA: Final[int] if sys.platform != "win32": - AF_ROUTE: int + AF_ROUTE: Final[int] if sys.platform == "darwin": - AF_SYSTEM: int + AF_SYSTEM: Final[int] if sys.platform != "darwin": - AF_IRDA: int + AF_IRDA: Final[int] if sys.platform != "win32" and sys.platform != "darwin": - AF_ASH: int - AF_ATMPVC: int - AF_ATMSVC: int - AF_AX25: int - AF_BRIDGE: int - AF_ECONET: int - AF_KEY: int - AF_LLC: int - AF_NETBEUI: int - AF_NETROM: int - AF_PPPOX: int - AF_ROSE: int - AF_SECURITY: int - AF_WANPIPE: int - AF_X25: int + AF_ASH: Final[int] + AF_ATMPVC: Final[int] + AF_ATMSVC: Final[int] + AF_AX25: Final[int] + AF_BRIDGE: Final[int] + AF_ECONET: Final[int] + AF_KEY: Final[int] + AF_LLC: Final[int] + AF_NETBEUI: Final[int] + AF_NETROM: Final[int] + AF_PPPOX: Final[int] + AF_ROSE: Final[int] + AF_SECURITY: Final[int] + AF_WANPIPE: Final[int] + AF_X25: Final[int] # Miscellaneous undocumented if sys.platform != "win32" and sys.platform != "linux": - LOCAL_PEERCRED: int + LOCAL_PEERCRED: Final[int] if sys.platform != "win32" and sys.platform != "darwin": # Defined in linux socket.h, but this isn't always present for # some reason. - IPX_TYPE: int + IPX_TYPE: Final[int] # ===== Classes ===== @@ -743,10 +743,10 @@ class socket: def timeout(self) -> float | None: ... # noqa: F811 if sys.platform == "win32": def __init__( - self, family: int = ..., type: int = ..., proto: int = ..., fileno: SupportsIndex | bytes | None = ... + self, family: int = ..., type: int = ..., proto: int = ..., fileno: SupportsIndex | bytes | None = None ) -> None: ... else: - def __init__(self, family: int = ..., type: int = ..., proto: int = ..., fileno: SupportsIndex | None = ...) -> None: ... + def __init__(self, family: int = ..., type: int = ..., proto: int = ..., fileno: SupportsIndex | None = None) -> None: ... def bind(self, address: _Address, /) -> None: ... def close(self) -> None: ... @@ -766,18 +766,18 @@ class socket: def ioctl(self, control: int, option: int | tuple[int, int, int] | bool, /) -> None: ... def listen(self, backlog: int = ..., /) -> None: ... - def recv(self, bufsize: int, flags: int = ..., /) -> bytes: ... - def recvfrom(self, bufsize: int, flags: int = ..., /) -> tuple[bytes, _RetAddress]: ... + def recv(self, bufsize: int, flags: int = 0, /) -> bytes: ... + def recvfrom(self, bufsize: int, flags: int = 0, /) -> tuple[bytes, _RetAddress]: ... if sys.platform != "win32": - def recvmsg(self, bufsize: int, ancbufsize: int = ..., flags: int = ..., /) -> tuple[bytes, list[_CMSG], int, Any]: ... + def recvmsg(self, bufsize: int, ancbufsize: int = 0, flags: int = 0, /) -> tuple[bytes, list[_CMSG], int, Any]: ... def recvmsg_into( - self, buffers: Iterable[WriteableBuffer], ancbufsize: int = ..., flags: int = ..., / + self, buffers: Iterable[WriteableBuffer], ancbufsize: int = 0, flags: int = 0, / ) -> tuple[int, list[_CMSG], int, Any]: ... - def recvfrom_into(self, buffer: WriteableBuffer, nbytes: int = ..., flags: int = ...) -> tuple[int, _RetAddress]: ... - def recv_into(self, buffer: WriteableBuffer, nbytes: int = ..., flags: int = ...) -> int: ... - def send(self, data: ReadableBuffer, flags: int = ..., /) -> int: ... - def sendall(self, data: ReadableBuffer, flags: int = ..., /) -> None: ... + def recvfrom_into(self, buffer: WriteableBuffer, nbytes: int = 0, flags: int = 0) -> tuple[int, _RetAddress]: ... + def recv_into(self, buffer: WriteableBuffer, nbytes: int = 0, flags: int = 0) -> int: ... + def send(self, data: ReadableBuffer, flags: int = 0, /) -> int: ... + def sendall(self, data: ReadableBuffer, flags: int = 0, /) -> None: ... @overload def sendto(self, data: ReadableBuffer, address: _Address, /) -> int: ... @overload @@ -787,13 +787,13 @@ class socket: self, buffers: Iterable[ReadableBuffer], ancdata: Iterable[_CMSGArg] = ..., - flags: int = ..., - address: _Address | None = ..., + flags: int = 0, + address: _Address | None = None, /, ) -> int: ... if sys.platform == "linux": def sendmsg_afalg( - self, msg: Iterable[ReadableBuffer] = ..., *, op: int, iv: Any = ..., assoclen: int = ..., flags: int = ... + self, msg: Iterable[ReadableBuffer] = ..., *, op: int, iv: Any = ..., assoclen: int = ..., flags: int = 0 ) -> int: ... def setblocking(self, flag: bool, /) -> None: ... @@ -816,12 +816,7 @@ def dup(fd: SupportsIndex, /) -> int: ... # the 5th tuple item is an address def getaddrinfo( - host: bytes | str | None, - port: bytes | str | int | None, - family: int = ..., - type: int = ..., - proto: int = ..., - flags: int = ..., + host: bytes | str | None, port: bytes | str | int | None, family: int = ..., type: int = 0, proto: int = 0, flags: int = 0 ) -> list[tuple[int, int, int, str, tuple[str, int] | tuple[str, int, int, int] | tuple[int, bytes]]]: ... def gethostbyname(hostname: str, /) -> str: ... def gethostbyname_ex(hostname: str, /) -> tuple[str, list[str], list[str]]: ... @@ -848,7 +843,7 @@ if sys.platform != "win32": def sethostname(name: str, /) -> None: ... def CMSG_LEN(length: int, /) -> int: ... def CMSG_SPACE(length: int, /) -> int: ... - def socketpair(family: int = ..., type: int = ..., proto: int = ..., /) -> tuple[socket, socket]: ... + def socketpair(family: int = ..., type: int = ..., proto: int = 0, /) -> tuple[socket, socket]: ... def if_nameindex() -> list[tuple[int, str]]: ... def if_nametoindex(oname: str, /) -> int: ... diff --git a/mypy/typeshed/stdlib/_ssl.pyi b/mypy/typeshed/stdlib/_ssl.pyi index 88dd067809046..8afa3e5297bda 100644 --- a/mypy/typeshed/stdlib/_ssl.pyi +++ b/mypy/typeshed/stdlib/_ssl.pyi @@ -12,8 +12,8 @@ from ssl import ( SSLWantWriteError as SSLWantWriteError, SSLZeroReturnError as SSLZeroReturnError, ) -from typing import Any, ClassVar, Literal, TypedDict, final, overload, type_check_only -from typing_extensions import NotRequired, Self, TypeAlias +from typing import Any, ClassVar, Final, Literal, TypedDict, final, overload, type_check_only +from typing_extensions import NotRequired, Self, TypeAlias, deprecated _PasswordType: TypeAlias = Callable[[], str | bytes | bytearray] | str | bytes | bytearray _PCTRTT: TypeAlias = tuple[tuple[str, str], ...] @@ -51,6 +51,7 @@ def RAND_add(string: str | ReadableBuffer, entropy: float, /) -> None: ... def RAND_bytes(n: int, /) -> bytes: ... if sys.version_info < (3, 12): + @deprecated("Deprecated since Python 3.6; removed in Python 3.12. Use `ssl.RAND_bytes()` instead.") def RAND_pseudo_bytes(n: int, /) -> tuple[bytes, bool]: ... if sys.version_info < (3, 10): @@ -161,135 +162,134 @@ if sys.version_info < (3, 12): err_names_to_codes: dict[str, tuple[int, int]] lib_codes_to_names: dict[int, str] -_DEFAULT_CIPHERS: str +_DEFAULT_CIPHERS: Final[str] # SSL error numbers -SSL_ERROR_ZERO_RETURN: int -SSL_ERROR_WANT_READ: int -SSL_ERROR_WANT_WRITE: int -SSL_ERROR_WANT_X509_LOOKUP: int -SSL_ERROR_SYSCALL: int -SSL_ERROR_SSL: int -SSL_ERROR_WANT_CONNECT: int -SSL_ERROR_EOF: int -SSL_ERROR_INVALID_ERROR_CODE: int +SSL_ERROR_ZERO_RETURN: Final = 6 +SSL_ERROR_WANT_READ: Final = 2 +SSL_ERROR_WANT_WRITE: Final = 3 +SSL_ERROR_WANT_X509_LOOKUP: Final = 4 +SSL_ERROR_SYSCALL: Final = 5 +SSL_ERROR_SSL: Final = 1 +SSL_ERROR_WANT_CONNECT: Final = 7 +SSL_ERROR_EOF: Final = 8 +SSL_ERROR_INVALID_ERROR_CODE: Final = 10 # verify modes -CERT_NONE: int -CERT_OPTIONAL: int -CERT_REQUIRED: int +CERT_NONE: Final = 0 +CERT_OPTIONAL: Final = 1 +CERT_REQUIRED: Final = 2 # verify flags -VERIFY_DEFAULT: int -VERIFY_CRL_CHECK_LEAF: int -VERIFY_CRL_CHECK_CHAIN: int -VERIFY_X509_STRICT: int -VERIFY_X509_TRUSTED_FIRST: int +VERIFY_DEFAULT: Final = 0 +VERIFY_CRL_CHECK_LEAF: Final = 0x4 +VERIFY_CRL_CHECK_CHAIN: Final = 0x8 +VERIFY_X509_STRICT: Final = 0x20 +VERIFY_X509_TRUSTED_FIRST: Final = 0x8000 if sys.version_info >= (3, 10): - VERIFY_ALLOW_PROXY_CERTS: int - VERIFY_X509_PARTIAL_CHAIN: int + VERIFY_ALLOW_PROXY_CERTS: Final = 0x40 + VERIFY_X509_PARTIAL_CHAIN: Final = 0x80000 # alert descriptions -ALERT_DESCRIPTION_CLOSE_NOTIFY: int -ALERT_DESCRIPTION_UNEXPECTED_MESSAGE: int -ALERT_DESCRIPTION_BAD_RECORD_MAC: int -ALERT_DESCRIPTION_RECORD_OVERFLOW: int -ALERT_DESCRIPTION_DECOMPRESSION_FAILURE: int -ALERT_DESCRIPTION_HANDSHAKE_FAILURE: int -ALERT_DESCRIPTION_BAD_CERTIFICATE: int -ALERT_DESCRIPTION_UNSUPPORTED_CERTIFICATE: int -ALERT_DESCRIPTION_CERTIFICATE_REVOKED: int -ALERT_DESCRIPTION_CERTIFICATE_EXPIRED: int -ALERT_DESCRIPTION_CERTIFICATE_UNKNOWN: int -ALERT_DESCRIPTION_ILLEGAL_PARAMETER: int -ALERT_DESCRIPTION_UNKNOWN_CA: int -ALERT_DESCRIPTION_ACCESS_DENIED: int -ALERT_DESCRIPTION_DECODE_ERROR: int -ALERT_DESCRIPTION_DECRYPT_ERROR: int -ALERT_DESCRIPTION_PROTOCOL_VERSION: int -ALERT_DESCRIPTION_INSUFFICIENT_SECURITY: int -ALERT_DESCRIPTION_INTERNAL_ERROR: int -ALERT_DESCRIPTION_USER_CANCELLED: int -ALERT_DESCRIPTION_NO_RENEGOTIATION: int -ALERT_DESCRIPTION_UNSUPPORTED_EXTENSION: int -ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE: int -ALERT_DESCRIPTION_UNRECOGNIZED_NAME: int -ALERT_DESCRIPTION_BAD_CERTIFICATE_STATUS_RESPONSE: int -ALERT_DESCRIPTION_BAD_CERTIFICATE_HASH_VALUE: int -ALERT_DESCRIPTION_UNKNOWN_PSK_IDENTITY: int +ALERT_DESCRIPTION_CLOSE_NOTIFY: Final = 0 +ALERT_DESCRIPTION_UNEXPECTED_MESSAGE: Final = 10 +ALERT_DESCRIPTION_BAD_RECORD_MAC: Final = 20 +ALERT_DESCRIPTION_RECORD_OVERFLOW: Final = 22 +ALERT_DESCRIPTION_DECOMPRESSION_FAILURE: Final = 30 +ALERT_DESCRIPTION_HANDSHAKE_FAILURE: Final = 40 +ALERT_DESCRIPTION_BAD_CERTIFICATE: Final = 42 +ALERT_DESCRIPTION_UNSUPPORTED_CERTIFICATE: Final = 43 +ALERT_DESCRIPTION_CERTIFICATE_REVOKED: Final = 44 +ALERT_DESCRIPTION_CERTIFICATE_EXPIRED: Final = 45 +ALERT_DESCRIPTION_CERTIFICATE_UNKNOWN: Final = 46 +ALERT_DESCRIPTION_ILLEGAL_PARAMETER: Final = 47 +ALERT_DESCRIPTION_UNKNOWN_CA: Final = 48 +ALERT_DESCRIPTION_ACCESS_DENIED: Final = 49 +ALERT_DESCRIPTION_DECODE_ERROR: Final = 50 +ALERT_DESCRIPTION_DECRYPT_ERROR: Final = 51 +ALERT_DESCRIPTION_PROTOCOL_VERSION: Final = 70 +ALERT_DESCRIPTION_INSUFFICIENT_SECURITY: Final = 71 +ALERT_DESCRIPTION_INTERNAL_ERROR: Final = 80 +ALERT_DESCRIPTION_USER_CANCELLED: Final = 90 +ALERT_DESCRIPTION_NO_RENEGOTIATION: Final = 100 +ALERT_DESCRIPTION_UNSUPPORTED_EXTENSION: Final = 110 +ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE: Final = 111 +ALERT_DESCRIPTION_UNRECOGNIZED_NAME: Final = 112 +ALERT_DESCRIPTION_BAD_CERTIFICATE_STATUS_RESPONSE: Final = 113 +ALERT_DESCRIPTION_BAD_CERTIFICATE_HASH_VALUE: Final = 114 +ALERT_DESCRIPTION_UNKNOWN_PSK_IDENTITY: Final = 115 # protocol versions -PROTOCOL_SSLv23: int -PROTOCOL_TLS: int -PROTOCOL_TLS_CLIENT: int -PROTOCOL_TLS_SERVER: int -PROTOCOL_TLSv1: int -PROTOCOL_TLSv1_1: int -PROTOCOL_TLSv1_2: int +PROTOCOL_SSLv23: Final = 2 +PROTOCOL_TLS: Final = 2 +PROTOCOL_TLS_CLIENT: Final = 16 +PROTOCOL_TLS_SERVER: Final = 17 +PROTOCOL_TLSv1: Final = 3 +PROTOCOL_TLSv1_1: Final = 4 +PROTOCOL_TLSv1_2: Final = 5 # protocol options -OP_ALL: int -OP_NO_SSLv2: int -OP_NO_SSLv3: int -OP_NO_TLSv1: int -OP_NO_TLSv1_1: int -OP_NO_TLSv1_2: int -OP_NO_TLSv1_3: int -OP_CIPHER_SERVER_PREFERENCE: int -OP_SINGLE_DH_USE: int -OP_NO_TICKET: int -OP_SINGLE_ECDH_USE: int -OP_NO_COMPRESSION: int -OP_ENABLE_MIDDLEBOX_COMPAT: int -OP_NO_RENEGOTIATION: int +OP_ALL: Final = 0x80000050 +OP_NO_SSLv2: Final = 0x0 +OP_NO_SSLv3: Final = 0x2000000 +OP_NO_TLSv1: Final = 0x4000000 +OP_NO_TLSv1_1: Final = 0x10000000 +OP_NO_TLSv1_2: Final = 0x8000000 +OP_NO_TLSv1_3: Final = 0x20000000 +OP_CIPHER_SERVER_PREFERENCE: Final = 0x400000 +OP_SINGLE_DH_USE: Final = 0x0 +OP_NO_TICKET: Final = 0x4000 +OP_SINGLE_ECDH_USE: Final = 0x0 +OP_NO_COMPRESSION: Final = 0x20000 +OP_ENABLE_MIDDLEBOX_COMPAT: Final = 0x100000 +OP_NO_RENEGOTIATION: Final = 0x40000000 if sys.version_info >= (3, 11) or sys.platform == "linux": - OP_IGNORE_UNEXPECTED_EOF: int + OP_IGNORE_UNEXPECTED_EOF: Final = 0x80 if sys.version_info >= (3, 12): - OP_LEGACY_SERVER_CONNECT: int - OP_ENABLE_KTLS: int + OP_LEGACY_SERVER_CONNECT: Final = 0x4 + OP_ENABLE_KTLS: Final = 0x8 # host flags -HOSTFLAG_ALWAYS_CHECK_SUBJECT: int -HOSTFLAG_NEVER_CHECK_SUBJECT: int -HOSTFLAG_NO_WILDCARDS: int -HOSTFLAG_NO_PARTIAL_WILDCARDS: int -HOSTFLAG_MULTI_LABEL_WILDCARDS: int -HOSTFLAG_SINGLE_LABEL_SUBDOMAINS: int +HOSTFLAG_ALWAYS_CHECK_SUBJECT: Final = 0x1 +HOSTFLAG_NEVER_CHECK_SUBJECT: Final = 0x20 +HOSTFLAG_NO_WILDCARDS: Final = 0x2 +HOSTFLAG_NO_PARTIAL_WILDCARDS: Final = 0x4 +HOSTFLAG_MULTI_LABEL_WILDCARDS: Final = 0x8 +HOSTFLAG_SINGLE_LABEL_SUBDOMAINS: Final = 0x10 if sys.version_info >= (3, 10): # certificate file types - # Typed as Literal so the overload on Certificate.public_bytes can work properly. - ENCODING_PEM: Literal[1] - ENCODING_DER: Literal[2] + ENCODING_PEM: Final = 1 + ENCODING_DER: Final = 2 # protocol versions -PROTO_MINIMUM_SUPPORTED: int -PROTO_MAXIMUM_SUPPORTED: int -PROTO_SSLv3: int -PROTO_TLSv1: int -PROTO_TLSv1_1: int -PROTO_TLSv1_2: int -PROTO_TLSv1_3: int +PROTO_MINIMUM_SUPPORTED: Final = -2 +PROTO_MAXIMUM_SUPPORTED: Final = -1 +PROTO_SSLv3: Final[int] +PROTO_TLSv1: Final[int] +PROTO_TLSv1_1: Final[int] +PROTO_TLSv1_2: Final[int] +PROTO_TLSv1_3: Final[int] # feature support -HAS_SNI: bool -HAS_TLS_UNIQUE: bool -HAS_ECDH: bool -HAS_NPN: bool +HAS_SNI: Final[bool] +HAS_TLS_UNIQUE: Final[bool] +HAS_ECDH: Final[bool] +HAS_NPN: Final[bool] if sys.version_info >= (3, 13): - HAS_PSK: bool -HAS_ALPN: bool -HAS_SSLv2: bool -HAS_SSLv3: bool -HAS_TLSv1: bool -HAS_TLSv1_1: bool -HAS_TLSv1_2: bool -HAS_TLSv1_3: bool + HAS_PSK: Final[bool] +HAS_ALPN: Final[bool] +HAS_SSLv2: Final[bool] +HAS_SSLv3: Final[bool] +HAS_TLSv1: Final[bool] +HAS_TLSv1_1: Final[bool] +HAS_TLSv1_2: Final[bool] +HAS_TLSv1_3: Final[bool] if sys.version_info >= (3, 14): - HAS_PHA: bool + HAS_PHA: Final[bool] # version info -OPENSSL_VERSION_NUMBER: int -OPENSSL_VERSION_INFO: tuple[int, int, int, int, int] -OPENSSL_VERSION: str -_OPENSSL_API_VERSION: tuple[int, int, int, int, int] +OPENSSL_VERSION_NUMBER: Final[int] +OPENSSL_VERSION_INFO: Final[tuple[int, int, int, int, int]] +OPENSSL_VERSION: Final[str] +_OPENSSL_API_VERSION: Final[tuple[int, int, int, int, int]] diff --git a/mypy/typeshed/stdlib/_tkinter.pyi b/mypy/typeshed/stdlib/_tkinter.pyi index 08eb00ca442bf..46366ccc17405 100644 --- a/mypy/typeshed/stdlib/_tkinter.pyi +++ b/mypy/typeshed/stdlib/_tkinter.pyi @@ -1,7 +1,7 @@ import sys from collections.abc import Callable from typing import Any, ClassVar, Final, final -from typing_extensions import TypeAlias +from typing_extensions import TypeAlias, deprecated # _tkinter is meant to be only used internally by tkinter, but some tkinter # functions e.g. return _tkinter.Tcl_Obj objects. Tcl_Obj represents a Tcl @@ -84,6 +84,7 @@ class TkappType: def record(self, script, /): ... def setvar(self, *ags, **kwargs): ... if sys.version_info < (3, 11): + @deprecated("Deprecated since Python 3.9; removed in Python 3.11. Use `splitlist()` instead.") def split(self, arg, /): ... def splitlist(self, arg, /): ... diff --git a/mypy/typeshed/stdlib/array.pyi b/mypy/typeshed/stdlib/array.pyi index bd96c9bc2d317..ccb7f1b98ef33 100644 --- a/mypy/typeshed/stdlib/array.pyi +++ b/mypy/typeshed/stdlib/array.pyi @@ -3,11 +3,14 @@ from _typeshed import ReadableBuffer, SupportsRead, SupportsWrite from collections.abc import Iterable, MutableSequence from types import GenericAlias from typing import Any, ClassVar, Literal, SupportsIndex, TypeVar, overload -from typing_extensions import Self, TypeAlias +from typing_extensions import Self, TypeAlias, deprecated _IntTypeCode: TypeAlias = Literal["b", "B", "h", "H", "i", "I", "l", "L", "q", "Q"] _FloatTypeCode: TypeAlias = Literal["f", "d"] -_UnicodeTypeCode: TypeAlias = Literal["u"] +if sys.version_info >= (3, 13): + _UnicodeTypeCode: TypeAlias = Literal["u", "w"] +else: + _UnicodeTypeCode: TypeAlias = Literal["u"] _TypeCode: TypeAlias = _IntTypeCode | _FloatTypeCode | _UnicodeTypeCode _T = TypeVar("_T", int, float, str) @@ -27,10 +30,23 @@ class array(MutableSequence[_T]): def __new__( cls: type[array[float]], typecode: _FloatTypeCode, initializer: bytes | bytearray | Iterable[float] = ..., / ) -> array[float]: ... - @overload - def __new__( - cls: type[array[str]], typecode: _UnicodeTypeCode, initializer: bytes | bytearray | Iterable[str] = ..., / - ) -> array[str]: ... + if sys.version_info >= (3, 13): + @overload + def __new__( + cls: type[array[str]], typecode: Literal["w"], initializer: bytes | bytearray | Iterable[str] = ..., / + ) -> array[str]: ... + @overload + @deprecated("Deprecated since Python 3.3; will be removed in Python 3.16. Use 'w' typecode instead.") + def __new__( + cls: type[array[str]], typecode: Literal["u"], initializer: bytes | bytearray | Iterable[str] = ..., / + ) -> array[str]: ... + else: + @overload + @deprecated("Deprecated since Python 3.3; will be removed in Python 3.16.") + def __new__( + cls: type[array[str]], typecode: Literal["u"], initializer: bytes | bytearray | Iterable[str] = ..., / + ) -> array[str]: ... + @overload def __new__(cls, typecode: str, initializer: Iterable[_T], /) -> Self: ... @overload @@ -58,6 +74,7 @@ class array(MutableSequence[_T]): def tounicode(self) -> str: ... __hash__: ClassVar[None] # type: ignore[assignment] + def __contains__(self, value: object, /) -> bool: ... def __len__(self) -> int: ... @overload def __getitem__(self, key: SupportsIndex, /) -> _T: ... diff --git a/mypy/typeshed/stdlib/ast.pyi b/mypy/typeshed/stdlib/ast.pyi index 3ba56f55932ab..8ee8671163012 100644 --- a/mypy/typeshed/stdlib/ast.pyi +++ b/mypy/typeshed/stdlib/ast.pyi @@ -1097,15 +1097,17 @@ class Constant(expr): kind: str | None if sys.version_info < (3, 14): # Aliases for value, for backwards compatibility - @deprecated("Will be removed in Python 3.14; use value instead") @property + @deprecated("Will be removed in Python 3.14. Use `value` instead.") def n(self) -> _ConstantValue: ... @n.setter + @deprecated("Will be removed in Python 3.14. Use `value` instead.") def n(self, value: _ConstantValue) -> None: ... - @deprecated("Will be removed in Python 3.14; use value instead") @property + @deprecated("Will be removed in Python 3.14. Use `value` instead.") def s(self) -> _ConstantValue: ... @s.setter + @deprecated("Will be removed in Python 3.14. Use `value` instead.") def s(self, value: _ConstantValue) -> None: ... def __init__(self, value: _ConstantValue, kind: str | None = None, **kwargs: Unpack[_Attributes]) -> None: ... diff --git a/mypy/typeshed/stdlib/asyncio/coroutines.pyi b/mypy/typeshed/stdlib/asyncio/coroutines.pyi index 8ef30b3d31980..59212f4ec398b 100644 --- a/mypy/typeshed/stdlib/asyncio/coroutines.pyi +++ b/mypy/typeshed/stdlib/asyncio/coroutines.pyi @@ -1,7 +1,7 @@ import sys from collections.abc import Awaitable, Callable, Coroutine from typing import Any, TypeVar, overload -from typing_extensions import ParamSpec, TypeGuard, TypeIs +from typing_extensions import ParamSpec, TypeGuard, TypeIs, deprecated # Keep asyncio.__all__ updated with any changes to __all__ here if sys.version_info >= (3, 11): @@ -14,6 +14,7 @@ _FunctionT = TypeVar("_FunctionT", bound=Callable[..., Any]) _P = ParamSpec("_P") if sys.version_info < (3, 11): + @deprecated("Deprecated since Python 3.8; removed in Python 3.11. Use `async def` instead.") def coroutine(func: _FunctionT) -> _FunctionT: ... @overload diff --git a/mypy/typeshed/stdlib/asyncio/trsock.pyi b/mypy/typeshed/stdlib/asyncio/trsock.pyi index e74cf6fd4e052..4dacbbd493991 100644 --- a/mypy/typeshed/stdlib/asyncio/trsock.pyi +++ b/mypy/typeshed/stdlib/asyncio/trsock.pyi @@ -5,7 +5,7 @@ from builtins import type as Type # alias to avoid name clashes with property n from collections.abc import Iterable from types import TracebackType from typing import Any, BinaryIO, NoReturn, overload -from typing_extensions import TypeAlias +from typing_extensions import TypeAlias, deprecated # These are based in socket, maybe move them out into _typeshed.pyi or such _Address: TypeAlias = socket._Address @@ -42,53 +42,82 @@ class TransportSocket: def setblocking(self, flag: bool) -> None: ... if sys.version_info < (3, 11): def _na(self, what: str) -> None: ... + @deprecated("Removed in Python 3.11") def accept(self) -> tuple[socket.socket, _RetAddress]: ... + @deprecated("Removed in Python 3.11") def connect(self, address: _Address) -> None: ... + @deprecated("Removed in Python 3.11") def connect_ex(self, address: _Address) -> int: ... + @deprecated("Removed in Python 3.11") def bind(self, address: _Address) -> None: ... if sys.platform == "win32": + @deprecated("Removed in Python 3.11") def ioctl(self, control: int, option: int | tuple[int, int, int] | bool) -> None: ... else: + @deprecated("Removed in Python 3.11") def ioctl(self, control: int, option: int | tuple[int, int, int] | bool) -> NoReturn: ... + @deprecated("Removed in Python 3.11") def listen(self, backlog: int = ..., /) -> None: ... + @deprecated("Removed in Python 3.11") def makefile(self) -> BinaryIO: ... + @deprecated("Rmoved in Python 3.11") def sendfile(self, file: BinaryIO, offset: int = ..., count: int | None = ...) -> int: ... + @deprecated("Removed in Python 3.11") def close(self) -> None: ... + @deprecated("Removed in Python 3.11") def detach(self) -> int: ... if sys.platform == "linux": + @deprecated("Removed in Python 3.11") def sendmsg_afalg( self, msg: Iterable[ReadableBuffer] = ..., *, op: int, iv: Any = ..., assoclen: int = ..., flags: int = ... ) -> int: ... else: + @deprecated("Removed in Python 3.11.") def sendmsg_afalg( self, msg: Iterable[ReadableBuffer] = ..., *, op: int, iv: Any = ..., assoclen: int = ..., flags: int = ... ) -> NoReturn: ... + @deprecated("Removed in Python 3.11.") def sendmsg( self, buffers: Iterable[ReadableBuffer], ancdata: Iterable[_CMSG] = ..., flags: int = ..., address: _Address = ..., / ) -> int: ... @overload + @deprecated("Removed in Python 3.11.") def sendto(self, data: ReadableBuffer, address: _Address) -> int: ... @overload + @deprecated("Removed in Python 3.11.") def sendto(self, data: ReadableBuffer, flags: int, address: _Address) -> int: ... + @deprecated("Removed in Python 3.11.") def send(self, data: ReadableBuffer, flags: int = ...) -> int: ... + @deprecated("Removed in Python 3.11.") def sendall(self, data: ReadableBuffer, flags: int = ...) -> None: ... + @deprecated("Removed in Python 3.11.") def set_inheritable(self, inheritable: bool) -> None: ... if sys.platform == "win32": + @deprecated("Removed in Python 3.11.") def share(self, process_id: int) -> bytes: ... else: + @deprecated("Removed in Python 3.11.") def share(self, process_id: int) -> NoReturn: ... + @deprecated("Removed in Python 3.11.") def recv_into(self, buffer: _WriteBuffer, nbytes: int = ..., flags: int = ...) -> int: ... + @deprecated("Removed in Python 3.11.") def recvfrom_into(self, buffer: _WriteBuffer, nbytes: int = ..., flags: int = ...) -> tuple[int, _RetAddress]: ... + @deprecated("Removed in Python 3.11.") def recvmsg_into( self, buffers: Iterable[_WriteBuffer], ancbufsize: int = ..., flags: int = ..., / ) -> tuple[int, list[_CMSG], int, Any]: ... + @deprecated("Removed in Python 3.11.") def recvmsg(self, bufsize: int, ancbufsize: int = ..., flags: int = ..., /) -> tuple[bytes, list[_CMSG], int, Any]: ... + @deprecated("Removed in Python 3.11.") def recvfrom(self, bufsize: int, flags: int = ...) -> tuple[bytes, _RetAddress]: ... + @deprecated("Removed in Python 3.11.") def recv(self, bufsize: int, flags: int = ...) -> bytes: ... + @deprecated("Removed in Python 3.11.") def __enter__(self) -> socket.socket: ... + @deprecated("Removed in Python 3.11.") def __exit__( self, exc_type: Type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None ) -> None: ... diff --git a/mypy/typeshed/stdlib/binascii.pyi b/mypy/typeshed/stdlib/binascii.pyi index 32e018c653cbc..e09d335596fc3 100644 --- a/mypy/typeshed/stdlib/binascii.pyi +++ b/mypy/typeshed/stdlib/binascii.pyi @@ -1,6 +1,6 @@ import sys from _typeshed import ReadableBuffer -from typing_extensions import TypeAlias +from typing_extensions import TypeAlias, deprecated # Many functions in binascii accept buffer objects # or ASCII-only strings. @@ -20,9 +20,13 @@ def a2b_qp(data: _AsciiBuffer, header: bool = False) -> bytes: ... def b2a_qp(data: ReadableBuffer, quotetabs: bool = False, istext: bool = True, header: bool = False) -> bytes: ... if sys.version_info < (3, 11): + @deprecated("Deprecated since Python 3.9; removed in Python 3.11.") def a2b_hqx(data: _AsciiBuffer, /) -> bytes: ... + @deprecated("Deprecated since Python 3.9; removed in Python 3.11.") def rledecode_hqx(data: ReadableBuffer, /) -> bytes: ... + @deprecated("Deprecated since Python 3.9; removed in Python 3.11.") def rlecode_hqx(data: ReadableBuffer, /) -> bytes: ... + @deprecated("Deprecated since Python 3.9; removed in Python 3.11.") def b2a_hqx(data: ReadableBuffer, /) -> bytes: ... def crc_hqx(data: ReadableBuffer, crc: int, /) -> int: ... diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi index baf399e7bb774..d7c0fe27c1ee3 100644 --- a/mypy/typeshed/stdlib/builtins.pyi +++ b/mypy/typeshed/stdlib/builtins.pyi @@ -348,6 +348,7 @@ class int: def __hash__(self) -> int: ... def __bool__(self) -> bool: ... def __index__(self) -> int: ... + def __format__(self, format_spec: str, /) -> str: ... class float: def __new__(cls, x: ConvertibleToFloat = ..., /) -> Self: ... @@ -409,6 +410,7 @@ class float: def __abs__(self) -> float: ... def __hash__(self) -> int: ... def __bool__(self) -> bool: ... + def __format__(self, format_spec: str, /) -> str: ... if sys.version_info >= (3, 14): @classmethod def from_number(cls, number: float | SupportsIndex | SupportsFloat, /) -> Self: ... @@ -445,6 +447,7 @@ class complex: def __abs__(self) -> float: ... def __hash__(self) -> int: ... def __bool__(self) -> bool: ... + def __format__(self, format_spec: str, /) -> str: ... if sys.version_info >= (3, 11): def __complex__(self) -> complex: ... if sys.version_info >= (3, 14): @@ -544,6 +547,7 @@ class str(Sequence[str]): def __ne__(self, value: object, /) -> bool: ... def __rmul__(self, value: SupportsIndex, /) -> str: ... # type: ignore[misc] def __getnewargs__(self) -> tuple[str]: ... + def __format__(self, format_spec: str, /) -> str: ... class bytes(Sequence[int]): @overload diff --git a/mypy/typeshed/stdlib/concurrent/futures/interpreter.pyi b/mypy/typeshed/stdlib/concurrent/futures/interpreter.pyi index 9c1078983d8cf..e101022babcb6 100644 --- a/mypy/typeshed/stdlib/concurrent/futures/interpreter.pyi +++ b/mypy/typeshed/stdlib/concurrent/futures/interpreter.pyi @@ -1,10 +1,13 @@ import sys -from collections.abc import Callable, Mapping +from collections.abc import Callable from concurrent.futures import ThreadPoolExecutor -from typing import Literal, Protocol, overload, type_check_only +from typing import Any, Literal, Protocol, overload, type_check_only from typing_extensions import ParamSpec, Self, TypeAlias, TypeVar, TypeVarTuple, Unpack _Task: TypeAlias = tuple[bytes, Literal["function", "script"]] +_Ts = TypeVarTuple("_Ts") +_P = ParamSpec("_P") +_R = TypeVar("_R") @type_check_only class _TaskFunc(Protocol): @@ -13,62 +16,41 @@ class _TaskFunc(Protocol): @overload def __call__(self, fn: str) -> tuple[bytes, Literal["script"]]: ... -_Ts = TypeVarTuple("_Ts") -_P = ParamSpec("_P") -_R = TypeVar("_R") - -# A `type.simplenamespace` with `__name__` attribute. -@type_check_only -class _HasName(Protocol): - __name__: str - -# `_interpreters.exec` technically gives us a simple namespace. -@type_check_only -class _ExcInfo(Protocol): - formatted: str - msg: str - type: _HasName - if sys.version_info >= (3, 14): from concurrent.futures.thread import BrokenThreadPool, WorkerContext as ThreadWorkerContext + from concurrent.interpreters import Interpreter, Queue - from _interpreters import InterpreterError - - class ExecutionFailed(InterpreterError): - def __init__(self, excinfo: _ExcInfo) -> None: ... # type: ignore[override] + def do_call(results: Queue, func: Callable[..., _R], args: tuple[Any, ...], kwargs: dict[str, Any]) -> _R: ... class WorkerContext(ThreadWorkerContext): - # Parent class doesn't have `shared` argument, - @overload # type: ignore[override] + interp: Interpreter | None + results: Queue | None + @overload # type: ignore[override] @classmethod def prepare( - cls, initializer: Callable[[Unpack[_Ts]], object], initargs: tuple[Unpack[_Ts]], shared: Mapping[str, object] + cls, initializer: Callable[[Unpack[_Ts]], object], initargs: tuple[Unpack[_Ts]] ) -> tuple[Callable[[], Self], _TaskFunc]: ... - @overload # type: ignore[override] + @overload @classmethod - def prepare( - cls, initializer: Callable[[], object], initargs: tuple[()], shared: Mapping[str, object] - ) -> tuple[Callable[[], Self], _TaskFunc]: ... - def __init__( - self, initdata: tuple[bytes, Literal["function", "script"]], shared: Mapping[str, object] | None = None - ) -> None: ... # type: ignore[override] + def prepare(cls, initializer: Callable[[], object], initargs: tuple[()]) -> tuple[Callable[[], Self], _TaskFunc]: ... + def __init__(self, initdata: _Task) -> None: ... def __del__(self) -> None: ... - def run(self, task: _Task) -> None: ... # type: ignore[override] + def run(self, task: _Task) -> None: ... # type: ignore[override] class BrokenInterpreterPool(BrokenThreadPool): ... class InterpreterPoolExecutor(ThreadPoolExecutor): BROKEN: type[BrokenInterpreterPool] - @overload # type: ignore[override] + @overload # type: ignore[override] @classmethod def prepare_context( - cls, initializer: Callable[[], object], initargs: tuple[()], shared: Mapping[str, object] + cls, initializer: Callable[[], object], initargs: tuple[()] ) -> tuple[Callable[[], WorkerContext], _TaskFunc]: ... - @overload # type: ignore[override] + @overload @classmethod def prepare_context( - cls, initializer: Callable[[Unpack[_Ts]], object], initargs: tuple[Unpack[_Ts]], shared: Mapping[str, object] + cls, initializer: Callable[[Unpack[_Ts]], object], initargs: tuple[Unpack[_Ts]] ) -> tuple[Callable[[], WorkerContext], _TaskFunc]: ... @overload def __init__( @@ -77,7 +59,6 @@ if sys.version_info >= (3, 14): thread_name_prefix: str = "", initializer: Callable[[], object] | None = None, initargs: tuple[()] = (), - shared: Mapping[str, object] | None = None, ) -> None: ... @overload def __init__( @@ -87,7 +68,6 @@ if sys.version_info >= (3, 14): *, initializer: Callable[[Unpack[_Ts]], object], initargs: tuple[Unpack[_Ts]], - shared: Mapping[str, object] | None = None, ) -> None: ... @overload def __init__( @@ -96,5 +76,4 @@ if sys.version_info >= (3, 14): thread_name_prefix: str, initializer: Callable[[Unpack[_Ts]], object], initargs: tuple[Unpack[_Ts]], - shared: Mapping[str, object] | None = None, ) -> None: ... diff --git a/mypy/typeshed/stdlib/configparser.pyi b/mypy/typeshed/stdlib/configparser.pyi index fb02701e3711d..b3a4210030266 100644 --- a/mypy/typeshed/stdlib/configparser.pyi +++ b/mypy/typeshed/stdlib/configparser.pyi @@ -3,7 +3,7 @@ from _typeshed import MaybeNone, StrOrBytesPath, SupportsWrite from collections.abc import Callable, ItemsView, Iterable, Iterator, Mapping, MutableMapping, Sequence from re import Pattern from typing import Any, ClassVar, Final, Literal, TypeVar, overload, type_check_only -from typing_extensions import TypeAlias +from typing_extensions import TypeAlias, deprecated if sys.version_info >= (3, 14): __all__ = ( @@ -137,6 +137,9 @@ class BasicInterpolation(Interpolation): ... class ExtendedInterpolation(Interpolation): ... if sys.version_info < (3, 13): + @deprecated( + "Deprecated since Python 3.2; removed in Python 3.13. Use `BasicInterpolation` or `ExtendedInterpolation` instead." + ) class LegacyInterpolation(Interpolation): def before_get(self, parser: _Parser, section: _SectionName, option: str, value: str, vars: _Section) -> str: ... @@ -271,6 +274,7 @@ class RawConfigParser(_Parser): def read_string(self, string: str, source: str = "") -> None: ... def read_dict(self, dictionary: Mapping[str, Mapping[str, Any]], source: str = "") -> None: ... if sys.version_info < (3, 12): + @deprecated("Deprecated since Python 3.2; removed in Python 3.12. Use `parser.read_file()` instead.") def readfp(self, fp: Iterable[str], filename: str | None = None) -> None: ... # These get* methods are partially applied (with the same names) in # SectionProxy; the stubs should be kept updated together @@ -331,7 +335,8 @@ class ConfigParser(RawConfigParser): ) -> str | _T: ... if sys.version_info < (3, 12): - class SafeConfigParser(ConfigParser): ... # deprecated alias + @deprecated("Deprecated since Python 3.2; removed in Python 3.12. Use `ConfigParser` instead.") + class SafeConfigParser(ConfigParser): ... class SectionProxy(MutableMapping[str, str]): def __init__(self, parser: RawConfigParser, name: str) -> None: ... @@ -443,10 +448,22 @@ class ParsingError(Error): elif sys.version_info >= (3, 12): def __init__(self, source: str) -> None: ... else: - def __init__(self, source: str | None = None, filename: str | None = None) -> None: ... + @overload + def __init__(self, source: str, filename: None = None) -> None: ... + @overload + @deprecated("The `filename` parameter removed in Python 3.12. Use `source` instead.") + def __init__(self, source: None = None, filename: str = ...) -> None: ... def append(self, lineno: int, line: str) -> None: ... + if sys.version_info < (3, 12): + @property + @deprecated("Deprecated since Python 3.2; removed in Python 3.12. Use `source` instead.") + def filename(self) -> str: ... + @filename.setter + @deprecated("Deprecated since Python 3.2; removed in Python 3.12. Use `source` instead.") + def filename(self, value: str) -> None: ... + class MissingSectionHeaderError(ParsingError): lineno: int line: str diff --git a/mypy/typeshed/stdlib/fcntl.pyi b/mypy/typeshed/stdlib/fcntl.pyi index 2fe64eb532013..5a3e89b0c6766 100644 --- a/mypy/typeshed/stdlib/fcntl.pyi +++ b/mypy/typeshed/stdlib/fcntl.pyi @@ -4,107 +4,107 @@ from typing import Any, Final, Literal, overload from typing_extensions import Buffer if sys.platform != "win32": - FASYNC: int - FD_CLOEXEC: int - F_DUPFD: int - F_DUPFD_CLOEXEC: int - F_GETFD: int - F_GETFL: int - F_GETLK: int - F_GETOWN: int - F_RDLCK: int - F_SETFD: int - F_SETFL: int - F_SETLK: int - F_SETLKW: int - F_SETOWN: int - F_UNLCK: int - F_WRLCK: int + FASYNC: Final[int] + FD_CLOEXEC: Final[int] + F_DUPFD: Final[int] + F_DUPFD_CLOEXEC: Final[int] + F_GETFD: Final[int] + F_GETFL: Final[int] + F_GETLK: Final[int] + F_GETOWN: Final[int] + F_RDLCK: Final[int] + F_SETFD: Final[int] + F_SETFL: Final[int] + F_SETLK: Final[int] + F_SETLKW: Final[int] + F_SETOWN: Final[int] + F_UNLCK: Final[int] + F_WRLCK: Final[int] - F_GETLEASE: int - F_SETLEASE: int + F_GETLEASE: Final[int] + F_SETLEASE: Final[int] if sys.platform == "darwin": - F_FULLFSYNC: int - F_NOCACHE: int - F_GETPATH: int + F_FULLFSYNC: Final[int] + F_NOCACHE: Final[int] + F_GETPATH: Final[int] if sys.platform == "linux": - F_SETLKW64: int - F_SETSIG: int - F_SHLCK: int - F_SETLK64: int - F_GETSIG: int - F_NOTIFY: int - F_EXLCK: int - F_GETLK64: int - F_ADD_SEALS: int - F_GET_SEALS: int - F_SEAL_GROW: int - F_SEAL_SEAL: int - F_SEAL_SHRINK: int - F_SEAL_WRITE: int + F_SETLKW64: Final[int] + F_SETSIG: Final[int] + F_SHLCK: Final[int] + F_SETLK64: Final[int] + F_GETSIG: Final[int] + F_NOTIFY: Final[int] + F_EXLCK: Final[int] + F_GETLK64: Final[int] + F_ADD_SEALS: Final[int] + F_GET_SEALS: Final[int] + F_SEAL_GROW: Final[int] + F_SEAL_SEAL: Final[int] + F_SEAL_SHRINK: Final[int] + F_SEAL_WRITE: Final[int] F_OFD_GETLK: Final[int] F_OFD_SETLK: Final[int] F_OFD_SETLKW: Final[int] if sys.version_info >= (3, 10): - F_GETPIPE_SZ: int - F_SETPIPE_SZ: int + F_GETPIPE_SZ: Final[int] + F_SETPIPE_SZ: Final[int] - DN_ACCESS: int - DN_ATTRIB: int - DN_CREATE: int - DN_DELETE: int - DN_MODIFY: int - DN_MULTISHOT: int - DN_RENAME: int + DN_ACCESS: Final[int] + DN_ATTRIB: Final[int] + DN_CREATE: Final[int] + DN_DELETE: Final[int] + DN_MODIFY: Final[int] + DN_MULTISHOT: Final[int] + DN_RENAME: Final[int] - LOCK_EX: int - LOCK_NB: int - LOCK_SH: int - LOCK_UN: int + LOCK_EX: Final[int] + LOCK_NB: Final[int] + LOCK_SH: Final[int] + LOCK_UN: Final[int] if sys.platform == "linux": - LOCK_MAND: int - LOCK_READ: int - LOCK_RW: int - LOCK_WRITE: int + LOCK_MAND: Final[int] + LOCK_READ: Final[int] + LOCK_RW: Final[int] + LOCK_WRITE: Final[int] if sys.platform == "linux": # Constants for the POSIX STREAMS interface. Present in glibc until 2.29 (released February 2019). # Never implemented on BSD, and considered "obsolescent" starting in POSIX 2008. # Probably still used on Solaris. - I_ATMARK: int - I_CANPUT: int - I_CKBAND: int - I_FDINSERT: int - I_FIND: int - I_FLUSH: int - I_FLUSHBAND: int - I_GETBAND: int - I_GETCLTIME: int - I_GETSIG: int - I_GRDOPT: int - I_GWROPT: int - I_LINK: int - I_LIST: int - I_LOOK: int - I_NREAD: int - I_PEEK: int - I_PLINK: int - I_POP: int - I_PUNLINK: int - I_PUSH: int - I_RECVFD: int - I_SENDFD: int - I_SETCLTIME: int - I_SETSIG: int - I_SRDOPT: int - I_STR: int - I_SWROPT: int - I_UNLINK: int + I_ATMARK: Final[int] + I_CANPUT: Final[int] + I_CKBAND: Final[int] + I_FDINSERT: Final[int] + I_FIND: Final[int] + I_FLUSH: Final[int] + I_FLUSHBAND: Final[int] + I_GETBAND: Final[int] + I_GETCLTIME: Final[int] + I_GETSIG: Final[int] + I_GRDOPT: Final[int] + I_GWROPT: Final[int] + I_LINK: Final[int] + I_LIST: Final[int] + I_LOOK: Final[int] + I_NREAD: Final[int] + I_PEEK: Final[int] + I_PLINK: Final[int] + I_POP: Final[int] + I_PUNLINK: Final[int] + I_PUSH: Final[int] + I_RECVFD: Final[int] + I_SENDFD: Final[int] + I_SETCLTIME: Final[int] + I_SETSIG: Final[int] + I_SRDOPT: Final[int] + I_STR: Final[int] + I_SWROPT: Final[int] + I_UNLINK: Final[int] if sys.version_info >= (3, 12) and sys.platform == "linux": - FICLONE: int - FICLONERANGE: int + FICLONE: Final[int] + FICLONERANGE: Final[int] if sys.version_info >= (3, 13) and sys.platform == "linux": F_OWNER_TID: Final = 0 diff --git a/mypy/typeshed/stdlib/gettext.pyi b/mypy/typeshed/stdlib/gettext.pyi index 5ff98b052cdbe..937aece034375 100644 --- a/mypy/typeshed/stdlib/gettext.pyi +++ b/mypy/typeshed/stdlib/gettext.pyi @@ -3,6 +3,7 @@ import sys from _typeshed import StrPath from collections.abc import Callable, Container, Iterable, Sequence from typing import Any, Final, Literal, Protocol, TypeVar, overload, type_check_only +from typing_extensions import deprecated __all__ = [ "NullTranslations", @@ -43,9 +44,13 @@ class NullTranslations: def info(self) -> dict[str, str]: ... def charset(self) -> str | None: ... if sys.version_info < (3, 11): + @deprecated("Deprecated since Python 3.8; removed in Python 3.11.") def output_charset(self) -> str | None: ... + @deprecated("Deprecated since Python 3.8; removed in Python 3.11.") def set_output_charset(self, charset: str) -> None: ... + @deprecated("Deprecated since Python 3.8; removed in Python 3.11. Use `gettext()` instead.") def lgettext(self, message: str) -> str: ... + @deprecated("Deprecated since Python 3.8; removed in Python 3.11. Use `ngettext()` instead.") def lngettext(self, msgid1: str, msgid2: str, n: int) -> str: ... def install(self, names: Container[str] | None = None) -> None: ... @@ -145,9 +150,16 @@ else: fallback: bool = False, codeset: str | None = None, ) -> NullTranslations: ... + @overload def install( - domain: str, localedir: StrPath | None = None, codeset: str | None = None, names: Container[str] | None = None + domain: str, localedir: StrPath | None = None, codeset: None = None, names: Container[str] | None = None ) -> None: ... + @overload + @deprecated("The `codeset` parameter is deprecated since Python 3.8; removed in Python 3.11.") + def install(domain: str, localedir: StrPath | None, codeset: str, /, names: Container[str] | None = None) -> None: ... + @overload + @deprecated("The `codeset` parameter is deprecated since Python 3.8; removed in Python 3.11.") + def install(domain: str, localedir: StrPath | None = None, *, codeset: str, names: Container[str] | None = None) -> None: ... def textdomain(domain: str | None = None) -> str: ... def bindtextdomain(domain: str, localedir: StrPath | None = None) -> str: ... @@ -161,10 +173,15 @@ def npgettext(context: str, msgid1: str, msgid2: str, n: int) -> str: ... def dnpgettext(domain: str, context: str, msgid1: str, msgid2: str, n: int) -> str: ... if sys.version_info < (3, 11): + @deprecated("Deprecated since Python 3.8; removed in Python 3.11. Use `gettext()` instead.") def lgettext(message: str) -> str: ... + @deprecated("Deprecated since Python 3.8; removed in Python 3.11. Use `dgettext()` instead.") def ldgettext(domain: str, message: str) -> str: ... + @deprecated("Deprecated since Python 3.8; removed in Python 3.11. Use `ngettext()` instead.") def lngettext(msgid1: str, msgid2: str, n: int) -> str: ... + @deprecated("Deprecated since Python 3.8; removed in Python 3.11. Use `dngettext()` instead.") def ldngettext(domain: str, msgid1: str, msgid2: str, n: int) -> str: ... + @deprecated("Deprecated since Python 3.8; removed in Python 3.11. Use `bindtextdomain()` instead.") def bind_textdomain_codeset(domain: str, codeset: str | None = None) -> str: ... Catalog = translation diff --git a/mypy/typeshed/stdlib/glob.pyi b/mypy/typeshed/stdlib/glob.pyi index 03cb5418e2565..63069d8009c8d 100644 --- a/mypy/typeshed/stdlib/glob.pyi +++ b/mypy/typeshed/stdlib/glob.pyi @@ -2,14 +2,22 @@ import sys from _typeshed import StrOrBytesPath from collections.abc import Iterator, Sequence from typing import AnyStr +from typing_extensions import deprecated __all__ = ["escape", "glob", "iglob"] if sys.version_info >= (3, 13): __all__ += ["translate"] -def glob0(dirname: AnyStr, pattern: AnyStr) -> list[AnyStr]: ... -def glob1(dirname: AnyStr, pattern: AnyStr) -> list[AnyStr]: ... +if sys.version_info >= (3, 10): + @deprecated("Will be removed in Python 3.15; Use `glob.glob` and pass *root_dir* argument instead.") + def glob0(dirname: AnyStr, pattern: AnyStr) -> list[AnyStr]: ... + @deprecated("Will be removed in Python 3.15; Use `glob.glob` and pass *root_dir* argument instead.") + def glob1(dirname: AnyStr, pattern: AnyStr) -> list[AnyStr]: ... + +else: + def glob0(dirname: AnyStr, pattern: AnyStr) -> list[AnyStr]: ... + def glob1(dirname: AnyStr, pattern: AnyStr) -> list[AnyStr]: ... if sys.version_info >= (3, 11): def glob( diff --git a/mypy/typeshed/stdlib/gzip.pyi b/mypy/typeshed/stdlib/gzip.pyi index 06f5e2880bd5a..b18f76f06e3ee 100644 --- a/mypy/typeshed/stdlib/gzip.pyi +++ b/mypy/typeshed/stdlib/gzip.pyi @@ -3,7 +3,7 @@ import zlib from _typeshed import ReadableBuffer, SizedBuffer, StrOrBytesPath, WriteableBuffer from io import FileIO, TextIOWrapper from typing import Final, Literal, Protocol, overload, type_check_only -from typing_extensions import TypeAlias +from typing_extensions import TypeAlias, deprecated if sys.version_info >= (3, 14): from compression._common._streams import BaseStream, DecompressReader @@ -143,6 +143,7 @@ class GzipFile(BaseStream): ) -> None: ... if sys.version_info < (3, 12): @property + @deprecated("Deprecated since Python 2.6; removed in Python 3.12. Use `name` attribute instead.") def filename(self) -> str: ... @property diff --git a/mypy/typeshed/stdlib/html/parser.pyi b/mypy/typeshed/stdlib/html/parser.pyi index 45336f03aaa7a..8b3fce0010b78 100644 --- a/mypy/typeshed/stdlib/html/parser.pyi +++ b/mypy/typeshed/stdlib/html/parser.pyi @@ -7,7 +7,8 @@ __all__ = ["HTMLParser"] class HTMLParser(ParserBase): CDATA_CONTENT_ELEMENTS: Final[tuple[str, ...]] - if sys.version_info >= (3, 14): + if sys.version_info >= (3, 13): + # Added in 3.13.6 RCDATA_CONTENT_ELEMENTS: Final[tuple[str, ...]] def __init__(self, *, convert_charrefs: bool = True) -> None: ... @@ -31,7 +32,8 @@ class HTMLParser(ParserBase): def parse_html_declaration(self, i: int) -> int: ... # undocumented def parse_pi(self, i: int) -> int: ... # undocumented def parse_starttag(self, i: int) -> int: ... # undocumented - if sys.version_info >= (3, 14): + if sys.version_info >= (3, 13): + # `escapable` parameter added in 3.13.6 def set_cdata_mode(self, elem: str, *, escapable: bool = False) -> None: ... # undocumented else: def set_cdata_mode(self, elem: str) -> None: ... # undocumented diff --git a/mypy/typeshed/stdlib/importlib/__init__.pyi b/mypy/typeshed/stdlib/importlib/__init__.pyi index cab81512e92f2..d60f90adee19c 100644 --- a/mypy/typeshed/stdlib/importlib/__init__.pyi +++ b/mypy/typeshed/stdlib/importlib/__init__.pyi @@ -2,6 +2,7 @@ import sys from importlib._bootstrap import __import__ as __import__ from importlib.abc import Loader from types import ModuleType +from typing_extensions import deprecated __all__ = ["__import__", "import_module", "invalidate_caches", "reload"] @@ -9,6 +10,7 @@ __all__ = ["__import__", "import_module", "invalidate_caches", "reload"] def import_module(name: str, package: str | None = None) -> ModuleType: ... if sys.version_info < (3, 12): + @deprecated("Deprecated since Python 3.4; removed in Python 3.12. Use `importlib.util.find_spec()` instead.") def find_loader(name: str, path: str | None = None) -> Loader | None: ... def invalidate_caches() -> None: ... diff --git a/mypy/typeshed/stdlib/importlib/_abc.pyi b/mypy/typeshed/stdlib/importlib/_abc.pyi index 1a21b9a72cd85..90ab340219172 100644 --- a/mypy/typeshed/stdlib/importlib/_abc.pyi +++ b/mypy/typeshed/stdlib/importlib/_abc.pyi @@ -2,11 +2,16 @@ import sys import types from abc import ABCMeta from importlib.machinery import ModuleSpec +from typing_extensions import deprecated if sys.version_info >= (3, 10): class Loader(metaclass=ABCMeta): def load_module(self, fullname: str) -> types.ModuleType: ... if sys.version_info < (3, 12): + @deprecated( + "Deprecated since Python 3.4; removed in Python 3.12. " + "The module spec is now used by the import machinery to generate a module repr." + ) def module_repr(self, module: types.ModuleType) -> str: ... def create_module(self, spec: ModuleSpec) -> types.ModuleType | None: ... diff --git a/mypy/typeshed/stdlib/importlib/abc.pyi b/mypy/typeshed/stdlib/importlib/abc.pyi index cf0fd0807b7b8..ef87663cb72dd 100644 --- a/mypy/typeshed/stdlib/importlib/abc.pyi +++ b/mypy/typeshed/stdlib/importlib/abc.pyi @@ -37,6 +37,7 @@ else: def exec_module(self, module: types.ModuleType) -> None: ... if sys.version_info < (3, 12): + @deprecated("Deprecated since Python 3.3; removed in Python 3.12. Use `MetaPathFinder` or `PathEntryFinder` instead.") class Finder(metaclass=ABCMeta): ... @deprecated("Deprecated as of Python 3.7: Use importlib.resources.abc.TraversableResources instead.") @@ -71,6 +72,7 @@ if sys.version_info >= (3, 10): # Please keep in sync with _typeshed.importlib.MetaPathFinderProtocol class MetaPathFinder(metaclass=ABCMeta): if sys.version_info < (3, 12): + @deprecated("Deprecated since Python 3.4; removed in Python 3.12. Use `MetaPathFinder.find_spec()` instead.") def find_module(self, fullname: str, path: Sequence[str] | None) -> Loader | None: ... def invalidate_caches(self) -> None: ... @@ -81,7 +83,9 @@ if sys.version_info >= (3, 10): class PathEntryFinder(metaclass=ABCMeta): if sys.version_info < (3, 12): + @deprecated("Deprecated since Python 3.4; removed in Python 3.12. Use `PathEntryFinder.find_spec()` instead.") def find_module(self, fullname: str) -> Loader | None: ... + @deprecated("Deprecated since Python 3.4; removed in Python 3.12. Use `find_spec()` instead.") def find_loader(self, fullname: str) -> tuple[Loader | None, Sequence[str]]: ... def invalidate_caches(self) -> None: ... diff --git a/mypy/typeshed/stdlib/importlib/metadata/__init__.pyi b/mypy/typeshed/stdlib/importlib/metadata/__init__.pyi index 789878382ceb8..d1315b2eb2f11 100644 --- a/mypy/typeshed/stdlib/importlib/metadata/__init__.pyi +++ b/mypy/typeshed/stdlib/importlib/metadata/__init__.pyi @@ -11,7 +11,7 @@ from os import PathLike from pathlib import Path from re import Pattern from typing import Any, ClassVar, Generic, NamedTuple, TypeVar, overload -from typing_extensions import Self, TypeAlias +from typing_extensions import Self, TypeAlias, deprecated _T = TypeVar("_T") _KT = TypeVar("_KT") @@ -89,8 +89,8 @@ class EntryPoint(_EntryPointBase): ) -> bool: ... # undocumented def __hash__(self) -> int: ... - def __eq__(self, other: object) -> bool: ... if sys.version_info >= (3, 11): + def __eq__(self, other: object) -> bool: ... def __lt__(self, other: object) -> bool: ... if sys.version_info < (3, 12): def __iter__(self) -> Iterator[Any]: ... # result of iter((str, Self)), really @@ -148,6 +148,7 @@ if sys.version_info >= (3, 10) and sys.version_info < (3, 12): def keys(self) -> dict_keys[_KT, _VT]: ... def values(self) -> dict_values[_KT, _VT]: ... + @deprecated("Deprecated since Python 3.10; removed in Python 3.12. Use `select` instead.") class SelectableGroups(Deprecated[str, EntryPoints], dict[str, EntryPoints]): # use as dict is deprecated since 3.10 @classmethod def load(cls, eps: Iterable[EntryPoint]) -> Self: ... diff --git a/mypy/typeshed/stdlib/importlib/util.pyi b/mypy/typeshed/stdlib/importlib/util.pyi index 370a08623842e..05c4d0d1edb30 100644 --- a/mypy/typeshed/stdlib/importlib/util.pyi +++ b/mypy/typeshed/stdlib/importlib/util.pyi @@ -12,13 +12,25 @@ from importlib._bootstrap_external import ( spec_from_file_location as spec_from_file_location, ) from importlib.abc import Loader -from typing_extensions import ParamSpec +from typing_extensions import ParamSpec, deprecated _P = ParamSpec("_P") if sys.version_info < (3, 12): + @deprecated( + "Deprecated since Python 3.4; removed in Python 3.12. " + "`__name__`, `__package__` and `__loader__` are now set automatically." + ) def module_for_loader(fxn: Callable[_P, types.ModuleType]) -> Callable[_P, types.ModuleType]: ... + @deprecated( + "Deprecated since Python 3.4; removed in Python 3.12. " + "`__name__`, `__package__` and `__loader__` are now set automatically." + ) def set_loader(fxn: Callable[_P, types.ModuleType]) -> Callable[_P, types.ModuleType]: ... + @deprecated( + "Deprecated since Python 3.4; removed in Python 3.12. " + "`__name__`, `__package__` and `__loader__` are now set automatically." + ) def set_package(fxn: Callable[_P, types.ModuleType]) -> Callable[_P, types.ModuleType]: ... def resolve_name(name: str, package: str | None) -> str: ... diff --git a/mypy/typeshed/stdlib/inspect.pyi b/mypy/typeshed/stdlib/inspect.pyi index e73f9e75838d0..f8ec6cad01603 100644 --- a/mypy/typeshed/stdlib/inspect.pyi +++ b/mypy/typeshed/stdlib/inspect.pyi @@ -26,7 +26,7 @@ from types import ( WrapperDescriptorType, ) from typing import Any, ClassVar, Final, Literal, NamedTuple, Protocol, TypeVar, overload, type_check_only -from typing_extensions import ParamSpec, Self, TypeAlias, TypeGuard, TypeIs +from typing_extensions import ParamSpec, Self, TypeAlias, TypeGuard, TypeIs, deprecated if sys.version_info >= (3, 14): from annotationlib import Format @@ -476,12 +476,14 @@ class Arguments(NamedTuple): def getargs(co: CodeType) -> Arguments: ... if sys.version_info < (3, 11): + @deprecated("Deprecated since Python 3.0; removed in Python 3.11.") class ArgSpec(NamedTuple): args: list[str] varargs: str | None keywords: str | None defaults: tuple[Any, ...] + @deprecated("Deprecated since Python 3.0; removed in Python 3.11. Use `inspect.signature()` instead.") def getargspec(func: object) -> ArgSpec: ... class FullArgSpec(NamedTuple): @@ -512,6 +514,9 @@ else: def formatannotationrelativeto(object: object) -> Callable[[object], str]: ... if sys.version_info < (3, 11): + @deprecated( + "Deprecated since Python 3.5; removed in Python 3.11. Use `inspect.signature()` and the `Signature` class instead." + ) def formatargspec( args: list[str], varargs: str | None = None, diff --git a/mypy/typeshed/stdlib/locale.pyi b/mypy/typeshed/stdlib/locale.pyi index 58de654495723..fae9f849b6373 100644 --- a/mypy/typeshed/stdlib/locale.pyi +++ b/mypy/typeshed/stdlib/locale.pyi @@ -18,6 +18,7 @@ from builtins import str as _str from collections.abc import Callable, Iterable from decimal import Decimal from typing import Any +from typing_extensions import deprecated if sys.version_info >= (3, 11): from _locale import getencoding as getencoding @@ -137,9 +138,14 @@ def getpreferredencoding(do_setlocale: bool = True) -> _str: ... def normalize(localename: _str) -> _str: ... if sys.version_info < (3, 13): - def resetlocale(category: int = ...) -> None: ... + if sys.version_info >= (3, 11): + @deprecated("Deprecated since Python 3.11; removed in Python 3.13. Use `locale.setlocale(locale.LC_ALL, '')` instead.") + def resetlocale(category: int = ...) -> None: ... + else: + def resetlocale(category: int = ...) -> None: ... if sys.version_info < (3, 12): + @deprecated("Deprecated since Python 3.7; removed in Python 3.12. Use `locale.format_string()` instead.") def format( percent: _str, value: float | Decimal, grouping: bool = False, monetary: bool = False, *additional: Any ) -> _str: ... diff --git a/mypy/typeshed/stdlib/pathlib/__init__.pyi b/mypy/typeshed/stdlib/pathlib/__init__.pyi index 774478bb2ff42..4858f8db1ed09 100644 --- a/mypy/typeshed/stdlib/pathlib/__init__.pyi +++ b/mypy/typeshed/stdlib/pathlib/__init__.pyi @@ -245,9 +245,13 @@ class Path(PurePath): self, mode: str, buffering: int = -1, encoding: str | None = None, errors: str | None = None, newline: str | None = None ) -> IO[Any]: ... - # These methods do "exist" on Windows on <3.13, but they always raise NotImplementedError. + # These methods do "exist" on Windows, but they always raise NotImplementedError. if sys.platform == "win32": - if sys.version_info < (3, 13): + if sys.version_info >= (3, 13): + # raises UnsupportedOperation: + def owner(self: Never, *, follow_symlinks: bool = True) -> str: ... # type: ignore[misc] + def group(self: Never, *, follow_symlinks: bool = True) -> str: ... # type: ignore[misc] + else: def owner(self: Never) -> str: ... # type: ignore[misc] def group(self: Never) -> str: ... # type: ignore[misc] else: @@ -297,7 +301,7 @@ class Path(PurePath): def write_text(self, data: str, encoding: str | None = None, errors: str | None = None) -> int: ... if sys.version_info < (3, 12): if sys.version_info >= (3, 10): - @deprecated("Deprecated as of Python 3.10 and removed in Python 3.12. Use hardlink_to() instead.") + @deprecated("Deprecated since Python 3.10; removed in Python 3.12. Use `hardlink_to()` instead.") def link_to(self, target: StrOrBytesPath) -> None: ... else: def link_to(self, target: StrOrBytesPath) -> None: ... diff --git a/mypy/typeshed/stdlib/pkgutil.pyi b/mypy/typeshed/stdlib/pkgutil.pyi index e764d08e79f80..7c70dcc4c5ab1 100644 --- a/mypy/typeshed/stdlib/pkgutil.pyi +++ b/mypy/typeshed/stdlib/pkgutil.pyi @@ -30,17 +30,23 @@ class ModuleInfo(NamedTuple): def extend_path(path: _PathT, name: str) -> _PathT: ... if sys.version_info < (3, 12): + @deprecated("Deprecated since Python 3.3; removed in Python 3.12. Use the `importlib` module instead.") class ImpImporter: def __init__(self, path: StrOrBytesPath | None = None) -> None: ... + @deprecated("Deprecated since Python 3.3; removed in Python 3.12. Use the `importlib` module instead.") class ImpLoader: def __init__(self, fullname: str, file: IO[str], filename: StrOrBytesPath, etc: tuple[str, str, int]) -> None: ... if sys.version_info < (3, 14): - @deprecated("Use importlib.util.find_spec() instead. Will be removed in Python 3.14.") - def find_loader(fullname: str) -> LoaderProtocol | None: ... - @deprecated("Use importlib.util.find_spec() instead. Will be removed in Python 3.14.") - def get_loader(module_or_name: str) -> LoaderProtocol | None: ... + if sys.version_info >= (3, 12): + @deprecated("Deprecated since Python 3.12; removed in Python 3.14. Use `importlib.util.find_spec()` instead.") + def find_loader(fullname: str) -> LoaderProtocol | None: ... + @deprecated("Deprecated since Python 3.12; removed in Python 3.14. Use `importlib.util.find_spec()` instead.") + def get_loader(module_or_name: str) -> LoaderProtocol | None: ... + else: + def find_loader(fullname: str) -> LoaderProtocol | None: ... + def get_loader(module_or_name: str) -> LoaderProtocol | None: ... def get_importer(path_item: StrOrBytesPath) -> PathEntryFinderProtocol | None: ... def iter_importers(fullname: str = "") -> Iterator[MetaPathFinderProtocol | PathEntryFinderProtocol]: ... diff --git a/mypy/typeshed/stdlib/pty.pyi b/mypy/typeshed/stdlib/pty.pyi index 941915179c4a5..d1c78f9e3dd67 100644 --- a/mypy/typeshed/stdlib/pty.pyi +++ b/mypy/typeshed/stdlib/pty.pyi @@ -15,10 +15,14 @@ if sys.platform != "win32": def openpty() -> tuple[int, int]: ... if sys.version_info < (3, 14): - @deprecated("Deprecated in 3.12, to be removed in 3.14; use openpty() instead") - def master_open() -> tuple[int, str]: ... - @deprecated("Deprecated in 3.12, to be removed in 3.14; use openpty() instead") - def slave_open(tty_name: str) -> int: ... + if sys.version_info >= (3, 12): + @deprecated("Deprecated since Python 3.12; removed in Python 3.14. Use `openpty()` instead.") + def master_open() -> tuple[int, str]: ... + @deprecated("Deprecated since Python 3.12; removed in Python 3.14. Use `openpty()` instead.") + def slave_open(tty_name: str) -> int: ... + else: + def master_open() -> tuple[int, str]: ... + def slave_open(tty_name: str) -> int: ... def fork() -> tuple[int, int]: ... def spawn(argv: str | Iterable[str], master_read: _Reader = ..., stdin_read: _Reader = ...) -> int: ... diff --git a/mypy/typeshed/stdlib/re.pyi b/mypy/typeshed/stdlib/re.pyi index b080626c5802f..fb2a06d5e4c81 100644 --- a/mypy/typeshed/stdlib/re.pyi +++ b/mypy/typeshed/stdlib/re.pyi @@ -6,7 +6,7 @@ from _typeshed import MaybeNone, ReadableBuffer from collections.abc import Callable, Iterator, Mapping from types import GenericAlias from typing import Any, AnyStr, Final, Generic, Literal, TypeVar, final, overload -from typing_extensions import TypeAlias +from typing_extensions import TypeAlias, deprecated __all__ = [ "match", @@ -307,4 +307,8 @@ def escape(pattern: AnyStr) -> AnyStr: ... def purge() -> None: ... if sys.version_info < (3, 13): - def template(pattern: AnyStr | Pattern[AnyStr], flags: _FlagsType = 0) -> Pattern[AnyStr]: ... + if sys.version_info >= (3, 11): + @deprecated("Deprecated since Python 3.11; removed in Python 3.13. Use `re.compile()` instead.") + def template(pattern: AnyStr | Pattern[AnyStr], flags: _FlagsType = 0) -> Pattern[AnyStr]: ... # undocumented + else: + def template(pattern: AnyStr | Pattern[AnyStr], flags: _FlagsType = 0) -> Pattern[AnyStr]: ... # undocumented diff --git a/mypy/typeshed/stdlib/smtpd.pyi b/mypy/typeshed/stdlib/smtpd.pyi index 7392bd51627d9..dee7e949f42fa 100644 --- a/mypy/typeshed/stdlib/smtpd.pyi +++ b/mypy/typeshed/stdlib/smtpd.pyi @@ -4,7 +4,7 @@ import socket import sys from collections import defaultdict from typing import Any -from typing_extensions import TypeAlias +from typing_extensions import TypeAlias, deprecated if sys.version_info >= (3, 11): __all__ = ["SMTPChannel", "SMTPServer", "DebuggingServer", "PureProxy"] @@ -87,5 +87,6 @@ class PureProxy(SMTPServer): def process_message(self, peer: _Address, mailfrom: str, rcpttos: list[str], data: bytes | str) -> str | None: ... # type: ignore[override] if sys.version_info < (3, 11): + @deprecated("Deprecated since Python 3.9; removed in Python 3.11.") class MailmanProxy(PureProxy): def process_message(self, peer: _Address, mailfrom: str, rcpttos: list[str], data: bytes | str) -> str | None: ... # type: ignore[override] diff --git a/mypy/typeshed/stdlib/socket.pyi b/mypy/typeshed/stdlib/socket.pyi index 491551dd52b15..d62f4c228151c 100644 --- a/mypy/typeshed/stdlib/socket.pyi +++ b/mypy/typeshed/stdlib/socket.pyi @@ -1051,14 +1051,12 @@ if sys.version_info >= (3, 14): if sys.platform == "linux": from _socket import ( - CAN_RAW_ERR_FILTER as CAN_RAW_ERR_FILTER, IP_FREEBIND as IP_FREEBIND, IP_RECVORIGDSTADDR as IP_RECVORIGDSTADDR, - SO_ORIGINAL_DST as SO_ORIGINAL_DST, VMADDR_CID_LOCAL as VMADDR_CID_LOCAL, ) - __all__ += ["CAN_RAW_ERR_FILTER", "IP_FREEBIND", "IP_RECVORIGDSTADDR", "VMADDR_CID_LOCAL"] + __all__ += ["IP_FREEBIND", "IP_RECVORIGDSTADDR", "VMADDR_CID_LOCAL"] # Re-exported from errno EBADF: int diff --git a/mypy/typeshed/stdlib/ssl.pyi b/mypy/typeshed/stdlib/ssl.pyi index 1b8631d3fb12b..f1893ec3194f0 100644 --- a/mypy/typeshed/stdlib/ssl.pyi +++ b/mypy/typeshed/stdlib/ssl.pyi @@ -27,7 +27,7 @@ from _ssl import ( ) from _typeshed import ReadableBuffer, StrOrBytesPath, WriteableBuffer from collections.abc import Callable, Iterable -from typing import Any, Literal, NamedTuple, TypedDict, overload, type_check_only +from typing import Any, Final, Literal, NamedTuple, TypedDict, overload, type_check_only from typing_extensions import Never, Self, TypeAlias, deprecated if sys.version_info >= (3, 13): @@ -125,9 +125,9 @@ class VerifyMode(enum.IntEnum): CERT_OPTIONAL = 1 CERT_REQUIRED = 2 -CERT_NONE: VerifyMode -CERT_OPTIONAL: VerifyMode -CERT_REQUIRED: VerifyMode +CERT_NONE: Final = VerifyMode.CERT_NONE +CERT_OPTIONAL: Final = VerifyMode.CERT_OPTIONAL +CERT_REQUIRED: Final = VerifyMode.CERT_REQUIRED class VerifyFlags(enum.IntFlag): VERIFY_DEFAULT = 0 @@ -139,15 +139,15 @@ class VerifyFlags(enum.IntFlag): VERIFY_ALLOW_PROXY_CERTS = 64 VERIFY_X509_PARTIAL_CHAIN = 524288 -VERIFY_DEFAULT: VerifyFlags -VERIFY_CRL_CHECK_LEAF: VerifyFlags -VERIFY_CRL_CHECK_CHAIN: VerifyFlags -VERIFY_X509_STRICT: VerifyFlags -VERIFY_X509_TRUSTED_FIRST: VerifyFlags +VERIFY_DEFAULT: Final = VerifyFlags.VERIFY_DEFAULT +VERIFY_CRL_CHECK_LEAF: Final = VerifyFlags.VERIFY_CRL_CHECK_LEAF +VERIFY_CRL_CHECK_CHAIN: Final = VerifyFlags.VERIFY_CRL_CHECK_CHAIN +VERIFY_X509_STRICT: Final = VerifyFlags.VERIFY_X509_STRICT +VERIFY_X509_TRUSTED_FIRST: Final = VerifyFlags.VERIFY_X509_TRUSTED_FIRST if sys.version_info >= (3, 10): - VERIFY_ALLOW_PROXY_CERTS: VerifyFlags - VERIFY_X509_PARTIAL_CHAIN: VerifyFlags + VERIFY_ALLOW_PROXY_CERTS: Final = VerifyFlags.VERIFY_ALLOW_PROXY_CERTS + VERIFY_X509_PARTIAL_CHAIN: Final = VerifyFlags.VERIFY_X509_PARTIAL_CHAIN class _SSLMethod(enum.IntEnum): PROTOCOL_SSLv23 = 2 @@ -160,15 +160,15 @@ class _SSLMethod(enum.IntEnum): PROTOCOL_TLS_CLIENT = 16 PROTOCOL_TLS_SERVER = 17 -PROTOCOL_SSLv23: _SSLMethod -PROTOCOL_SSLv2: _SSLMethod -PROTOCOL_SSLv3: _SSLMethod -PROTOCOL_TLSv1: _SSLMethod -PROTOCOL_TLSv1_1: _SSLMethod -PROTOCOL_TLSv1_2: _SSLMethod -PROTOCOL_TLS: _SSLMethod -PROTOCOL_TLS_CLIENT: _SSLMethod -PROTOCOL_TLS_SERVER: _SSLMethod +PROTOCOL_SSLv23: Final = _SSLMethod.PROTOCOL_SSLv23 +PROTOCOL_SSLv2: Final = _SSLMethod.PROTOCOL_SSLv2 +PROTOCOL_SSLv3: Final = _SSLMethod.PROTOCOL_SSLv3 +PROTOCOL_TLSv1: Final = _SSLMethod.PROTOCOL_TLSv1 +PROTOCOL_TLSv1_1: Final = _SSLMethod.PROTOCOL_TLSv1_1 +PROTOCOL_TLSv1_2: Final = _SSLMethod.PROTOCOL_TLSv1_2 +PROTOCOL_TLS: Final = _SSLMethod.PROTOCOL_TLS +PROTOCOL_TLS_CLIENT: Final = _SSLMethod.PROTOCOL_TLS_CLIENT +PROTOCOL_TLS_SERVER: Final = _SSLMethod.PROTOCOL_TLS_SERVER class Options(enum.IntFlag): OP_ALL = 2147483728 @@ -191,29 +191,29 @@ class Options(enum.IntFlag): if sys.version_info >= (3, 11) or sys.platform == "linux": OP_IGNORE_UNEXPECTED_EOF = 128 -OP_ALL: Options -OP_NO_SSLv2: Options -OP_NO_SSLv3: Options -OP_NO_TLSv1: Options -OP_NO_TLSv1_1: Options -OP_NO_TLSv1_2: Options -OP_NO_TLSv1_3: Options -OP_CIPHER_SERVER_PREFERENCE: Options -OP_SINGLE_DH_USE: Options -OP_SINGLE_ECDH_USE: Options -OP_NO_COMPRESSION: Options -OP_NO_TICKET: Options -OP_NO_RENEGOTIATION: Options -OP_ENABLE_MIDDLEBOX_COMPAT: Options +OP_ALL: Final = Options.OP_ALL +OP_NO_SSLv2: Final = Options.OP_NO_SSLv2 +OP_NO_SSLv3: Final = Options.OP_NO_SSLv3 +OP_NO_TLSv1: Final = Options.OP_NO_TLSv1 +OP_NO_TLSv1_1: Final = Options.OP_NO_TLSv1_1 +OP_NO_TLSv1_2: Final = Options.OP_NO_TLSv1_2 +OP_NO_TLSv1_3: Final = Options.OP_NO_TLSv1_3 +OP_CIPHER_SERVER_PREFERENCE: Final = Options.OP_CIPHER_SERVER_PREFERENCE +OP_SINGLE_DH_USE: Final = Options.OP_SINGLE_DH_USE +OP_SINGLE_ECDH_USE: Final = Options.OP_SINGLE_ECDH_USE +OP_NO_COMPRESSION: Final = Options.OP_NO_COMPRESSION +OP_NO_TICKET: Final = Options.OP_NO_TICKET +OP_NO_RENEGOTIATION: Final = Options.OP_NO_RENEGOTIATION +OP_ENABLE_MIDDLEBOX_COMPAT: Final = Options.OP_ENABLE_MIDDLEBOX_COMPAT if sys.version_info >= (3, 12): - OP_LEGACY_SERVER_CONNECT: Options - OP_ENABLE_KTLS: Options + OP_LEGACY_SERVER_CONNECT: Final = Options.OP_LEGACY_SERVER_CONNECT + OP_ENABLE_KTLS: Final = Options.OP_ENABLE_KTLS if sys.version_info >= (3, 11) or sys.platform == "linux": - OP_IGNORE_UNEXPECTED_EOF: Options + OP_IGNORE_UNEXPECTED_EOF: Final = Options.OP_IGNORE_UNEXPECTED_EOF -HAS_NEVER_CHECK_COMMON_NAME: bool +HAS_NEVER_CHECK_COMMON_NAME: Final[bool] -CHANNEL_BINDING_TYPES: list[str] +CHANNEL_BINDING_TYPES: Final[list[str]] class AlertDescription(enum.IntEnum): ALERT_DESCRIPTION_ACCESS_DENIED = 49 @@ -244,33 +244,33 @@ class AlertDescription(enum.IntEnum): ALERT_DESCRIPTION_UNSUPPORTED_EXTENSION = 110 ALERT_DESCRIPTION_USER_CANCELLED = 90 -ALERT_DESCRIPTION_HANDSHAKE_FAILURE: AlertDescription -ALERT_DESCRIPTION_INTERNAL_ERROR: AlertDescription -ALERT_DESCRIPTION_ACCESS_DENIED: AlertDescription -ALERT_DESCRIPTION_BAD_CERTIFICATE: AlertDescription -ALERT_DESCRIPTION_BAD_CERTIFICATE_HASH_VALUE: AlertDescription -ALERT_DESCRIPTION_BAD_CERTIFICATE_STATUS_RESPONSE: AlertDescription -ALERT_DESCRIPTION_BAD_RECORD_MAC: AlertDescription -ALERT_DESCRIPTION_CERTIFICATE_EXPIRED: AlertDescription -ALERT_DESCRIPTION_CERTIFICATE_REVOKED: AlertDescription -ALERT_DESCRIPTION_CERTIFICATE_UNKNOWN: AlertDescription -ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE: AlertDescription -ALERT_DESCRIPTION_CLOSE_NOTIFY: AlertDescription -ALERT_DESCRIPTION_DECODE_ERROR: AlertDescription -ALERT_DESCRIPTION_DECOMPRESSION_FAILURE: AlertDescription -ALERT_DESCRIPTION_DECRYPT_ERROR: AlertDescription -ALERT_DESCRIPTION_ILLEGAL_PARAMETER: AlertDescription -ALERT_DESCRIPTION_INSUFFICIENT_SECURITY: AlertDescription -ALERT_DESCRIPTION_NO_RENEGOTIATION: AlertDescription -ALERT_DESCRIPTION_PROTOCOL_VERSION: AlertDescription -ALERT_DESCRIPTION_RECORD_OVERFLOW: AlertDescription -ALERT_DESCRIPTION_UNEXPECTED_MESSAGE: AlertDescription -ALERT_DESCRIPTION_UNKNOWN_CA: AlertDescription -ALERT_DESCRIPTION_UNKNOWN_PSK_IDENTITY: AlertDescription -ALERT_DESCRIPTION_UNRECOGNIZED_NAME: AlertDescription -ALERT_DESCRIPTION_UNSUPPORTED_CERTIFICATE: AlertDescription -ALERT_DESCRIPTION_UNSUPPORTED_EXTENSION: AlertDescription -ALERT_DESCRIPTION_USER_CANCELLED: AlertDescription +ALERT_DESCRIPTION_HANDSHAKE_FAILURE: Final = AlertDescription.ALERT_DESCRIPTION_HANDSHAKE_FAILURE +ALERT_DESCRIPTION_INTERNAL_ERROR: Final = AlertDescription.ALERT_DESCRIPTION_INTERNAL_ERROR +ALERT_DESCRIPTION_ACCESS_DENIED: Final = AlertDescription.ALERT_DESCRIPTION_ACCESS_DENIED +ALERT_DESCRIPTION_BAD_CERTIFICATE: Final = AlertDescription.ALERT_DESCRIPTION_BAD_CERTIFICATE +ALERT_DESCRIPTION_BAD_CERTIFICATE_HASH_VALUE: Final = AlertDescription.ALERT_DESCRIPTION_BAD_CERTIFICATE_HASH_VALUE +ALERT_DESCRIPTION_BAD_CERTIFICATE_STATUS_RESPONSE: Final = AlertDescription.ALERT_DESCRIPTION_BAD_CERTIFICATE_STATUS_RESPONSE +ALERT_DESCRIPTION_BAD_RECORD_MAC: Final = AlertDescription.ALERT_DESCRIPTION_BAD_RECORD_MAC +ALERT_DESCRIPTION_CERTIFICATE_EXPIRED: Final = AlertDescription.ALERT_DESCRIPTION_CERTIFICATE_EXPIRED +ALERT_DESCRIPTION_CERTIFICATE_REVOKED: Final = AlertDescription.ALERT_DESCRIPTION_CERTIFICATE_REVOKED +ALERT_DESCRIPTION_CERTIFICATE_UNKNOWN: Final = AlertDescription.ALERT_DESCRIPTION_CERTIFICATE_UNKNOWN +ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE: Final = AlertDescription.ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +ALERT_DESCRIPTION_CLOSE_NOTIFY: Final = AlertDescription.ALERT_DESCRIPTION_CLOSE_NOTIFY +ALERT_DESCRIPTION_DECODE_ERROR: Final = AlertDescription.ALERT_DESCRIPTION_DECODE_ERROR +ALERT_DESCRIPTION_DECOMPRESSION_FAILURE: Final = AlertDescription.ALERT_DESCRIPTION_DECOMPRESSION_FAILURE +ALERT_DESCRIPTION_DECRYPT_ERROR: Final = AlertDescription.ALERT_DESCRIPTION_DECRYPT_ERROR +ALERT_DESCRIPTION_ILLEGAL_PARAMETER: Final = AlertDescription.ALERT_DESCRIPTION_ILLEGAL_PARAMETER +ALERT_DESCRIPTION_INSUFFICIENT_SECURITY: Final = AlertDescription.ALERT_DESCRIPTION_INSUFFICIENT_SECURITY +ALERT_DESCRIPTION_NO_RENEGOTIATION: Final = AlertDescription.ALERT_DESCRIPTION_NO_RENEGOTIATION +ALERT_DESCRIPTION_PROTOCOL_VERSION: Final = AlertDescription.ALERT_DESCRIPTION_PROTOCOL_VERSION +ALERT_DESCRIPTION_RECORD_OVERFLOW: Final = AlertDescription.ALERT_DESCRIPTION_RECORD_OVERFLOW +ALERT_DESCRIPTION_UNEXPECTED_MESSAGE: Final = AlertDescription.ALERT_DESCRIPTION_UNEXPECTED_MESSAGE +ALERT_DESCRIPTION_UNKNOWN_CA: Final = AlertDescription.ALERT_DESCRIPTION_UNKNOWN_CA +ALERT_DESCRIPTION_UNKNOWN_PSK_IDENTITY: Final = AlertDescription.ALERT_DESCRIPTION_UNKNOWN_PSK_IDENTITY +ALERT_DESCRIPTION_UNRECOGNIZED_NAME: Final = AlertDescription.ALERT_DESCRIPTION_UNRECOGNIZED_NAME +ALERT_DESCRIPTION_UNSUPPORTED_CERTIFICATE: Final = AlertDescription.ALERT_DESCRIPTION_UNSUPPORTED_CERTIFICATE +ALERT_DESCRIPTION_UNSUPPORTED_EXTENSION: Final = AlertDescription.ALERT_DESCRIPTION_UNSUPPORTED_EXTENSION +ALERT_DESCRIPTION_USER_CANCELLED: Final = AlertDescription.ALERT_DESCRIPTION_USER_CANCELLED # This class is not exposed. It calls itself ssl._ASN1Object. @type_check_only @@ -518,20 +518,20 @@ class SSLErrorNumber(enum.IntEnum): SSL_ERROR_WANT_X509_LOOKUP = 4 SSL_ERROR_ZERO_RETURN = 6 -SSL_ERROR_EOF: SSLErrorNumber # undocumented -SSL_ERROR_INVALID_ERROR_CODE: SSLErrorNumber # undocumented -SSL_ERROR_SSL: SSLErrorNumber # undocumented -SSL_ERROR_SYSCALL: SSLErrorNumber # undocumented -SSL_ERROR_WANT_CONNECT: SSLErrorNumber # undocumented -SSL_ERROR_WANT_READ: SSLErrorNumber # undocumented -SSL_ERROR_WANT_WRITE: SSLErrorNumber # undocumented -SSL_ERROR_WANT_X509_LOOKUP: SSLErrorNumber # undocumented -SSL_ERROR_ZERO_RETURN: SSLErrorNumber # undocumented +SSL_ERROR_EOF: Final = SSLErrorNumber.SSL_ERROR_EOF # undocumented +SSL_ERROR_INVALID_ERROR_CODE: Final = SSLErrorNumber.SSL_ERROR_INVALID_ERROR_CODE # undocumented +SSL_ERROR_SSL: Final = SSLErrorNumber.SSL_ERROR_SSL # undocumented +SSL_ERROR_SYSCALL: Final = SSLErrorNumber.SSL_ERROR_SYSCALL # undocumented +SSL_ERROR_WANT_CONNECT: Final = SSLErrorNumber.SSL_ERROR_WANT_CONNECT # undocumented +SSL_ERROR_WANT_READ: Final = SSLErrorNumber.SSL_ERROR_WANT_READ # undocumented +SSL_ERROR_WANT_WRITE: Final = SSLErrorNumber.SSL_ERROR_WANT_WRITE # undocumented +SSL_ERROR_WANT_X509_LOOKUP: Final = SSLErrorNumber.SSL_ERROR_WANT_X509_LOOKUP # undocumented +SSL_ERROR_ZERO_RETURN: Final = SSLErrorNumber.SSL_ERROR_ZERO_RETURN # undocumented def get_protocol_name(protocol_code: int) -> str: ... -PEM_FOOTER: str -PEM_HEADER: str -SOCK_STREAM: int -SOL_SOCKET: int -SO_TYPE: int +PEM_FOOTER: Final[str] +PEM_HEADER: Final[str] +SOCK_STREAM: Final = socket.SOCK_STREAM +SOL_SOCKET: Final = socket.SOL_SOCKET +SO_TYPE: Final = socket.SO_TYPE diff --git a/mypy/typeshed/stdlib/sys/__init__.pyi b/mypy/typeshed/stdlib/sys/__init__.pyi index 149f374d6e179..b16e7c0abd052 100644 --- a/mypy/typeshed/stdlib/sys/__init__.pyi +++ b/mypy/typeshed/stdlib/sys/__init__.pyi @@ -181,6 +181,14 @@ class _flags(_UninstantiableStructseq, tuple[int, ...]): if sys.version_info >= (3, 11): @property def safe_path(self) -> bool: ... + if sys.version_info >= (3, 13): + @property + def gil(self) -> Literal[0, 1]: ... + if sys.version_info >= (3, 14): + @property + def thread_inherit_context(self) -> Literal[0, 1]: ... + @property + def context_aware_warnings(self) -> Literal[0, 1]: ... # Whether or not this exists on lower versions of Python # may depend on which patch release you're using # (it was backported to all Python versions on 3.8+ as a security fix) diff --git a/mypy/typeshed/stdlib/tkinter/__init__.pyi b/mypy/typeshed/stdlib/tkinter/__init__.pyi index b802d5e97c840..76b2ddcf17df1 100644 --- a/mypy/typeshed/stdlib/tkinter/__init__.pyi +++ b/mypy/typeshed/stdlib/tkinter/__init__.pyi @@ -1029,6 +1029,7 @@ class Tk(Misc, Wm): def loadtk(self) -> None: ... def record(self, script, /): ... if sys.version_info < (3, 11): + @deprecated("Deprecated since Python 3.9; removed in Python 3.11. Use `splitlist()` instead.") def split(self, arg, /): ... def splitlist(self, arg, /): ... diff --git a/mypy/typeshed/stdlib/turtle.pyi b/mypy/typeshed/stdlib/turtle.pyi index 7d39026b80413..e41476b73b8c9 100644 --- a/mypy/typeshed/stdlib/turtle.pyi +++ b/mypy/typeshed/stdlib/turtle.pyi @@ -4,7 +4,7 @@ from collections.abc import Callable, Generator, Sequence from contextlib import contextmanager from tkinter import Canvas, Frame, Misc, PhotoImage, Scrollbar from typing import Any, ClassVar, Literal, TypedDict, overload, type_check_only -from typing_extensions import Self, TypeAlias +from typing_extensions import Self, TypeAlias, deprecated __all__ = [ "ScrolledCanvas", @@ -426,6 +426,7 @@ class RawTurtle(TPen, TNavigator): # type: ignore[misc] # Conflicting methods def get_shapepoly(self) -> _PolygonCoords | None: ... if sys.version_info < (3, 13): + @deprecated("Deprecated since Python 3.1; removed in Python 3.13. Use `tiltangle()` instead.") def settiltangle(self, angle: float) -> None: ... @overload @@ -707,6 +708,7 @@ def shapetransform( def get_shapepoly() -> _PolygonCoords | None: ... if sys.version_info < (3, 13): + @deprecated("Deprecated since Python 3.1; removed in Python 3.13. Use `tiltangle()` instead.") def settiltangle(angle: float) -> None: ... @overload diff --git a/mypy/typeshed/stdlib/typing.pyi b/mypy/typeshed/stdlib/typing.pyi index a85aa2e2dc83a..fd9da29addbf4 100644 --- a/mypy/typeshed/stdlib/typing.pyi +++ b/mypy/typeshed/stdlib/typing.pyi @@ -207,12 +207,12 @@ class TypeVar: contravariant: bool = False, ) -> None: ... if sys.version_info >= (3, 10): - def __or__(self, right: Any) -> _SpecialForm: ... # AnnotationForm - def __ror__(self, left: Any) -> _SpecialForm: ... # AnnotationForm + def __or__(self, right: Any, /) -> _SpecialForm: ... # AnnotationForm + def __ror__(self, left: Any, /) -> _SpecialForm: ... # AnnotationForm if sys.version_info >= (3, 11): - def __typing_subst__(self, arg: Any) -> Any: ... + def __typing_subst__(self, arg: Any, /) -> Any: ... if sys.version_info >= (3, 13): - def __typing_prepare_subst__(self, alias: Any, args: Any) -> tuple[Any, ...]: ... + def __typing_prepare_subst__(self, alias: Any, args: Any, /) -> tuple[Any, ...]: ... def has_default(self) -> bool: ... if sys.version_info >= (3, 14): @property @@ -273,8 +273,8 @@ if sys.version_info >= (3, 11): def __init__(self, name: str) -> None: ... def __iter__(self) -> Any: ... - def __typing_subst__(self, arg: Never) -> Never: ... - def __typing_prepare_subst__(self, alias: Any, args: Any) -> tuple[Any, ...]: ... + def __typing_subst__(self, arg: Never, /) -> Never: ... + def __typing_prepare_subst__(self, alias: Any, args: Any, /) -> tuple[Any, ...]: ... if sys.version_info >= (3, 14): @property def evaluate_default(self) -> EvaluateFunc | None: ... @@ -289,7 +289,7 @@ if sys.version_info >= (3, 10): else: def __init__(self, origin: ParamSpec) -> None: ... - def __eq__(self, other: object) -> bool: ... + def __eq__(self, other: object, /) -> bool: ... __hash__: ClassVar[None] # type: ignore[assignment] @final @@ -301,7 +301,7 @@ if sys.version_info >= (3, 10): else: def __init__(self, origin: ParamSpec) -> None: ... - def __eq__(self, other: object) -> bool: ... + def __eq__(self, other: object, /) -> bool: ... __hash__: ClassVar[None] # type: ignore[assignment] @final @@ -365,11 +365,11 @@ if sys.version_info >= (3, 10): @property def kwargs(self) -> ParamSpecKwargs: ... if sys.version_info >= (3, 11): - def __typing_subst__(self, arg: Any) -> Any: ... - def __typing_prepare_subst__(self, alias: Any, args: Any) -> tuple[Any, ...]: ... + def __typing_subst__(self, arg: Any, /) -> Any: ... + def __typing_prepare_subst__(self, alias: Any, args: Any, /) -> tuple[Any, ...]: ... - def __or__(self, right: Any) -> _SpecialForm: ... - def __ror__(self, left: Any) -> _SpecialForm: ... + def __or__(self, right: Any, /) -> _SpecialForm: ... + def __ror__(self, left: Any, /) -> _SpecialForm: ... if sys.version_info >= (3, 13): def has_default(self) -> bool: ... if sys.version_info >= (3, 14): @@ -710,7 +710,7 @@ class ItemsView(MappingView, AbstractSet[tuple[_KT_co, _VT_co]], Generic[_KT_co, def __init__(self, mapping: Mapping[_KT_co, _VT_co]) -> None: ... # undocumented def __and__(self, other: Iterable[Any]) -> set[tuple[_KT_co, _VT_co]]: ... def __rand__(self, other: Iterable[_T]) -> set[_T]: ... - def __contains__(self, item: object) -> bool: ... + def __contains__(self, item: tuple[object, object]) -> bool: ... # type: ignore[override] def __iter__(self) -> Iterator[tuple[_KT_co, _VT_co]]: ... def __or__(self, other: Iterable[_T]) -> set[tuple[_KT_co, _VT_co] | _T]: ... def __ror__(self, other: Iterable[_T]) -> set[tuple[_KT_co, _VT_co] | _T]: ... @@ -1115,9 +1115,9 @@ if sys.version_info >= (3, 12): # It's writable on types, but not on instances of TypeAliasType. @property def __module__(self) -> str | None: ... # type: ignore[override] - def __getitem__(self, parameters: Any) -> GenericAlias: ... # AnnotationForm - def __or__(self, right: Any) -> _SpecialForm: ... - def __ror__(self, left: Any) -> _SpecialForm: ... + def __getitem__(self, parameters: Any, /) -> GenericAlias: ... # AnnotationForm + def __or__(self, right: Any, /) -> _SpecialForm: ... + def __ror__(self, left: Any, /) -> _SpecialForm: ... if sys.version_info >= (3, 14): @property def evaluate_value(self) -> EvaluateFunc: ... diff --git a/mypy/typeshed/stdlib/typing_extensions.pyi b/mypy/typeshed/stdlib/typing_extensions.pyi index 22b6ada8ffb7b..71bf3d87d4996 100644 --- a/mypy/typeshed/stdlib/typing_extensions.pyi +++ b/mypy/typeshed/stdlib/typing_extensions.pyi @@ -593,8 +593,8 @@ else: def __getitem__(self, parameters: Incomplete | tuple[Incomplete, ...]) -> AnnotationForm: ... def __init_subclass__(cls, *args: Unused, **kwargs: Unused) -> NoReturn: ... if sys.version_info >= (3, 10): - def __or__(self, right: Any) -> _SpecialForm: ... - def __ror__(self, left: Any) -> _SpecialForm: ... + def __or__(self, right: Any, /) -> _SpecialForm: ... + def __ror__(self, left: Any, /) -> _SpecialForm: ... # PEP 727 class Doc: diff --git a/mypy/typeshed/stdlib/urllib/request.pyi b/mypy/typeshed/stdlib/urllib/request.pyi index b99577c1cf71b..876b9d3f165cd 100644 --- a/mypy/typeshed/stdlib/urllib/request.pyi +++ b/mypy/typeshed/stdlib/urllib/request.pyi @@ -325,7 +325,7 @@ def urlretrieve( def urlcleanup() -> None: ... if sys.version_info < (3, 14): - @deprecated("Deprecated since Python 3.3; Removed in 3.14; Use newer urlopen functions and methods.") + @deprecated("Deprecated since Python 3.3; removed in Python 3.14. Use newer `urlopen` functions and methods.") class URLopener: version: ClassVar[str] def __init__(self, proxies: dict[str, str] | None = None, **x509: str) -> None: ... @@ -356,7 +356,7 @@ if sys.version_info < (3, 14): def open_unknown_proxy(self, proxy: str, fullurl: str, data: ReadableBuffer | None = None) -> None: ... # undocumented def __del__(self) -> None: ... - @deprecated("Deprecated since Python 3.3; Removed in 3.14; Use newer urlopen functions and methods.") + @deprecated("Deprecated since Python 3.3; removed in Python 3.14. Use newer `urlopen` functions and methods.") class FancyURLopener(URLopener): def prompt_user_passwd(self, host: str, realm: str) -> tuple[str, str]: ... def get_user_passwd(self, host: str, realm: str, clear_cache: int = 0) -> tuple[str, str]: ... # undocumented diff --git a/mypy/typeshed/stdlib/winreg.pyi b/mypy/typeshed/stdlib/winreg.pyi index d4d04817d7e09..0a22bb23d8f66 100644 --- a/mypy/typeshed/stdlib/winreg.pyi +++ b/mypy/typeshed/stdlib/winreg.pyi @@ -123,7 +123,7 @@ if sys.platform == "win32": def __int__(self) -> int: ... def __enter__(self) -> Self: ... def __exit__( - self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: TracebackType | None + self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: TracebackType | None, / ) -> bool | None: ... def Close(self) -> None: ... def Detach(self) -> int: ... diff --git a/mypy/typeshed/stdlib/zipimport.pyi b/mypy/typeshed/stdlib/zipimport.pyi index 4aab318e7c71d..22af3c272759b 100644 --- a/mypy/typeshed/stdlib/zipimport.pyi +++ b/mypy/typeshed/stdlib/zipimport.pyi @@ -27,8 +27,14 @@ class zipimporter(_LoaderBasics): def __init__(self, path: StrOrBytesPath) -> None: ... if sys.version_info < (3, 12): - def find_loader(self, fullname: str, path: str | None = None) -> tuple[zipimporter | None, list[str]]: ... # undocumented - def find_module(self, fullname: str, path: str | None = None) -> zipimporter | None: ... + if sys.version_info >= (3, 10): + @deprecated("Deprecated since Python 3.10; removed in Python 3.12. Use `find_spec()` instead.") + def find_loader(self, fullname: str, path: str | None = None) -> tuple[zipimporter | None, list[str]]: ... + @deprecated("Deprecated since Python 3.10; removed in Python 3.12. Use `find_spec()` instead.") + def find_module(self, fullname: str, path: str | None = None) -> zipimporter | None: ... + else: + def find_loader(self, fullname: str, path: str | None = None) -> tuple[zipimporter | None, list[str]]: ... + def find_module(self, fullname: str, path: str | None = None) -> zipimporter | None: ... def get_code(self, fullname: str) -> CodeType: ... def get_data(self, pathname: str) -> bytes: ... @@ -42,10 +48,12 @@ class zipimporter(_LoaderBasics): def get_source(self, fullname: str) -> str | None: ... def is_package(self, fullname: str) -> bool: ... - @deprecated("Deprecated since 3.10; use exec_module() instead") - def load_module(self, fullname: str) -> ModuleType: ... if sys.version_info >= (3, 10): + @deprecated("Deprecated since Python 3.10; removed in Python 3.15. Use `exec_module()` instead.") + def load_module(self, fullname: str) -> ModuleType: ... def exec_module(self, module: ModuleType) -> None: ... def create_module(self, spec: ModuleSpec) -> None: ... def find_spec(self, fullname: str, target: ModuleType | None = None) -> ModuleSpec | None: ... def invalidate_caches(self) -> None: ... + else: + def load_module(self, fullname: str) -> ModuleType: ... diff --git a/mypy/typeshed/stdlib/zlib.pyi b/mypy/typeshed/stdlib/zlib.pyi index 7cafb44b34a7b..4e410fdd18ad9 100644 --- a/mypy/typeshed/stdlib/zlib.pyi +++ b/mypy/typeshed/stdlib/zlib.pyi @@ -4,11 +4,11 @@ from typing import Any, Final, final, type_check_only from typing_extensions import Self DEFLATED: Final = 8 -DEF_MEM_LEVEL: int # can change +DEF_MEM_LEVEL: Final[int] DEF_BUF_SIZE: Final = 16384 -MAX_WBITS: int -ZLIB_VERSION: str # can change -ZLIB_RUNTIME_VERSION: str # can change +MAX_WBITS: Final[int] +ZLIB_VERSION: Final[str] +ZLIB_RUNTIME_VERSION: Final[str] Z_NO_COMPRESSION: Final = 0 Z_PARTIAL_FLUSH: Final = 1 Z_BEST_COMPRESSION: Final = 9 @@ -26,6 +26,10 @@ Z_RLE: Final = 3 Z_SYNC_FLUSH: Final = 2 Z_TREES: Final = 6 +if sys.version_info >= (3, 14) and sys.platform == "win32": + # Available when zlib was built with zlib-ng, usually only on Windows + ZLIBNG_VERSION: Final[str] + class error(Exception): ... # This class is not exposed at runtime. It calls itself zlib.Compress. From abf61fbd49ff244e86867d60de0945162328faf9 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 18 Aug 2025 13:30:51 +0100 Subject: [PATCH 177/424] [mypyc] Refactor building IR for "in" against a tuple (#19679) This covers `x in (a, b)` and `x in [a, b]`. Also add an irbuild test for a use case that is currently inefficient. This is in preparation for improving IR building for "in" against tuple. --- mypyc/irbuild/expression.py | 128 +++++++++++++++-------------- mypyc/test-data/irbuild-tuple.test | 30 +++++++ 2 files changed, 95 insertions(+), 63 deletions(-) diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index c3d863fa96dee..a600afff4bc94 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -701,24 +701,70 @@ def transform_comparison_expr(builder: IRBuilder, e: ComparisonExpr) -> Value: # x in (...)/[...] # x not in (...)/[...] first_op = e.operators[0] - if ( - first_op in ["in", "not in"] - and len(e.operators) == 1 - and isinstance(e.operands[1], (TupleExpr, ListExpr)) - ): - items = e.operands[1].items + if first_op in ["in", "not in"] and len(e.operators) == 1: + result = try_specialize_in_expr(builder, first_op, e.operands[0], e.operands[1], e.line) + if result is not None: + return result + + if len(e.operators) == 1: + # Special some common simple cases + if first_op in ("is", "is not"): + right_expr = e.operands[1] + if isinstance(right_expr, NameExpr) and right_expr.fullname == "builtins.None": + # Special case 'is None' / 'is not None'. + return translate_is_none(builder, e.operands[0], negated=first_op != "is") + left_expr = e.operands[0] + if is_int_rprimitive(builder.node_type(left_expr)): + right_expr = e.operands[1] + if is_int_rprimitive(builder.node_type(right_expr)): + if first_op in int_borrow_friendly_op: + borrow_left = is_borrow_friendly_expr(builder, right_expr) + left = builder.accept(left_expr, can_borrow=borrow_left) + right = builder.accept(right_expr, can_borrow=True) + return builder.binary_op(left, right, first_op, e.line) + + # TODO: Don't produce an expression when used in conditional context + # All of the trickiness here is due to support for chained conditionals + # (`e1 < e2 > e3`, etc). `e1 < e2 > e3` is approximately equivalent to + # `e1 < e2 and e2 > e3` except that `e2` is only evaluated once. + expr_type = builder.node_type(e) + + # go(i, prev) generates code for `ei opi e{i+1} op{i+1} ... en`, + # assuming that prev contains the value of `ei`. + def go(i: int, prev: Value) -> Value: + if i == len(e.operators) - 1: + return transform_basic_comparison( + builder, e.operators[i], prev, builder.accept(e.operands[i + 1]), e.line + ) + + next = builder.accept(e.operands[i + 1]) + return builder.builder.shortcircuit_helper( + "and", + expr_type, + lambda: transform_basic_comparison(builder, e.operators[i], prev, next, e.line), + lambda: go(i + 1, next), + e.line, + ) + + return go(0, builder.accept(e.operands[0])) + + +def try_specialize_in_expr( + builder: IRBuilder, op: str, lhs: Expression, rhs: Expression, line: int +) -> Value | None: + if isinstance(rhs, (TupleExpr, ListExpr)): + items = rhs.items n_items = len(items) # x in y -> x == y[0] or ... or x == y[n] # x not in y -> x != y[0] and ... and x != y[n] # 16 is arbitrarily chosen to limit code size if 1 < n_items < 16: - if e.operators[0] == "in": + if op == "in": bin_op = "or" cmp_op = "==" else: bin_op = "and" cmp_op = "!=" - lhs = e.operands[0] mypy_file = builder.graph["builtins"].tree assert mypy_file is not None info = mypy_file.names["bool"].node @@ -738,78 +784,34 @@ def transform_comparison_expr(builder: IRBuilder, e: ComparisonExpr) -> Value: # x in [y]/(y) -> x == y # x not in [y]/(y) -> x != y elif n_items == 1: - if e.operators[0] == "in": + if op == "in": cmp_op = "==" else: cmp_op = "!=" - e.operators = [cmp_op] - e.operands[1] = items[0] + left = builder.accept(lhs) + right = builder.accept(items[0]) + return transform_basic_comparison(builder, cmp_op, left, right, line) # x in []/() -> False # x not in []/() -> True elif n_items == 0: - if e.operators[0] == "in": + if op == "in": return builder.false() else: return builder.true() # x in {...} # x not in {...} - if ( - first_op in ("in", "not in") - and len(e.operators) == 1 - and isinstance(e.operands[1], SetExpr) - ): - set_literal = precompute_set_literal(builder, e.operands[1]) + if isinstance(rhs, SetExpr): + set_literal = precompute_set_literal(builder, rhs) if set_literal is not None: - lhs = e.operands[0] result = builder.builder.primitive_op( - set_in_op, [builder.accept(lhs), set_literal], e.line, bool_rprimitive + set_in_op, [builder.accept(lhs), set_literal], line, bool_rprimitive ) - if first_op == "not in": - return builder.unary_op(result, "not", e.line) + if op == "not in": + return builder.unary_op(result, "not", line) return result - if len(e.operators) == 1: - # Special some common simple cases - if first_op in ("is", "is not"): - right_expr = e.operands[1] - if isinstance(right_expr, NameExpr) and right_expr.fullname == "builtins.None": - # Special case 'is None' / 'is not None'. - return translate_is_none(builder, e.operands[0], negated=first_op != "is") - left_expr = e.operands[0] - if is_int_rprimitive(builder.node_type(left_expr)): - right_expr = e.operands[1] - if is_int_rprimitive(builder.node_type(right_expr)): - if first_op in int_borrow_friendly_op: - borrow_left = is_borrow_friendly_expr(builder, right_expr) - left = builder.accept(left_expr, can_borrow=borrow_left) - right = builder.accept(right_expr, can_borrow=True) - return builder.binary_op(left, right, first_op, e.line) - - # TODO: Don't produce an expression when used in conditional context - # All of the trickiness here is due to support for chained conditionals - # (`e1 < e2 > e3`, etc). `e1 < e2 > e3` is approximately equivalent to - # `e1 < e2 and e2 > e3` except that `e2` is only evaluated once. - expr_type = builder.node_type(e) - - # go(i, prev) generates code for `ei opi e{i+1} op{i+1} ... en`, - # assuming that prev contains the value of `ei`. - def go(i: int, prev: Value) -> Value: - if i == len(e.operators) - 1: - return transform_basic_comparison( - builder, e.operators[i], prev, builder.accept(e.operands[i + 1]), e.line - ) - - next = builder.accept(e.operands[i + 1]) - return builder.builder.shortcircuit_helper( - "and", - expr_type, - lambda: transform_basic_comparison(builder, e.operators[i], prev, next, e.line), - lambda: go(i + 1, next), - e.line, - ) - - return go(0, builder.accept(e.operands[0])) + return None def translate_is_none(builder: IRBuilder, expr: Expression, negated: bool) -> Value: diff --git a/mypyc/test-data/irbuild-tuple.test b/mypyc/test-data/irbuild-tuple.test index 0342ec304c25c..712b9c26355ab 100644 --- a/mypyc/test-data/irbuild-tuple.test +++ b/mypyc/test-data/irbuild-tuple.test @@ -209,6 +209,36 @@ L5: L6: return r3 +[case testTupleOperatorInFinalTuple] +from typing import Final + +tt: Final = (1, 2) + +def f(x: int) -> bool: + return x in tt +[out] +def f(x): + x :: int + r0 :: tuple[int, int] + r1 :: bool + r2, r3 :: object + r4 :: i32 + r5 :: bit + r6 :: bool +L0: + r0 = __main__.tt :: static + if is_error(r0) goto L1 else goto L2 +L1: + r1 = raise NameError('value for final name "tt" was not set') + unreachable +L2: + r2 = box(int, x) + r3 = box(tuple[int, int], r0) + r4 = PySequence_Contains(r3, r2) + r5 = r4 >= 0 :: signed + r6 = truncate r4: i32 to builtins.bool + return r6 + [case testTupleBuiltFromList] def f(val: int) -> bool: return val % 2 == 0 From 1f9505c13657c6028553e2628b8194b8e8d247d2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 18 Aug 2025 15:11:24 +0100 Subject: [PATCH 178/424] [mypyc] Speed up "in" against final fixed-length tuple (#19682) Previously the `in` operation here boxed the tuple: ``` TUP: Final = ('x', 'y') ... if s in TUP: ... ``` Now we don't box the tuple and inline the comparisons against each tuple item instead, which is more efficient. Also make the semantics closer to Python and add tests. --- mypyc/irbuild/expression.py | 59 ++++++++----- mypyc/test-data/irbuild-tuple.test | 88 ++++++++++++++------ mypyc/test-data/run-lists.test | 128 ++++++++++++++--------------- mypyc/test-data/run-tuples.test | 83 +++++++++++++++++++ 4 files changed, 247 insertions(+), 111 deletions(-) diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index a600afff4bc94..2df47680d27ce 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -752,35 +752,51 @@ def go(i: int, prev: Value) -> Value: def try_specialize_in_expr( builder: IRBuilder, op: str, lhs: Expression, rhs: Expression, line: int ) -> Value | None: + left: Value | None = None + items: list[Value] | None = None + if isinstance(rhs, (TupleExpr, ListExpr)): - items = rhs.items + left = builder.accept(lhs) + items = [builder.accept(item) for item in rhs.items] + elif isinstance(builder.node_type(rhs), RTuple): + left = builder.accept(lhs) + tuple_val = builder.accept(rhs) + assert isinstance(tuple_val.type, RTuple) + items = [builder.add(TupleGet(tuple_val, i)) for i in range(len(tuple_val.type.types))] + + if items is not None: + assert left is not None n_items = len(items) # x in y -> x == y[0] or ... or x == y[n] # x not in y -> x != y[0] and ... and x != y[n] - # 16 is arbitrarily chosen to limit code size - if 1 < n_items < 16: + if n_items > 1: if op == "in": - bin_op = "or" cmp_op = "==" else: - bin_op = "and" cmp_op = "!=" - mypy_file = builder.graph["builtins"].tree - assert mypy_file is not None - info = mypy_file.names["bool"].node - assert isinstance(info, TypeInfo), info - bool_type = Instance(info, []) - exprs = [] + out = BasicBlock() for item in items: - expr = ComparisonExpr([cmp_op], [lhs, item]) - builder.types[expr] = bool_type - exprs.append(expr) - - or_expr: Expression = exprs.pop(0) - for expr in exprs: - or_expr = OpExpr(bin_op, or_expr, expr) - builder.types[or_expr] = bool_type - return builder.accept(or_expr) + cmp = transform_basic_comparison(builder, cmp_op, left, item, line) + bool_val = builder.builder.bool_value(cmp) + next_block = BasicBlock() + if op == "in": + builder.add_bool_branch(bool_val, out, next_block) + else: + builder.add_bool_branch(bool_val, next_block, out) + builder.activate_block(next_block) + result_reg = Register(bool_rprimitive) + end = BasicBlock() + if op == "in": + values = builder.false(), builder.true() + else: + values = builder.true(), builder.false() + builder.assign(result_reg, values[0], line) + builder.goto(end) + builder.activate_block(out) + builder.assign(result_reg, values[1], line) + builder.goto(end) + builder.activate_block(end) + return result_reg # x in [y]/(y) -> x == y # x not in [y]/(y) -> x != y elif n_items == 1: @@ -788,8 +804,7 @@ def try_specialize_in_expr( cmp_op = "==" else: cmp_op = "!=" - left = builder.accept(lhs) - right = builder.accept(items[0]) + right = items[0] return transform_basic_comparison(builder, cmp_op, left, right, line) # x in []/() -> False # x not in []/() -> True diff --git a/mypyc/test-data/irbuild-tuple.test b/mypyc/test-data/irbuild-tuple.test index 712b9c26355ab..00ea7f074a5d8 100644 --- a/mypyc/test-data/irbuild-tuple.test +++ b/mypyc/test-data/irbuild-tuple.test @@ -184,31 +184,66 @@ def f(i: int) -> bool: [out] def f(i): i :: int - r0 :: bit - r1 :: bool - r2 :: bit + r0, r1, r2 :: bit r3 :: bool - r4 :: bit L0: r0 = int_eq i, 2 - if r0 goto L1 else goto L2 :: bool + if r0 goto L4 else goto L1 :: bool L1: - r1 = r0 - goto L3 + r1 = int_eq i, 4 + if r1 goto L4 else goto L2 :: bool L2: - r2 = int_eq i, 4 - r1 = r2 + r2 = int_eq i, 6 + if r2 goto L4 else goto L3 :: bool L3: - if r1 goto L4 else goto L5 :: bool + r3 = 0 + goto L5 L4: - r3 = r1 - goto L6 + r3 = 1 L5: - r4 = int_eq i, 6 - r3 = r4 -L6: return r3 +[case testTupleOperatorNotIn] +def x() -> int: + return 1 +def y() -> int: + return 2 +def z() -> int: + return 3 + +def f() -> bool: + return z() not in (x(), y()) +[out] +def x(): +L0: + return 2 +def y(): +L0: + return 4 +def z(): +L0: + return 6 +def f(): + r0, r1, r2 :: int + r3, r4 :: bit + r5 :: bool +L0: + r0 = z() + r1 = x() + r2 = y() + r3 = int_ne r0, r1 + if r3 goto L1 else goto L3 :: bool +L1: + r4 = int_ne r0, r2 + if r4 goto L2 else goto L3 :: bool +L2: + r5 = 1 + goto L4 +L3: + r5 = 0 +L4: + return r5 + [case testTupleOperatorInFinalTuple] from typing import Final @@ -221,9 +256,8 @@ def f(x): x :: int r0 :: tuple[int, int] r1 :: bool - r2, r3 :: object - r4 :: i32 - r5 :: bit + r2, r3 :: int + r4, r5 :: bit r6 :: bool L0: r0 = __main__.tt :: static @@ -232,11 +266,19 @@ L1: r1 = raise NameError('value for final name "tt" was not set') unreachable L2: - r2 = box(int, x) - r3 = box(tuple[int, int], r0) - r4 = PySequence_Contains(r3, r2) - r5 = r4 >= 0 :: signed - r6 = truncate r4: i32 to builtins.bool + r2 = r0[0] + r3 = r0[1] + r4 = int_eq x, r2 + if r4 goto L5 else goto L3 :: bool +L3: + r5 = int_eq x, r3 + if r5 goto L5 else goto L4 :: bool +L4: + r6 = 0 + goto L6 +L5: + r6 = 1 +L6: return r6 [case testTupleBuiltFromList] diff --git a/mypyc/test-data/run-lists.test b/mypyc/test-data/run-lists.test index 54bcc03846049..1569579c1156d 100644 --- a/mypyc/test-data/run-lists.test +++ b/mypyc/test-data/run-lists.test @@ -364,7 +364,6 @@ def test_multiply() -> None: assert l1 == [1, 1, 1] [case testOperatorInExpression] - def tuple_in_int0(i: int) -> bool: return i in [] @@ -416,71 +415,68 @@ def list_not_in_str(s: "str") -> bool: def list_in_mixed(i: object): return i in [[], (), "", 0, 0.0, False, 0j, {}, set(), type] -[file driver.py] - -from native import * - -assert not tuple_in_int0(0) -assert not tuple_in_int1(0) -assert tuple_in_int1(1) -assert not tuple_in_int3(0) -assert tuple_in_int3(1) -assert tuple_in_int3(2) -assert tuple_in_int3(3) -assert not tuple_in_int3(4) - -assert tuple_not_in_int0(0) -assert tuple_not_in_int1(0) -assert not tuple_not_in_int1(1) -assert tuple_not_in_int3(0) -assert not tuple_not_in_int3(1) -assert not tuple_not_in_int3(2) -assert not tuple_not_in_int3(3) -assert tuple_not_in_int3(4) - -assert tuple_in_str("foo") -assert tuple_in_str("bar") -assert tuple_in_str("baz") -assert not tuple_in_str("apple") -assert not tuple_in_str("pie") -assert not tuple_in_str("\0") -assert not tuple_in_str("") - -assert not list_in_int0(0) -assert not list_in_int1(0) -assert list_in_int1(1) -assert not list_in_int3(0) -assert list_in_int3(1) -assert list_in_int3(2) -assert list_in_int3(3) -assert not list_in_int3(4) - -assert list_not_in_int0(0) -assert list_not_in_int1(0) -assert not list_not_in_int1(1) -assert list_not_in_int3(0) -assert not list_not_in_int3(1) -assert not list_not_in_int3(2) -assert not list_not_in_int3(3) -assert list_not_in_int3(4) - -assert list_in_str("foo") -assert list_in_str("bar") -assert list_in_str("baz") -assert not list_in_str("apple") -assert not list_in_str("pie") -assert not list_in_str("\0") -assert not list_in_str("") - -assert list_in_mixed(0) -assert list_in_mixed([]) -assert list_in_mixed({}) -assert list_in_mixed(()) -assert list_in_mixed(False) -assert list_in_mixed(0.0) -assert not list_in_mixed([1]) -assert not list_in_mixed(object) -assert list_in_mixed(type) +def test_in_operator_various_cases() -> None: + assert not tuple_in_int0(0) + assert not tuple_in_int1(0) + assert tuple_in_int1(1) + assert not tuple_in_int3(0) + assert tuple_in_int3(1) + assert tuple_in_int3(2) + assert tuple_in_int3(3) + assert not tuple_in_int3(4) + + assert tuple_not_in_int0(0) + assert tuple_not_in_int1(0) + assert not tuple_not_in_int1(1) + assert tuple_not_in_int3(0) + assert not tuple_not_in_int3(1) + assert not tuple_not_in_int3(2) + assert not tuple_not_in_int3(3) + assert tuple_not_in_int3(4) + + assert tuple_in_str("foo") + assert tuple_in_str("bar") + assert tuple_in_str("baz") + assert not tuple_in_str("apple") + assert not tuple_in_str("pie") + assert not tuple_in_str("\0") + assert not tuple_in_str("") + + assert not list_in_int0(0) + assert not list_in_int1(0) + assert list_in_int1(1) + assert not list_in_int3(0) + assert list_in_int3(1) + assert list_in_int3(2) + assert list_in_int3(3) + assert not list_in_int3(4) + + assert list_not_in_int0(0) + assert list_not_in_int1(0) + assert not list_not_in_int1(1) + assert list_not_in_int3(0) + assert not list_not_in_int3(1) + assert not list_not_in_int3(2) + assert not list_not_in_int3(3) + assert list_not_in_int3(4) + + assert list_in_str("foo") + assert list_in_str("bar") + assert list_in_str("baz") + assert not list_in_str("apple") + assert not list_in_str("pie") + assert not list_in_str("\0") + assert not list_in_str("") + + assert list_in_mixed(0) + assert list_in_mixed([]) + assert list_in_mixed({}) + assert list_in_mixed(()) + assert list_in_mixed(False) + assert list_in_mixed(0.0) + assert not list_in_mixed([1]) + assert not list_in_mixed(object) + assert list_in_mixed(type) [case testListBuiltFromGenerator] def test_from_gen() -> None: diff --git a/mypyc/test-data/run-tuples.test b/mypyc/test-data/run-tuples.test index 5d9485288cfb0..f5e1733d429b2 100644 --- a/mypyc/test-data/run-tuples.test +++ b/mypyc/test-data/run-tuples.test @@ -292,6 +292,89 @@ TUPLE: Final[Tuple[str, ...]] = ('x', 'y') def test_final_boxed_tuple() -> None: t = TUPLE assert t == ('x', 'y') + assert 'x' in TUPLE + assert 'y' in TUPLE + b: object = 'z' in TUPLE + assert not b + assert 'z' not in TUPLE + b2: object = 'x' not in TUPLE + assert not b2 + b3: object = 'y' not in TUPLE + assert not b3 + +TUP2: Final = ('x', 'y') +TUP1: Final = ('x',) +TUP0: Final = () + +def test_final_tuple_in() -> None: + assert 'x' + str() in TUP2 + assert 'y' + str() in TUP2 + b: object = 'z' + str() in TUP2 + assert not b + + assert 'x' + str() in TUP1 + b2: object = 'y' in TUP1 + assert not b2 + + b3: object = 'x' in TUP0 + assert not b3 + +def test_final_tuple_not_in() -> None: + assert 'z' + str() not in TUP2 + b: object = 'x' + str() not in TUP2 + assert not b + b2: object = 'y' + str() not in TUP2 + assert not b2 + + assert 'y' + str() not in TUP1 + b3: object = 'x' not in TUP1 + assert not b2 + + assert 'x' not in TUP0 + +log = [] + +def f_a() -> str: + log.append('f_a') + return 'a' + +def f_a2() -> str: + log.append('f_a2') + return 'a' + +def f_b() -> str: + log.append('f_b') + return 'b' + +def f_c() -> str: + log.append('f_c') + return 'c' + +def test_tuple_in_order_of_evaluation() -> None: + log.clear() + assert f_a() in (f_b(), f_a2()) + assert log ==["f_a", "f_b", "f_a2"] + + log.clear() + assert f_a() not in (f_b(), f_c()) + assert log ==["f_a", "f_b", "f_c"] + + log.clear() + assert f_a() in (f_b(), f_a2(), f_c()) + assert log ==["f_a", "f_b", "f_a2", "f_c"] + +def f_t() -> tuple[str, ...]: + log.append('f_t') + return ('x', 'a') + +def test_tuple_in_non_specialized() -> None: + log.clear() + assert f_a() in f_t() + assert log == ["f_a", "f_t"] + + log.clear() + assert f_b() not in f_t() + assert log == ["f_b", "f_t"] def test_add() -> None: res = (1, 2, 3, 4) From 38eeff85a8c290371648477a07dd38fbb4508018 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 18 Aug 2025 16:43:31 +0100 Subject: [PATCH 179/424] [mypyc] Add primitive for .__name__ (#19683) This seems quite common in real-world code, including in performance-critical functions. Python 3.11 added a C API function for this, which we use here. The primitive works for arbtirary objects, but only type objects have a specialized code path. Other use cases of `__name__` seem typically less performance-sensitive. This PR makes this micro-benchmark 2.0x faster on Python 3.13: ``` from typing import Iterator class FooBar: pass def foo(x: type[object], n: int) -> str: for a in range(n): s = x.__name__ return s def bench(n: int) -> None: for i in range(n): foo(FooBar, 1000) from time import time bench(50 * 1000) t0 = time() bench(50 * 1000) print(time() - t0) ``` --- mypyc/irbuild/expression.py | 10 +++++- mypyc/lib-rt/CPy.h | 4 +++ mypyc/lib-rt/misc_ops.c | 14 ++++++++ mypyc/lib-rt/mypyc_util.h | 1 + mypyc/primitives/generic_ops.py | 10 ++++++ mypyc/test-data/irbuild-classes.test | 48 ++++++++++++++++++++++++++++ mypyc/test-data/run-classes.test | 48 ++++++++++++++++++++++++++++ 7 files changed, 134 insertions(+), 1 deletion(-) diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index 2df47680d27ce..312ba98e3c5d5 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -76,6 +76,7 @@ is_int_rprimitive, is_list_rprimitive, is_none_rprimitive, + is_object_rprimitive, object_rprimitive, set_rprimitive, ) @@ -98,7 +99,7 @@ from mypyc.irbuild.specialize import apply_function_specialization, apply_method_specialization from mypyc.primitives.bytes_ops import bytes_slice_op from mypyc.primitives.dict_ops import dict_get_item_op, dict_new_op, exact_dict_set_item_op -from mypyc.primitives.generic_ops import iter_op +from mypyc.primitives.generic_ops import iter_op, name_op from mypyc.primitives.list_ops import list_append_op, list_extend_op, list_slice_op from mypyc.primitives.misc_ops import ellipsis_op, get_module_dict_op, new_slice_op, type_op from mypyc.primitives.registry import builtin_names @@ -218,6 +219,13 @@ def transform_member_expr(builder: IRBuilder, expr: MemberExpr) -> Value: obj = builder.accept(expr.expr, can_borrow=can_borrow) rtype = builder.node_type(expr) + if ( + is_object_rprimitive(obj.type) + and expr.name == "__name__" + and builder.options.capi_version >= (3, 11) + ): + return builder.primitive_op(name_op, [obj], expr.line) + # Special case: for named tuples transform attribute access to faster index access. typ = get_proper_type(builder.types.get(expr.expr)) if isinstance(typ, TupleType) and typ.partial_fallback.type.is_named_tuple: diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index 1881aa97f3084..ffa4f6a363d2f 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -931,6 +931,10 @@ PyObject *CPy_GetANext(PyObject *aiter); void CPy_SetTypeAliasTypeComputeFunction(PyObject *alias, PyObject *compute_value); void CPyTrace_LogEvent(const char *location, const char *line, const char *op, const char *details); +#if CPY_3_11_FEATURES +PyObject *CPy_GetName(PyObject *obj); +#endif + #if CPY_3_14_FEATURES void CPy_SetImmortal(PyObject *obj); #endif diff --git a/mypyc/lib-rt/misc_ops.c b/mypyc/lib-rt/misc_ops.c index 0c9d7812ac6c7..b7593491a6e61 100644 --- a/mypyc/lib-rt/misc_ops.c +++ b/mypyc/lib-rt/misc_ops.c @@ -1045,6 +1045,20 @@ PyObject *CPy_GetANext(PyObject *aiter) return NULL; } +#if CPY_3_11_FEATURES + +// Return obj.__name__ (specialized to type objects, which are the most common target). +PyObject *CPy_GetName(PyObject *obj) { + if (PyType_Check(obj)) { + return PyType_GetName((PyTypeObject *)obj); + } + _Py_IDENTIFIER(__name__); + PyObject *name = _PyUnicode_FromId(&PyId___name__); /* borrowed */ + return PyObject_GetAttr(obj, name); +} + +#endif + #ifdef MYPYC_LOG_TRACE // This is only compiled in if trace logging is enabled by user diff --git a/mypyc/lib-rt/mypyc_util.h b/mypyc/lib-rt/mypyc_util.h index f200d4f90defa..4168d3c53ee28 100644 --- a/mypyc/lib-rt/mypyc_util.h +++ b/mypyc/lib-rt/mypyc_util.h @@ -140,6 +140,7 @@ static inline CPyTagged CPyTagged_ShortFromSsize_t(Py_ssize_t x) { } // Are we targeting Python 3.X or newer? +#define CPY_3_11_FEATURES (PY_VERSION_HEX >= 0x030b0000) #define CPY_3_12_FEATURES (PY_VERSION_HEX >= 0x030c0000) #define CPY_3_14_FEATURES (PY_VERSION_HEX >= 0x030e0000) diff --git a/mypyc/primitives/generic_ops.py b/mypyc/primitives/generic_ops.py index 54510d99cf87a..4a95be4e5d4e2 100644 --- a/mypyc/primitives/generic_ops.py +++ b/mypyc/primitives/generic_ops.py @@ -26,6 +26,7 @@ ERR_NEG_INT, binary_op, custom_op, + custom_primitive_op, function_op, method_op, unary_op, @@ -382,3 +383,12 @@ c_function_name="CPy_GetANext", error_kind=ERR_MAGIC, ) + +# x.__name__ (requires Python 3.11+) +name_op = custom_primitive_op( + name="__name__", + arg_types=[object_rprimitive], + return_type=object_rprimitive, + c_function_name="CPy_GetName", + error_kind=ERR_MAGIC, +) diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index c7bf5de852a85..2bdbb42b8d23e 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -1468,3 +1468,51 @@ L1: L2: r2 = self == r0 return r2 + +[case testTypeObjectName_python3_11] +from typing import Any + +class C: pass +class D(C): pass + +def n1(t: type[object]) -> str: + return t.__name__ + +def n2(t: Any) -> str: + return t.__name__ + +def n3() -> str: + return C.__name__ + +def n4(t: type[C]) -> str: + return t.__name__ +[out] +def n1(t): + t, r0 :: object + r1 :: str +L0: + r0 = CPy_GetName(t) + r1 = cast(str, r0) + return r1 +def n2(t): + t, r0 :: object + r1 :: str +L0: + r0 = CPy_GetName(t) + r1 = cast(str, r0) + return r1 +def n3(): + r0, r1 :: object + r2 :: str +L0: + r0 = __main__.C :: type + r1 = CPy_GetName(r0) + r2 = cast(str, r1) + return r2 +def n4(t): + t, r0 :: object + r1 :: str +L0: + r0 = CPy_GetName(t) + r1 = cast(str, r0) + return r1 diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 1481f3e068715..9582eec07b1a2 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -3115,3 +3115,51 @@ f(C(1, "yes")) [out] B 1 C yes + +[case testTypeObjectName] +from typing import Any +import re + +from dynamic import E, foo, Thing + +class C: pass +class D(C): pass + +def type_name(t: type[object]) -> str: + return t.__name__ + +def any_name(x: Any) -> str: + return x.__name__ + +def assert_type_name(x: Any) -> None: + assert type_name(x) == getattr(x, "__name__") + assert any_name(x) == getattr(x, "__name__") + +def assert_any_name(x: Any) -> None: + assert any_name(x) == getattr(x, "__name__") + +def test_type_name() -> None: + assert_type_name(C) + assert_type_name(D) + assert_type_name(int) + assert_type_name(E) + assert_type_name(re.Pattern) + +def test_module_name() -> None: + assert_any_name(re) + +def test_function_name() -> None: + assert_any_name(any_name) + assert_any_name(foo) + +def test_obj_name() -> None: + assert_any_name(Thing()) + +[file dynamic.py] +class E: pass + +def foo(): pass + +class Thing: + def __init__(self): + self.__name__ = "xyz" From 91487cbb6ba81954aac315932615afde7f05171d Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Mon, 18 Aug 2025 12:29:23 -0400 Subject: [PATCH 180/424] [mypyc] feat: extend stararg fastpath from #19623 to handle lists and generic sequences (#19629) This PR extends #19623 with additional logic for handling non-tuple star inputs Now, we can use the fast path for any arbitrary sequence, in addition to tuples. I opted to separate this PR from 19623 to keep them smaller and easier to review, and to declutter the changes in the IR. --- mypyc/irbuild/ll_builder.py | 34 +++- mypyc/primitives/tuple_ops.py | 2 +- mypyc/test-data/irbuild-basic.test | 281 +++++++++++++++++++++++++++-- 3 files changed, 294 insertions(+), 23 deletions(-) diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index c5f9503b8c663..f244e2f05e054 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -184,7 +184,12 @@ str_ssize_t_size_op, unicode_compare, ) -from mypyc.primitives.tuple_ops import list_tuple_op, new_tuple_op, new_tuple_with_length_op +from mypyc.primitives.tuple_ops import ( + list_tuple_op, + new_tuple_op, + new_tuple_with_length_op, + sequence_tuple_op, +) from mypyc.rt_subtype import is_runtime_subtype from mypyc.sametype import is_same_type from mypyc.subtype import is_subtype @@ -789,16 +794,25 @@ def _construct_varargs( for value, kind, name in args: if kind == ARG_STAR: if star_result is None: - # fast path if star expr is a tuple: - # we can pass the immutable tuple straight into the function call. - if is_tuple_rprimitive(value.type): - if len(args) == 1: - # fn(*args) - return value, self._create_dict([], [], line) - elif len(args) == 2 and args[1][1] == ARG_STAR2: - # fn(*args, **kwargs) + # star args fastpath + if len(args) == 1: + # fn(*args) + if is_list_rprimitive(value.type): + value = self.primitive_op(list_tuple_op, [value], line) + elif not is_tuple_rprimitive(value.type) and not isinstance( + value.type, RTuple + ): + value = self.primitive_op(sequence_tuple_op, [value], line) + return value, self._create_dict([], [], line) + elif len(args) == 2 and args[1][1] == ARG_STAR2: + # fn(*args, **kwargs) + if is_tuple_rprimitive(value.type) or isinstance(value.type, RTuple): star_result = value - continue + elif is_list_rprimitive(value.type): + star_result = self.primitive_op(list_tuple_op, [value], line) + else: + star_result = self.primitive_op(sequence_tuple_op, [value], line) + continue # elif ...: TODO extend this to optimize fn(*args, k=1, **kwargs) case # TODO optimize this case using the length utils - currently in review star_result = self.new_list_op(star_values, line) diff --git a/mypyc/primitives/tuple_ops.py b/mypyc/primitives/tuple_ops.py index d95161acf853e..f262dec8b05ae 100644 --- a/mypyc/primitives/tuple_ops.py +++ b/mypyc/primitives/tuple_ops.py @@ -76,7 +76,7 @@ ) # Construct tuple from an arbitrary (iterable) object. -function_op( +sequence_tuple_op = function_op( name="builtins.tuple", arg_types=[object_rprimitive], return_type=tuple_rprimitive, diff --git a/mypyc/test-data/irbuild-basic.test b/mypyc/test-data/irbuild-basic.test index 8d981db2b3915..f52e1af03b528 100644 --- a/mypyc/test-data/irbuild-basic.test +++ b/mypyc/test-data/irbuild-basic.test @@ -1675,25 +1675,19 @@ def g(): r1 :: dict r2 :: str r3 :: object - r4 :: list + r4 :: dict r5, r6 :: object - r7 :: tuple - r8 :: dict - r9 :: object - r10 :: tuple[int, int, int] + r7 :: tuple[int, int, int] L0: r0 = (2, 4, 6) r1 = __main__.globals :: static r2 = 'f' r3 = CPyDict_GetItem(r1, r2) - r4 = PyList_New(0) + r4 = PyDict_New() r5 = box(tuple[int, int, int], r0) - r6 = CPyList_Extend(r4, r5) - r7 = PyList_AsTuple(r4) - r8 = PyDict_New() - r9 = PyObject_Call(r3, r7, r8) - r10 = unbox(tuple[int, int, int], r9) - return r10 + r6 = PyObject_Call(r3, r5, r4) + r7 = unbox(tuple[int, int, int], r6) + return r7 def h(): r0 :: tuple[int, int] r1 :: dict @@ -3546,3 +3540,266 @@ L0: r2 = PyObject_Vectorcall(r1, 0, 0, 0) r3 = box(None, 1) return r3 + +[case testStarArgFastPathTuple] +from typing import Any, Callable +def deco(fn: Callable[..., Any]) -> Callable[..., Any]: + def wrapper(*args: Any) -> Any: + return fn(*args) + return wrapper + +[out] +def wrapper_deco_obj.__get__(__mypyc_self__, instance, owner): + __mypyc_self__, instance, owner, r0 :: object + r1 :: bit + r2 :: object +L0: + r0 = load_address _Py_NoneStruct + r1 = instance == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return __mypyc_self__ +L2: + r2 = PyMethod_New(__mypyc_self__, instance) + return r2 +def wrapper_deco_obj.__call__(__mypyc_self__, args): + __mypyc_self__ :: __main__.wrapper_deco_obj + args :: tuple + r0 :: __main__.deco_env + r1 :: object + r2 :: dict + r3 :: object +L0: + r0 = __mypyc_self__.__mypyc_env__ + r1 = r0.fn + r2 = PyDict_New() + r3 = PyObject_Call(r1, args, r2) + return r3 +def deco(fn): + fn :: object + r0 :: __main__.deco_env + r1 :: bool + r2 :: __main__.wrapper_deco_obj + r3 :: bool + wrapper :: object +L0: + r0 = deco_env() + r0.fn = fn; r1 = is_error + r2 = wrapper_deco_obj() + r2.__mypyc_env__ = r0; r3 = is_error + wrapper = r2 + return wrapper + +[case testStarArgFastPathList] +from typing import Any, Callable +def deco(fn: Callable[..., Any]) -> Callable[[list[Any]], Any]: + def wrapper(args: list[Any]) -> Any: + return fn(*args) + return wrapper + +[out] +def wrapper_deco_obj.__get__(__mypyc_self__, instance, owner): + __mypyc_self__, instance, owner, r0 :: object + r1 :: bit + r2 :: object +L0: + r0 = load_address _Py_NoneStruct + r1 = instance == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return __mypyc_self__ +L2: + r2 = PyMethod_New(__mypyc_self__, instance) + return r2 +def wrapper_deco_obj.__call__(__mypyc_self__, args): + __mypyc_self__ :: __main__.wrapper_deco_obj + args :: list + r0 :: __main__.deco_env + r1 :: object + r2 :: tuple + r3 :: dict + r4 :: object +L0: + r0 = __mypyc_self__.__mypyc_env__ + r1 = r0.fn + r2 = PyList_AsTuple(args) + r3 = PyDict_New() + r4 = PyObject_Call(r1, r2, r3) + return r4 +def deco(fn): + fn :: object + r0 :: __main__.deco_env + r1 :: bool + r2 :: __main__.wrapper_deco_obj + r3 :: bool + wrapper :: object +L0: + r0 = deco_env() + r0.fn = fn; r1 = is_error + r2 = wrapper_deco_obj() + r2.__mypyc_env__ = r0; r3 = is_error + wrapper = r2 + return wrapper + +[case testStarArgFastPathListWithKwargs] +from typing import Any, Callable +def deco(fn: Callable[..., Any]) -> Callable[..., Any]: + def wrapper(lst: list[Any], kwargs: dict[str, Any]) -> Any: + return fn(*lst, **kwargs) + return wrapper + +[out] +def wrapper_deco_obj.__get__(__mypyc_self__, instance, owner): + __mypyc_self__, instance, owner, r0 :: object + r1 :: bit + r2 :: object +L0: + r0 = load_address _Py_NoneStruct + r1 = instance == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return __mypyc_self__ +L2: + r2 = PyMethod_New(__mypyc_self__, instance) + return r2 +def wrapper_deco_obj.__call__(__mypyc_self__, lst, kwargs): + __mypyc_self__ :: __main__.wrapper_deco_obj + lst :: list + kwargs :: dict + r0 :: __main__.deco_env + r1 :: object + r2 :: tuple + r3 :: dict + r4 :: i32 + r5 :: bit + r6 :: object +L0: + r0 = __mypyc_self__.__mypyc_env__ + r1 = r0.fn + r2 = PyList_AsTuple(lst) + r3 = PyDict_New() + r4 = CPyDict_UpdateInDisplay(r3, kwargs) + r5 = r4 >= 0 :: signed + r6 = PyObject_Call(r1, r2, r3) + return r6 +def deco(fn): + fn :: object + r0 :: __main__.deco_env + r1 :: bool + r2 :: __main__.wrapper_deco_obj + r3 :: bool + wrapper :: object +L0: + r0 = deco_env() + r0.fn = fn; r1 = is_error + r2 = wrapper_deco_obj() + r2.__mypyc_env__ = r0; r3 = is_error + wrapper = r2 + return wrapper + +[case testStarArgFastPathSequence] +from typing import Any, Callable +def deco(fn: Callable[[Any], Any]) -> Callable[[Any], Any]: + def wrapper(args: Any) -> Any: + return fn(*args) + return wrapper + +[out] +def wrapper_deco_obj.__get__(__mypyc_self__, instance, owner): + __mypyc_self__, instance, owner, r0 :: object + r1 :: bit + r2 :: object +L0: + r0 = load_address _Py_NoneStruct + r1 = instance == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return __mypyc_self__ +L2: + r2 = PyMethod_New(__mypyc_self__, instance) + return r2 +def wrapper_deco_obj.__call__(__mypyc_self__, args): + __mypyc_self__ :: __main__.wrapper_deco_obj + args :: object + r0 :: __main__.deco_env + r1 :: object + r2 :: tuple + r3 :: dict + r4 :: object +L0: + r0 = __mypyc_self__.__mypyc_env__ + r1 = r0.fn + r2 = PySequence_Tuple(args) + r3 = PyDict_New() + r4 = PyObject_Call(r1, r2, r3) + return r4 +def deco(fn): + fn :: object + r0 :: __main__.deco_env + r1 :: bool + r2 :: __main__.wrapper_deco_obj + r3 :: bool + wrapper :: object +L0: + r0 = deco_env() + r0.fn = fn; r1 = is_error + r2 = wrapper_deco_obj() + r2.__mypyc_env__ = r0; r3 = is_error + wrapper = r2 + return wrapper + +[case testStarArgFastPathSequenceWithKwargs] +from typing import Any, Callable +def deco(fn: Callable[[Any], Any]) -> Callable[[Any], Any]: + def wrapper(args: Any, **kwargs: Any) -> Any: + return fn(*args, **kwargs) + return wrapper + +[out] +def wrapper_deco_obj.__get__(__mypyc_self__, instance, owner): + __mypyc_self__, instance, owner, r0 :: object + r1 :: bit + r2 :: object +L0: + r0 = load_address _Py_NoneStruct + r1 = instance == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return __mypyc_self__ +L2: + r2 = PyMethod_New(__mypyc_self__, instance) + return r2 +def wrapper_deco_obj.__call__(__mypyc_self__, args, kwargs): + __mypyc_self__ :: __main__.wrapper_deco_obj + args :: object + kwargs :: dict + r0 :: __main__.deco_env + r1 :: object + r2 :: tuple + r3 :: dict + r4 :: i32 + r5 :: bit + r6 :: object +L0: + r0 = __mypyc_self__.__mypyc_env__ + r1 = r0.fn + r2 = PySequence_Tuple(args) + r3 = PyDict_New() + r4 = CPyDict_UpdateInDisplay(r3, kwargs) + r5 = r4 >= 0 :: signed + r6 = PyObject_Call(r1, r2, r3) + return r6 +def deco(fn): + fn :: object + r0 :: __main__.deco_env + r1 :: bool + r2 :: __main__.wrapper_deco_obj + r3 :: bool + wrapper :: object +L0: + r0 = deco_env() + r0.fn = fn; r1 = is_error + r2 = wrapper_deco_obj() + r2.__mypyc_env__ = r0; r3 = is_error + wrapper = r2 + return wrapper From 6c5b13ccbad7bcdb4b45b12d9ae1d0af8cb9a0e5 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 19 Aug 2025 11:48:51 +0100 Subject: [PATCH 181/424] Force all deserialized objects to the oldest GC generation (#19681) This is a hack, but it gives ~30% perf win for `mypy -c 'import torch'` on a warm run. This should not increase memory consumption too much, since we shouldn't create any cyclic garbage during deserialization (we do create some cyclic references, like `TypeInfo` -> `SymbolTable` -> `Instance` -> `TypeInfo`, but those are genuine long-living objects). --- mypy/build.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/mypy/build.py b/mypy/build.py index 71575de9d8773..883ae1f22f196 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -116,6 +116,8 @@ "abc", } +# We are careful now, we can increase this in future if safe/useful. +MAX_GC_FREEZE_CYCLES = 1 Graph: _TypeAlias = dict[str, "State"] @@ -707,6 +709,8 @@ def __init__( # new file can be processed O(n**2) times. This cache # avoids most of this redundant work. self.ast_cache: dict[str, tuple[MypyFile, list[ErrorInfo]]] = {} + # Number of times we used GC optimization hack for fresh SCCs. + self.gc_freeze_cycles = 0 def dump_stats(self) -> None: if self.options.dump_build_stats: @@ -3326,8 +3330,29 @@ def process_graph(graph: Graph, manager: BuildManager) -> None: # # TODO: see if it's possible to determine if we need to process only a # _subset_ of the past SCCs instead of having to process them all. + if ( + platform.python_implementation() == "CPython" + and manager.gc_freeze_cycles < MAX_GC_FREEZE_CYCLES + ): + # When deserializing cache we create huge amount of new objects, so even + # with our generous GC thresholds, GC is still doing a lot of pointless + # work searching for garbage. So, we temporarily disable it when + # processing fresh SCCs, and then move all the new objects to the oldest + # generation with the freeze()/unfreeze() trick below. This is arguably + # a hack, but it gives huge performance wins for large third-party + # libraries, like torch. + gc.collect() + gc.disable() for prev_scc in fresh_scc_queue: process_fresh_modules(graph, prev_scc, manager) + if ( + platform.python_implementation() == "CPython" + and manager.gc_freeze_cycles < MAX_GC_FREEZE_CYCLES + ): + manager.gc_freeze_cycles += 1 + gc.freeze() + gc.unfreeze() + gc.enable() fresh_scc_queue = [] size = len(scc) if size == 1: From 68809c0056c3bc6446481ec7429d48f6fcc1f5c2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 19 Aug 2025 14:32:50 +0100 Subject: [PATCH 182/424] [mypyc] Specialize bytes.decode calls with common encodings (#19688) This is similar to #18232, which specialized `encode`. A micro-benchmark that calls `decode` repeatedly was up to 45% faster. --- mypyc/irbuild/specialize.py | 65 ++++++++++++++++++++++++++++ mypyc/lib-rt/CPy.h | 3 ++ mypyc/lib-rt/str_ops.c | 39 +++++++++++++++++ mypyc/primitives/str_ops.py | 26 ++++++++++- mypyc/test-data/fixtures/ir.py | 2 +- mypyc/test-data/irbuild-str.test | 38 +++++++++++++--- mypyc/test-data/run-strings.test | 74 +++++++++++++++++++++++++++++--- 7 files changed, 233 insertions(+), 14 deletions(-) diff --git a/mypyc/irbuild/specialize.py b/mypyc/irbuild/specialize.py index 8459682222343..0880c62bc7a58 100644 --- a/mypyc/irbuild/specialize.py +++ b/mypyc/irbuild/specialize.py @@ -50,6 +50,7 @@ RTuple, RType, bool_rprimitive, + bytes_rprimitive, c_int_rprimitive, dict_rprimitive, int16_rprimitive, @@ -98,6 +99,9 @@ from mypyc.primitives.misc_ops import isinstance_bool from mypyc.primitives.set_ops import isinstance_frozenset, isinstance_set from mypyc.primitives.str_ops import ( + bytes_decode_ascii_strict, + bytes_decode_latin1_strict, + bytes_decode_utf8_strict, isinstance_str, str_encode_ascii_strict, str_encode_latin1_strict, @@ -787,6 +791,67 @@ def str_encode_fast_path(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> return None +@specialize_function("decode", bytes_rprimitive) +def bytes_decode_fast_path(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: + """Specialize common cases of obj.decode for most used encodings and strict errors.""" + + if not isinstance(callee, MemberExpr): + return None + + # We can only specialize if we have string literals as args + if len(expr.arg_kinds) > 0 and not isinstance(expr.args[0], StrExpr): + return None + if len(expr.arg_kinds) > 1 and not isinstance(expr.args[1], StrExpr): + return None + + encoding = "utf8" + errors = "strict" + if len(expr.arg_kinds) > 0 and isinstance(expr.args[0], StrExpr): + if expr.arg_kinds[0] == ARG_NAMED: + if expr.arg_names[0] == "encoding": + encoding = expr.args[0].value + elif expr.arg_names[0] == "errors": + errors = expr.args[0].value + elif expr.arg_kinds[0] == ARG_POS: + encoding = expr.args[0].value + else: + return None + if len(expr.arg_kinds) > 1 and isinstance(expr.args[1], StrExpr): + if expr.arg_kinds[1] == ARG_NAMED: + if expr.arg_names[1] == "encoding": + encoding = expr.args[1].value + elif expr.arg_names[1] == "errors": + errors = expr.args[1].value + elif expr.arg_kinds[1] == ARG_POS: + errors = expr.args[1].value + else: + return None + + if errors != "strict": + # We can only specialize strict errors + return None + + encoding = encoding.lower().replace("_", "-") # normalize + # Specialized encodings and their accepted aliases + if encoding in ["u8", "utf", "utf8", "utf-8", "cp65001"]: + return builder.call_c(bytes_decode_utf8_strict, [builder.accept(callee.expr)], expr.line) + elif encoding in ["646", "ascii", "usascii", "us-ascii"]: + return builder.call_c(bytes_decode_ascii_strict, [builder.accept(callee.expr)], expr.line) + elif encoding in [ + "iso8859-1", + "iso-8859-1", + "8859", + "cp819", + "latin", + "latin1", + "latin-1", + "l1", + ]: + return builder.call_c(bytes_decode_latin1_strict, [builder.accept(callee.expr)], expr.line) + + return None + + @specialize_function("mypy_extensions.i64") def translate_i64(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: if len(expr.args) != 1 or expr.arg_kinds[0] != ARG_POS: diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index ffa4f6a363d2f..aea5db25f29fa 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -752,6 +752,9 @@ PyObject *CPyStr_Removesuffix(PyObject *self, PyObject *suffix); bool CPyStr_IsTrue(PyObject *obj); Py_ssize_t CPyStr_Size_size_t(PyObject *str); PyObject *CPy_Decode(PyObject *obj, PyObject *encoding, PyObject *errors); +PyObject *CPy_DecodeUTF8(PyObject *bytes); +PyObject *CPy_DecodeASCII(PyObject *bytes); +PyObject *CPy_DecodeLatin1(PyObject *bytes); PyObject *CPy_Encode(PyObject *obj, PyObject *encoding, PyObject *errors); Py_ssize_t CPyStr_Count(PyObject *unicode, PyObject *substring, CPyTagged start); Py_ssize_t CPyStr_CountFull(PyObject *unicode, PyObject *substring, CPyTagged start, CPyTagged end); diff --git a/mypyc/lib-rt/str_ops.c b/mypyc/lib-rt/str_ops.c index a2d10aacea46e..337ef14fc955f 100644 --- a/mypyc/lib-rt/str_ops.c +++ b/mypyc/lib-rt/str_ops.c @@ -513,6 +513,45 @@ PyObject *CPy_Decode(PyObject *obj, PyObject *encoding, PyObject *errors) { } } +PyObject *CPy_DecodeUTF8(PyObject *bytes) { + if (PyBytes_CheckExact(bytes)) { + char *buffer = PyBytes_AsString(bytes); // Borrowed reference + if (buffer == NULL) { + return NULL; + } + Py_ssize_t size = PyBytes_Size(bytes); + return PyUnicode_DecodeUTF8(buffer, size, "strict"); + } else { + return PyUnicode_FromEncodedObject(bytes, "utf-8", "strict"); + } +} + +PyObject *CPy_DecodeASCII(PyObject *bytes) { + if (PyBytes_CheckExact(bytes)) { + char *buffer = PyBytes_AsString(bytes); // Borrowed reference + if (buffer == NULL) { + return NULL; + } + Py_ssize_t size = PyBytes_Size(bytes); + return PyUnicode_DecodeASCII(buffer, size, "strict");; + } else { + return PyUnicode_FromEncodedObject(bytes, "ascii", "strict"); + } +} + +PyObject *CPy_DecodeLatin1(PyObject *bytes) { + if (PyBytes_CheckExact(bytes)) { + char *buffer = PyBytes_AsString(bytes); // Borrowed reference + if (buffer == NULL) { + return NULL; + } + Py_ssize_t size = PyBytes_Size(bytes); + return PyUnicode_DecodeLatin1(buffer, size, "strict"); + } else { + return PyUnicode_FromEncodedObject(bytes, "latin1", "strict"); + } +} + PyObject *CPy_Encode(PyObject *obj, PyObject *encoding, PyObject *errors) { const char *enc = NULL; const char *err = NULL; diff --git a/mypyc/primitives/str_ops.py b/mypyc/primitives/str_ops.py index f07081c6aaa50..a8f4e4df74c2d 100644 --- a/mypyc/primitives/str_ops.py +++ b/mypyc/primitives/str_ops.py @@ -387,7 +387,7 @@ extra_int_constants=[(0, pointer_rprimitive)], ) -# obj.decode(encoding, errors) +# bytes.decode(encoding, errors) method_op( name="decode", arg_types=[bytes_rprimitive, str_rprimitive, str_rprimitive], @@ -396,6 +396,30 @@ error_kind=ERR_MAGIC, ) +# bytes.decode(encoding) - utf8 strict specialization +bytes_decode_utf8_strict = custom_op( + arg_types=[bytes_rprimitive], + return_type=str_rprimitive, + c_function_name="CPy_DecodeUTF8", + error_kind=ERR_MAGIC, +) + +# bytes.decode(encoding) - ascii strict specialization +bytes_decode_ascii_strict = custom_op( + arg_types=[bytes_rprimitive], + return_type=str_rprimitive, + c_function_name="CPy_DecodeASCII", + error_kind=ERR_MAGIC, +) + +# bytes.decode(encoding) - latin1 strict specialization +bytes_decode_latin1_strict = custom_op( + arg_types=[bytes_rprimitive], + return_type=str_rprimitive, + c_function_name="CPy_DecodeLatin1", + error_kind=ERR_MAGIC, +) + # str.encode() method_op( name="encode", diff --git a/mypyc/test-data/fixtures/ir.py b/mypyc/test-data/fixtures/ir.py index 661ae50fd5f3c..28c9244b4b27b 100644 --- a/mypyc/test-data/fixtures/ir.py +++ b/mypyc/test-data/fixtures/ir.py @@ -171,7 +171,7 @@ def __getitem__(self, i: int) -> int: ... @overload def __getitem__(self, i: slice) -> bytes: ... def join(self, x: Iterable[object]) -> bytes: ... - def decode(self, x: str=..., y: str=...) -> str: ... + def decode(self, encoding: str=..., errors: str=...) -> str: ... def __iter__(self) -> Iterator[int]: ... class bytearray: diff --git a/mypyc/test-data/irbuild-str.test b/mypyc/test-data/irbuild-str.test index 24807510193d7..245acf7402a11 100644 --- a/mypyc/test-data/irbuild-str.test +++ b/mypyc/test-data/irbuild-str.test @@ -325,19 +325,43 @@ L0: [case testDecode] def f(b: bytes) -> None: b.decode() + b.decode('Utf_8') b.decode('utf-8') + b.decode('UTF8') + b.decode('latin1') + b.decode('Latin-1') + b.decode('ascii') + encoding = 'utf-8' + b.decode(encoding) b.decode('utf-8', 'backslashreplace') +def variants(b: bytes) -> None: + b.decode(encoding="UTF_8") + b.decode("ascii", errors="strict") [out] def f(b): b :: bytes - r0, r1, r2, r3, r4, r5 :: str + r0, r1, r2, r3, r4, r5, r6, r7, encoding, r8, r9, r10, r11 :: str L0: - r0 = CPy_Decode(b, 0, 0) - r1 = 'utf-8' - r2 = CPy_Decode(b, r1, 0) - r3 = 'utf-8' - r4 = 'backslashreplace' - r5 = CPy_Decode(b, r3, r4) + r0 = CPy_DecodeUTF8(b) + r1 = CPy_DecodeUTF8(b) + r2 = CPy_DecodeUTF8(b) + r3 = CPy_DecodeUTF8(b) + r4 = CPy_DecodeLatin1(b) + r5 = CPy_DecodeLatin1(b) + r6 = CPy_DecodeASCII(b) + r7 = 'utf-8' + encoding = r7 + r8 = CPy_Decode(b, encoding, 0) + r9 = 'utf-8' + r10 = 'backslashreplace' + r11 = CPy_Decode(b, r9, r10) + return 1 +def variants(b): + b :: bytes + r0, r1 :: str +L0: + r0 = CPy_DecodeUTF8(b) + r1 = CPy_DecodeASCII(b) return 1 [case testEncode_64bit] diff --git a/mypyc/test-data/run-strings.test b/mypyc/test-data/run-strings.test index 8a914c08bfb21..b4f3ebd669104 100644 --- a/mypyc/test-data/run-strings.test +++ b/mypyc/test-data/run-strings.test @@ -792,14 +792,23 @@ def test_ord() -> None: ord('') [case testDecode] +from testutil import assertRaises + def test_decode() -> None: assert "\N{GREEK CAPITAL LETTER DELTA}" == '\u0394' assert "\u0394" == "\u0394" assert "\U00000394" == '\u0394' assert b'\x80abc'.decode('utf-8', 'replace') == '\ufffdabc' assert b'\x80abc'.decode('utf-8', 'backslashreplace') == '\\x80abc' + assert b''.decode() == '' + assert b'a'.decode() == 'a' assert b'abc'.decode() == 'abc' assert b'abc'.decode('utf-8') == 'abc' + assert b'abc'.decode('utf-8' + str()) == 'abc' + assert b'abc\x00\xce'.decode('latin-1') == 'abc\x00\xce' + assert b'abc\x00\xce'.decode('latin-1' + str()) == 'abc\x00\xce' + assert b'abc\x00\x7f'.decode('ascii') == 'abc\x00\x7f' + assert b'abc\x00\x7f'.decode('ascii' + str()) == 'abc\x00\x7f' assert b'\x80abc'.decode('utf-8', 'ignore') == 'abc' assert b'\x80abc'.decode('UTF-8', 'ignore') == 'abc' assert b'\x80abc'.decode('Utf-8', 'ignore') == 'abc' @@ -808,16 +817,71 @@ def test_decode() -> None: assert b'\xd2\xbb\xb6\xfe\xc8\xfd'.decode('gbk', 'ignore') == '一二三' assert b'\xd2\xbb\xb6\xfe\xc8\xfd'.decode('latin1', 'ignore') == 'Ò»¶þÈý' assert b'Z\xc3\xbcrich'.decode("utf-8") == 'Zürich' - try: - b'Z\xc3\xbcrich'.decode('ascii') - assert False - except UnicodeDecodeError: - pass + assert b'Z\xc3\xbcrich'.decode("utf-8" + str()) == 'Zürich' + assert bytearray(range(5)).decode() == '\x00\x01\x02\x03\x04' b = bytearray(b'\xe4\xbd\xa0\xe5\xa5\xbd') assert b.decode() == '你好' assert b.decode('gbk') == '浣犲ソ' assert b.decode('latin1') == 'ä½\xa0好' + assert b.decode('latin1' + str()) == 'ä½\xa0好' + +def test_decode_error() -> None: + try: + b'Z\xc3\xbcrich'.decode('ascii') + assert False + except UnicodeDecodeError: + pass + try: + b'Z\xc3\xbcrich'.decode('ascii' + str()) + assert False + except UnicodeDecodeError: + pass + try: + b'Z\xc3y'.decode('utf8') + assert False + except UnicodeDecodeError: + pass + try: + b'Z\xc3y'.decode('utf8' + str()) + assert False + except UnicodeDecodeError: + pass + +def test_decode_bytearray() -> None: + b: bytes = bytearray(b'foo\x00bar') + assert b.decode() == 'foo\x00bar' + assert b.decode('utf-8') == 'foo\x00bar' + assert b.decode('latin-1') == 'foo\x00bar' + assert b.decode('ascii') == 'foo\x00bar' + assert b.decode('utf-8' + str()) == 'foo\x00bar' + assert b.decode('latin-1' + str()) == 'foo\x00bar' + assert b.decode('ascii' + str()) == 'foo\x00bar' + b2: bytes = bytearray(b'foo\x00bar\xbe') + assert b2.decode('latin-1') == 'foo\x00bar\xbe' + with assertRaises(UnicodeDecodeError): + b2.decode('ascii') + with assertRaises(UnicodeDecodeError): + b2.decode('ascii' + str()) + with assertRaises(UnicodeDecodeError): + b2.decode('utf-8') + with assertRaises(UnicodeDecodeError): + b2.decode('utf-8' + str()) + b3: bytes = bytearray(b'Z\xc3\xbcrich') + assert b3.decode("utf-8") == 'Zürich' + +def test_invalid_encoding() -> None: + try: + b"foo".decode("ut-f-8") + assert False + except Exception as e: + assert repr(e).startswith("LookupError") + try: + encoding = "ut-f-8" + b"foo".decode(encoding) + assert False + except Exception as e: + assert repr(e).startswith("LookupError") [case testEncode] from testutil import assertRaises From 722f4dd92d368e059f616a143def40b6d2eaa8d0 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 19 Aug 2025 16:32:57 +0100 Subject: [PATCH 183/424] [mypyc] Optimize type(x) and x.__class__ (#19691) Using `Py_TYPE` avoids a function call. Also specialize `x.__class__` for instances of native classes -- it's treated as equivalent to `type(x)`. This is not generally possible, so only do it for native instances where we can make more assumptions. --- mypyc/irbuild/expression.py | 6 ++++ mypyc/lib-rt/CPy.h | 6 ++++ mypyc/primitives/misc_ops.py | 2 +- mypyc/test-data/fixtures/ir.py | 1 + mypyc/test-data/irbuild-classes.test | 42 +++++++++++++++++++++++++ mypyc/test-data/irbuild-try.test | 2 +- mypyc/test-data/run-classes.test | 46 ++++++++++++++++++++++++++++ 7 files changed, 103 insertions(+), 2 deletions(-) diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index 312ba98e3c5d5..e82203021ae31 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -69,6 +69,7 @@ Value, ) from mypyc.ir.rtypes import ( + RInstance, RTuple, bool_rprimitive, int_rprimitive, @@ -226,6 +227,11 @@ def transform_member_expr(builder: IRBuilder, expr: MemberExpr) -> Value: ): return builder.primitive_op(name_op, [obj], expr.line) + if isinstance(obj.type, RInstance) and expr.name == "__class__": + # A non-native class could override "__class__" using "__getattribute__", so + # only apply to RInstance types. + return builder.primitive_op(type_op, [obj], expr.line) + # Special case: for named tuples transform attribute access to faster index access. typ = get_proper_type(builder.types.get(expr.expr)) if isinstance(typ, TupleType) and typ.partial_fallback.type.is_named_tuple: diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index aea5db25f29fa..8cd141545bbbd 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -877,6 +877,12 @@ static inline bool CPy_TypeCheck(PyObject *o, PyObject *type) { return PyObject_TypeCheck(o, (PyTypeObject *)type); } +static inline PyObject *CPy_TYPE(PyObject *obj) { + PyObject *result = (PyObject *)Py_TYPE(obj); + Py_INCREF(result); + return result; +} + PyObject *CPy_CalculateMetaclass(PyObject *type, PyObject *o); PyObject *CPy_GetCoro(PyObject *obj); PyObject *CPyIter_Send(PyObject *iter, PyObject *val); diff --git a/mypyc/primitives/misc_ops.py b/mypyc/primitives/misc_ops.py index e3d59f53ed761..a13f87cc94e92 100644 --- a/mypyc/primitives/misc_ops.py +++ b/mypyc/primitives/misc_ops.py @@ -214,7 +214,7 @@ type_op = function_op( name="builtins.type", arg_types=[object_rprimitive], - c_function_name="PyObject_Type", + c_function_name="CPy_TYPE", return_type=object_rprimitive, error_kind=ERR_NEVER, ) diff --git a/mypyc/test-data/fixtures/ir.py b/mypyc/test-data/fixtures/ir.py index 28c9244b4b27b..c041c661741c2 100644 --- a/mypyc/test-data/fixtures/ir.py +++ b/mypyc/test-data/fixtures/ir.py @@ -39,6 +39,7 @@ def __pow__(self, other: T_contra, modulo: _M) -> T_co: ... ] class object: + __class__: type def __init__(self) -> None: pass def __eq__(self, x: object) -> bool: pass def __ne__(self, x: object) -> bool: pass diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index 2bdbb42b8d23e..4bb20ee9c65cc 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -1516,3 +1516,45 @@ L0: r0 = CPy_GetName(t) r1 = cast(str, r0) return r1 + +[case testTypeOfObject] +class C: pass +class D(C): pass + +def generic_type(x: object) -> type[object]: + return type(x) + +def generic_class(x: object) -> type[object]: + return x.__class__ + +def native_type(x: C) -> type[object]: + return type(x) + +def native_class(x: C) -> type[object]: + return x.__class__ +[out] +def generic_type(x): + x, r0 :: object +L0: + r0 = CPy_TYPE(x) + return r0 +def generic_class(x): + x :: object + r0 :: str + r1 :: object +L0: + r0 = '__class__' + r1 = CPyObject_GetAttr(x, r0) + return r1 +def native_type(x): + x :: __main__.C + r0 :: object +L0: + r0 = CPy_TYPE(x) + return r0 +def native_class(x): + x :: __main__.C + r0 :: object +L0: + r0 = CPy_TYPE(x) + return r0 diff --git a/mypyc/test-data/irbuild-try.test b/mypyc/test-data/irbuild-try.test index ad1aa78c0554b..ec470eae1e88e 100644 --- a/mypyc/test-data/irbuild-try.test +++ b/mypyc/test-data/irbuild-try.test @@ -412,7 +412,7 @@ def foo(x): r36 :: bit L0: r0 = PyObject_Vectorcall(x, 0, 0, 0) - r1 = PyObject_Type(r0) + r1 = CPy_TYPE(r0) r2 = '__exit__' r3 = CPyObject_GetAttr(r1, r2) r4 = '__enter__' diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 9582eec07b1a2..fed5cfb658707 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -3163,3 +3163,49 @@ def foo(): pass class Thing: def __init__(self): self.__name__ = "xyz" + +[case testTypeOfObject] +from typing import Any + +from dynamic import Dyn + +class Foo: pass +class Bar(Foo): pass + +def generic_type(x) -> type[object]: + return x.__class__ + +def test_built_in_type() -> None: + i: Any = int + l: Any = list + assert type(i()) is i().__class__ + assert type(i()) is int + assert type(l()) is list + n = 5 + assert n.__class__ is i + +def test_native_class() -> None: + f_any: Any = Foo() + b_any: Any = Bar() + f: Foo = f_any + b: Foo = b_any + if int("1"): # use int("1") to avoid constant folding + assert type(f) is Foo + assert type(b) is Bar + if int("2"): + assert f.__class__ is Foo + assert b.__class__ is Bar + if int("3"): + assert f_any.__class__ is Foo + assert b_any.__class__ is Bar + if int("4"): + assert type(f_any) is Foo + assert type(b_any) is Bar + +def test_python_class() -> None: + d = Dyn() + assert type(d) is Dyn + assert d.__class__ is Dyn + +[file dynamic.py] +class Dyn: pass From 0d23c61f067c56ef51d3c00b902c7582af1d9263 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 19 Aug 2025 08:43:49 -0700 Subject: [PATCH 184/424] Implement PEP 800 (@disjoint_base) (#19678) https://peps.python.org/pep-0800/ - Recognize the @disjoint_base decorator - Error if a class definition has incompatible disjoint bases - Recognize that classes with incompatible disjoint bases cannot exist - Check in stubtest that @disjoint_base is correctly applied - The self check found a line of dead code in mypy itself, due to classes that are disjoint bases from `__slots__`. --- mypy/checker.py | 7 ++ mypy/checkmember.py | 8 +-- mypy/message_registry.py | 3 + mypy/nodes.py | 4 ++ mypy/semanal.py | 3 + mypy/stubtest.py | 63 ++++++++++++++++++ mypy/test/teststubtest.py | 65 ++++++++++++++++++- mypy/typeops.py | 51 +++++++++++++++ mypy/types.py | 5 +- test-data/unit/check-classes.test | 2 +- test-data/unit/check-final.test | 64 ++++++++++++++++++ test-data/unit/check-incremental.test | 6 +- test-data/unit/check-isinstance.test | 27 ++++++++ test-data/unit/check-slots.test | 1 + test-data/unit/lib-stub/typing_extensions.pyi | 1 + 15 files changed, 296 insertions(+), 14 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 47c72924bf3c6..0fe77e953d066 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -171,6 +171,7 @@ from mypy.typeanal import check_for_explicit_any, has_any_from_unimported_type, make_optional_type from mypy.typeops import ( bind_self, + can_have_shared_disjoint_base, coerce_to_literal, custom_special_method, erase_def_to_union_or_bound, @@ -2658,6 +2659,8 @@ def visit_class_def(self, defn: ClassDef) -> None: for base in typ.mro[1:]: if base.is_final: self.fail(message_registry.CANNOT_INHERIT_FROM_FINAL.format(base.name), defn) + if not can_have_shared_disjoint_base(typ.bases): + self.fail(message_registry.INCOMPATIBLE_DISJOINT_BASES.format(typ.name), defn) with self.tscope.class_scope(defn.info), self.enter_partial_types(is_class=True): old_binder = self.binder self.binder = ConditionalTypeBinder(self.options) @@ -5826,6 +5829,10 @@ def _make_fake_typeinfo_and_full_name( format_type_distinctly(*base_classes, options=self.options, bare=True), "and" ) + if not can_have_shared_disjoint_base(base_classes): + errors.append((pretty_names_list, "have distinct disjoint bases")) + return None + new_errors = [] for base in base_classes: if base.type.is_final: diff --git a/mypy/checkmember.py b/mypy/checkmember.py index 2c41f2e273cc0..e7de1b7a304fe 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -1484,13 +1484,7 @@ def analyze_decorator_or_funcbase_access( if isinstance(defn, Decorator): return analyze_var(name, defn.var, itype, mx) typ = function_type(defn, mx.chk.named_type("builtins.function")) - is_trivial_self = False - if isinstance(defn, Decorator): - # Use fast path if there are trivial decorators like @classmethod or @property - is_trivial_self = defn.func.is_trivial_self and not defn.decorators - elif isinstance(defn, (FuncDef, OverloadedFuncDef)): - is_trivial_self = defn.is_trivial_self - if is_trivial_self: + if isinstance(defn, (FuncDef, OverloadedFuncDef)) and defn.is_trivial_self: return bind_self_fast(typ, mx.self_type) typ = check_self_arg(typ, mx.self_type, defn.is_class, mx.context, name, mx.msg) return bind_self(typ, original_type=mx.self_type, is_classmethod=defn.is_class) diff --git a/mypy/message_registry.py b/mypy/message_registry.py index 381aedfca0598..09004322aee9f 100644 --- a/mypy/message_registry.py +++ b/mypy/message_registry.py @@ -239,6 +239,9 @@ def with_additional_msg(self, info: str) -> ErrorMessage: ) CANNOT_MAKE_DELETABLE_FINAL: Final = ErrorMessage("Deletable attribute cannot be final") +# Disjoint bases +INCOMPATIBLE_DISJOINT_BASES: Final = ErrorMessage('Class "{}" has incompatible disjoint bases') + # Enum ENUM_MEMBERS_ATTR_WILL_BE_OVERRIDDEN: Final = ErrorMessage( 'Assigned "__members__" will be overridden by "Enum" internally' diff --git a/mypy/nodes.py b/mypy/nodes.py index 99b9bf72c9484..8c2110b156f1e 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -3004,6 +3004,7 @@ class is generic then it will be a type constructor of higher kind. "_mro_refs", "bad_mro", "is_final", + "is_disjoint_base", "declared_metaclass", "metaclass_type", "names", @@ -3055,6 +3056,7 @@ class is generic then it will be a type constructor of higher kind. _mro_refs: list[str] | None bad_mro: bool # Could not construct full MRO is_final: bool + is_disjoint_base: bool declared_metaclass: mypy.types.Instance | None metaclass_type: mypy.types.Instance | None @@ -3209,6 +3211,7 @@ class is generic then it will be a type constructor of higher kind. "is_protocol", "runtime_protocol", "is_final", + "is_disjoint_base", "is_intersection", ] @@ -3241,6 +3244,7 @@ def __init__(self, names: SymbolTable, defn: ClassDef, module_name: str) -> None self.type_var_tuple_suffix: int | None = None self.add_type_vars() self.is_final = False + self.is_disjoint_base = False self.is_enum = False self.fallback_to_any = False self.meta_fallback_to_any = False diff --git a/mypy/semanal.py b/mypy/semanal.py index eef658d9300b2..e8426a4e48858 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -254,6 +254,7 @@ ASSERT_TYPE_NAMES, DATACLASS_TRANSFORM_NAMES, DEPRECATED_TYPE_NAMES, + DISJOINT_BASE_DECORATOR_NAMES, FINAL_DECORATOR_NAMES, FINAL_TYPE_NAMES, IMPORTED_REVEAL_TYPE_NAMES, @@ -2188,6 +2189,8 @@ def analyze_class_decorator_common( """ if refers_to_fullname(decorator, FINAL_DECORATOR_NAMES): info.is_final = True + elif refers_to_fullname(decorator, DISJOINT_BASE_DECORATOR_NAMES): + info.is_disjoint_base = True elif refers_to_fullname(decorator, TYPE_CHECK_ONLY_NAMES): info.is_type_check_only = True elif (deprecated := self.get_deprecated(decorator)) is not None: diff --git a/mypy/stubtest.py b/mypy/stubtest.py index ef8c8dc318e1a..43da0518b3f9d 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -17,6 +17,7 @@ import os import pkgutil import re +import struct import symtable import sys import traceback @@ -466,6 +467,67 @@ class SubClass(runtime): # type: ignore[misc] ) +SIZEOF_PYOBJECT = struct.calcsize("P") + + +def _shape_differs(t1: type[object], t2: type[object]) -> bool: + """Check whether two types differ in shape. + + Mirrors the shape_differs() function in typeobject.c in CPython.""" + if sys.version_info >= (3, 12): + return t1.__basicsize__ != t2.__basicsize__ or t1.__itemsize__ != t2.__itemsize__ + else: + # CPython had more complicated logic before 3.12: + # https://github.com/python/cpython/blob/f3c6f882cddc8dc30320d2e73edf019e201394fc/Objects/typeobject.c#L2224 + # We attempt to mirror it here well enough to support the most common cases. + if t1.__itemsize__ or t2.__itemsize__: + return t1.__basicsize__ != t2.__basicsize__ or t1.__itemsize__ != t2.__itemsize__ + t_size = t1.__basicsize__ + if not t2.__weakrefoffset__ and t1.__weakrefoffset__ + SIZEOF_PYOBJECT == t_size: + t_size -= SIZEOF_PYOBJECT + if not t2.__dictoffset__ and t1.__dictoffset__ + SIZEOF_PYOBJECT == t_size: + t_size -= SIZEOF_PYOBJECT + if not t2.__weakrefoffset__ and t2.__weakrefoffset__ == t_size: + t_size -= SIZEOF_PYOBJECT + return t_size != t2.__basicsize__ + + +def _is_disjoint_base(typ: type[object]) -> bool: + """Return whether a type is a disjoint base at runtime, mirroring CPython's logic in typeobject.c. + + See PEP 800.""" + if typ is object: + return True + base = typ.__base__ + assert base is not None, f"Type {typ} has no base" + return _shape_differs(typ, base) + + +def _verify_disjoint_base( + stub: nodes.TypeInfo, runtime: type[object], object_path: list[str] +) -> Iterator[Error]: + # If it's final, doesn't matter whether it's a disjoint base or not + if stub.is_final: + return + is_disjoint_runtime = _is_disjoint_base(runtime) + if is_disjoint_runtime and not stub.is_disjoint_base: + yield Error( + object_path, + "is a disjoint base at runtime, but isn't marked with @disjoint_base in the stub", + stub, + runtime, + stub_desc=repr(stub), + ) + elif not is_disjoint_runtime and stub.is_disjoint_base: + yield Error( + object_path, + "is marked with @disjoint_base in the stub, but isn't a disjoint base at runtime", + stub, + runtime, + stub_desc=repr(stub), + ) + + def _verify_metaclass( stub: nodes.TypeInfo, runtime: type[Any], object_path: list[str], *, is_runtime_typeddict: bool ) -> Iterator[Error]: @@ -534,6 +596,7 @@ def verify_typeinfo( return yield from _verify_final(stub, runtime, object_path) + yield from _verify_disjoint_base(stub, runtime, object_path) is_runtime_typeddict = stub.typeddict_type is not None and is_typeddict(runtime) yield from _verify_metaclass( stub, runtime, object_path, is_runtime_typeddict=is_runtime_typeddict diff --git a/mypy/test/teststubtest.py b/mypy/test/teststubtest.py index b071c0ee8ab6e..69e2abe62f85f 100644 --- a/mypy/test/teststubtest.py +++ b/mypy/test/teststubtest.py @@ -1405,9 +1405,16 @@ def spam(x=Flags4(0)): pass ) yield Case( stub=""" + import sys from typing import Final, Literal - class BytesEnum(bytes, enum.Enum): - a = b'foo' + from typing_extensions import disjoint_base + if sys.version_info >= (3, 12): + class BytesEnum(bytes, enum.Enum): + a = b'foo' + else: + @disjoint_base + class BytesEnum(bytes, enum.Enum): + a = b'foo' FOO: Literal[BytesEnum.a] BAR: Final = BytesEnum.a BAZ: BytesEnum @@ -1613,6 +1620,60 @@ def test_not_subclassable(self) -> Iterator[Case]: error="CannotBeSubclassed", ) + @collect_cases + def test_disjoint_base(self) -> Iterator[Case]: + yield Case( + stub=""" + class A: pass + """, + runtime=""" + class A: pass + """, + error=None, + ) + yield Case( + stub=""" + from typing_extensions import disjoint_base + + @disjoint_base + class B: pass + """, + runtime=""" + class B: pass + """, + error="test_module.B", + ) + yield Case( + stub=""" + from typing_extensions import Self + + class mytakewhile: + def __new__(cls, predicate: object, iterable: object, /) -> Self: ... + def __iter__(self) -> Self: ... + def __next__(self) -> object: ... + """, + runtime=""" + from itertools import takewhile as mytakewhile + """, + # Should have @disjoint_base + error="test_module.mytakewhile", + ) + yield Case( + stub=""" + from typing_extensions import disjoint_base, Self + + @disjoint_base + class mycorrecttakewhile: + def __new__(cls, predicate: object, iterable: object, /) -> Self: ... + def __iter__(self) -> Self: ... + def __next__(self) -> object: ... + """, + runtime=""" + from itertools import takewhile as mycorrecttakewhile + """, + error=None, + ) + @collect_cases def test_has_runtime_final_decorator(self) -> Iterator[Case]: yield Case( diff --git a/mypy/typeops.py b/mypy/typeops.py index 88b3c5da48ce0..0cb6018d01fd3 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -1253,3 +1253,54 @@ def named_type(fullname: str) -> Instance: ) ) return subtype + + +def _is_disjoint_base(info: TypeInfo) -> bool: + # It either has the @disjoint_base decorator or defines nonempty __slots__. + if info.is_disjoint_base: + return True + if not info.slots: + return False + own_slots = { + slot + for slot in info.slots + if not any( + base_info.type.slots is not None and slot in base_info.type.slots + for base_info in info.bases + ) + } + return bool(own_slots) + + +def _get_disjoint_base_of(instance: Instance) -> TypeInfo | None: + """Returns the disjoint base of the given instance, if it exists.""" + if _is_disjoint_base(instance.type): + return instance.type + for base in instance.type.mro: + if _is_disjoint_base(base): + return base + return None + + +def can_have_shared_disjoint_base(instances: list[Instance]) -> bool: + """Returns whether the given instances can share a disjoint base. + + This means that a child class of these classes can exist at runtime. + """ + # Ignore None disjoint bases (which are `object`). + disjoint_bases = [ + base for instance in instances if (base := _get_disjoint_base_of(instance)) is not None + ] + if not disjoint_bases: + # All are `object`. + return True + + candidate = disjoint_bases[0] + for base in disjoint_bases[1:]: + if candidate.has_base(base.fullname): + continue + elif base.has_base(candidate.fullname): + candidate = base + else: + return False + return True diff --git a/mypy/types.py b/mypy/types.py index 26c5b474ba6cf..4fa8b8e64703a 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -119,9 +119,12 @@ # Supported Unpack type names. UNPACK_TYPE_NAMES: Final = ("typing.Unpack", "typing_extensions.Unpack") -# Supported @deprecated type names +# Supported @deprecated decorator names DEPRECATED_TYPE_NAMES: Final = ("warnings.deprecated", "typing_extensions.deprecated") +# Supported @disjoint_base decorator names +DISJOINT_BASE_DECORATOR_NAMES: Final = ("typing.disjoint_base", "typing_extensions.disjoint_base") + # We use this constant in various places when checking `tuple` subtyping: TUPLE_LIKE_INSTANCE_NAMES: Final = ( "builtins.tuple", diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index 23dbe2bc07afb..5cc4910fb2653 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -6084,7 +6084,7 @@ class B(A): __slots__ = ('a', 'b') class C: __slots__ = ('x',) -class D(B, C): +class D(B, C): # E: Class "D" has incompatible disjoint bases __slots__ = ('aa', 'bb', 'cc') [builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-final.test b/test-data/unit/check-final.test index d23199dc8b331..e3fc4614fc069 100644 --- a/test-data/unit/check-final.test +++ b/test-data/unit/check-final.test @@ -1272,3 +1272,67 @@ if FOO is not None: def func() -> int: return FOO + +[case testDisjointBase] +from typing_extensions import disjoint_base + +@disjoint_base +class Disjoint1: pass + +@disjoint_base +class Disjoint2: pass + +@disjoint_base +class DisjointChild(Disjoint1): pass + +class C1: pass +class C2(Disjoint1, C1): pass +class C3(DisjointChild, Disjoint1): pass + +class C4(Disjoint1, Disjoint2): # E: Class "C4" has incompatible disjoint bases + pass + +class C5(Disjoint2, Disjoint1): # E: Class "C5" has incompatible disjoint bases + pass + +class C6(Disjoint2, DisjointChild): # E: Class "C6" has incompatible disjoint bases + pass + +class C7(DisjointChild, Disjoint2): # E: Class "C7" has incompatible disjoint bases + pass + +class C8(DisjointChild, Disjoint1, Disjoint2): # E: Class "C8" has incompatible disjoint bases + pass + +class C9(C2, Disjoint2): # E: Class "C9" has incompatible disjoint bases + pass + +class C10(C3, Disjoint2): # E: Class "C10" has incompatible disjoint bases + pass + +[builtins fixtures/tuple.pyi] +[case testDisjointBaseSlots] +class S1: + __slots__ = ("a",) + +class S2: + __slots__ = ("b",) + +class S3: + __slots__ = () + +class S4(S1): + __slots__ = ("c",) + +class S5(S1, S2): # E: Class "S5" has incompatible disjoint bases + pass + +class S6(S1, S3): pass # OK +class S7(S3, S1): pass # OK + +class S8(S4, S1): pass # OK + +class S9(S2, S4): # E: Class "S9" has incompatible disjoint bases + pass + +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 7d791319537f8..c3fe98e69d95f 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -3170,13 +3170,13 @@ C(5, 'foo', True) [file a.py] import attrs -@attrs.define +@attrs.define(slots=False) class A: a: int [file b.py] import attrs -@attrs.define +@attrs.define(slots=False) class B: b: str @@ -3184,7 +3184,7 @@ class B: from a import A from b import B import attrs -@attrs.define +@attrs.define(slots=False) class C(A, B): c: bool diff --git a/test-data/unit/check-isinstance.test b/test-data/unit/check-isinstance.test index 640fc10915d10..5043d54221086 100644 --- a/test-data/unit/check-isinstance.test +++ b/test-data/unit/check-isinstance.test @@ -2551,6 +2551,33 @@ def f2(x: T2) -> T2: return C() [builtins fixtures/isinstance.pyi] +[case testIsInstanceDisjointBase] +# flags: --warn-unreachable +from typing_extensions import disjoint_base + +@disjoint_base +class Disjoint1: pass +@disjoint_base +class Disjoint2: pass +@disjoint_base +class Disjoint3(Disjoint1): pass +class Child(Disjoint1): pass +class Unrelated: pass + +def f(d1: Disjoint1, u: Unrelated, c: Child) -> object: + if isinstance(d1, Disjoint2): # E: Subclass of "Disjoint1" and "Disjoint2" cannot exist: have distinct disjoint bases + return u # E: Statement is unreachable + if isinstance(u, Disjoint1): # OK + return d1 + if isinstance(c, Disjoint3): # OK + return c + if isinstance(c, Disjoint2): # E: Subclass of "Child" and "Disjoint2" cannot exist: have distinct disjoint bases + return c # E: Statement is unreachable + return d1 + +[builtins fixtures/isinstance.pyi] + + [case testIsInstanceAdHocIntersectionUsage] # flags: --warn-unreachable class A: pass diff --git a/test-data/unit/check-slots.test b/test-data/unit/check-slots.test index 10b664bffb119..25dd630e1cbed 100644 --- a/test-data/unit/check-slots.test +++ b/test-data/unit/check-slots.test @@ -180,6 +180,7 @@ b.m = 2 b.b = 2 b._two = 2 [out] +main:5: error: Class "B" has incompatible disjoint bases main:11: error: Trying to assign name "_one" that is not in "__slots__" of type "__main__.B" main:16: error: "B" has no attribute "b" main:17: error: "B" has no attribute "_two" diff --git a/test-data/unit/lib-stub/typing_extensions.pyi b/test-data/unit/lib-stub/typing_extensions.pyi index cb054b0e6b4fc..6158a0c9ebbc3 100644 --- a/test-data/unit/lib-stub/typing_extensions.pyi +++ b/test-data/unit/lib-stub/typing_extensions.pyi @@ -93,5 +93,6 @@ def dataclass_transform( def override(__arg: _T) -> _T: ... def deprecated(__msg: str) -> Callable[[_T], _T]: ... +def disjoint_base(__arg: _T) -> _T: ... _FutureFeatureFixture = 0 From 657bdd84cff780a4b96c04525e2693c074a3fbf3 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 19 Aug 2025 20:33:09 +0100 Subject: [PATCH 185/424] More efficient (fixed-format) serialization (#19668) This makes deserialization ~2.5x faster compared to `orjson`. This is fully functional, but still PoC in terms of distribution logic. Some comments: * In you want to try this in compiled mode, simply install mypy using `MYPY_USE_MYPYC=1`, this will install the extension automatically (for now) * If you want to just play with the extension, or use it in interpreted mode, use `pip install mypyc/lib-rt` * I translated (de-)serialization logic from JSON methods almost verbatim (including comments) * This may be still not the most efficient way to do this, but I wanted to write something simple, that probably still gets us 90% there in terms of performance. I am still open to suggestions however * Please forgive me if the PR looks not very polished, I feel tired, but needed some kind of closure on this :-) Some technical notes: * The huge `try/except` import blob in `mypy/cache.py` is temporary, it is needed for now to be able to run tests without installing mypy itself (only with `test-requirements.txt`). * There is certain asymmetry with read/write for literals, this is intentional because we allow `complex` and/or `None` in some cases, but not in other cases. * General convention is that during deserialization the type/symbol marker is consumer by the caller (except for `MypyFile`, which is special). There is no convention for few classes that are not types/symbols. * I add new primitive type for `native_internal.Buffer` (and possible more type in future from `native`) for better/automatic method call specializations. If this feels wrong/risky, I can convert this to a more ad-hoc logic in `transform_call_expr()` Related issue: #3456 --- mypy/build.py | 49 +- mypy/cache.py | 153 ++++++ mypy/fixup.py | 2 + mypy/main.py | 3 + mypy/modulefinder.py | 7 +- mypy/nodes.py | 520 +++++++++++++++++- mypy/options.py | 2 + mypy/types.py | 465 +++++++++++++++- mypy/typeshed/stubs/mypy-native/METADATA.toml | 1 + .../stubs/mypy-native/native_internal.pyi | 12 + mypyc/analysis/ircheck.py | 3 +- mypyc/build.py | 25 + mypyc/codegen/emit.py | 3 +- mypyc/codegen/emitmodule.py | 8 +- mypyc/ir/rtypes.py | 9 + mypyc/irbuild/mapper.py | 3 + mypyc/lib-rt/native_internal.c | 510 +++++++++++++++++ mypyc/lib-rt/native_internal.h | 52 ++ mypyc/lib-rt/setup.py | 96 ++-- mypyc/options.py | 5 + mypyc/primitives/misc_ops.py | 97 ++++ mypyc/test-data/irbuild-classes.test | 49 ++ mypyc/test-data/run-classes.test | 68 +++ mypyc/test/test_external.py | 1 + mypyc/test/test_run.py | 7 +- setup.py | 4 + test-data/unit/lib-stub/native_internal.pyi | 12 + 27 files changed, 2099 insertions(+), 67 deletions(-) create mode 100644 mypy/cache.py create mode 100644 mypy/typeshed/stubs/mypy-native/METADATA.toml create mode 100644 mypy/typeshed/stubs/mypy-native/native_internal.pyi create mode 100644 mypyc/lib-rt/native_internal.c create mode 100644 mypyc/lib-rt/native_internal.h create mode 100644 test-data/unit/lib-stub/native_internal.pyi diff --git a/mypy/build.py b/mypy/build.py index 883ae1f22f196..4f22e0703d97d 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -40,6 +40,7 @@ from typing_extensions import TypeAlias as _TypeAlias import mypy.semanal_main +from mypy.cache import Buffer from mypy.checker import TypeChecker from mypy.error_formatter import OUTPUT_CHOICES, ErrorFormatter from mypy.errors import CompileError, ErrorInfo, Errors, report_internal_error @@ -1143,6 +1144,17 @@ def read_deps_cache(manager: BuildManager, graph: Graph) -> dict[str, FgDepMeta] return module_deps_metas +def _load_ff_file(file: str, manager: BuildManager, log_error: str) -> bytes | None: + t0 = time.time() + try: + data = manager.metastore.read(file) + except OSError: + manager.log(log_error + file) + return None + manager.add_stats(metastore_read_time=time.time() - t0) + return data + + def _load_json_file( file: str, manager: BuildManager, log_success: str, log_error: str ) -> dict[str, Any] | None: @@ -1263,7 +1275,11 @@ def get_cache_names(id: str, path: str, options: Options) -> tuple[str, str, str deps_json = None if options.cache_fine_grained: deps_json = prefix + ".deps.json" - return (prefix + ".meta.json", prefix + ".data.json", deps_json) + if options.fixed_format_cache: + data_suffix = ".data.ff" + else: + data_suffix = ".data.json" + return (prefix + ".meta.json", prefix + data_suffix, deps_json) def find_cache_meta(id: str, path: str, manager: BuildManager) -> CacheMeta | None: @@ -1563,8 +1579,13 @@ def write_cache( tree.path = path # Serialize data and analyze interface - data = tree.serialize() - data_bytes = json_dumps(data, manager.options.debug_cache) + if manager.options.fixed_format_cache: + data_io = Buffer() + tree.write(data_io) + data_bytes = data_io.getvalue() + else: + data = tree.serialize() + data_bytes = json_dumps(data, manager.options.debug_cache) interface_hash = hash_digest(data_bytes) plugin_data = manager.plugin.report_config_data(ReportConfigContext(id, path, is_check=False)) @@ -2089,15 +2110,23 @@ def load_tree(self, temporary: bool = False) -> None: self.meta is not None ), "Internal error: this method must be called only for cached modules" - data = _load_json_file( - self.meta.data_json, self.manager, "Load tree ", "Could not load tree: " - ) + data: bytes | dict[str, Any] | None + if self.options.fixed_format_cache: + data = _load_ff_file(self.meta.data_json, self.manager, "Could not load tree: ") + else: + data = _load_json_file( + self.meta.data_json, self.manager, "Load tree ", "Could not load tree: " + ) if data is None: return t0 = time.time() # TODO: Assert data file wasn't changed. - self.tree = MypyFile.deserialize(data) + if isinstance(data, bytes): + data_io = Buffer(data) + self.tree = MypyFile.read(data_io) + else: + self.tree = MypyFile.deserialize(data) t1 = time.time() self.manager.add_stats(deserialize_time=t1 - t0) if not temporary: @@ -2485,7 +2514,11 @@ def write_cache(self) -> None: ): if self.options.debug_serialize: try: - self.tree.serialize() + if self.manager.options.fixed_format_cache: + data = Buffer() + self.tree.write(data) + else: + self.tree.serialize() except Exception: print(f"Error serializing {self.id}", file=self.manager.stdout) raise # Propagate to display traceback diff --git a/mypy/cache.py b/mypy/cache.py new file mode 100644 index 0000000000000..49f568c1f3c19 --- /dev/null +++ b/mypy/cache.py @@ -0,0 +1,153 @@ +from __future__ import annotations + +from collections.abc import Sequence +from typing import TYPE_CHECKING, Final + +try: + from native_internal import ( + Buffer as Buffer, + read_bool as read_bool, + read_float as read_float, + read_int as read_int, + read_str as read_str, + write_bool as write_bool, + write_float as write_float, + write_int as write_int, + write_str as write_str, + ) +except ImportError: + # TODO: temporary, remove this after we publish mypy-native on PyPI. + if not TYPE_CHECKING: + + class Buffer: + def __init__(self, source: bytes = b"") -> None: + raise NotImplementedError + + def getvalue(self) -> bytes: + raise NotImplementedError + + def read_int(data: Buffer) -> int: + raise NotImplementedError + + def write_int(data: Buffer, value: int) -> None: + raise NotImplementedError + + def read_str(data: Buffer) -> str: + raise NotImplementedError + + def write_str(data: Buffer, value: str) -> None: + raise NotImplementedError + + def read_bool(data: Buffer) -> bool: + raise NotImplementedError + + def write_bool(data: Buffer, value: bool) -> None: + raise NotImplementedError + + def read_float(data: Buffer) -> float: + raise NotImplementedError + + def write_float(data: Buffer, value: float) -> None: + raise NotImplementedError + + +LITERAL_INT: Final = 1 +LITERAL_STR: Final = 2 +LITERAL_BOOL: Final = 3 +LITERAL_FLOAT: Final = 4 +LITERAL_COMPLEX: Final = 5 +LITERAL_NONE: Final = 6 + + +def read_literal(data: Buffer, marker: int) -> int | str | bool | float: + if marker == LITERAL_INT: + return read_int(data) + elif marker == LITERAL_STR: + return read_str(data) + elif marker == LITERAL_BOOL: + return read_bool(data) + elif marker == LITERAL_FLOAT: + return read_float(data) + assert False, f"Unknown literal marker {marker}" + + +def write_literal(data: Buffer, value: int | str | bool | float | complex | None) -> None: + if isinstance(value, bool): + write_int(data, LITERAL_BOOL) + write_bool(data, value) + elif isinstance(value, int): + write_int(data, LITERAL_INT) + write_int(data, value) + elif isinstance(value, str): + write_int(data, LITERAL_STR) + write_str(data, value) + elif isinstance(value, float): + write_int(data, LITERAL_FLOAT) + write_float(data, value) + elif isinstance(value, complex): + write_int(data, LITERAL_COMPLEX) + write_float(data, value.real) + write_float(data, value.imag) + else: + write_int(data, LITERAL_NONE) + + +def read_int_opt(data: Buffer) -> int | None: + if read_bool(data): + return read_int(data) + return None + + +def write_int_opt(data: Buffer, value: int | None) -> None: + if value is not None: + write_bool(data, True) + write_int(data, value) + else: + write_bool(data, False) + + +def read_str_opt(data: Buffer) -> str | None: + if read_bool(data): + return read_str(data) + return None + + +def write_str_opt(data: Buffer, value: str | None) -> None: + if value is not None: + write_bool(data, True) + write_str(data, value) + else: + write_bool(data, False) + + +def read_int_list(data: Buffer) -> list[int]: + size = read_int(data) + return [read_int(data) for _ in range(size)] + + +def write_int_list(data: Buffer, value: list[int]) -> None: + write_int(data, len(value)) + for item in value: + write_int(data, item) + + +def read_str_list(data: Buffer) -> list[str]: + size = read_int(data) + return [read_str(data) for _ in range(size)] + + +def write_str_list(data: Buffer, value: Sequence[str]) -> None: + write_int(data, len(value)) + for item in value: + write_str(data, item) + + +def read_str_opt_list(data: Buffer) -> list[str | None]: + size = read_int(data) + return [read_str_opt(data) for _ in range(size)] + + +def write_str_opt_list(data: Buffer, value: list[str | None]) -> None: + write_int(data, len(value)) + for item in value: + write_str_opt(data, item) diff --git a/mypy/fixup.py b/mypy/fixup.py index 18bdc1c6f497f..bec5929ad4b14 100644 --- a/mypy/fixup.py +++ b/mypy/fixup.py @@ -97,6 +97,8 @@ def visit_type_info(self, info: TypeInfo) -> None: info.declared_metaclass.accept(self.type_fixer) if info.metaclass_type: info.metaclass_type.accept(self.type_fixer) + if info.self_type: + info.self_type.accept(self.type_fixer) if info.alt_promote: info.alt_promote.accept(self.type_fixer) instance = Instance(info, []) diff --git a/mypy/main.py b/mypy/main.py index fd50c7677a112..0f70eb41bb14c 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -1056,6 +1056,9 @@ def add_invertible_flag( action="store_true", help="Include fine-grained dependency information in the cache for the mypy daemon", ) + incremental_group.add_argument( + "--fixed-format-cache", action="store_true", help=argparse.SUPPRESS + ) incremental_group.add_argument( "--skip-version-check", action="store_true", diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index d159736078ebf..d61c9ee3ec3fd 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -796,6 +796,7 @@ def default_lib_path( custom_typeshed_dir = os.path.abspath(custom_typeshed_dir) typeshed_dir = os.path.join(custom_typeshed_dir, "stdlib") mypy_extensions_dir = os.path.join(custom_typeshed_dir, "stubs", "mypy-extensions") + mypy_native_dir = os.path.join(custom_typeshed_dir, "stubs", "mypy-native") versions_file = os.path.join(typeshed_dir, "VERSIONS") if not os.path.isdir(typeshed_dir) or not os.path.isfile(versions_file): print( @@ -811,11 +812,13 @@ def default_lib_path( data_dir = auto typeshed_dir = os.path.join(data_dir, "typeshed", "stdlib") mypy_extensions_dir = os.path.join(data_dir, "typeshed", "stubs", "mypy-extensions") + mypy_native_dir = os.path.join(data_dir, "typeshed", "stubs", "mypy-native") path.append(typeshed_dir) - # Get mypy-extensions stubs from typeshed, since we treat it as an - # "internal" library, similar to typing and typing-extensions. + # Get mypy-extensions and mypy-native stubs from typeshed, since we treat them as + # "internal" libraries, similar to typing and typing-extensions. path.append(mypy_extensions_dir) + path.append(mypy_native_dir) # Add fallback path that can be used if we have a broken installation. if sys.platform != "win32": diff --git a/mypy/nodes.py b/mypy/nodes.py index 8c2110b156f1e..b9c08f02f3168 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import os from abc import abstractmethod from collections import defaultdict @@ -13,6 +14,30 @@ from mypy_extensions import trait import mypy.strconv +from mypy.cache import ( + LITERAL_COMPLEX, + LITERAL_NONE, + Buffer, + read_bool, + read_float, + read_int, + read_int_list, + read_int_opt, + read_literal, + read_str, + read_str_list, + read_str_opt, + read_str_opt_list, + write_bool, + write_int, + write_int_list, + write_int_opt, + write_literal, + write_str, + write_str_list, + write_str_opt, + write_str_opt_list, +) from mypy.options import Options from mypy.util import is_sunder, is_typeshed_file, short_type from mypy.visitor import ExpressionVisitor, NodeVisitor, StatementVisitor @@ -240,6 +265,13 @@ def deserialize(cls, data: JsonDict) -> SymbolNode: return method(data) raise NotImplementedError(f"unexpected .class {classname}") + def write(self, data: Buffer) -> None: + raise NotImplementedError(f"Cannot serialize {self.__class__.__name__} instance") + + @classmethod + def read(cls, data: Buffer) -> SymbolNode: + raise NotImplementedError(f"Cannot deserialize {cls.__name__} instance") + # Items: fullname, related symbol table node, surrounding type (if any) Definition: _TypeAlias = tuple[str, "SymbolTableNode", Optional["TypeInfo"]] @@ -368,7 +400,7 @@ def serialize(self) -> JsonDict: "is_stub": self.is_stub, "path": self.path, "is_partial_stub_package": self.is_partial_stub_package, - "future_import_flags": list(self.future_import_flags), + "future_import_flags": sorted(self.future_import_flags), } @classmethod @@ -384,6 +416,28 @@ def deserialize(cls, data: JsonDict) -> MypyFile: tree.future_import_flags = set(data["future_import_flags"]) return tree + def write(self, data: Buffer) -> None: + write_int(data, MYPY_FILE) + write_str(data, self._fullname) + self.names.write(data, self._fullname) + write_bool(data, self.is_stub) + write_str(data, self.path) + write_bool(data, self.is_partial_stub_package) + write_str_list(data, sorted(self.future_import_flags)) + + @classmethod + def read(cls, data: Buffer) -> MypyFile: + assert read_int(data) == MYPY_FILE + tree = MypyFile([], []) + tree._fullname = read_str(data) + tree.names = SymbolTable.read(data) + tree.is_stub = read_bool(data) + tree.path = read_str(data) + tree.is_partial_stub_package = read_bool(data) + tree.future_import_flags = set(read_str_list(data)) + tree.is_cache_skeleton = True + return tree + class ImportBase(Statement): """Base class for all import statements.""" @@ -656,6 +710,41 @@ def deserialize(cls, data: JsonDict) -> OverloadedFuncDef: # NOTE: res.info will be set in the fixup phase. return res + def write(self, data: Buffer) -> None: + write_int(data, OVERLOADED_FUNC_DEF) + write_int(data, len(self.items)) + for item in self.items: + item.write(data) + mypy.types.write_type_opt(data, self.type) + write_str(data, self._fullname) + if self.impl is None: + write_bool(data, False) + else: + write_bool(data, True) + self.impl.write(data) + write_flags(data, self, FUNCBASE_FLAGS) + write_str_opt(data, self.deprecated) + write_int_opt(data, self.setter_index) + + @classmethod + def read(cls, data: Buffer) -> OverloadedFuncDef: + res = OverloadedFuncDef([read_overload_part(data) for _ in range(read_int(data))]) + typ = mypy.types.read_type_opt(data) + if typ is not None: + assert isinstance(typ, mypy.types.ProperType) + res.type = typ + res._fullname = read_str(data) + if read_bool(data): + res.impl = read_overload_part(data) + # set line for empty overload items, as not set in __init__ + if len(res.items) > 0: + res.set_line(res.impl.line) + read_flags(data, res, FUNCBASE_FLAGS) + res.deprecated = read_str_opt(data) + res.setter_index = read_int_opt(data) + # NOTE: res.info will be set in the fixup phase. + return res + def is_dynamic(self) -> bool: return all(item.is_dynamic() for item in self.items) @@ -932,6 +1021,46 @@ def deserialize(cls, data: JsonDict) -> FuncDef: del ret.min_args return ret + def write(self, data: Buffer) -> None: + write_int(data, FUNC_DEF) + write_str(data, self._name) + mypy.types.write_type_opt(data, self.type) + write_str(data, self._fullname) + write_flags(data, self, FUNCDEF_FLAGS) + write_str_opt_list(data, self.arg_names) + write_int_list(data, [int(ak.value) for ak in self.arg_kinds]) + write_int(data, self.abstract_status) + if self.dataclass_transform_spec is None: + write_bool(data, False) + else: + write_bool(data, True) + self.dataclass_transform_spec.write(data) + write_str_opt(data, self.deprecated) + write_str_opt(data, self.original_first_arg) + + @classmethod + def read(cls, data: Buffer) -> FuncDef: + name = read_str(data) + typ: mypy.types.FunctionLike | None = None + if read_bool(data): + typ = mypy.types.read_function_like(data) + ret = FuncDef(name, [], Block([]), typ) + ret._fullname = read_str(data) + read_flags(data, ret, FUNCDEF_FLAGS) + # NOTE: ret.info is set in the fixup phase. + ret.arg_names = read_str_opt_list(data) + ret.arg_kinds = [ARG_KINDS[ak] for ak in read_int_list(data)] + ret.abstract_status = read_int(data) + if read_bool(data): + ret.dataclass_transform_spec = DataclassTransformSpec.read(data) + ret.deprecated = read_str_opt(data) + ret.original_first_arg = read_str_opt(data) + # Leave these uninitialized so that future uses will trigger an error + del ret.arguments + del ret.max_pos + del ret.min_args + return ret + # All types that are both SymbolNodes and FuncBases. See the FuncBase # docstring for the rationale. @@ -1004,6 +1133,22 @@ def deserialize(cls, data: JsonDict) -> Decorator: dec.is_overload = data["is_overload"] return dec + def write(self, data: Buffer) -> None: + write_int(data, DECORATOR) + self.func.write(data) + self.var.write(data) + write_bool(data, self.is_overload) + + @classmethod + def read(cls, data: Buffer) -> Decorator: + assert read_int(data) == FUNC_DEF + func = FuncDef.read(data) + assert read_int(data) == VAR + var = Var.read(data) + dec = Decorator(func, [], var) + dec.is_overload = read_bool(data) + return dec + def is_dynamic(self) -> bool: return self.func.is_dynamic() @@ -1180,6 +1325,35 @@ def deserialize(cls, data: JsonDict) -> Var: v.final_value = data.get("final_value") return v + def write(self, data: Buffer) -> None: + write_int(data, VAR) + write_str(data, self._name) + mypy.types.write_type_opt(data, self.type) + mypy.types.write_type_opt(data, self.setter_type) + write_str(data, self._fullname) + write_flags(data, self, VAR_FLAGS) + write_literal(data, self.final_value) + + @classmethod + def read(cls, data: Buffer) -> Var: + name = read_str(data) + typ = mypy.types.read_type_opt(data) + v = Var(name, typ) + setter_type: mypy.types.CallableType | None = None + if read_bool(data): + assert read_int(data) == mypy.types.CALLABLE_TYPE + setter_type = mypy.types.CallableType.read(data) + v.setter_type = setter_type + v.is_ready = False # Override True default set in __init__ + v._fullname = read_str(data) + read_flags(data, v, VAR_FLAGS) + marker = read_int(data) + if marker == LITERAL_COMPLEX: + v.final_value = complex(read_float(data), read_float(data)) + elif marker != LITERAL_NONE: + v.final_value = read_literal(data, marker) + return v + class ClassDef(Statement): """Class definition""" @@ -1290,6 +1464,22 @@ def deserialize(cls, data: JsonDict) -> ClassDef: res.fullname = data["fullname"] return res + def write(self, data: Buffer) -> None: + write_int(data, CLASS_DEF) + write_str(data, self.name) + mypy.types.write_type_list(data, self.type_vars) + write_str(data, self.fullname) + + @classmethod + def read(cls, data: Buffer) -> ClassDef: + res = ClassDef( + read_str(data), + Block([]), + [mypy.types.read_type_var_like(data) for _ in range(read_int(data))], + ) + res.fullname = read_str(data) + return res + class GlobalDecl(Statement): """Declaration global x, y, ...""" @@ -2707,6 +2897,26 @@ def deserialize(cls, data: JsonDict) -> TypeVarExpr: data["variance"], ) + def write(self, data: Buffer) -> None: + write_int(data, TYPE_VAR_EXPR) + write_str(data, self._name) + write_str(data, self._fullname) + mypy.types.write_type_list(data, self.values) + self.upper_bound.write(data) + self.default.write(data) + write_int(data, self.variance) + + @classmethod + def read(cls, data: Buffer) -> TypeVarExpr: + return TypeVarExpr( + read_str(data), + read_str(data), + mypy.types.read_type_list(data), + mypy.types.read_type(data), + mypy.types.read_type(data), + read_int(data), + ) + class ParamSpecExpr(TypeVarLikeExpr): __slots__ = () @@ -2737,6 +2947,24 @@ def deserialize(cls, data: JsonDict) -> ParamSpecExpr: data["variance"], ) + def write(self, data: Buffer) -> None: + write_int(data, PARAM_SPEC_EXPR) + write_str(data, self._name) + write_str(data, self._fullname) + self.upper_bound.write(data) + self.default.write(data) + write_int(data, self.variance) + + @classmethod + def read(cls, data: Buffer) -> ParamSpecExpr: + return ParamSpecExpr( + read_str(data), + read_str(data), + mypy.types.read_type(data), + mypy.types.read_type(data), + read_int(data), + ) + class TypeVarTupleExpr(TypeVarLikeExpr): """Type variable tuple expression TypeVarTuple(...).""" @@ -2787,6 +3015,28 @@ def deserialize(cls, data: JsonDict) -> TypeVarTupleExpr: data["variance"], ) + def write(self, data: Buffer) -> None: + write_int(data, TYPE_VAR_TUPLE_EXPR) + self.tuple_fallback.write(data) + write_str(data, self._name) + write_str(data, self._fullname) + self.upper_bound.write(data) + self.default.write(data) + write_int(data, self.variance) + + @classmethod + def read(cls, data: Buffer) -> TypeVarTupleExpr: + assert read_int(data) == mypy.types.INSTANCE + fallback = mypy.types.Instance.read(data) + return TypeVarTupleExpr( + read_str(data), + read_str(data), + mypy.types.read_type(data), + fallback, + mypy.types.read_type(data), + read_int(data), + ) + class TypeAliasExpr(Expression): """Type alias expression (rvalue).""" @@ -3598,7 +3848,6 @@ def deserialize(cls, data: JsonDict) -> TypeInfo: module_name = data["module_name"] ti = TypeInfo(names, defn, module_name) ti._fullname = data["fullname"] - # TODO: Is there a reason to reconstruct ti.subtypes? ti.abstract_attributes = [(attr[0], attr[1]) for attr in data["abstract_attributes"]] ti.type_vars = data["type_vars"] ti.has_param_spec_type = data["has_param_spec_type"] @@ -3658,6 +3907,100 @@ def deserialize(cls, data: JsonDict) -> TypeInfo: ti.deprecated = data.get("deprecated") return ti + def write(self, data: Buffer) -> None: + write_int(data, TYPE_INFO) + self.names.write(data, self.fullname) + self.defn.write(data) + write_str(data, self.module_name) + write_str(data, self.fullname) + write_str_list(data, [a for a, _ in self.abstract_attributes]) + write_int_list(data, [s for _, s in self.abstract_attributes]) + write_str_list(data, self.type_vars) + write_bool(data, self.has_param_spec_type) + mypy.types.write_type_list(data, self.bases) + write_str_list(data, [c.fullname for c in self.mro]) + mypy.types.write_type_list(data, self._promote) + mypy.types.write_type_opt(data, self.alt_promote) + mypy.types.write_type_opt(data, self.declared_metaclass) + mypy.types.write_type_opt(data, self.metaclass_type) + mypy.types.write_type_opt(data, self.tuple_type) + mypy.types.write_type_opt(data, self.typeddict_type) + write_flags(data, self, TypeInfo.FLAGS) + write_str(data, json.dumps(self.metadata)) + if self.slots is None: + write_bool(data, False) + else: + write_bool(data, True) + write_str_list(data, sorted(self.slots)) + write_str_list(data, self.deletable_attributes) + mypy.types.write_type_opt(data, self.self_type) + if self.dataclass_transform_spec is None: + write_bool(data, False) + else: + write_bool(data, True) + self.dataclass_transform_spec.write(data) + write_str_opt(data, self.deprecated) + + @classmethod + def read(cls, data: Buffer) -> TypeInfo: + names = SymbolTable.read(data) + assert read_int(data) == CLASS_DEF + defn = ClassDef.read(data) + module_name = read_str(data) + ti = TypeInfo(names, defn, module_name) + ti._fullname = read_str(data) + attrs = read_str_list(data) + statuses = read_int_list(data) + ti.abstract_attributes = list(zip(attrs, statuses)) + ti.type_vars = read_str_list(data) + ti.has_param_spec_type = read_bool(data) + num_bases = read_int(data) + ti.bases = [] + for _ in range(num_bases): + assert read_int(data) == mypy.types.INSTANCE + ti.bases.append(mypy.types.Instance.read(data)) + # NOTE: ti.mro will be set in the fixup phase based on these + # names. The reason we need to store the mro instead of just + # recomputing it from base classes has to do with a subtle + # point about fine-grained incremental: the cache files might + # not be loaded until after a class in the mro has changed its + # bases, which causes the mro to change. If we recomputed our + # mro, we would compute the *new* mro, which leaves us with no + # way to detect that the mro has changed! Thus, we need to make + # sure to load the original mro so that once the class is + # rechecked, it can tell that the mro has changed. + ti._mro_refs = read_str_list(data) + ti._promote = cast(list[mypy.types.ProperType], mypy.types.read_type_list(data)) + if read_bool(data): + assert read_int(data) == mypy.types.INSTANCE + ti.alt_promote = mypy.types.Instance.read(data) + if read_bool(data): + assert read_int(data) == mypy.types.INSTANCE + ti.declared_metaclass = mypy.types.Instance.read(data) + if read_bool(data): + assert read_int(data) == mypy.types.INSTANCE + ti.metaclass_type = mypy.types.Instance.read(data) + if read_bool(data): + assert read_int(data) == mypy.types.TUPLE_TYPE + ti.tuple_type = mypy.types.TupleType.read(data) + if read_bool(data): + assert read_int(data) == mypy.types.TYPED_DICT_TYPE + ti.typeddict_type = mypy.types.TypedDictType.read(data) + read_flags(data, ti, TypeInfo.FLAGS) + metadata = read_str(data) + if metadata != "{}": + ti.metadata = json.loads(metadata) + if read_bool(data): + ti.slots = set(read_str_list(data)) + ti.deletable_attributes = read_str_list(data) + if read_bool(data): + assert read_int(data) == mypy.types.TYPE_VAR_TYPE + ti.self_type = mypy.types.TypeVarType.read(data) + if read_bool(data): + ti.dataclass_transform_spec = DataclassTransformSpec.read(data) + ti.deprecated = read_str_opt(data) + return ti + class FakeInfo(TypeInfo): __slots__ = ("msg",) @@ -3886,6 +4229,9 @@ def fullname(self) -> str: def has_param_spec_type(self) -> bool: return any(isinstance(v, mypy.types.ParamSpecType) for v in self.alias_tvars) + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_type_alias(self) + def serialize(self) -> JsonDict: data: JsonDict = { ".class": "TypeAlias", @@ -3900,9 +4246,6 @@ def serialize(self) -> JsonDict: } return data - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_type_alias(self) - @classmethod def deserialize(cls, data: JsonDict) -> TypeAlias: assert data[".class"] == "TypeAlias" @@ -3926,6 +4269,33 @@ def deserialize(cls, data: JsonDict) -> TypeAlias: python_3_12_type_alias=python_3_12_type_alias, ) + def write(self, data: Buffer) -> None: + write_int(data, TYPE_ALIAS) + write_str(data, self._fullname) + self.target.write(data) + mypy.types.write_type_list(data, self.alias_tvars) + write_int(data, self.line) + write_int(data, self.column) + write_bool(data, self.no_args) + write_bool(data, self.normalized) + write_bool(data, self.python_3_12_type_alias) + + @classmethod + def read(cls, data: Buffer) -> TypeAlias: + fullname = read_str(data) + target = mypy.types.read_type(data) + alias_tvars = [mypy.types.read_type_var_like(data) for _ in range(read_int(data))] + return TypeAlias( + target, + fullname, + read_int(data), + read_int(data), + alias_tvars=alias_tvars, + no_args=read_bool(data), + normalized=read_bool(data), + python_3_12_type_alias=read_bool(data), + ) + class PlaceholderNode(SymbolNode): """Temporary symbol node that will later become a real SymbolNode. @@ -4184,6 +4554,49 @@ def deserialize(cls, data: JsonDict) -> SymbolTableNode: stnode.plugin_generated = data["plugin_generated"] return stnode + def write(self, data: Buffer, prefix: str, name: str) -> None: + write_int(data, self.kind) + write_bool(data, self.module_hidden) + write_bool(data, self.module_public) + write_bool(data, self.implicit) + write_bool(data, self.plugin_generated) + + cross_ref = None + if isinstance(self.node, MypyFile): + cross_ref = self.node.fullname + else: + assert self.node is not None, f"{prefix}:{name}" + if prefix is not None: + fullname = self.node.fullname + if ( + "." in fullname + and fullname != prefix + "." + name + and not (isinstance(self.node, Var) and self.node.from_module_getattr) + ): + assert not isinstance( + self.node, PlaceholderNode + ), f"Definition of {fullname} is unexpectedly incomplete" + cross_ref = fullname + + write_str_opt(data, cross_ref) + if cross_ref is None: + assert self.node is not None + self.node.write(data) + + @classmethod + def read(cls, data: Buffer) -> SymbolTableNode: + sym = SymbolTableNode(read_int(data), None) + sym.module_hidden = read_bool(data) + sym.module_public = read_bool(data) + sym.implicit = read_bool(data) + sym.plugin_generated = read_bool(data) + cross_ref = read_str_opt(data) + if cross_ref is None: + sym.node = read_symbol(data) + else: + sym.cross_ref = cross_ref + return sym + class SymbolTable(dict[str, SymbolTableNode]): """Static representation of a namespace dictionary. @@ -4235,6 +4648,29 @@ def deserialize(cls, data: JsonDict) -> SymbolTable: st[key] = SymbolTableNode.deserialize(value) return st + def write(self, data: Buffer, fullname: str) -> None: + size = 0 + for key, value in self.items(): + # Skip __builtins__: it's a reference to the builtins + # module that gets added to every module by + # SemanticAnalyzerPass2.visit_file(), but it shouldn't be + # accessed by users of the module. + if key == "__builtins__" or value.no_serialize: + continue + size += 1 + write_int(data, size) + for key in sorted(self): + value = self[key] + if key == "__builtins__" or value.no_serialize: + continue + write_str(data, key) + value.write(data, fullname, key) + + @classmethod + def read(cls, data: Buffer) -> SymbolTable: + size = read_int(data) + return SymbolTable([(read_str(data), SymbolTableNode.read(data)) for _ in range(size)]) + class DataclassTransformSpec: """Specifies how a dataclass-like transform should be applied. The fields here are based on the @@ -4285,6 +4721,23 @@ def deserialize(cls, data: JsonDict) -> DataclassTransformSpec: field_specifiers=tuple(data.get("field_specifiers", [])), ) + def write(self, data: Buffer) -> None: + write_bool(data, self.eq_default) + write_bool(data, self.order_default) + write_bool(data, self.kw_only_default) + write_bool(data, self.frozen_default) + write_str_list(data, self.field_specifiers) + + @classmethod + def read(cls, data: Buffer) -> DataclassTransformSpec: + return DataclassTransformSpec( + eq_default=read_bool(data), + order_default=read_bool(data), + kw_only_default=read_bool(data), + frozen_default=read_bool(data), + field_specifiers=tuple(read_str_list(data)), + ) + def get_flags(node: Node, names: list[str]) -> list[str]: return [name for name in names if getattr(node, name)] @@ -4295,6 +4748,17 @@ def set_flags(node: Node, flags: list[str]) -> None: setattr(node, name, True) +def write_flags(data: Buffer, node: SymbolNode, flags: list[str]) -> None: + for flag in flags: + write_bool(data, getattr(node, flag)) + + +def read_flags(data: Buffer, node: SymbolNode, flags: list[str]) -> None: + for flag in flags: + if read_bool(data): + setattr(node, flag, True) + + def get_member_expr_fullname(expr: MemberExpr) -> str | None: """Return the qualified name representation of a member expression. @@ -4410,3 +4874,49 @@ def local_definitions( yield fullname, symnode, info if isinstance(node, TypeInfo): yield from local_definitions(node.names, fullname, node) + + +MYPY_FILE: Final = 0 +OVERLOADED_FUNC_DEF: Final = 1 +FUNC_DEF: Final = 2 +DECORATOR: Final = 3 +VAR: Final = 4 +TYPE_VAR_EXPR: Final = 5 +PARAM_SPEC_EXPR: Final = 6 +TYPE_VAR_TUPLE_EXPR: Final = 7 +TYPE_INFO: Final = 8 +TYPE_ALIAS: Final = 9 +CLASS_DEF: Final = 10 + + +def read_symbol(data: Buffer) -> mypy.nodes.SymbolNode: + marker = read_int(data) + # The branches here are ordered manually by type "popularity". + if marker == VAR: + return mypy.nodes.Var.read(data) + if marker == FUNC_DEF: + return mypy.nodes.FuncDef.read(data) + if marker == DECORATOR: + return mypy.nodes.Decorator.read(data) + if marker == TYPE_INFO: + return mypy.nodes.TypeInfo.read(data) + if marker == OVERLOADED_FUNC_DEF: + return mypy.nodes.OverloadedFuncDef.read(data) + if marker == TYPE_VAR_EXPR: + return mypy.nodes.TypeVarExpr.read(data) + if marker == TYPE_ALIAS: + return mypy.nodes.TypeAlias.read(data) + if marker == PARAM_SPEC_EXPR: + return mypy.nodes.ParamSpecExpr.read(data) + if marker == TYPE_VAR_TUPLE_EXPR: + return mypy.nodes.TypeVarTupleExpr.read(data) + assert False, f"Unknown symbol marker {marker}" + + +def read_overload_part(data: Buffer) -> OverloadPart: + marker = read_int(data) + if marker == DECORATOR: + return Decorator.read(data) + if marker == FUNC_DEF: + return FuncDef.read(data) + assert False, f"Invalid marker for an OverloadPart {marker}" diff --git a/mypy/options.py b/mypy/options.py index 6d7eca7728883..ad4b26cca095f 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -72,6 +72,7 @@ class BuildType: "disable_bytearray_promotion", "disable_memoryview_promotion", "strict_bytes", + "fixed_format_cache", } ) - {"debug_cache"} @@ -286,6 +287,7 @@ def __init__(self) -> None: self.incremental = True self.cache_dir = defaults.CACHE_DIR self.sqlite_cache = False + self.fixed_format_cache = False self.debug_cache = False self.skip_version_check = False self.skip_cache_mtime_checks = False diff --git a/mypy/types.py b/mypy/types.py index 4fa8b8e64703a..b48e0ef4d9855 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -10,6 +10,25 @@ import mypy.nodes from mypy.bogus_type import Bogus +from mypy.cache import ( + Buffer, + read_bool, + read_int, + read_int_list, + read_literal, + read_str, + read_str_list, + read_str_opt, + read_str_opt_list, + write_bool, + write_int, + write_int_list, + write_literal, + write_str, + write_str_list, + write_str_opt, + write_str_opt_list, +) from mypy.nodes import ARG_KINDS, ARG_POS, ARG_STAR, ARG_STAR2, INVARIANT, ArgKind, SymbolNode from mypy.options import Options from mypy.state import state @@ -276,6 +295,13 @@ def serialize(self) -> JsonDict | str: def deserialize(cls, data: JsonDict) -> Type: raise NotImplementedError(f"Cannot deserialize {cls.__name__} instance") + def write(self, data: Buffer) -> None: + raise NotImplementedError(f"Cannot serialize {self.__class__.__name__} instance") + + @classmethod + def read(cls, data: Buffer) -> Type: + raise NotImplementedError(f"Cannot deserialize {cls.__name__} instance") + def is_singleton_type(self) -> bool: return False @@ -391,6 +417,11 @@ def can_be_false_default(self) -> bool: return self.alias.target.can_be_false return super().can_be_false_default() + def copy_modified(self, *, args: list[Type] | None = None) -> TypeAliasType: + return TypeAliasType( + self.alias, args if args is not None else self.args.copy(), self.line, self.column + ) + def accept(self, visitor: TypeVisitor[T]) -> T: return visitor.visit_type_alias_type(self) @@ -424,10 +455,17 @@ def deserialize(cls, data: JsonDict) -> TypeAliasType: alias.type_ref = data["type_ref"] return alias - def copy_modified(self, *, args: list[Type] | None = None) -> TypeAliasType: - return TypeAliasType( - self.alias, args if args is not None else self.args.copy(), self.line, self.column - ) + def write(self, data: Buffer) -> None: + write_int(data, TYPE_ALIAS_TYPE) + write_type_list(data, self.args) + assert self.alias is not None + write_str(data, self.alias.fullname) + + @classmethod + def read(cls, data: Buffer) -> TypeAliasType: + alias = TypeAliasType(None, read_type_list(data)) + alias.type_ref = read_str(data) + return alias class TypeGuardedType(Type): @@ -696,6 +734,29 @@ def deserialize(cls, data: JsonDict) -> TypeVarType: variance=data["variance"], ) + def write(self, data: Buffer) -> None: + write_int(data, TYPE_VAR_TYPE) + write_str(data, self.name) + write_str(data, self.fullname) + write_int(data, self.id.raw_id) + write_str(data, self.id.namespace) + write_type_list(data, self.values) + self.upper_bound.write(data) + self.default.write(data) + write_int(data, self.variance) + + @classmethod + def read(cls, data: Buffer) -> TypeVarType: + return TypeVarType( + read_str(data), + read_str(data), + TypeVarId(read_int(data), namespace=read_str(data)), + read_type_list(data), + read_type(data), + read_type(data), + read_int(data), + ) + class ParamSpecFlavor: # Simple ParamSpec reference such as "P" @@ -825,6 +886,31 @@ def deserialize(cls, data: JsonDict) -> ParamSpecType: prefix=Parameters.deserialize(data["prefix"]), ) + def write(self, data: Buffer) -> None: + write_int(data, PARAM_SPEC_TYPE) + self.prefix.write(data) + write_str(data, self.name) + write_str(data, self.fullname) + write_int(data, self.id.raw_id) + write_str(data, self.id.namespace) + write_int(data, self.flavor) + self.upper_bound.write(data) + self.default.write(data) + + @classmethod + def read(cls, data: Buffer) -> ParamSpecType: + assert read_int(data) == PARAMETERS + prefix = Parameters.read(data) + return ParamSpecType( + read_str(data), + read_str(data), + TypeVarId(read_int(data), namespace=read_str(data)), + read_int(data), + read_type(data), + read_type(data), + prefix=prefix, + ) + class TypeVarTupleType(TypeVarLikeType): """Type that refers to a TypeVarTuple. @@ -880,6 +966,31 @@ def deserialize(cls, data: JsonDict) -> TypeVarTupleType: min_len=data["min_len"], ) + def write(self, data: Buffer) -> None: + write_int(data, TYPE_VAR_TUPLE_TYPE) + self.tuple_fallback.write(data) + write_str(data, self.name) + write_str(data, self.fullname) + write_int(data, self.id.raw_id) + write_str(data, self.id.namespace) + self.upper_bound.write(data) + self.default.write(data) + write_int(data, self.min_len) + + @classmethod + def read(cls, data: Buffer) -> TypeVarTupleType: + assert read_int(data) == INSTANCE + fallback = Instance.read(data) + return TypeVarTupleType( + read_str(data), + read_str(data), + TypeVarId(read_int(data), namespace=read_str(data)), + read_type(data), + fallback, + read_type(data), + min_len=read_int(data), + ) + def accept(self, visitor: TypeVisitor[T]) -> T: return visitor.visit_type_var_tuple(self) @@ -1011,6 +1122,22 @@ def deserialize(cls, data: JsonDict) -> UnboundType: original_str_fallback=data["expr_fallback"], ) + def write(self, data: Buffer) -> None: + write_int(data, UNBOUND_TYPE) + write_str(data, self.name) + write_type_list(data, self.args) + write_str_opt(data, self.original_str_expr) + write_str_opt(data, self.original_str_fallback) + + @classmethod + def read(cls, data: Buffer) -> UnboundType: + return UnboundType( + read_str(data), + read_type_list(data), + original_str_expr=read_str_opt(data), + original_str_fallback=read_str_opt(data), + ) + class CallableArgument(ProperType): """Represents a Arg(type, 'name') inside a Callable's type list. @@ -1105,6 +1232,14 @@ def accept(self, visitor: TypeVisitor[T]) -> T: def serialize(self) -> JsonDict: return {".class": "UnpackType", "type": self.type.serialize()} + def write(self, data: Buffer) -> None: + write_int(data, UNPACK_TYPE) + self.type.write(data) + + @classmethod + def read(cls, data: Buffer) -> UnpackType: + return UnpackType(read_type(data)) + @classmethod def deserialize(cls, data: JsonDict) -> UnpackType: assert data[".class"] == "UnpackType" @@ -1206,6 +1341,21 @@ def deserialize(cls, data: JsonDict) -> AnyType: data["missing_import_name"], ) + def write(self, data: Buffer) -> None: + write_int(data, ANY_TYPE) + write_type_opt(data, self.source_any) + write_int(data, self.type_of_any) + write_str_opt(data, self.missing_import_name) + + @classmethod + def read(cls, data: Buffer) -> AnyType: + if read_bool(data): + assert read_int(data) == ANY_TYPE + source_any = AnyType.read(data) + else: + source_any = None + return AnyType(read_int(data), source_any, read_str_opt(data)) + class UninhabitedType(ProperType): """This type has no members. @@ -1252,6 +1402,13 @@ def deserialize(cls, data: JsonDict) -> UninhabitedType: assert data[".class"] == "UninhabitedType" return UninhabitedType() + def write(self, data: Buffer) -> None: + write_int(data, UNINHABITED_TYPE) + + @classmethod + def read(cls, data: Buffer) -> UninhabitedType: + return UninhabitedType() + class NoneType(ProperType): """The type of 'None'. @@ -1284,6 +1441,13 @@ def deserialize(cls, data: JsonDict) -> NoneType: assert data[".class"] == "NoneType" return NoneType() + def write(self, data: Buffer) -> None: + write_int(data, NONE_TYPE) + + @classmethod + def read(cls, data: Buffer) -> NoneType: + return NoneType() + def is_singleton_type(self) -> bool: return True @@ -1331,6 +1495,14 @@ def deserialize(cls, data: JsonDict) -> DeletedType: assert data[".class"] == "DeletedType" return DeletedType(data["source"]) + def write(self, data: Buffer) -> None: + write_int(data, DELETED_TYPE) + write_str_opt(data, self.source) + + @classmethod + def read(cls, data: Buffer) -> DeletedType: + return DeletedType(read_str_opt(data)) + # Fake TypeInfo to be used as a placeholder during Instance de-serialization. NOT_READY: Final = mypy.nodes.FakeInfo("De-serialization failure: TypeInfo not fixed") @@ -1373,7 +1545,7 @@ def serialize(self) -> JsonDict: return { ".class": "ExtraAttrs", "attrs": {k: v.serialize() for k, v in self.attrs.items()}, - "immutable": list(self.immutable), + "immutable": sorted(self.immutable), "mod_name": self.mod_name, } @@ -1386,6 +1558,15 @@ def deserialize(cls, data: JsonDict) -> ExtraAttrs: data["mod_name"], ) + def write(self, data: Buffer) -> None: + write_type_map(data, self.attrs) + write_str_list(data, sorted(self.immutable)) + write_str_opt(data, self.mod_name) + + @classmethod + def read(cls, data: Buffer) -> ExtraAttrs: + return ExtraAttrs(read_type_map(data), set(read_str_list(data)), read_str_opt(data)) + class Instance(ProperType): """An instance type of form C[T1, ..., Tn]. @@ -1522,6 +1703,29 @@ def deserialize(cls, data: JsonDict | str) -> Instance: inst.extra_attrs = ExtraAttrs.deserialize(data["extra_attrs"]) return inst + def write(self, data: Buffer) -> None: + write_int(data, INSTANCE) + write_str(data, self.type.fullname) + write_type_list(data, self.args) + write_type_opt(data, self.last_known_value) + if self.extra_attrs is None: + write_bool(data, False) + else: + write_bool(data, True) + self.extra_attrs.write(data) + + @classmethod + def read(cls, data: Buffer) -> Instance: + type_ref = read_str(data) + inst = Instance(NOT_READY, read_type_list(data)) + inst.type_ref = type_ref + if read_bool(data): + assert read_int(data) == LITERAL_TYPE + inst.last_known_value = LiteralType.read(data) + if read_bool(data): + inst.extra_attrs = ExtraAttrs.read(data) + return inst + def copy_modified( self, *, @@ -1798,6 +2002,26 @@ def deserialize(cls, data: JsonDict) -> Parameters: imprecise_arg_kinds=data["imprecise_arg_kinds"], ) + def write(self, data: Buffer) -> None: + write_int(data, PARAMETERS) + write_type_list(data, self.arg_types) + write_int_list(data, [int(x.value) for x in self.arg_kinds]) + write_str_opt_list(data, self.arg_names) + write_type_list(data, self.variables) + write_bool(data, self.imprecise_arg_kinds) + + @classmethod + def read(cls, data: Buffer) -> Parameters: + return Parameters( + read_type_list(data), + # This is a micro-optimization until mypyc gets dedicated enum support. Otherwise, + # we would spend ~20% of types deserialization time in Enum.__call__(). + [ARG_KINDS[ak] for ak in read_int_list(data)], + read_str_opt_list(data), + variables=[read_type_var_like(data) for _ in range(read_int(data))], + imprecise_arg_kinds=read_bool(data), + ) + def __hash__(self) -> int: return hash( ( @@ -2300,6 +2524,46 @@ def deserialize(cls, data: JsonDict) -> CallableType: unpack_kwargs=data["unpack_kwargs"], ) + def write(self, data: Buffer) -> None: + write_int(data, CALLABLE_TYPE) + self.fallback.write(data) + write_type_list(data, self.arg_types) + write_int_list(data, [int(x.value) for x in self.arg_kinds]) + write_str_opt_list(data, self.arg_names) + self.ret_type.write(data) + write_str_opt(data, self.name) + write_type_list(data, self.variables) + write_bool(data, self.is_ellipsis_args) + write_bool(data, self.implicit) + write_bool(data, self.is_bound) + write_type_opt(data, self.type_guard) + write_type_opt(data, self.type_is) + write_bool(data, self.from_concatenate) + write_bool(data, self.imprecise_arg_kinds) + write_bool(data, self.unpack_kwargs) + + @classmethod + def read(cls, data: Buffer) -> CallableType: + assert read_int(data) == INSTANCE + fallback = Instance.read(data) + return CallableType( + read_type_list(data), + [ARG_KINDS[ak] for ak in read_int_list(data)], + read_str_opt_list(data), + read_type(data), + fallback, + name=read_str_opt(data), + variables=[read_type_var_like(data) for _ in range(read_int(data))], + is_ellipsis_args=read_bool(data), + implicit=read_bool(data), + is_bound=read_bool(data), + type_guard=read_type_opt(data), + type_is=read_type_opt(data), + from_concatenate=read_bool(data), + imprecise_arg_kinds=read_bool(data), + unpack_kwargs=read_bool(data), + ) + # This is a little safety net to prevent reckless special-casing of callables # that can potentially break Unpack[...] with **kwargs. @@ -2375,6 +2639,19 @@ def deserialize(cls, data: JsonDict) -> Overloaded: assert data[".class"] == "Overloaded" return Overloaded([CallableType.deserialize(t) for t in data["items"]]) + def write(self, data: Buffer) -> None: + write_int(data, OVERLOADED) + write_type_list(data, self.items) + + @classmethod + def read(cls, data: Buffer) -> Overloaded: + items = [] + num_overloads = read_int(data) + for _ in range(num_overloads): + assert read_int(data) == CALLABLE_TYPE + items.append(CallableType.read(data)) + return Overloaded(items) + class TupleType(ProperType): """The tuple type Tuple[T1, ..., Tn] (at least one type argument). @@ -2471,6 +2748,18 @@ def deserialize(cls, data: JsonDict) -> TupleType: implicit=data["implicit"], ) + def write(self, data: Buffer) -> None: + write_int(data, TUPLE_TYPE) + self.partial_fallback.write(data) + write_type_list(data, self.items) + write_bool(data, self.implicit) + + @classmethod + def read(cls, data: Buffer) -> TupleType: + assert read_int(data) == INSTANCE + fallback = Instance.read(data) + return TupleType(read_type_list(data), fallback, implicit=read_bool(data)) + def copy_modified( self, *, fallback: Instance | None = None, items: list[Type] | None = None ) -> TupleType: @@ -2641,6 +2930,21 @@ def deserialize(cls, data: JsonDict) -> TypedDictType: Instance.deserialize(data["fallback"]), ) + def write(self, data: Buffer) -> None: + write_int(data, TYPED_DICT_TYPE) + self.fallback.write(data) + write_type_map(data, self.items) + write_str_list(data, sorted(self.required_keys)) + write_str_list(data, sorted(self.readonly_keys)) + + @classmethod + def read(cls, data: Buffer) -> TypedDictType: + assert read_int(data) == INSTANCE + fallback = Instance.read(data) + return TypedDictType( + read_type_map(data), set(read_str_list(data)), set(read_str_list(data)), fallback + ) + @property def is_final(self) -> bool: return self.fallback.type.is_final @@ -2889,6 +3193,18 @@ def deserialize(cls, data: JsonDict) -> LiteralType: assert data[".class"] == "LiteralType" return LiteralType(value=data["value"], fallback=Instance.deserialize(data["fallback"])) + def write(self, data: Buffer) -> None: + write_int(data, LITERAL_TYPE) + self.fallback.write(data) + write_literal(data, self.value) + + @classmethod + def read(cls, data: Buffer) -> LiteralType: + assert read_int(data) == INSTANCE + fallback = Instance.read(data) + marker = read_int(data) + return LiteralType(read_literal(data, marker), fallback) + def is_singleton_type(self) -> bool: return self.is_enum_literal() or isinstance(self.value, bool) @@ -2990,6 +3306,15 @@ def deserialize(cls, data: JsonDict) -> UnionType: uses_pep604_syntax=data["uses_pep604_syntax"], ) + def write(self, data: Buffer) -> None: + write_int(data, UNION_TYPE) + write_type_list(data, self.items) + write_bool(data, self.uses_pep604_syntax) + + @classmethod + def read(cls, data: Buffer) -> UnionType: + return UnionType(read_type_list(data), uses_pep604_syntax=read_bool(data)) + class PartialType(ProperType): """Type such as List[?] where type arguments are unknown, or partial None type. @@ -3126,6 +3451,14 @@ def deserialize(cls, data: JsonDict) -> Type: assert data[".class"] == "TypeType" return TypeType.make_normalized(deserialize_type(data["item"])) + def write(self, data: Buffer) -> None: + write_int(data, TYPE_TYPE) + self.item.write(data) + + @classmethod + def read(cls, data: Buffer) -> Type: + return TypeType.make_normalized(read_type(data)) + class PlaceholderType(ProperType): """Temporary, yet-unknown type during semantic analysis. @@ -3786,6 +4119,128 @@ def type_vars_as_args(type_vars: Sequence[TypeVarLikeType]) -> tuple[Type, ...]: return tuple(args) +TYPE_ALIAS_TYPE: Final = 1 +TYPE_VAR_TYPE: Final = 2 +PARAM_SPEC_TYPE: Final = 3 +TYPE_VAR_TUPLE_TYPE: Final = 4 +UNBOUND_TYPE: Final = 5 +UNPACK_TYPE: Final = 6 +ANY_TYPE: Final = 7 +UNINHABITED_TYPE: Final = 8 +NONE_TYPE: Final = 9 +DELETED_TYPE: Final = 10 +INSTANCE: Final = 11 +CALLABLE_TYPE: Final = 12 +OVERLOADED: Final = 13 +TUPLE_TYPE: Final = 14 +TYPED_DICT_TYPE: Final = 15 +LITERAL_TYPE: Final = 16 +UNION_TYPE: Final = 17 +TYPE_TYPE: Final = 18 +PARAMETERS: Final = 19 + + +def read_type(data: Buffer) -> Type: + marker = read_int(data) + # The branches here are ordered manually by type "popularity". + if marker == INSTANCE: + return Instance.read(data) + if marker == ANY_TYPE: + return AnyType.read(data) + if marker == TYPE_VAR_TYPE: + return TypeVarType.read(data) + if marker == CALLABLE_TYPE: + return CallableType.read(data) + if marker == NONE_TYPE: + return NoneType.read(data) + if marker == UNION_TYPE: + return UnionType.read(data) + if marker == LITERAL_TYPE: + return LiteralType.read(data) + if marker == TYPE_ALIAS_TYPE: + return TypeAliasType.read(data) + if marker == TUPLE_TYPE: + return TupleType.read(data) + if marker == TYPED_DICT_TYPE: + return TypedDictType.read(data) + if marker == TYPE_TYPE: + return TypeType.read(data) + if marker == OVERLOADED: + return Overloaded.read(data) + if marker == PARAM_SPEC_TYPE: + return ParamSpecType.read(data) + if marker == TYPE_VAR_TUPLE_TYPE: + return TypeVarTupleType.read(data) + if marker == UNPACK_TYPE: + return UnpackType.read(data) + if marker == PARAMETERS: + return Parameters.read(data) + if marker == UNINHABITED_TYPE: + return UninhabitedType.read(data) + if marker == UNBOUND_TYPE: + return UnboundType.read(data) + if marker == DELETED_TYPE: + return DeletedType.read(data) + assert False, f"Unknown type marker {marker}" + + +def read_function_like(data: Buffer) -> FunctionLike: + marker = read_int(data) + if marker == CALLABLE_TYPE: + return CallableType.read(data) + if marker == OVERLOADED: + return Overloaded.read(data) + assert False, f"Invalid type marker for FunctionLike {marker}" + + +def read_type_var_like(data: Buffer) -> TypeVarLikeType: + marker = read_int(data) + if marker == TYPE_VAR_TYPE: + return TypeVarType.read(data) + if marker == PARAM_SPEC_TYPE: + return ParamSpecType.read(data) + if marker == TYPE_VAR_TUPLE_TYPE: + return TypeVarTupleType.read(data) + assert False, f"Invalid type marker for TypeVarLikeType {marker}" + + +def read_type_opt(data: Buffer) -> Type | None: + if read_bool(data): + return read_type(data) + return None + + +def write_type_opt(data: Buffer, value: Type | None) -> None: + if value is not None: + write_bool(data, True) + value.write(data) + else: + write_bool(data, False) + + +def read_type_list(data: Buffer) -> list[Type]: + size = read_int(data) + return [read_type(data) for _ in range(size)] + + +def write_type_list(data: Buffer, value: Sequence[Type]) -> None: + write_int(data, len(value)) + for item in value: + item.write(data) + + +def read_type_map(data: Buffer) -> dict[str, Type]: + size = read_int(data) + return {read_str(data): read_type(data) for _ in range(size)} + + +def write_type_map(data: Buffer, value: dict[str, Type]) -> None: + write_int(data, len(value)) + for key in sorted(value): + write_str(data, key) + value[key].write(data) + + # This cyclic import is unfortunate, but to avoid it we would need to move away all uses # of get_proper_type() from types.py. Majority of them have been removed, but few remaining # are quite tricky to get rid of, but ultimately we want to do it at some point. diff --git a/mypy/typeshed/stubs/mypy-native/METADATA.toml b/mypy/typeshed/stubs/mypy-native/METADATA.toml new file mode 100644 index 0000000000000..76574b01cb4b1 --- /dev/null +++ b/mypy/typeshed/stubs/mypy-native/METADATA.toml @@ -0,0 +1 @@ +version = "0.0.*" diff --git a/mypy/typeshed/stubs/mypy-native/native_internal.pyi b/mypy/typeshed/stubs/mypy-native/native_internal.pyi new file mode 100644 index 0000000000000..bc1f570a8e9c1 --- /dev/null +++ b/mypy/typeshed/stubs/mypy-native/native_internal.pyi @@ -0,0 +1,12 @@ +class Buffer: + def __init__(self, source: bytes = ...) -> None: ... + def getvalue(self) -> bytes: ... + +def write_bool(data: Buffer, value: bool) -> None: ... +def read_bool(data: Buffer) -> bool: ... +def write_str(data: Buffer, value: str) -> None: ... +def read_str(data: Buffer) -> str: ... +def write_float(data: Buffer, value: float) -> None: ... +def read_float(data: Buffer) -> float: ... +def write_int(data: Buffer, value: int) -> None: ... +def read_int(data: Buffer) -> int: ... diff --git a/mypyc/analysis/ircheck.py b/mypyc/analysis/ircheck.py index 4ad2a52c1036b..6980c9cee4191 100644 --- a/mypyc/analysis/ircheck.py +++ b/mypyc/analysis/ircheck.py @@ -56,6 +56,7 @@ ) from mypyc.ir.pprint import format_func from mypyc.ir.rtypes import ( + KNOWN_NATIVE_TYPES, RArray, RInstance, RPrimitive, @@ -181,7 +182,7 @@ def check_op_sources_valid(fn: FuncIR) -> list[FnError]: set_rprimitive.name, tuple_rprimitive.name, range_rprimitive.name, -} +} | set(KNOWN_NATIVE_TYPES) def can_coerce_to(src: RType, dest: RType) -> bool: diff --git a/mypyc/build.py b/mypyc/build.py index 4a2d703b9f108..efbd0dce31db8 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -492,6 +492,8 @@ def mypycify( strict_dunder_typing: bool = False, group_name: str | None = None, log_trace: bool = False, + depends_on_native_internal: bool = False, + install_native_libs: bool = False, ) -> list[Extension]: """Main entry point to building using mypyc. @@ -542,6 +544,11 @@ def mypycify( mypyc_trace.txt (derived from executed operations). This is useful for performance analysis, such as analyzing which primitive ops are used the most and on which lines. + depends_on_native_internal: This is True only for mypy itself. + install_native_libs: If True, also build the native extension modules. Normally, + those are build and published on PyPI separately, but during + tests, we want to use their development versions (i.e. from + current commit). """ # Figure out our configuration @@ -555,6 +562,7 @@ def mypycify( strict_dunder_typing=strict_dunder_typing, group_name=group_name, log_trace=log_trace, + depends_on_native_internal=depends_on_native_internal, ) # Generate all the actual important C code @@ -653,4 +661,21 @@ def mypycify( build_single_module(group_sources, cfilenames + shared_cfilenames, cflags) ) + if install_native_libs: + for name in ["native_internal.c"] + RUNTIME_C_FILES: + rt_file = os.path.join(build_dir, name) + with open(os.path.join(include_dir(), name), encoding="utf-8") as f: + write_file(rt_file, f.read()) + extensions.append( + get_extension()( + "native_internal", + sources=[ + os.path.join(build_dir, file) + for file in ["native_internal.c"] + RUNTIME_C_FILES + ], + include_dirs=[include_dir()], + extra_compile_args=cflags, + ) + ) + return extensions diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index 8c4a69cfa3cbb..9ca761bd8ac55 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -39,6 +39,7 @@ is_int64_rprimitive, is_int_rprimitive, is_list_rprimitive, + is_native_rprimitive, is_none_rprimitive, is_object_rprimitive, is_optional_type, @@ -704,7 +705,7 @@ def emit_cast( self.emit_lines(f" {dest} = {src};", "else {") self.emit_cast_error_handler(error, src, dest, typ, raise_exception) self.emit_line("}") - elif is_object_rprimitive(typ): + elif is_object_rprimitive(typ) or is_native_rprimitive(typ): if declare_dest: self.emit_line(f"PyObject *{dest};") self.emit_arg_check(src, dest, typ, "", optional) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 1e49b1320b26d..e31fcf8ea0c9f 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -601,6 +601,8 @@ def generate_c_for_modules(self) -> list[tuple[str, str]]: ext_declarations.emit_line(f"#define MYPYC_NATIVE{self.group_suffix}_H") ext_declarations.emit_line("#include ") ext_declarations.emit_line("#include ") + if self.compiler_options.depends_on_native_internal: + ext_declarations.emit_line("#include ") declarations = Emitter(self.context) declarations.emit_line(f"#ifndef MYPYC_NATIVE_INTERNAL{self.group_suffix}_H") @@ -1027,6 +1029,10 @@ def emit_module_exec_func( declaration = f"int CPyExec_{exported_name(module_name)}(PyObject *module)" module_static = self.module_internal_static_name(module_name, emitter) emitter.emit_lines(declaration, "{") + if self.compiler_options.depends_on_native_internal: + emitter.emit_line("if (import_native_internal() < 0) {") + emitter.emit_line("return -1;") + emitter.emit_line("}") emitter.emit_line("PyObject* modname = NULL;") if self.multi_phase_init: emitter.emit_line(f"{module_static} = module;") @@ -1187,7 +1193,7 @@ def declare_internal_globals(self, module_name: str, emitter: Emitter) -> None: self.declare_global("PyObject *", static_name) def module_internal_static_name(self, module_name: str, emitter: Emitter) -> str: - return emitter.static_name(module_name + "_internal", None, prefix=MODULE_PREFIX) + return emitter.static_name(module_name + "__internal", None, prefix=MODULE_PREFIX) def declare_module(self, module_name: str, emitter: Emitter) -> None: # We declare two globals for each compiled module: diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 3c2fbfec10356..667ff60b0204a 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -512,6 +512,15 @@ def __hash__(self) -> int: # Python range object. range_rprimitive: Final = RPrimitive("builtins.range", is_unboxed=False, is_refcounted=True) +KNOWN_NATIVE_TYPES: Final = { + name: RPrimitive(name, is_unboxed=False, is_refcounted=True) + for name in ["native_internal.Buffer"] +} + + +def is_native_rprimitive(rtype: RType) -> bool: + return isinstance(rtype, RPrimitive) and rtype.name in KNOWN_NATIVE_TYPES + def is_tagged(rtype: RType) -> TypeGuard[RPrimitive]: return rtype is int_rprimitive or rtype is short_int_rprimitive diff --git a/mypyc/irbuild/mapper.py b/mypyc/irbuild/mapper.py index 815688d90fb6f..05aa0e45c569a 100644 --- a/mypyc/irbuild/mapper.py +++ b/mypyc/irbuild/mapper.py @@ -25,6 +25,7 @@ from mypyc.ir.class_ir import ClassIR from mypyc.ir.func_ir import FuncDecl, FuncSignature, RuntimeArg from mypyc.ir.rtypes import ( + KNOWN_NATIVE_TYPES, RInstance, RTuple, RType, @@ -119,6 +120,8 @@ def type_to_rtype(self, typ: Type | None) -> RType: return int16_rprimitive elif typ.type.fullname == "mypy_extensions.u8": return uint8_rprimitive + elif typ.type.fullname in KNOWN_NATIVE_TYPES: + return KNOWN_NATIVE_TYPES[typ.type.fullname] else: return object_rprimitive elif isinstance(typ, TupleType): diff --git a/mypyc/lib-rt/native_internal.c b/mypyc/lib-rt/native_internal.c new file mode 100644 index 0000000000000..11a3fafee56f7 --- /dev/null +++ b/mypyc/lib-rt/native_internal.c @@ -0,0 +1,510 @@ +#define PY_SSIZE_T_CLEAN +#include +#include "CPy.h" +#define NATIVE_INTERNAL_MODULE +#include "native_internal.h" + +#define START_SIZE 512 + +typedef struct { + PyObject_HEAD + Py_ssize_t pos; + Py_ssize_t end; + Py_ssize_t size; + char *buf; + PyObject *source; +} BufferObject; + +static PyTypeObject BufferType; + +static PyObject* +Buffer_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + if (type != &BufferType) { + PyErr_SetString(PyExc_TypeError, "Buffer should not be subclassed"); + return NULL; + } + + BufferObject *self = (BufferObject *)type->tp_alloc(type, 0); + if (self != NULL) { + self->pos = 0; + self->end = 0; + self->size = 0; + self->buf = NULL; + } + return (PyObject *) self; +} + + +static int +Buffer_init_internal(BufferObject *self, PyObject *source) { + if (source) { + if (!PyBytes_Check(source)) { + PyErr_SetString(PyExc_TypeError, "source must be a bytes object"); + return -1; + } + self->size = PyBytes_GET_SIZE(source); + self->end = self->size; + // This returns a pointer to internal bytes data, so make our own copy. + char *buf = PyBytes_AsString(source); + self->buf = PyMem_Malloc(self->size); + memcpy(self->buf, buf, self->size); + } else { + self->buf = PyMem_Malloc(START_SIZE); + self->size = START_SIZE; + } + return 0; +} + +static PyObject* +Buffer_internal(PyObject *source) { + BufferObject *self = (BufferObject *)BufferType.tp_alloc(&BufferType, 0); + if (self == NULL) + return NULL; + self->pos = 0; + self->end = 0; + self->size = 0; + self->buf = NULL; + if (Buffer_init_internal(self, source) == -1) { + Py_DECREF(self); + return NULL; + } + return (PyObject *)self; +} + +static PyObject* +Buffer_internal_empty(void) { + return Buffer_internal(NULL); +} + +static int +Buffer_init(BufferObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"source", NULL}; + PyObject *source = NULL; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O", kwlist, &source)) + return -1; + + return Buffer_init_internal(self, source); +} + +static void +Buffer_dealloc(BufferObject *self) +{ + PyMem_Free(self->buf); + Py_TYPE(self)->tp_free((PyObject *)self); +} + +static PyObject* +Buffer_getvalue_internal(PyObject *self) +{ + return PyBytes_FromStringAndSize(((BufferObject *)self)->buf, ((BufferObject *)self)->end); +} + +static PyObject* +Buffer_getvalue(BufferObject *self, PyObject *Py_UNUSED(ignored)) +{ + return PyBytes_FromStringAndSize(self->buf, self->end); +} + +static PyMethodDef Buffer_methods[] = { + {"getvalue", (PyCFunction) Buffer_getvalue, METH_NOARGS, + "Return the buffer content as bytes object" + }, + {NULL} /* Sentinel */ +}; + +static PyTypeObject BufferType = { + .ob_base = PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "Buffer", + .tp_doc = PyDoc_STR("Mypy cache buffer objects"), + .tp_basicsize = sizeof(BufferObject), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_new = Buffer_new, + .tp_init = (initproc) Buffer_init, + .tp_dealloc = (destructor) Buffer_dealloc, + .tp_methods = Buffer_methods, +}; + +static inline char +_check_buffer(PyObject *data) { + if (Py_TYPE(data) != &BufferType) { + PyErr_Format( + PyExc_TypeError, "data must be a Buffer object, got %s", Py_TYPE(data)->tp_name + ); + return 2; + } + return 1; +} + +static inline char +_check_size(BufferObject *data, Py_ssize_t need) { + Py_ssize_t target = data->pos + need; + if (target <= data->size) + return 1; + do + data->size *= 2; + while (target >= data->size); + data->buf = PyMem_Realloc(data->buf, data->size); + if (!data->buf) { + PyErr_NoMemory(); + return 2; + } + return 1; +} + +static inline char +_check_read(BufferObject *data, Py_ssize_t need) { + if (data->pos + need > data->end) { + PyErr_SetString(PyExc_ValueError, "reading past the buffer end"); + return 2; + } + return 1; +} + +static char +read_bool_internal(PyObject *data) { + if (_check_buffer(data) == 2) + return 2; + + if (_check_read((BufferObject *)data, 1) == 2) + return 2; + char *buf = ((BufferObject *)data)->buf; + char res = buf[((BufferObject *)data)->pos]; + ((BufferObject *)data)->pos += 1; + return res; +} + +static PyObject* +read_bool(PyObject *self, PyObject *args, PyObject *kwds) { + static char *kwlist[] = {"data", NULL}; + PyObject *data = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O", kwlist, &data)) + return NULL; + char res = read_bool_internal(data); + if (res == 2) + return NULL; + PyObject *retval = res ? Py_True : Py_False; + Py_INCREF(retval); + return retval; +} + +static char +write_bool_internal(PyObject *data, char value) { + if (_check_buffer(data) == 2) + return 2; + + if (_check_size((BufferObject *)data, 1) == 2) + return 2; + char *buf = ((BufferObject *)data)->buf; + buf[((BufferObject *)data)->pos] = value; + ((BufferObject *)data)->pos += 1; + ((BufferObject *)data)->end += 1; + return 1; +} + +static PyObject* +write_bool(PyObject *self, PyObject *args, PyObject *kwds) { + static char *kwlist[] = {"data", "value", NULL}; + PyObject *data = NULL; + PyObject *value = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO", kwlist, &data, &value)) + return NULL; + if (!PyBool_Check(value)) { + PyErr_SetString(PyExc_TypeError, "value must be a bool"); + return NULL; + } + if (write_bool_internal(data, value == Py_True) == 2) { + return NULL; + } + Py_INCREF(Py_None); + return Py_None; +} + +static PyObject* +read_str_internal(PyObject *data) { + if (_check_buffer(data) == 2) + return NULL; + + if (_check_read((BufferObject *)data, sizeof(Py_ssize_t)) == 2) + return NULL; + char *buf = ((BufferObject *)data)->buf; + // Read string length. + Py_ssize_t size = *(Py_ssize_t *)(buf + ((BufferObject *)data)->pos); + ((BufferObject *)data)->pos += sizeof(Py_ssize_t); + if (_check_read((BufferObject *)data, size) == 2) + return NULL; + // Read string content. + PyObject *res = PyUnicode_FromStringAndSize( + buf + ((BufferObject *)data)->pos, (Py_ssize_t)size + ); + if (!res) + return NULL; + ((BufferObject *)data)->pos += size; + return res; +} + +static PyObject* +read_str(PyObject *self, PyObject *args, PyObject *kwds) { + static char *kwlist[] = {"data", NULL}; + PyObject *data = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O", kwlist, &data)) + return NULL; + return read_str_internal(data); +} + +static char +write_str_internal(PyObject *data, PyObject *value) { + if (_check_buffer(data) == 2) + return 2; + + Py_ssize_t size; + const char *chunk = PyUnicode_AsUTF8AndSize(value, &size); + if (!chunk) + return 2; + Py_ssize_t need = size + sizeof(Py_ssize_t); + if (_check_size((BufferObject *)data, need) == 2) + return 2; + + char *buf = ((BufferObject *)data)->buf; + // Write string length. + *(Py_ssize_t *)(buf + ((BufferObject *)data)->pos) = size; + ((BufferObject *)data)->pos += sizeof(Py_ssize_t); + // Write string content. + memcpy(buf + ((BufferObject *)data)->pos, chunk, size); + ((BufferObject *)data)->pos += size; + ((BufferObject *)data)->end += need; + return 1; +} + +static PyObject* +write_str(PyObject *self, PyObject *args, PyObject *kwds) { + static char *kwlist[] = {"data", "value", NULL}; + PyObject *data = NULL; + PyObject *value = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO", kwlist, &data, &value)) + return NULL; + if (!PyUnicode_Check(value)) { + PyErr_SetString(PyExc_TypeError, "value must be a str"); + return NULL; + } + if (write_str_internal(data, value) == 2) { + return NULL; + } + Py_INCREF(Py_None); + return Py_None; +} + +static double +read_float_internal(PyObject *data) { + if (_check_buffer(data) == 2) + return CPY_FLOAT_ERROR; + + if (_check_read((BufferObject *)data, sizeof(double)) == 2) + return CPY_FLOAT_ERROR; + char *buf = ((BufferObject *)data)->buf; + double res = *(double *)(buf + ((BufferObject *)data)->pos); + ((BufferObject *)data)->pos += sizeof(double); + return res; +} + +static PyObject* +read_float(PyObject *self, PyObject *args, PyObject *kwds) { + static char *kwlist[] = {"data", NULL}; + PyObject *data = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O", kwlist, &data)) + return NULL; + double retval = read_float_internal(data); + if (retval == CPY_FLOAT_ERROR && PyErr_Occurred()) { + return NULL; + } + return PyFloat_FromDouble(retval); +} + +static char +write_float_internal(PyObject *data, double value) { + if (_check_buffer(data) == 2) + return 2; + + if (_check_size((BufferObject *)data, sizeof(double)) == 2) + return 2; + char *buf = ((BufferObject *)data)->buf; + *(double *)(buf + ((BufferObject *)data)->pos) = value; + ((BufferObject *)data)->pos += sizeof(double); + ((BufferObject *)data)->end += sizeof(double); + return 1; +} + +static PyObject* +write_float(PyObject *self, PyObject *args, PyObject *kwds) { + static char *kwlist[] = {"data", "value", NULL}; + PyObject *data = NULL; + PyObject *value = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO", kwlist, &data, &value)) + return NULL; + if (!PyFloat_Check(value)) { + PyErr_SetString(PyExc_TypeError, "value must be a float"); + return NULL; + } + if (write_float_internal(data, PyFloat_AsDouble(value)) == 2) { + return NULL; + } + Py_INCREF(Py_None); + return Py_None; +} + +static CPyTagged +read_int_internal(PyObject *data) { + if (_check_buffer(data) == 2) + return CPY_INT_TAG; + + if (_check_read((BufferObject *)data, sizeof(CPyTagged)) == 2) + return CPY_INT_TAG; + char *buf = ((BufferObject *)data)->buf; + + CPyTagged ret = *(CPyTagged *)(buf + ((BufferObject *)data)->pos); + ((BufferObject *)data)->pos += sizeof(CPyTagged); + if ((ret & CPY_INT_TAG) == 0) + return ret; + // People who have literal ints not fitting in size_t should be punished :-) + PyObject *str_ret = read_str_internal(data); + if (str_ret == NULL) + return CPY_INT_TAG; + PyObject* ret_long = PyLong_FromUnicodeObject(str_ret, 10); + Py_DECREF(str_ret); + return ((CPyTagged)ret_long) | CPY_INT_TAG; +} + +static PyObject* +read_int(PyObject *self, PyObject *args, PyObject *kwds) { + static char *kwlist[] = {"data", NULL}; + PyObject *data = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O", kwlist, &data)) + return NULL; + CPyTagged retval = read_int_internal(data); + if (retval == CPY_INT_TAG) { + return NULL; + } + return CPyTagged_StealAsObject(retval); +} + +static char +write_int_internal(PyObject *data, CPyTagged value) { + if (_check_buffer(data) == 2) + return 2; + + if (_check_size((BufferObject *)data, sizeof(CPyTagged)) == 2) + return 2; + char *buf = ((BufferObject *)data)->buf; + if ((value & CPY_INT_TAG) == 0) { + *(CPyTagged *)(buf + ((BufferObject *)data)->pos) = value; + } else { + *(CPyTagged *)(buf + ((BufferObject *)data)->pos) = CPY_INT_TAG; + } + ((BufferObject *)data)->pos += sizeof(CPyTagged); + ((BufferObject *)data)->end += sizeof(CPyTagged); + if ((value & CPY_INT_TAG) != 0) { + PyObject *str_value = PyObject_Str(CPyTagged_LongAsObject(value)); + if (str_value == NULL) + return 2; + char res = write_str_internal(data, str_value); + Py_DECREF(str_value); + if (res == 2) + return 2; + } + return 1; +} + +static PyObject* +write_int(PyObject *self, PyObject *args, PyObject *kwds) { + static char *kwlist[] = {"data", "value", NULL}; + PyObject *data = NULL; + PyObject *value = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO", kwlist, &data, &value)) + return NULL; + if (!PyLong_Check(value)) { + PyErr_SetString(PyExc_TypeError, "value must be an int"); + return NULL; + } + CPyTagged tagged_value = CPyTagged_BorrowFromObject(value); + if (write_int_internal(data, tagged_value) == 2) { + return NULL; + } + Py_INCREF(Py_None); + return Py_None; +} + +static PyMethodDef native_internal_module_methods[] = { + // TODO: switch public wrappers to METH_FASTCALL. + {"write_bool", (PyCFunction)write_bool, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("write a bool")}, + {"read_bool", (PyCFunction)read_bool, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("read a bool")}, + {"write_str", (PyCFunction)write_str, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("write a string")}, + {"read_str", (PyCFunction)read_str, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("read a string")}, + {"write_float", (PyCFunction)write_float, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("write a float")}, + {"read_float", (PyCFunction)read_float, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("read a float")}, + {"write_int", (PyCFunction)write_int, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("write an int")}, + {"read_int", (PyCFunction)read_int, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("read an int")}, + {NULL, NULL, 0, NULL} +}; + +static int +NativeInternal_ABI_Version(void) { + return NATIVE_INTERNAL_ABI_VERSION; +} + +static int +native_internal_module_exec(PyObject *m) +{ + if (PyType_Ready(&BufferType) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "Buffer", (PyObject *) &BufferType) < 0) { + return -1; + } + + // Export mypy internal C API, be careful with the order! + static void *NativeInternal_API[12] = { + (void *)Buffer_internal, + (void *)Buffer_internal_empty, + (void *)Buffer_getvalue_internal, + (void *)write_bool_internal, + (void *)read_bool_internal, + (void *)write_str_internal, + (void *)read_str_internal, + (void *)write_float_internal, + (void *)read_float_internal, + (void *)write_int_internal, + (void *)read_int_internal, + (void *)NativeInternal_ABI_Version, + }; + PyObject *c_api_object = PyCapsule_New((void *)NativeInternal_API, "native_internal._C_API", NULL); + if (PyModule_Add(m, "_C_API", c_api_object) < 0) { + return -1; + } + return 0; +} + +static PyModuleDef_Slot native_internal_module_slots[] = { + {Py_mod_exec, native_internal_module_exec}, +#ifdef Py_MOD_GIL_NOT_USED + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, +#endif + {0, NULL} +}; + +static PyModuleDef native_internal_module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "native_internal", + .m_doc = "Mypy cache serialization utils", + .m_size = 0, + .m_methods = native_internal_module_methods, + .m_slots = native_internal_module_slots, +}; + +PyMODINIT_FUNC +PyInit_native_internal(void) +{ + return PyModuleDef_Init(&native_internal_module); +} diff --git a/mypyc/lib-rt/native_internal.h b/mypyc/lib-rt/native_internal.h new file mode 100644 index 0000000000000..3bd3dd1bbb337 --- /dev/null +++ b/mypyc/lib-rt/native_internal.h @@ -0,0 +1,52 @@ +#ifndef NATIVE_INTERNAL_H +#define NATIVE_INTERNAL_H + +#define NATIVE_INTERNAL_ABI_VERSION 0 + +#ifdef NATIVE_INTERNAL_MODULE + +static PyObject *Buffer_internal(PyObject *source); +static PyObject *Buffer_internal_empty(void); +static PyObject *Buffer_getvalue_internal(PyObject *self); +static char write_bool_internal(PyObject *data, char value); +static char read_bool_internal(PyObject *data); +static char write_str_internal(PyObject *data, PyObject *value); +static PyObject *read_str_internal(PyObject *data); +static char write_float_internal(PyObject *data, double value); +static double read_float_internal(PyObject *data); +static char write_int_internal(PyObject *data, CPyTagged value); +static CPyTagged read_int_internal(PyObject *data); +static int NativeInternal_ABI_Version(void); + +#else + +static void **NativeInternal_API; + +#define Buffer_internal (*(PyObject* (*)(PyObject *source)) NativeInternal_API[0]) +#define Buffer_internal_empty (*(PyObject* (*)(void)) NativeInternal_API[1]) +#define Buffer_getvalue_internal (*(PyObject* (*)(PyObject *source)) NativeInternal_API[2]) +#define write_bool_internal (*(char (*)(PyObject *source, char value)) NativeInternal_API[3]) +#define read_bool_internal (*(char (*)(PyObject *source)) NativeInternal_API[4]) +#define write_str_internal (*(char (*)(PyObject *source, PyObject *value)) NativeInternal_API[5]) +#define read_str_internal (*(PyObject* (*)(PyObject *source)) NativeInternal_API[6]) +#define write_float_internal (*(char (*)(PyObject *source, double value)) NativeInternal_API[7]) +#define read_float_internal (*(double (*)(PyObject *source)) NativeInternal_API[8]) +#define write_int_internal (*(char (*)(PyObject *source, CPyTagged value)) NativeInternal_API[9]) +#define read_int_internal (*(CPyTagged (*)(PyObject *source)) NativeInternal_API[10]) +#define NativeInternal_ABI_Version (*(int (*)(void)) NativeInternal_API[11]) + +static int +import_native_internal(void) +{ + NativeInternal_API = (void **)PyCapsule_Import("native_internal._C_API", 0); + if (NativeInternal_API == NULL) + return -1; + if (NativeInternal_ABI_Version() != NATIVE_INTERNAL_ABI_VERSION) { + PyErr_SetString(PyExc_ValueError, "ABI version conflict for native_internal"); + return -1; + } + return 0; +} + +#endif +#endif // NATIVE_INTERNAL_H diff --git a/mypyc/lib-rt/setup.py b/mypyc/lib-rt/setup.py index 1faacc8fc136b..5b7a2919c0fd2 100644 --- a/mypyc/lib-rt/setup.py +++ b/mypyc/lib-rt/setup.py @@ -12,60 +12,74 @@ from distutils.core import Extension, setup from typing import Any -kwargs: dict[str, Any] -if sys.platform == "darwin": - kwargs = {"language": "c++"} - compile_args = [] -else: - kwargs = {} - compile_args = ["--std=c++11"] +C_APIS_TO_TEST = [ + "init.c", + "int_ops.c", + "float_ops.c", + "list_ops.c", + "exc_ops.c", + "generic_ops.c", + "pythonsupport.c", +] -class build_ext_custom(build_ext): # noqa: N801 - def get_library_names(self): +class BuildExtGtest(build_ext): + def get_library_names(self) -> list[str]: return ["gtest"] - def run(self): + def run(self) -> None: + # Build Google Test, the C++ framework we use for testing C code. + # The source code for Google Test is copied to this repository. gtest_dir = os.path.abspath( os.path.join(os.path.dirname(__file__), "..", "external", "googletest") ) - os.makedirs(self.build_temp, exist_ok=True) - - # Build Google Test, the C++ framework we use for testing C code. - # The source code for Google Test is copied to this repository. subprocess.check_call( ["make", "-f", os.path.join(gtest_dir, "make", "Makefile"), f"GTEST_DIR={gtest_dir}"], cwd=self.build_temp, ) - self.library_dirs = [self.build_temp] - return build_ext.run(self) -setup( - name="test_capi", - version="0.1", - ext_modules=[ - Extension( - "test_capi", - [ - "test_capi.cc", - "init.c", - "int_ops.c", - "float_ops.c", - "list_ops.c", - "exc_ops.c", - "generic_ops.c", - "pythonsupport.c", - ], - depends=["CPy.h", "mypyc_util.h", "pythonsupport.h"], - extra_compile_args=["-Wno-unused-function", "-Wno-sign-compare"] + compile_args, - libraries=["gtest"], - include_dirs=["../external/googletest", "../external/googletest/include"], - **kwargs, - ) - ], - cmdclass={"build_ext": build_ext_custom}, -) +if "--run-capi-tests" in sys.argv: + sys.argv.pop() + + kwargs: dict[str, Any] + if sys.platform == "darwin": + kwargs = {"language": "c++"} + compile_args = [] + else: + kwargs = {} + compile_args = ["--std=c++11"] + + setup( + name="test_capi", + version="0.1", + ext_modules=[ + Extension( + "test_capi", + ["test_capi.cc"] + C_APIS_TO_TEST, + depends=["CPy.h", "mypyc_util.h", "pythonsupport.h"], + extra_compile_args=["-Wno-unused-function", "-Wno-sign-compare"] + compile_args, + libraries=["gtest"], + include_dirs=["../external/googletest", "../external/googletest/include"], + **kwargs, + ) + ], + cmdclass={"build_ext": BuildExtGtest}, + ) +else: + # TODO: we need a way to share our preferred C flags and get_extension() logic with + # mypyc/build.py without code duplication. + setup( + name="mypy-native", + version="0.0.1", + ext_modules=[ + Extension( + "native_internal", + ["native_internal.c", "init.c", "int_ops.c", "exc_ops.c", "pythonsupport.c"], + include_dirs=["."], + ) + ], + ) diff --git a/mypyc/options.py b/mypyc/options.py index 50c76d3c0656e..c009d3c6a7a45 100644 --- a/mypyc/options.py +++ b/mypyc/options.py @@ -17,6 +17,7 @@ def __init__( strict_dunder_typing: bool = False, group_name: str | None = None, log_trace: bool = False, + depends_on_native_internal: bool = False, ) -> None: self.strip_asserts = strip_asserts self.multi_file = multi_file @@ -50,3 +51,7 @@ def __init__( # mypyc_trace.txt when compiled module is executed. This is useful for # performance analysis. self.log_trace = log_trace + # If enabled, add capsule imports of native_internal API. This should be used + # only for mypy itself, third-party code compiled with mypyc should not use + # native_internal. + self.depends_on_native_internal = depends_on_native_internal diff --git a/mypyc/primitives/misc_ops.py b/mypyc/primitives/misc_ops.py index a13f87cc94e92..8738255081e2b 100644 --- a/mypyc/primitives/misc_ops.py +++ b/mypyc/primitives/misc_ops.py @@ -4,14 +4,18 @@ from mypyc.ir.ops import ERR_FALSE, ERR_MAGIC, ERR_NEVER from mypyc.ir.rtypes import ( + KNOWN_NATIVE_TYPES, bit_rprimitive, bool_rprimitive, + bytes_rprimitive, c_int_rprimitive, c_pointer_rprimitive, c_pyssize_t_rprimitive, cstring_rprimitive, dict_rprimitive, + float_rprimitive, int_rprimitive, + none_rprimitive, object_pointer_rprimitive, object_rprimitive, pointer_rprimitive, @@ -24,6 +28,7 @@ custom_primitive_op, function_op, load_address_op, + method_op, ) # Get the 'bool' type object. @@ -326,3 +331,95 @@ return_type=void_rtype, error_kind=ERR_NEVER, ) + +buffer_rprimitive = KNOWN_NATIVE_TYPES["native_internal.Buffer"] + +# Buffer(source) +function_op( + name="native_internal.Buffer", + arg_types=[bytes_rprimitive], + return_type=buffer_rprimitive, + c_function_name="Buffer_internal", + error_kind=ERR_MAGIC, +) + +# Buffer() +function_op( + name="native_internal.Buffer", + arg_types=[], + return_type=buffer_rprimitive, + c_function_name="Buffer_internal_empty", + error_kind=ERR_MAGIC, +) + +method_op( + name="getvalue", + arg_types=[buffer_rprimitive], + return_type=bytes_rprimitive, + c_function_name="Buffer_getvalue_internal", + error_kind=ERR_MAGIC, +) + +function_op( + name="native_internal.write_bool", + arg_types=[object_rprimitive, bool_rprimitive], + return_type=none_rprimitive, + c_function_name="write_bool_internal", + error_kind=ERR_MAGIC, +) + +function_op( + name="native_internal.read_bool", + arg_types=[object_rprimitive], + return_type=bool_rprimitive, + c_function_name="read_bool_internal", + error_kind=ERR_MAGIC, +) + +function_op( + name="native_internal.write_str", + arg_types=[object_rprimitive, str_rprimitive], + return_type=none_rprimitive, + c_function_name="write_str_internal", + error_kind=ERR_MAGIC, +) + +function_op( + name="native_internal.read_str", + arg_types=[object_rprimitive], + return_type=str_rprimitive, + c_function_name="read_str_internal", + error_kind=ERR_MAGIC, +) + +function_op( + name="native_internal.write_float", + arg_types=[object_rprimitive, float_rprimitive], + return_type=none_rprimitive, + c_function_name="write_float_internal", + error_kind=ERR_MAGIC, +) + +function_op( + name="native_internal.read_float", + arg_types=[object_rprimitive], + return_type=float_rprimitive, + c_function_name="read_float_internal", + error_kind=ERR_MAGIC, +) + +function_op( + name="native_internal.write_int", + arg_types=[object_rprimitive, int_rprimitive], + return_type=none_rprimitive, + c_function_name="write_int_internal", + error_kind=ERR_MAGIC, +) + +function_op( + name="native_internal.read_int", + arg_types=[object_rprimitive], + return_type=int_rprimitive, + c_function_name="read_int_internal", + error_kind=ERR_MAGIC, +) diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index 4bb20ee9c65cc..68bc18c7bdeb0 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -1409,6 +1409,55 @@ class TestOverload: def __mypyc_generator_helper__(self, x: Any) -> Any: return x +[case testNativeBufferFastPath] +from native_internal import ( + Buffer, write_bool, read_bool, write_str, read_str, write_float, read_float, write_int, read_int +) + +def foo() -> None: + b = Buffer() + write_str(b, "foo") + write_bool(b, True) + write_float(b, 0.1) + write_int(b, 1) + + b = Buffer(b.getvalue()) + x = read_str(b) + y = read_bool(b) + z = read_float(b) + t = read_int(b) +[out] +def foo(): + r0, b :: native_internal.Buffer + r1 :: str + r2, r3, r4, r5 :: None + r6 :: bytes + r7 :: native_internal.Buffer + r8, x :: str + r9, y :: bool + r10, z :: float + r11, t :: int +L0: + r0 = Buffer_internal_empty() + b = r0 + r1 = 'foo' + r2 = write_str_internal(b, r1) + r3 = write_bool_internal(b, 1) + r4 = write_float_internal(b, 0.1) + r5 = write_int_internal(b, 2) + r6 = Buffer_getvalue_internal(b) + r7 = Buffer_internal(r6) + b = r7 + r8 = read_str_internal(b) + x = r8 + r9 = read_bool_internal(b) + y = r9 + r10 = read_float_internal(b) + z = r10 + r11 = read_int_internal(b) + t = r11 + return 1 + [case testEnumFastPath] from enum import Enum diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index fed5cfb658707..6f1217bd36e68 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -2710,6 +2710,74 @@ from native import Player [out] Player.MIN = +[case testBufferRoundTrip_native_libs] +from native_internal import ( + Buffer, write_bool, read_bool, write_str, read_str, write_float, read_float, write_int, read_int +) + +def test_buffer_basic() -> None: + b = Buffer(b"foo") + assert b.getvalue() == b"foo" + +def test_buffer_roundtrip() -> None: + b = Buffer() + write_str(b, "foo") + write_bool(b, True) + write_str(b, "bar" * 1000) + write_bool(b, False) + write_float(b, 0.1) + write_int(b, 0) + write_int(b, 1) + write_int(b, 2) + write_int(b, 2 ** 85) + + b = Buffer(b.getvalue()) + assert read_str(b) == "foo" + assert read_bool(b) is True + assert read_str(b) == "bar" * 1000 + assert read_bool(b) is False + assert read_float(b) == 0.1 + assert read_int(b) == 0 + assert read_int(b) == 1 + assert read_int(b) == 2 + assert read_int(b) == 2 ** 85 + +[file driver.py] +from native import * + +test_buffer_basic() +test_buffer_roundtrip() + +def test_buffer_basic_interpreted() -> None: + b = Buffer(b"foo") + assert b.getvalue() == b"foo" + +def test_buffer_roundtrip_interpreted() -> None: + b = Buffer() + write_str(b, "foo") + write_bool(b, True) + write_str(b, "bar" * 1000) + write_bool(b, False) + write_float(b, 0.1) + write_int(b, 0) + write_int(b, 1) + write_int(b, 2) + write_int(b, 2 ** 85) + + b = Buffer(b.getvalue()) + assert read_str(b) == "foo" + assert read_bool(b) is True + assert read_str(b) == "bar" * 1000 + assert read_bool(b) is False + assert read_float(b) == 0.1 + assert read_int(b) == 0 + assert read_int(b) == 1 + assert read_int(b) == 2 + assert read_int(b) == 2 ** 85 + +test_buffer_basic_interpreted() +test_buffer_roundtrip_interpreted() + [case testEnumMethodCalls] from enum import Enum from typing import overload, Optional, Union diff --git a/mypyc/test/test_external.py b/mypyc/test/test_external.py index 010c74dee42e5..a416cf2ee1300 100644 --- a/mypyc/test/test_external.py +++ b/mypyc/test/test_external.py @@ -34,6 +34,7 @@ def test_c_unit_test(self) -> None: "build_ext", f"--build-lib={tmpdir}", f"--build-temp={tmpdir}", + "--run-capi-tests", ], env=env, cwd=os.path.join(base_dir, "mypyc", "lib-rt"), diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py index 5078426b977f5..172a1016dd918 100644 --- a/mypyc/test/test_run.py +++ b/mypyc/test/test_run.py @@ -86,7 +86,7 @@ setup(name='test_run_output', ext_modules=mypycify({}, separate={}, skip_cgen_input={!r}, strip_asserts=False, - multi_file={}, opt_level='{}'), + multi_file={}, opt_level='{}', install_native_libs={}), ) """ @@ -239,11 +239,13 @@ def run_case_step(self, testcase: DataDrivenTestCase, incremental_step: int) -> groups = construct_groups(sources, separate, len(module_names) > 1, None) + native_libs = "_native_libs" in testcase.name try: compiler_options = CompilerOptions( multi_file=self.multi_file, separate=self.separate, strict_dunder_typing=self.strict_dunder_typing, + depends_on_native_internal=native_libs, ) result = emitmodule.parse_and_typecheck( sources=sources, @@ -270,14 +272,13 @@ def run_case_step(self, testcase: DataDrivenTestCase, incremental_step: int) -> check_serialization_roundtrip(ir) opt_level = int(os.environ.get("MYPYC_OPT_LEVEL", 0)) - debug_level = int(os.environ.get("MYPYC_DEBUG_LEVEL", 0)) setup_file = os.path.abspath(os.path.join(WORKDIR, "setup.py")) # We pass the C file information to the build script via setup.py unfortunately with open(setup_file, "w", encoding="utf-8") as f: f.write( setup_format.format( - module_paths, separate, cfiles, self.multi_file, opt_level, debug_level + module_paths, separate, cfiles, self.multi_file, opt_level, native_libs ) ) diff --git a/setup.py b/setup.py index e085b0be38464..798ff4f6c7101 100644 --- a/setup.py +++ b/setup.py @@ -154,6 +154,10 @@ def run(self) -> None: # our Appveyor builds run out of memory sometimes. multi_file=sys.platform == "win32" or force_multifile, log_trace=log_trace, + # Mypy itself is allowed to use native_internal extension. + depends_on_native_internal=True, + # TODO: temporary, remove this after we publish mypy-native on PyPI. + install_native_libs=True, ) else: diff --git a/test-data/unit/lib-stub/native_internal.pyi b/test-data/unit/lib-stub/native_internal.pyi new file mode 100644 index 0000000000000..bc1f570a8e9c1 --- /dev/null +++ b/test-data/unit/lib-stub/native_internal.pyi @@ -0,0 +1,12 @@ +class Buffer: + def __init__(self, source: bytes = ...) -> None: ... + def getvalue(self) -> bytes: ... + +def write_bool(data: Buffer, value: bool) -> None: ... +def read_bool(data: Buffer) -> bool: ... +def write_str(data: Buffer, value: str) -> None: ... +def read_str(data: Buffer) -> str: ... +def write_float(data: Buffer, value: float) -> None: ... +def read_float(data: Buffer) -> float: ... +def write_int(data: Buffer, value: int) -> None: ... +def read_int(data: Buffer) -> int: ... From d1c69046a9b54fff1219f14832116810456b29bb Mon Sep 17 00:00:00 2001 From: Brian Schubert Date: Wed, 20 Aug 2025 16:33:51 -0400 Subject: [PATCH 186/424] Don't expand PEP 695 aliases when checking node fullnames (#19699) Fixes #19698 --- mypy/semanal.py | 2 +- test-data/unit/check-python312.test | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index e8426a4e48858..77e6b0c005e2a 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -7678,7 +7678,7 @@ def refers_to_fullname(node: Expression, fullnames: str | tuple[str, ...]) -> bo return False if node.fullname in fullnames: return True - if isinstance(node.node, TypeAlias): + if isinstance(node.node, TypeAlias) and not node.node.python_3_12_type_alias: return is_named_instance(node.node.target, fullnames) return False diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index d275503dc411b..817184dc561cc 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -2095,3 +2095,19 @@ from dataclasses import dataclass class Test[*Ts, R]: a: Callable[[*Ts], R] [builtins fixtures/dict.pyi] + +[case testPEP695AliasDoesNotReferToFullname] +# https://github.com/python/mypy/issues/19698 +from typing import TypeAliasType +type D = dict +type T = type +type TA = TypeAliasType + +D() # E: "TypeAliasType" not callable +X = TA("Y") # E: "TypeAliasType" not callable + +x: object +if T(x) is str: # E: "TypeAliasType" not callable + reveal_type(x) # N: Revealed type is "builtins.object" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] From b99948bc13832666515bc11b1b8410890badebd0 Mon Sep 17 00:00:00 2001 From: Brian Schubert Date: Wed, 20 Aug 2025 22:07:01 -0400 Subject: [PATCH 187/424] Fail gracefully on unsupported template strings (PEP 750) (#19700) Refs #19329 Stopgap until proper support is added --- mypy/fastparse.py | 22 ++++++++++++++++++++++ mypy/test/testcheck.py | 2 ++ mypy/test/testparse.py | 4 ++++ test-data/unit/check-python314.test | 3 +++ test-data/unit/parse-python314.test | 5 +++++ 5 files changed, 36 insertions(+) create mode 100644 test-data/unit/check-python314.test create mode 100644 test-data/unit/parse-python314.test diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 0e1b66f0db59d..99d5c48c92d7e 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -188,6 +188,13 @@ def ast3_parse( ast_TypeVar = Any ast_TypeVarTuple = Any +if sys.version_info >= (3, 14): + ast_TemplateStr = ast3.TemplateStr + ast_Interpolation = ast3.Interpolation +else: + ast_TemplateStr = Any + ast_Interpolation = Any + N = TypeVar("N", bound=Node) # There is no way to create reasonable fallbacks at this stage, @@ -1705,6 +1712,21 @@ def visit_FormattedValue(self, n: ast3.FormattedValue) -> Expression: ) return self.set_line(result_expression, n) + # TemplateStr(expr* values) + def visit_TemplateStr(self, n: ast_TemplateStr) -> Expression: + self.fail( + ErrorMessage("PEP 750 template strings are not yet supported"), + n.lineno, + n.col_offset, + blocker=False, + ) + e = TempNode(AnyType(TypeOfAny.from_error)) + return self.set_line(e, n) + + # Interpolation(expr value, constant str, int conversion, expr? format_spec) + def visit_Interpolation(self, n: ast_Interpolation) -> Expression: + assert False, "Unreachable" + # Attribute(expr value, identifier attr, expr_context ctx) def visit_Attribute(self, n: Attribute) -> MemberExpr | SuperExpr: value = n.value diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index 206eab726a216..04ef5370d381f 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -47,6 +47,8 @@ typecheck_files.remove("check-python312.test") if sys.version_info < (3, 13): typecheck_files.remove("check-python313.test") +if sys.version_info < (3, 14): + typecheck_files.remove("check-python314.test") class TypeCheckSuite(DataSuite): diff --git a/mypy/test/testparse.py b/mypy/test/testparse.py index 027ca4dd28873..c8bcb5c0d807f 100644 --- a/mypy/test/testparse.py +++ b/mypy/test/testparse.py @@ -27,6 +27,8 @@ class ParserSuite(DataSuite): files.remove("parse-python312.test") if sys.version_info < (3, 13): files.remove("parse-python313.test") + if sys.version_info < (3, 14): + files.remove("parse-python314.test") def run_case(self, testcase: DataDrivenTestCase) -> None: test_parser(testcase) @@ -46,6 +48,8 @@ def test_parser(testcase: DataDrivenTestCase) -> None: options.python_version = (3, 12) elif testcase.file.endswith("python313.test"): options.python_version = (3, 13) + elif testcase.file.endswith("python314.test"): + options.python_version = (3, 14) else: options.python_version = defaults.PYTHON3_VERSION diff --git a/test-data/unit/check-python314.test b/test-data/unit/check-python314.test new file mode 100644 index 0000000000000..f1043aab860ae --- /dev/null +++ b/test-data/unit/check-python314.test @@ -0,0 +1,3 @@ +[case testTemplateString] +reveal_type(t"mypy") # E: PEP 750 template strings are not yet supported \ + # N: Revealed type is "Any" diff --git a/test-data/unit/parse-python314.test b/test-data/unit/parse-python314.test new file mode 100644 index 0000000000000..34fe753084f67 --- /dev/null +++ b/test-data/unit/parse-python314.test @@ -0,0 +1,5 @@ +[case testTemplateString] +x = 'mypy' +t'Hello {x}' +[out] +main:2: error: PEP 750 template strings are not yet supported From 8da097d1758bc403553d82b4838a6c9b76c6a627 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Thu, 21 Aug 2025 05:36:17 -0400 Subject: [PATCH 188/424] [mypyc] feat: extend stararg fastpath from #19629 with star2 fastpath (#19630) This PR further extends the stararg fastpath PRs (#19623 , #19629) with fastpath logic for star2 All 3 PRs were kept separate in order to make them easier to review and to make the changes in the IR more obvious in a diff. --- mypyc/irbuild/ll_builder.py | 25 ++++++++++++++- mypyc/primitives/dict_ops.py | 2 +- mypyc/test-data/irbuild-basic.test | 46 ++++++++++----------------- mypyc/test-data/irbuild-generics.test | 32 +++++++------------ mypyc/test-data/run-functions.test | 26 +++++++++++++++ 5 files changed, 80 insertions(+), 51 deletions(-) diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index f244e2f05e054..116a1bb4bae0e 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -128,6 +128,8 @@ from mypyc.primitives.bytes_ops import bytes_compare from mypyc.primitives.dict_ops import ( dict_build_op, + dict_copy, + dict_copy_op, dict_new_op, dict_ssize_t_size_op, dict_update_in_display_op, @@ -806,19 +808,40 @@ def _construct_varargs( return value, self._create_dict([], [], line) elif len(args) == 2 and args[1][1] == ARG_STAR2: # fn(*args, **kwargs) + # TODO: extend to cover(*args, **k, **w, **a, **r, **g, **s) if is_tuple_rprimitive(value.type) or isinstance(value.type, RTuple): star_result = value elif is_list_rprimitive(value.type): star_result = self.primitive_op(list_tuple_op, [value], line) else: star_result = self.primitive_op(sequence_tuple_op, [value], line) - continue + + star2_arg = args[1] + star2_value = star2_arg[0] + if is_dict_rprimitive(star2_value.type): + star2_fastpath_op = dict_copy_op + else: + star2_fastpath_op = dict_copy + return star_result, self.primitive_op( + star2_fastpath_op, [star2_value], line + ) # elif ...: TODO extend this to optimize fn(*args, k=1, **kwargs) case # TODO optimize this case using the length utils - currently in review star_result = self.new_list_op(star_values, line) self.primitive_op(list_extend_op, [star_result, value], line) elif kind == ARG_STAR2: if star2_result is None: + if len(args) == 1: + # early exit with fastpath if the only arg is ARG_STAR2 + # TODO: can we maintain an empty tuple in memory and just reuse it again and again? + if is_dict_rprimitive(value.type): + star2_fastpath_op = dict_copy_op + else: + star2_fastpath_op = dict_copy + return self.new_tuple([], line), self.primitive_op( + star2_fastpath_op, [value], line + ) + star2_result = self._create_dict(star2_keys, star2_values, line) self.call_c(dict_update_in_display_op, [star2_result, value], line=line) diff --git a/mypyc/primitives/dict_ops.py b/mypyc/primitives/dict_ops.py index 21f8a4badca33..f98bcc8ac2ec2 100644 --- a/mypyc/primitives/dict_ops.py +++ b/mypyc/primitives/dict_ops.py @@ -53,7 +53,7 @@ ) # Construct a dictionary from another dictionary. -function_op( +dict_copy_op = function_op( name="builtins.dict", arg_types=[dict_rprimitive], return_type=dict_rprimitive, diff --git a/mypyc/test-data/irbuild-basic.test b/mypyc/test-data/irbuild-basic.test index f52e1af03b528..63e4ef55d3fcc 100644 --- a/mypyc/test-data/irbuild-basic.test +++ b/mypyc/test-data/irbuild-basic.test @@ -1740,12 +1740,10 @@ def g(): r6, r7 :: dict r8 :: str r9 :: object - r10 :: dict - r11 :: i32 - r12 :: bit - r13 :: tuple - r14 :: object - r15 :: tuple[int, int, int] + r10 :: tuple + r11 :: dict + r12 :: object + r13 :: tuple[int, int, int] L0: r0 = 'a' r1 = 'b' @@ -1757,13 +1755,11 @@ L0: r7 = __main__.globals :: static r8 = 'f' r9 = CPyDict_GetItem(r7, r8) - r10 = PyDict_New() - r11 = CPyDict_UpdateInDisplay(r10, r6) - r12 = r11 >= 0 :: signed - r13 = PyTuple_Pack(0) - r14 = PyObject_Call(r9, r13, r10) - r15 = unbox(tuple[int, int, int], r14) - return r15 + r10 = PyTuple_Pack(0) + r11 = PyDict_Copy(r6) + r12 = PyObject_Call(r9, r10, r11) + r13 = unbox(tuple[int, int, int], r12) + return r13 def h(): r0, r1 :: str r2, r3 :: object @@ -3670,18 +3666,14 @@ def wrapper_deco_obj.__call__(__mypyc_self__, lst, kwargs): r1 :: object r2 :: tuple r3 :: dict - r4 :: i32 - r5 :: bit - r6 :: object + r4 :: object L0: r0 = __mypyc_self__.__mypyc_env__ r1 = r0.fn r2 = PyList_AsTuple(lst) - r3 = PyDict_New() - r4 = CPyDict_UpdateInDisplay(r3, kwargs) - r5 = r4 >= 0 :: signed - r6 = PyObject_Call(r1, r2, r3) - return r6 + r3 = PyDict_Copy(kwargs) + r4 = PyObject_Call(r1, r2, r3) + return r4 def deco(fn): fn :: object r0 :: __main__.deco_env @@ -3777,18 +3769,14 @@ def wrapper_deco_obj.__call__(__mypyc_self__, args, kwargs): r1 :: object r2 :: tuple r3 :: dict - r4 :: i32 - r5 :: bit - r6 :: object + r4 :: object L0: r0 = __mypyc_self__.__mypyc_env__ r1 = r0.fn r2 = PySequence_Tuple(args) - r3 = PyDict_New() - r4 = CPyDict_UpdateInDisplay(r3, kwargs) - r5 = r4 >= 0 :: signed - r6 = PyObject_Call(r1, r2, r3) - return r6 + r3 = PyDict_Copy(kwargs) + r4 = PyObject_Call(r1, r2, r3) + return r4 def deco(fn): fn :: object r0 :: __main__.deco_env diff --git a/mypyc/test-data/irbuild-generics.test b/mypyc/test-data/irbuild-generics.test index 783492e63e472..96437a0079c96 100644 --- a/mypyc/test-data/irbuild-generics.test +++ b/mypyc/test-data/irbuild-generics.test @@ -167,17 +167,13 @@ def execute(func, args, kwargs): func :: object args :: tuple kwargs, r0 :: dict - r1 :: i32 - r2 :: bit - r3 :: object - r4 :: int + r1 :: object + r2 :: int L0: - r0 = PyDict_New() - r1 = CPyDict_UpdateInDisplay(r0, kwargs) - r2 = r1 >= 0 :: signed - r3 = PyObject_Call(func, args, r0) - r4 = unbox(int, r3) - return r4 + r0 = PyDict_Copy(kwargs) + r1 = PyObject_Call(func, args, r0) + r2 = unbox(int, r1) + return r2 def f(x): x :: int L0: @@ -703,10 +699,8 @@ def inner_deco_obj.__call__(__mypyc_self__, args, kwargs): r22, can_iter, r23, can_use_keys, r24, can_use_values :: list r25 :: object r26 :: dict - r27 :: i32 - r28 :: bit - r29 :: object - r30 :: int + r27 :: object + r28 :: int L0: r0 = __mypyc_self__.__mypyc_env__ r1 = var_object_size args @@ -758,12 +752,10 @@ L9: r24 = CPyDict_Values(kwargs) can_use_values = r24 r25 = r0.func - r26 = PyDict_New() - r27 = CPyDict_UpdateInDisplay(r26, kwargs) - r28 = r27 >= 0 :: signed - r29 = PyObject_Call(r25, args, r26) - r30 = unbox(int, r29) - return r30 + r26 = PyDict_Copy(kwargs) + r27 = PyObject_Call(r25, args, r26) + r28 = unbox(int, r27) + return r28 def deco(func): func :: object r0 :: __main__.deco_env diff --git a/mypyc/test-data/run-functions.test b/mypyc/test-data/run-functions.test index 3d7f1f3cc7475..9bc5bb05c8d6a 100644 --- a/mypyc/test-data/run-functions.test +++ b/mypyc/test-data/run-functions.test @@ -1312,3 +1312,29 @@ from native import f print(f(1)) [out] 2 + +[case testStarArgFastPaths] +from typing import Any, Mapping +def fn(x: str, y: int) -> str: + return x * y +def star_tuple(*args: Any) -> str: + return fn(*args) +def star_list(args: list[Any]) -> str: + return fn(*args) +def star_generic(args: dict[Any, Any]) -> str: + return fn(*args) +def star2(**kwargs: Any) -> str: + return fn(**kwargs) +def star2_generic(kwargs: Mapping[Any, Any]) -> str: + return fn(**kwargs) + +def test_star_fastpath_tuple() -> None: + assert star_tuple("a", 3) == "aaa" +def test_star_fastpath_list() -> None: + assert star_list(["a", 3]) == "aaa" +def test_star_fastpath_generic() -> None: + assert star_generic({"a": None, 3: None}) == "aaa" +def test_star2_fastpath() -> None: + assert star2(x="a", y=3) == "aaa" +def test_star2_fastpath_generic() -> None: + assert star2_generic({"x": "a", "y": 3}) == "aaa" From 30a52639c7ac32e8d64906f3ba96d819f4785b3e Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 21 Aug 2025 06:51:08 -0700 Subject: [PATCH 189/424] stubtest: do not require @disjoint_base if there are __slots__ (#19701) --- mypy/stubtest.py | 4 +++- mypy/test/teststubtest.py | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index 43da0518b3f9d..f560049f2ec8a 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -510,7 +510,9 @@ def _verify_disjoint_base( if stub.is_final: return is_disjoint_runtime = _is_disjoint_base(runtime) - if is_disjoint_runtime and not stub.is_disjoint_base: + # Don't complain about missing @disjoint_base if there are __slots__, because + # in that case we can infer that it's a disjoint base. + if is_disjoint_runtime and not stub.is_disjoint_base and not runtime.__dict__.get("__slots__"): yield Error( object_path, "is a disjoint base at runtime, but isn't marked with @disjoint_base in the stub", diff --git a/mypy/test/teststubtest.py b/mypy/test/teststubtest.py index 69e2abe62f85f..e6bc2c8181647 100644 --- a/mypy/test/teststubtest.py +++ b/mypy/test/teststubtest.py @@ -1430,6 +1430,31 @@ class BytesEnum(bytes, enum.Enum): """, error=None, ) + yield Case( + stub=""" + class HasSlotsAndNothingElse: + __slots__ = ("x",) + x: int + + class HasInheritedSlots(HasSlotsAndNothingElse): + pass + + class HasEmptySlots: + __slots__ = () + """, + runtime=""" + class HasSlotsAndNothingElse: + __slots__ = ("x",) + x: int + + class HasInheritedSlots(HasSlotsAndNothingElse): + pass + + class HasEmptySlots: + __slots__ = () + """, + error=None, + ) @collect_cases def test_decorator(self) -> Iterator[Case]: From 13fa6c3066612fcb38672c1002809dd5b08ac8e9 Mon Sep 17 00:00:00 2001 From: Stephen Morton Date: Thu, 21 Aug 2025 08:52:55 -0700 Subject: [PATCH 190/424] stubtest: get better signatures for `__init__` of C classes (#18259) When an `__init__` method has the generic C-class signature, check the underlying class for a better signature. I was looking at `asyncio.futures.Future.__init__` and wanted to take advantage of the better `__text_signature__` on `asyncio.futures.Future`. The upside is that this clears several allowlist entries and surfaced several other cases where typeshed is currently incorrect, with no false positives. --- mypy/stubtest.py | 68 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index f560049f2ec8a..296550f949675 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -938,6 +938,7 @@ def from_overloadedfuncdef(stub: nodes.OverloadedFuncDef) -> Signature[nodes.Arg or arg.pos_only or assume_positional_only or arg.variable.name.strip("_") == "self" + or (index == 0 and arg.variable.name.strip("_") == "cls") else arg.variable.name ) all_args.setdefault(name, []).append((arg, index)) @@ -1008,6 +1009,7 @@ def _verify_signature( and not stub_arg.pos_only and not stub_arg.variable.name.startswith("__") and stub_arg.variable.name.strip("_") != "self" + and stub_arg.variable.name.strip("_") != "cls" and not is_dunder(function_name, exclude_special=True) # noisy for dunder methods ): yield ( @@ -1019,6 +1021,7 @@ def _verify_signature( and (stub_arg.pos_only or stub_arg.variable.name.startswith("__")) and not runtime_arg.name.startswith("__") and stub_arg.variable.name.strip("_") != "self" + and stub_arg.variable.name.strip("_") != "cls" and not is_dunder(function_name, exclude_special=True) # noisy for dunder methods ): yield ( @@ -1662,6 +1665,71 @@ def is_read_only_property(runtime: object) -> bool: def safe_inspect_signature(runtime: Any) -> inspect.Signature | None: + if ( + hasattr(runtime, "__name__") + and runtime.__name__ == "__init__" + and hasattr(runtime, "__text_signature__") + and runtime.__text_signature__ == "($self, /, *args, **kwargs)" + and hasattr(runtime, "__objclass__") + and hasattr(runtime.__objclass__, "__text_signature__") + and runtime.__objclass__.__text_signature__ is not None + ): + # This is an __init__ method with the generic C-class signature. + # In this case, the underlying class often has a better signature, + # which we can convert into an __init__ signature by adding in the + # self parameter. + try: + s = inspect.signature(runtime.__objclass__) + + parameter_kind: inspect._ParameterKind = inspect.Parameter.POSITIONAL_OR_KEYWORD + if s.parameters: + first_parameter = next(iter(s.parameters.values())) + if first_parameter.kind == inspect.Parameter.POSITIONAL_ONLY: + parameter_kind = inspect.Parameter.POSITIONAL_ONLY + return s.replace( + parameters=[inspect.Parameter("self", parameter_kind), *s.parameters.values()] + ) + except Exception: + pass + + if ( + hasattr(runtime, "__name__") + and runtime.__name__ == "__new__" + and hasattr(runtime, "__text_signature__") + and runtime.__text_signature__ == "($type, *args, **kwargs)" + and hasattr(runtime, "__self__") + and hasattr(runtime.__self__, "__text_signature__") + and runtime.__self__.__text_signature__ is not None + ): + # This is a __new__ method with the generic C-class signature. + # In this case, the underlying class often has a better signature, + # which we can convert into a __new__ signature by adding in the + # cls parameter. + + # If the attached class has a valid __init__, skip recovering a + # signature for this __new__ method. + has_init = False + if ( + hasattr(runtime.__self__, "__init__") + and hasattr(runtime.__self__.__init__, "__objclass__") + and runtime.__self__.__init__.__objclass__ is runtime.__self__ + ): + has_init = True + + if not has_init: + try: + s = inspect.signature(runtime.__self__) + parameter_kind = inspect.Parameter.POSITIONAL_OR_KEYWORD + if s.parameters: + first_parameter = next(iter(s.parameters.values())) + if first_parameter.kind == inspect.Parameter.POSITIONAL_ONLY: + parameter_kind = inspect.Parameter.POSITIONAL_ONLY + return s.replace( + parameters=[inspect.Parameter("cls", parameter_kind), *s.parameters.values()] + ) + except Exception: + pass + try: try: return inspect.signature(runtime) From 35a15b1302f3ca97492d5eda1c53b5034c0ba2f4 Mon Sep 17 00:00:00 2001 From: PrinceNaroliya Date: Fri, 22 Aug 2025 09:34:02 +0530 Subject: [PATCH 191/424] =?UTF-8?q?stubtest:=20correct=20"argument"=20?= =?UTF-8?q?=E2=86=92=20"parameter"=20terminology=20in=20error=20messages?= =?UTF-8?q?=20(#19707)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description This PR fixes the incorrect usage of the word **"argument"** in `stubtest` error messages and replaces it with the correct term **"parameter"**, as requested in issue #16508. According to convention: - **Parameter** → part of the function signature - **Argument** → actual value passed when calling the function Since `stubtest` deals only with *parameters*, this correction improves the accuracy of the error messages. ## Related Issue Fixes #16508 --- mypy/stubtest.py | 52 +++++++++++++++++++-------------------- mypy/test/teststubtest.py | 8 +++--- 2 files changed, 31 insertions(+), 29 deletions(-) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index 296550f949675..db902bae08c9b 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -747,8 +747,8 @@ def names_approx_match(a: str, b: str) -> bool: if stub_arg.variable.name == "_self": return yield ( - f'stub argument "{stub_arg.variable.name}" ' - f'differs from runtime argument "{runtime_arg.name}"' + f'stub parameter "{stub_arg.variable.name}" ' + f'differs from runtime parameter "{runtime_arg.name}"' ) @@ -759,8 +759,8 @@ def _verify_arg_default_value( if runtime_arg.default is not inspect.Parameter.empty: if stub_arg.kind.is_required(): yield ( - f'runtime argument "{runtime_arg.name}" ' - "has a default value but stub argument does not" + f'runtime parameter "{runtime_arg.name}" ' + "has a default value but stub parameter does not" ) else: runtime_type = get_mypy_type_of_runtime_value(runtime_arg.default) @@ -781,9 +781,9 @@ def _verify_arg_default_value( and not is_subtype_helper(runtime_type, stub_type) ): yield ( - f'runtime argument "{runtime_arg.name}" ' + f'runtime parameter "{runtime_arg.name}" ' f"has a default value of type {runtime_type}, " - f"which is incompatible with stub argument type {stub_type}" + f"which is incompatible with stub parameter type {stub_type}" ) if stub_arg.initializer is not None: stub_default = evaluate_expression(stub_arg.initializer) @@ -807,15 +807,15 @@ def _verify_arg_default_value( defaults_match = False if not defaults_match: yield ( - f'runtime argument "{runtime_arg.name}" ' + f'runtime parameter "{runtime_arg.name}" ' f"has a default value of {runtime_arg.default!r}, " - f"which is different from stub argument default {stub_default!r}" + f"which is different from stub parameter default {stub_default!r}" ) else: if stub_arg.kind.is_optional(): yield ( - f'stub argument "{stub_arg.variable.name}" has a default value ' - f"but runtime argument does not" + f'stub parameter "{stub_arg.variable.name}" has a default value ' + f"but runtime parameter does not" ) @@ -1013,7 +1013,7 @@ def _verify_signature( and not is_dunder(function_name, exclude_special=True) # noisy for dunder methods ): yield ( - f'stub argument "{stub_arg.variable.name}" should be positional-only ' + f'stub parameter "{stub_arg.variable.name}" should be positional-only ' f'(add "/", e.g. "{runtime_arg.name}, /")' ) if ( @@ -1025,7 +1025,7 @@ def _verify_signature( and not is_dunder(function_name, exclude_special=True) # noisy for dunder methods ): yield ( - f'stub argument "{stub_arg.variable.name}" should be positional or keyword ' + f'stub parameter "{stub_arg.variable.name}" should be positional or keyword ' '(remove "/")' ) @@ -1040,28 +1040,28 @@ def _verify_signature( # If the variable is in runtime.kwonly, it's just mislabelled as not a # keyword-only argument if stub_arg.variable.name not in runtime.kwonly: - msg = f'runtime does not have argument "{stub_arg.variable.name}"' + msg = f'runtime does not have parameter "{stub_arg.variable.name}"' if runtime.varkw is not None: msg += ". Maybe you forgot to make it keyword-only in the stub?" yield msg else: - yield f'stub argument "{stub_arg.variable.name}" is not keyword-only' + yield f'stub parameter "{stub_arg.variable.name}" is not keyword-only' if stub.varpos is not None: - yield f'runtime does not have *args argument "{stub.varpos.variable.name}"' + yield f'runtime does not have *args parameter "{stub.varpos.variable.name}"' elif len(stub.pos) < len(runtime.pos): for runtime_arg in runtime.pos[len(stub.pos) :]: if runtime_arg.name not in stub.kwonly: if not _is_private_parameter(runtime_arg): - yield f'stub does not have argument "{runtime_arg.name}"' + yield f'stub does not have parameter "{runtime_arg.name}"' else: - yield f'runtime argument "{runtime_arg.name}" is not keyword-only' + yield f'runtime parameter "{runtime_arg.name}" is not keyword-only' # Checks involving *args if len(stub.pos) <= len(runtime.pos) or runtime.varpos is None: if stub.varpos is None and runtime.varpos is not None: - yield f'stub does not have *args argument "{runtime.varpos.name}"' + yield f'stub does not have *args parameter "{runtime.varpos.name}"' if stub.varpos is not None and runtime.varpos is None: - yield f'runtime does not have *args argument "{stub.varpos.variable.name}"' + yield f'runtime does not have *args parameter "{stub.varpos.variable.name}"' # Check keyword-only args for arg in sorted(set(stub.kwonly) & set(runtime.kwonly)): @@ -1080,9 +1080,9 @@ def _verify_signature( if arg in {runtime_arg.name for runtime_arg in runtime.pos}: # Don't report this if we've reported it before if arg not in {runtime_arg.name for runtime_arg in runtime.pos[len(stub.pos) :]}: - yield f'runtime argument "{arg}" is not keyword-only' + yield f'runtime parameter "{arg}" is not keyword-only' else: - yield f'runtime does not have argument "{arg}"' + yield f'runtime does not have parameter "{arg}"' for arg in sorted(set(runtime.kwonly) - set(stub.kwonly)): if arg in {stub_arg.variable.name for stub_arg in stub.pos}: # Don't report this if we've reported it before @@ -1090,10 +1090,10 @@ def _verify_signature( runtime.varpos is None and arg in {stub_arg.variable.name for stub_arg in stub.pos[len(runtime.pos) :]} ): - yield f'stub argument "{arg}" is not keyword-only' + yield f'stub parameter "{arg}" is not keyword-only' else: if not _is_private_parameter(runtime.kwonly[arg]): - yield f'stub does not have argument "{arg}"' + yield f'stub does not have parameter "{arg}"' # Checks involving **kwargs if stub.varkw is None and runtime.varkw is not None: @@ -1103,9 +1103,9 @@ def _verify_signature( stub_pos_names = {stub_arg.variable.name for stub_arg in stub.pos} # Ideally we'd do a strict subset check, but in practice the errors from that aren't useful if not set(runtime.kwonly).issubset(set(stub.kwonly) | stub_pos_names): - yield f'stub does not have **kwargs argument "{runtime.varkw.name}"' + yield f'stub does not have **kwargs parameter "{runtime.varkw.name}"' if stub.varkw is not None and runtime.varkw is None: - yield f'runtime does not have **kwargs argument "{stub.varkw.variable.name}"' + yield f'runtime does not have **kwargs parameter "{stub.varkw.variable.name}"' def _is_private_parameter(arg: inspect.Parameter) -> bool: @@ -1425,7 +1425,7 @@ def apply_decorator_to_funcitem( if decorator.fullname == "builtins.classmethod": if func.arguments[0].variable.name not in ("cls", "mcs", "metacls"): raise StubtestFailure( - f"unexpected class argument name {func.arguments[0].variable.name!r} " + f"unexpected class parameter name {func.arguments[0].variable.name!r} " f"in {dec.fullname}" ) # FuncItem is written so that copy.copy() actually works, even when compiled diff --git a/mypy/test/teststubtest.py b/mypy/test/teststubtest.py index e6bc2c8181647..c9404e206e4f7 100644 --- a/mypy/test/teststubtest.py +++ b/mypy/test/teststubtest.py @@ -2583,8 +2583,8 @@ def test_output(self) -> None: options=[], ) expected = ( - f'error: {TEST_MODULE_NAME}.bad is inconsistent, stub argument "number" differs ' - 'from runtime argument "num"\n' + f'error: {TEST_MODULE_NAME}.bad is inconsistent, stub parameter "number" differs ' + 'from runtime parameter "num"\n' f"Stub: in file {TEST_MODULE_NAME}.pyi:1\n" "def (number: builtins.int, text: builtins.str)\n" f"Runtime: in file {TEST_MODULE_NAME}.py:1\ndef (num, text)\n\n" @@ -2599,7 +2599,9 @@ def test_output(self) -> None: ) expected = ( "{}.bad is inconsistent, " - 'stub argument "number" differs from runtime argument "num"\n'.format(TEST_MODULE_NAME) + 'stub parameter "number" differs from runtime parameter "num"\n'.format( + TEST_MODULE_NAME + ) ) assert output == expected From fa7fa7f7a6a632ff9fe37b3e7e659b97bb8a5e7b Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 22 Aug 2025 12:28:43 -0700 Subject: [PATCH 192/424] stubtest: flag redundant @disjoint_base decorators (#19715) Co-authored-by: Alex Waygood --- mypy/stubtest.py | 53 +++++++++++++++++++++++++++-------- mypy/test/teststubtest.py | 58 +++++++++++++++++++++++++++++++++------ 2 files changed, 91 insertions(+), 20 deletions(-) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index db902bae08c9b..482a14984950f 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -506,13 +506,16 @@ def _is_disjoint_base(typ: type[object]) -> bool: def _verify_disjoint_base( stub: nodes.TypeInfo, runtime: type[object], object_path: list[str] ) -> Iterator[Error]: - # If it's final, doesn't matter whether it's a disjoint base or not - if stub.is_final: - return is_disjoint_runtime = _is_disjoint_base(runtime) # Don't complain about missing @disjoint_base if there are __slots__, because # in that case we can infer that it's a disjoint base. - if is_disjoint_runtime and not stub.is_disjoint_base and not runtime.__dict__.get("__slots__"): + if ( + is_disjoint_runtime + and not stub.is_disjoint_base + and not runtime.__dict__.get("__slots__") + and not stub.is_final + and not (stub.is_enum and stub.enum_members) + ): yield Error( object_path, "is a disjoint base at runtime, but isn't marked with @disjoint_base in the stub", @@ -520,14 +523,40 @@ def _verify_disjoint_base( runtime, stub_desc=repr(stub), ) - elif not is_disjoint_runtime and stub.is_disjoint_base: - yield Error( - object_path, - "is marked with @disjoint_base in the stub, but isn't a disjoint base at runtime", - stub, - runtime, - stub_desc=repr(stub), - ) + elif stub.is_disjoint_base: + if not is_disjoint_runtime: + yield Error( + object_path, + "is marked with @disjoint_base in the stub, but isn't a disjoint base at runtime", + stub, + runtime, + stub_desc=repr(stub), + ) + if runtime.__dict__.get("__slots__"): + yield Error( + object_path, + "is marked as @disjoint_base, but also has slots; add __slots__ instead", + stub, + runtime, + stub_desc=repr(stub), + ) + elif stub.is_final: + yield Error( + object_path, + "is marked as @disjoint_base, but also marked as @final; remove @disjoint_base", + stub, + runtime, + stub_desc=repr(stub), + ) + elif stub.is_enum and stub.enum_members: + yield Error( + object_path, + "is marked as @disjoint_base, but is an enum with members, which is implicitly final; " + "remove @disjoint_base", + stub, + runtime, + stub_desc=repr(stub), + ) def _verify_metaclass( diff --git a/mypy/test/teststubtest.py b/mypy/test/teststubtest.py index c9404e206e4f7..2bf071d34d48b 100644 --- a/mypy/test/teststubtest.py +++ b/mypy/test/teststubtest.py @@ -1407,14 +1407,9 @@ def spam(x=Flags4(0)): pass stub=""" import sys from typing import Final, Literal - from typing_extensions import disjoint_base - if sys.version_info >= (3, 12): - class BytesEnum(bytes, enum.Enum): - a = b'foo' - else: - @disjoint_base - class BytesEnum(bytes, enum.Enum): - a = b'foo' + class BytesEnum(bytes, enum.Enum): + a = b'foo' + FOO: Literal[BytesEnum.a] BAR: Final = BytesEnum.a BAZ: BytesEnum @@ -1698,6 +1693,53 @@ def __next__(self) -> object: ... """, error=None, ) + yield Case( + runtime=""" + class IsDisjointBaseBecauseItHasSlots: + __slots__ = ("a",) + a: int + """, + stub=""" + from typing_extensions import disjoint_base + + @disjoint_base + class IsDisjointBaseBecauseItHasSlots: + a: int + """, + error="test_module.IsDisjointBaseBecauseItHasSlots", + ) + yield Case( + runtime=""" + class IsFinalSoDisjointBaseIsRedundant: ... + """, + stub=""" + from typing_extensions import disjoint_base, final + + @final + @disjoint_base + class IsFinalSoDisjointBaseIsRedundant: ... + """, + error="test_module.IsFinalSoDisjointBaseIsRedundant", + ) + yield Case( + runtime=""" + import enum + + class IsEnumWithMembersSoDisjointBaseIsRedundant(enum.Enum): + A = 1 + B = 2 + """, + stub=""" + from typing_extensions import disjoint_base + import enum + + @disjoint_base + class IsEnumWithMembersSoDisjointBaseIsRedundant(enum.Enum): + A = 1 + B = 2 + """, + error="test_module.IsEnumWithMembersSoDisjointBaseIsRedundant", + ) @collect_cases def test_has_runtime_final_decorator(self) -> Iterator[Case]: From 01e2a8c4a6851e3e5eff77b96f0f4a5d0b41079d Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Fri, 22 Aug 2025 22:55:44 +0200 Subject: [PATCH 193/424] Use union for captured vars in or pattern (#19710) Mypy creates a union type for the pattern subject in an or pattern already. Use union for captured variables as well. --- mypy/checkpattern.py | 2 +- test-data/unit/check-python310.test | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mypy/checkpattern.py b/mypy/checkpattern.py index 48840466f0d89..2b9e823c55b71 100644 --- a/mypy/checkpattern.py +++ b/mypy/checkpattern.py @@ -192,7 +192,7 @@ def visit_or_pattern(self, o: OrPattern) -> PatternType: for capture_list in capture_types.values(): typ = UninhabitedType() for _, other in capture_list: - typ = join_types(typ, other) + typ = make_simplified_union([typ, other]) captures[capture_list[0][0]] = typ diff --git a/test-data/unit/check-python310.test b/test-data/unit/check-python310.test index f264167cb0670..a4d6188136c67 100644 --- a/test-data/unit/check-python310.test +++ b/test-data/unit/check-python310.test @@ -1204,13 +1204,13 @@ match m: case 1 | "foo": reveal_type(m) # N: Revealed type is "Union[Literal[1], Literal['foo']]" -[case testMatchOrPatterCapturesMissing] +[case testMatchOrPatternCapturesMissing] from typing import List m: List[int] match m: case [x, y] | list(x): # E: Alternative patterns bind different names - reveal_type(x) # N: Revealed type is "builtins.object" + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.list[builtins.int]]" reveal_type(y) # N: Revealed type is "builtins.int" [builtins fixtures/list.pyi] @@ -1219,7 +1219,7 @@ m: object match m: case list(x) | dict(x): - reveal_type(x) # N: Revealed type is "typing.Iterable[Any]" + reveal_type(x) # N: Revealed type is "Union[builtins.list[Any], builtins.dict[Any, Any]]" [builtins fixtures/dict.pyi] -- Interactions -- @@ -1405,7 +1405,7 @@ m: Union[str, bytes, int] match m: case str(a) | bytes(a): - reveal_type(a) # N: Revealed type is "builtins.object" + reveal_type(a) # N: Revealed type is "Union[builtins.str, builtins.bytes]" reveal_type(m) # N: Revealed type is "Union[builtins.str, builtins.bytes]" case b: reveal_type(b) # N: Revealed type is "builtins.int" From ac4cacbba3c8511bb591fd11f813de66e79a7aa7 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Fri, 22 Aug 2025 17:29:26 -0700 Subject: [PATCH 194/424] Somewhat better support for isinstance on old-style unions (#19714) Partially fixes #17680 , remainder of the issue should probably be fixed in the isinstance stub --- mypy/checkexpr.py | 5 ++++- test-data/unit/check-unions.test | 27 +++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 9752a5e68638f..88b3005b13763 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -582,7 +582,10 @@ def visit_call_expr_inner(self, e: CallExpr, allow_none_return: bool = False) -> and not node.node.no_args and not ( isinstance(union_target := get_proper_type(node.node.target), UnionType) - and union_target.uses_pep604_syntax + and ( + union_target.uses_pep604_syntax + or self.chk.options.python_version >= (3, 10) + ) ) ): self.msg.type_arguments_not_allowed(e) diff --git a/test-data/unit/check-unions.test b/test-data/unit/check-unions.test index f8c894a7957bb..398b007ce57d0 100644 --- a/test-data/unit/check-unions.test +++ b/test-data/unit/check-unions.test @@ -1345,3 +1345,30 @@ x: Union[C1, C2, C3, C4, C5, C6, C7, C8, C9, C10, C11] y: Union[C1, C2, C3, C4, C5, C6, C7, C8, C9, C10, C11, None] x = y # E: Incompatible types in assignment (expression has type "Union[C1, C2, C3, C4, C5, <6 more items>, None]", variable has type "Union[C1, C2, C3, C4, C5, <6 more items>]") \ # N: Item in the first union not in the second: "None" + +[case testTypeAliasWithOldUnionIsInstance] +# flags: --python-version 3.10 +from typing import Union +SimpleAlias = Union[int, str] + +def foo(x: Union[int, str, tuple]): + # TODO: fix the typeshed stub for isinstance + if isinstance(x, SimpleAlias): # E: Argument 2 to "isinstance" has incompatible type ""; expected "type" + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str, builtins.tuple[Any, ...]]" + else: + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str, builtins.tuple[Any, ...]]" +[builtins fixtures/tuple.pyi] + + +[case testTypeAliasWithOldUnionIsInstancePython39] +# flags: --python-version 3.9 +from typing import Union +SimpleAlias = Union[int, str] + +def foo(x: Union[int, str, tuple]): + if isinstance(x, SimpleAlias): # E: Parameterized generics cannot be used with class or instance checks \ + # E: Argument 2 to "isinstance" has incompatible type ""; expected "type" + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str, builtins.tuple[Any, ...]]" + else: + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str, builtins.tuple[Any, ...]]" +[builtins fixtures/tuple.pyi] From f51b6995353626a56b8ae7cf078ef28119ae5eb5 Mon Sep 17 00:00:00 2001 From: Stephen Morton Date: Fri, 22 Aug 2025 18:12:26 -0700 Subject: [PATCH 195/424] stubtest: handle overloads with mixed pos-only params (#18287) Fixes #17023 Stubtest should only mangle positional-only parameter names if they're positional-only in all branches of the overload. The signatures get really ugly and wrong otherwise. I'm not sure if I did the new `test_overload_signature` in the best way. I couldn't figure out a way to get covert a string into a `nodes.OverloadedFuncDef` object with any of the techniques in existing tests in `teststubtest.py`. Maybe the new test case is sufficient, but I wanted to test the signature generation directly. --- mypy/stubtest.py | 30 +++++++++++++++++++-------- mypy/test/teststubtest.py | 43 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 8 deletions(-) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index 482a14984950f..884a442d15fbd 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -954,22 +954,36 @@ def from_overloadedfuncdef(stub: nodes.OverloadedFuncDef) -> Signature[nodes.Arg # For most dunder methods, just assume all args are positional-only assume_positional_only = is_dunder(stub.name, exclude_special=True) - all_args: dict[str, list[tuple[nodes.Argument, int]]] = {} + is_arg_pos_only: defaultdict[str, set[bool]] = defaultdict(set) for func in map(_resolve_funcitem_from_decorator, stub.items): assert func is not None, "Failed to resolve decorated overload" args = maybe_strip_cls(stub.name, func.arguments) for index, arg in enumerate(args): - # For positional-only args, we allow overloads to have different names for the same - # argument. To accomplish this, we just make up a fake index-based name. - name = ( - f"__{index}" - if arg.variable.name.startswith("__") + if ( + arg.variable.name.startswith("__") or arg.pos_only or assume_positional_only or arg.variable.name.strip("_") == "self" or (index == 0 and arg.variable.name.strip("_") == "cls") - else arg.variable.name - ) + ): + is_arg_pos_only[arg.variable.name].add(True) + else: + is_arg_pos_only[arg.variable.name].add(False) + + all_args: dict[str, list[tuple[nodes.Argument, int]]] = {} + for func in map(_resolve_funcitem_from_decorator, stub.items): + assert func is not None, "Failed to resolve decorated overload" + args = maybe_strip_cls(stub.name, func.arguments) + for index, arg in enumerate(args): + # For positional-only args, we allow overloads to have different names for the same + # argument. To accomplish this, we just make up a fake index-based name. + # We can only use the index-based name if the argument is always + # positional only. Sometimes overloads have an arg as positional-only + # in some but not all branches of the overload. + name = arg.variable.name + if is_arg_pos_only[name] == {True}: + name = f"__{index}" + all_args.setdefault(name, []).append((arg, index)) def get_position(arg_name: str) -> int: diff --git a/mypy/test/teststubtest.py b/mypy/test/teststubtest.py index 2bf071d34d48b..ee69d2077f0f3 100644 --- a/mypy/test/teststubtest.py +++ b/mypy/test/teststubtest.py @@ -13,7 +13,11 @@ from typing import Any, Callable import mypy.stubtest +from mypy import build, nodes +from mypy.modulefinder import BuildSource +from mypy.options import Options from mypy.stubtest import parse_options, test_stubs +from mypy.test.config import test_temp_dir from mypy.test.data import root_dir @@ -158,6 +162,14 @@ def __invert__(self: _T) -> _T: pass """ +def build_helper(source: str) -> build.BuildResult: + return build.build( + sources=[BuildSource("main.pyi", None, textwrap.dedent(source))], + options=Options(), + alt_lib_path=test_temp_dir, + ) + + def run_stubtest_with_stderr( stub: str, runtime: str, options: list[str], config_file: str | None = None ) -> tuple[str, str]: @@ -842,6 +854,18 @@ def f2(self, *a) -> int: ... """, error=None, ) + yield Case( + stub=""" + @overload + def f(a: int) -> int: ... + @overload + def f(a: int, b: str, /) -> str: ... + """, + runtime=""" + def f(a, *args): ... + """, + error=None, + ) @collect_cases def test_property(self) -> Iterator[Case]: @@ -2790,6 +2814,25 @@ def test_builtin_signature_with_unrepresentable_default(self) -> None: == "def (self, sep = ..., bytes_per_sep = ...)" ) + def test_overload_signature(self) -> None: + # The same argument as both positional-only and pos-or-kw in + # different overloads previously produced incorrect signatures + source = """ + from typing import overload + @overload + def myfunction(arg: int) -> None: ... + @overload + def myfunction(arg: str, /) -> None: ... + """ + result = build_helper(source) + stub = result.files["__main__"].names["myfunction"].node + assert isinstance(stub, nodes.OverloadedFuncDef) + sig = mypy.stubtest.Signature.from_overloadedfuncdef(stub) + if sys.version_info >= (3, 10): + assert str(sig) == "def (arg: builtins.int | builtins.str)" + else: + assert str(sig) == "def (arg: Union[builtins.int, builtins.str])" + def test_config_file(self) -> None: runtime = "temp = 5\n" stub = "from decimal import Decimal\ntemp: Decimal\n" From 15b8ca967cc6187effcab23e6613da2db4546584 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Sat, 23 Aug 2025 04:17:58 +0200 Subject: [PATCH 196/424] Remove unnecessary error message for match class patterns (#19708) Remove `Cannot determine type of ...` error for class patterns if the class to match cannot be resolved. In these cases a `Name ... is not defined` error is already emitted. The captured variable should simply be inferred as `Any`. Previously, this was especially an issue for class matches to a class from an untyped library together with a MemberExpr. An example from pylint / astroid which shouldn't emit any errors: ```py from typing import Any from astroid import nodes def func(var: Any) -> None: match var: case nodes.Assign(targets=t): reveal_type(t) # Any ``` --- mypy/checkpattern.py | 8 ++++---- test-data/unit/check-python310.test | 9 +++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/mypy/checkpattern.py b/mypy/checkpattern.py index 2b9e823c55b71..fc00e9f202915 100644 --- a/mypy/checkpattern.py +++ b/mypy/checkpattern.py @@ -539,12 +539,12 @@ def visit_class_pattern(self, o: ClassPattern) -> PatternType: # type_info = o.class_ref.node if type_info is None: - return PatternType(AnyType(TypeOfAny.from_error), AnyType(TypeOfAny.from_error), {}) - if isinstance(type_info, TypeAlias) and not type_info.no_args: + typ: Type = AnyType(TypeOfAny.from_error) + elif isinstance(type_info, TypeAlias) and not type_info.no_args: self.msg.fail(message_registry.CLASS_PATTERN_GENERIC_TYPE_ALIAS, o) return self.early_non_match() - if isinstance(type_info, TypeInfo): - typ: Type = fill_typevars_with_any(type_info) + elif isinstance(type_info, TypeInfo): + typ = fill_typevars_with_any(type_info) elif isinstance(type_info, TypeAlias): typ = type_info.target elif ( diff --git a/test-data/unit/check-python310.test b/test-data/unit/check-python310.test index a4d6188136c67..b49bb13d520f0 100644 --- a/test-data/unit/check-python310.test +++ b/test-data/unit/check-python310.test @@ -721,13 +721,14 @@ m: object match m: case xyz(y): # E: Name "xyz" is not defined reveal_type(m) # N: Revealed type is "Any" - reveal_type(y) # E: Cannot determine type of "y" \ - # N: Revealed type is "Any" + reveal_type(y) # N: Revealed type is "Any" match m: case xyz(z=x): # E: Name "xyz" is not defined - reveal_type(x) # E: Cannot determine type of "x" \ - # N: Revealed type is "Any" + reveal_type(x) # N: Revealed type is "Any" + case (xyz1() as n) | (xyz2(attr=n)): # E: Name "xyz1" is not defined \ + # E: Name "xyz2" is not defined + reveal_type(n) # N: Revealed type is "Any" [case testMatchClassPatternCaptureDataclass] from dataclasses import dataclass From 116b92bae7b5dbf5e6bd36fd9b0c6804973e5554 Mon Sep 17 00:00:00 2001 From: Stephen Morton Date: Sat, 23 Aug 2025 06:11:52 -0700 Subject: [PATCH 197/424] More detailed checking of type objects in stubtest (#18251) This uses `checkmember.type_object_type` and context to produce better types of type objects. --- mypy/stubtest.py | 69 +++++++++++++++++++++++++++++++++------ mypy/test/teststubtest.py | 25 ++++++++++++++ 2 files changed, 84 insertions(+), 10 deletions(-) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index 884a442d15fbd..ee15fed81b4b4 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -34,6 +34,9 @@ from typing_extensions import get_origin, is_typeddict import mypy.build +import mypy.checkexpr +import mypy.checkmember +import mypy.erasetype import mypy.modulefinder import mypy.nodes import mypy.state @@ -792,7 +795,11 @@ def _verify_arg_default_value( "has a default value but stub parameter does not" ) else: - runtime_type = get_mypy_type_of_runtime_value(runtime_arg.default) + type_context = stub_arg.variable.type + runtime_type = get_mypy_type_of_runtime_value( + runtime_arg.default, type_context=type_context + ) + # Fallback to the type annotation type if var type is missing. The type annotation # is an UnboundType, but I don't know enough to know what the pros and cons here are. # UnboundTypes have ugly question marks following them, so default to var type. @@ -1247,7 +1254,7 @@ def verify_var( ): yield Error(object_path, "is read-only at runtime but not in the stub", stub, runtime) - runtime_type = get_mypy_type_of_runtime_value(runtime) + runtime_type = get_mypy_type_of_runtime_value(runtime, type_context=stub.type) if ( runtime_type is not None and stub.type is not None @@ -1832,7 +1839,18 @@ def is_subtype_helper(left: mypy.types.Type, right: mypy.types.Type) -> bool: return mypy.subtypes.is_subtype(left, right) -def get_mypy_type_of_runtime_value(runtime: Any) -> mypy.types.Type | None: +def get_mypy_node_for_name(module: str, type_name: str) -> mypy.nodes.SymbolNode | None: + stub = get_stub(module) + if stub is None: + return None + if type_name not in stub.names: + return None + return stub.names[type_name].node + + +def get_mypy_type_of_runtime_value( + runtime: Any, type_context: mypy.types.Type | None = None +) -> mypy.types.Type | None: """Returns a mypy type object representing the type of ``runtime``. Returns None if we can't find something that works. @@ -1893,14 +1911,45 @@ def anytype() -> mypy.types.AnyType: is_ellipsis_args=True, ) - # Try and look up a stub for the runtime object - stub = get_stub(type(runtime).__module__) - if stub is None: - return None - type_name = type(runtime).__name__ - if type_name not in stub.names: + skip_type_object_type = False + if type_context: + # Don't attempt to process the type object when context is generic + # This is related to issue #3737 + type_context = mypy.types.get_proper_type(type_context) + # Callable types with a generic return value + if isinstance(type_context, mypy.types.CallableType): + if isinstance(type_context.ret_type, mypy.types.TypeVarType): + skip_type_object_type = True + # Type[x] where x is generic + if isinstance(type_context, mypy.types.TypeType): + if isinstance(type_context.item, mypy.types.TypeVarType): + skip_type_object_type = True + + if isinstance(runtime, type) and not skip_type_object_type: + + def _named_type(name: str) -> mypy.types.Instance: + parts = name.rsplit(".", maxsplit=1) + node = get_mypy_node_for_name(parts[0], parts[1]) + assert isinstance(node, nodes.TypeInfo) + any_type = mypy.types.AnyType(mypy.types.TypeOfAny.special_form) + return mypy.types.Instance(node, [any_type] * len(node.defn.type_vars)) + + # Try and look up a stub for the runtime object itself + # The logic here is similar to ExpressionChecker.analyze_ref_expr + type_info = get_mypy_node_for_name(runtime.__module__, runtime.__name__) + if isinstance(type_info, nodes.TypeInfo): + result: mypy.types.Type | None = None + result = mypy.typeops.type_object_type(type_info, _named_type) + if mypy.checkexpr.is_type_type_context(type_context): + # This is the type in a type[] expression, so substitute type + # variables with Any. + result = mypy.erasetype.erase_typevars(result) + return result + + # Try and look up a stub for the runtime object's type + type_info = get_mypy_node_for_name(type(runtime).__module__, type(runtime).__name__) + if type_info is None: return None - type_info = stub.names[type_name].node if isinstance(type_info, nodes.Var): return type_info.type if not isinstance(type_info, nodes.TypeInfo): diff --git a/mypy/test/teststubtest.py b/mypy/test/teststubtest.py index ee69d2077f0f3..28263e20099d7 100644 --- a/mypy/test/teststubtest.py +++ b/mypy/test/teststubtest.py @@ -2636,6 +2636,31 @@ class _X1: ... error=None, ) + @collect_cases + def test_type_default_protocol(self) -> Iterator[Case]: + yield Case( + stub=""" + from typing import Protocol + + class _FormatterClass(Protocol): + def __call__(self, *, prog: str) -> HelpFormatter: ... + + class ArgumentParser: + def __init__(self, formatter_class: _FormatterClass = ...) -> None: ... + + class HelpFormatter: + def __init__(self, prog: str, indent_increment: int = 2) -> None: ... + """, + runtime=""" + class HelpFormatter: + def __init__(self, prog, indent_increment=2) -> None: ... + + class ArgumentParser: + def __init__(self, formatter_class=HelpFormatter): ... + """, + error=None, + ) + def remove_color_code(s: str) -> str: return re.sub("\\x1b.*?m", "", s) # this works! From 1f89b1a04ddf39c6a346591535a080f719399e2e Mon Sep 17 00:00:00 2001 From: Brian Schubert Date: Mon, 25 Aug 2025 15:56:29 -0400 Subject: [PATCH 198/424] Include base mypy requirements in docs requirements (#19727) Attempt to fix #19726 After #19062, the docs build script attempts to import `mypy.main`, so building the docs now requires all of mypy's base requirements to be present as well. --- docs/requirements-docs.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 747f376a8f5a4..09062a635e63f 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,3 +1,4 @@ +-r ../mypy-requirements.txt sphinx>=8.1.0 furo>=2022.3.4 myst-parser>=4.0.0 From ffe2db864e3ea4a079b48892cd98ba2eabeda9a3 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Mon, 25 Aug 2025 21:57:46 +0200 Subject: [PATCH 199/424] Update dependencies (#19720) lxml 6.0.1 has wheels for Python 3.14 now. https://pypi.org/project/lxml/6.0.1/#files --- test-requirements.in | 2 +- test-requirements.txt | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/test-requirements.in b/test-requirements.in index 666dd9fc082c5..df074965a1e83 100644 --- a/test-requirements.in +++ b/test-requirements.in @@ -5,7 +5,7 @@ -r build-requirements.txt attrs>=18.0 filelock>=3.3.0 -lxml>=5.3.0; python_version<'3.14' +lxml>=5.3.0; python_version<'3.15' psutil>=4.0 pytest>=8.1.0 pytest-xdist>=1.34.0 diff --git a/test-requirements.txt b/test-requirements.txt index 11ac675eca15e..521208c5aa27d 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.13 # by the following command: # # pip-compile --allow-unsafe --output-file=test-requirements.txt --strip-extras test-requirements.in @@ -8,21 +8,21 @@ attrs==25.3.0 # via -r test-requirements.in cfgv==3.4.0 # via pre-commit -coverage==7.10.1 +coverage==7.10.5 # via pytest-cov distlib==0.4.0 # via virtualenv execnet==2.1.1 # via pytest-xdist -filelock==3.18.0 +filelock==3.19.1 # via # -r test-requirements.in # virtualenv -identify==2.6.12 +identify==2.6.13 # via pre-commit iniconfig==2.1.0 # via pytest -lxml==6.0.0 ; python_version < "3.14" +lxml==6.0.1 ; python_version < "3.15" # via -r test-requirements.in mypy-extensions==1.1.0 # via -r mypy-requirements.txt @@ -38,7 +38,7 @@ pluggy==1.6.0 # via # pytest # pytest-cov -pre-commit==4.2.0 +pre-commit==4.3.0 # via -r test-requirements.in psutil==7.0.0 # via -r test-requirements.in @@ -57,13 +57,13 @@ pyyaml==6.0.2 # via pre-commit tomli==2.2.1 # via -r test-requirements.in -types-psutil==7.0.0.20250601 +types-psutil==7.0.0.20250822 # via -r build-requirements.txt -types-setuptools==80.9.0.20250529 +types-setuptools==80.9.0.20250822 # via -r build-requirements.txt typing-extensions==4.14.1 # via -r mypy-requirements.txt -virtualenv==20.32.0 +virtualenv==20.34.0 # via pre-commit # The following packages are considered to be unsafe in a requirements file: From 881a35a6761b3b9a916ca159768ec68c3e745a05 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Tue, 26 Aug 2025 01:17:15 +0200 Subject: [PATCH 200/424] Revert "Enable colored output for argparse help in Python 3.14 (#19021)" (#19721) Reverts #19021 After the PR was merged, `color` was changed to `true` for Python 3.14. Setting it manually is no longer necessary. https://github.com/python/cpython/pull/136809 https://docs.python.org/3.14/library/argparse.html#color --- mypy/dmypy/client.py | 3 --- mypy/main.py | 2 -- mypy/stubgen.py | 2 -- mypy/stubtest.py | 2 -- 4 files changed, 9 deletions(-) diff --git a/mypy/dmypy/client.py b/mypy/dmypy/client.py index b34e9bf8ced28..3db47f80d01b7 100644 --- a/mypy/dmypy/client.py +++ b/mypy/dmypy/client.py @@ -36,9 +36,6 @@ def __init__(self, prog: str, **kwargs: Any) -> None: parser = argparse.ArgumentParser( prog="dmypy", description="Client for mypy daemon mode", fromfile_prefix_chars="@" ) -if sys.version_info >= (3, 14): - parser.color = True # Set as init arg in 3.14 - parser.set_defaults(action=None) parser.add_argument( "--status-file", default=DEFAULT_STATUS_FILE, help="status file to retrieve daemon details" diff --git a/mypy/main.py b/mypy/main.py index 0f70eb41bb14c..ad257bab69966 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -485,8 +485,6 @@ def define_options( stdout=stdout, stderr=stderr, ) - if sys.version_info >= (3, 14): - parser.color = True # Set as init arg in 3.14 strict_flag_names: list[str] = [] strict_flag_assignments: list[tuple[str, bool]] = [] diff --git a/mypy/stubgen.py b/mypy/stubgen.py index ece22ba235bf3..60fbd7f43c0fd 100755 --- a/mypy/stubgen.py +++ b/mypy/stubgen.py @@ -1899,8 +1899,6 @@ def parse_options(args: list[str]) -> Options: parser = argparse.ArgumentParser( prog="stubgen", usage=HEADER, description=DESCRIPTION, fromfile_prefix_chars="@" ) - if sys.version_info >= (3, 14): - parser.color = True # Set as init arg in 3.14 parser.add_argument( "--ignore-errors", diff --git a/mypy/stubtest.py b/mypy/stubtest.py index ee15fed81b4b4..31b3fd20b0025 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -2348,8 +2348,6 @@ def parse_options(args: list[str]) -> _Arguments: parser = argparse.ArgumentParser( description="Compares stubs to objects introspected from the runtime." ) - if sys.version_info >= (3, 14): - parser.color = True # Set as init arg in 3.14 parser.add_argument("modules", nargs="*", help="Modules to test") parser.add_argument( "--concise", From 50fc847a976484d02cf6132cec9dbce2d0e545d2 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Tue, 26 Aug 2025 02:37:56 +0200 Subject: [PATCH 201/424] Omit errors for class pattern matches against object (#19709) Since the class pattern matches any subclass, it can also be used to check whether the matched object has a specific attribute. Mypy should not emit an error for it. ```py match m: case object(foo=_): m.foo ``` Using `object` for it is recommended in [PEP 635](https://peps.python.org/pep-0635/#history-and-context) and more prominently in the precursor [PEP 622](https://peps.python.org/pep-0622/#class-patterns). --- mypy/checker_shared.py | 4 ++++ mypy/checkpattern.py | 15 +++++++++------ test-data/unit/check-python310.test | 9 +++++++++ 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/mypy/checker_shared.py b/mypy/checker_shared.py index 2a8fbdb0c9f19..0014d2c6fc880 100644 --- a/mypy/checker_shared.py +++ b/mypy/checker_shared.py @@ -272,6 +272,10 @@ def checking_await_set(self) -> Iterator[None]: def get_precise_awaitable_type(self, typ: Type, local_errors: ErrorWatcher) -> Type | None: raise NotImplementedError + @abstractmethod + def add_any_attribute_to_type(self, typ: Type, name: str) -> Type: + raise NotImplementedError + @abstractmethod def is_defined_in_stub(self, typ: Instance, /) -> bool: raise NotImplementedError diff --git a/mypy/checkpattern.py b/mypy/checkpattern.py index fc00e9f202915..f81684d2f44ae 100644 --- a/mypy/checkpattern.py +++ b/mypy/checkpattern.py @@ -671,12 +671,15 @@ def visit_class_pattern(self, o: ClassPattern) -> PatternType: has_local_errors = local_errors.has_new_errors() if has_local_errors or key_type is None: key_type = AnyType(TypeOfAny.from_error) - self.msg.fail( - message_registry.CLASS_PATTERN_UNKNOWN_KEYWORD.format( - typ.str_with_options(self.options), keyword - ), - pattern, - ) + if not (type_info and type_info.fullname == "builtins.object"): + self.msg.fail( + message_registry.CLASS_PATTERN_UNKNOWN_KEYWORD.format( + typ.str_with_options(self.options), keyword + ), + pattern, + ) + elif keyword is not None: + new_type = self.chk.add_any_attribute_to_type(new_type, keyword) inner_type, inner_rest_type, inner_captures = self.accept(pattern, key_type) if is_uninhabited(inner_type): diff --git a/test-data/unit/check-python310.test b/test-data/unit/check-python310.test index b49bb13d520f0..24bf2fdb8fb4e 100644 --- a/test-data/unit/check-python310.test +++ b/test-data/unit/check-python310.test @@ -1003,15 +1003,24 @@ match m: [builtins fixtures/tuple.pyi] [case testMatchClassPatternNonexistentKeyword] +from typing import Any class A: ... m: object +n: Any match m: case A(a=j): # E: Class "__main__.A" has no attribute "a" reveal_type(m) # N: Revealed type is "__main__.A" reveal_type(j) # N: Revealed type is "Any" +match n: + # Matching against object should not emit an error for non-existing keys + case object(a=k): + reveal_type(n) # N: Revealed type is "builtins.object" + reveal_type(n.a) # N: Revealed type is "Any" + reveal_type(k) # N: Revealed type is "Any" + [case testMatchClassPatternDuplicateKeyword] class A: a: str From d1dffe275b9ef0babb2df6ddf0a329b4304cb114 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Tue, 26 Aug 2025 13:36:23 -0400 Subject: [PATCH 202/424] [mypyc] feat: PyObject_CallObject op for fn(*args) fastpath (#19631) This PR adds a new custom_op for PyObject_CallObject which is more efficient than PyObject_Call in cases where there are no kwargs. posarg-only use cases are already optimized but this is helpful for patterns such as `fn(*args)` or `fn(a1, a2, *args)` This PR extends #19623 and #19629 , as this change will not be helpful until those PRs are merged. --- mypyc/annotate.py | 1 + mypyc/irbuild/ll_builder.py | 15 +++++---- mypyc/primitives/generic_ops.py | 9 ++++++ mypyc/test-data/irbuild-basic.test | 52 ++++++++++++------------------ 4 files changed, 39 insertions(+), 38 deletions(-) diff --git a/mypyc/annotate.py b/mypyc/annotate.py index 6736ca63c9e8e..bc282fc3ea6c1 100644 --- a/mypyc/annotate.py +++ b/mypyc/annotate.py @@ -77,6 +77,7 @@ def __init__(self, message: str, priority: int = 1) -> None: "PyNumber_Rshift": Annotation('Generic ">>" operation.'), "PyNumber_Invert": Annotation('Generic "~" operation.'), "PyObject_Call": Annotation("Generic call operation."), + "PyObject_CallObject": Annotation("Generic call operation."), "PyObject_RichCompare": Annotation("Generic comparison operation."), "PyObject_GetItem": Annotation("Generic indexing operation."), "PyObject_SetItem": Annotation("Generic indexed assignment."), diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index 116a1bb4bae0e..ba8ef94b00bdd 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -141,6 +141,7 @@ generic_ssize_t_len_op, py_call_op, py_call_with_kwargs_op, + py_call_with_posargs_op, py_getattr_op, py_method_call_op, py_vectorcall_method_op, @@ -805,7 +806,7 @@ def _construct_varargs( value.type, RTuple ): value = self.primitive_op(sequence_tuple_op, [value], line) - return value, self._create_dict([], [], line) + return value, None elif len(args) == 2 and args[1][1] == ARG_STAR2: # fn(*args, **kwargs) # TODO: extend to cover(*args, **k, **w, **a, **r, **g, **s) @@ -938,7 +939,7 @@ def _construct_varargs( elif not is_tuple_rprimitive(star_result.type): # if star_result is a tuple we took the fast path star_result = self.primitive_op(list_tuple_op, [star_result], line) - if has_star2 and star2_result is None: + if has_star2 and star2_result is None and len(star2_keys) > 0: # TODO: use dict_copy_op for simple cases of **kwargs star2_result = self._create_dict(star2_keys, star2_values, line) @@ -964,13 +965,16 @@ def py_call( if arg_kinds is None or all(kind == ARG_POS for kind in arg_kinds): return self.call_c(py_call_op, [function] + arg_values, line) - # Otherwise fallback to py_call_with_kwargs_op. + # Otherwise fallback to py_call_with_posargs_op or py_call_with_kwargs_op. assert arg_names is not None pos_args_tuple, kw_args_dict = self._construct_varargs( list(zip(arg_values, arg_kinds, arg_names)), line, has_star=True, has_star2=True ) - assert pos_args_tuple and kw_args_dict + assert pos_args_tuple + + if kw_args_dict is None: + return self.call_c(py_call_with_posargs_op, [function, pos_args_tuple], line) return self.call_c(py_call_with_kwargs_op, [function, pos_args_tuple, kw_args_dict], line) @@ -1169,8 +1173,7 @@ def native_args_to_positional( assert star_arg output_arg = star_arg elif arg.kind == ARG_STAR2: - assert star2_arg - output_arg = star2_arg + output_arg = star2_arg or self._create_dict([], [], line) elif not lst: if is_fixed_width_rtype(arg.type): output_arg = Integer(0, arg.type) diff --git a/mypyc/primitives/generic_ops.py b/mypyc/primitives/generic_ops.py index 4a95be4e5d4e2..8a4ddc3702808 100644 --- a/mypyc/primitives/generic_ops.py +++ b/mypyc/primitives/generic_ops.py @@ -308,6 +308,15 @@ error_kind=ERR_MAGIC, ) +# Call callable object with positional args only: func(*args) +# Arguments are (func, *args tuple). +py_call_with_posargs_op = custom_op( + arg_types=[object_rprimitive, object_rprimitive], + return_type=object_rprimitive, + c_function_name="PyObject_CallObject", + error_kind=ERR_MAGIC, +) + # Call method with positional arguments: obj.method(arg1, ...) # Arguments are (object, attribute name, arg1, ...). py_method_call_op = custom_op( diff --git a/mypyc/test-data/irbuild-basic.test b/mypyc/test-data/irbuild-basic.test index 63e4ef55d3fcc..4eeeca04719c6 100644 --- a/mypyc/test-data/irbuild-basic.test +++ b/mypyc/test-data/irbuild-basic.test @@ -1674,20 +1674,17 @@ def g(): r0 :: tuple[int, int, int] r1 :: dict r2 :: str - r3 :: object - r4 :: dict - r5, r6 :: object - r7 :: tuple[int, int, int] + r3, r4, r5 :: object + r6 :: tuple[int, int, int] L0: r0 = (2, 4, 6) r1 = __main__.globals :: static r2 = 'f' r3 = CPyDict_GetItem(r1, r2) - r4 = PyDict_New() - r5 = box(tuple[int, int, int], r0) - r6 = PyObject_Call(r3, r5, r4) - r7 = unbox(tuple[int, int, int], r6) - return r7 + r4 = box(tuple[int, int, int], r0) + r5 = PyObject_CallObject(r3, r4) + r6 = unbox(tuple[int, int, int], r5) + return r6 def h(): r0 :: tuple[int, int] r1 :: dict @@ -1698,9 +1695,8 @@ def h(): r6 :: ptr r7, r8 :: object r9 :: tuple - r10 :: dict - r11 :: object - r12 :: tuple[int, int, int] + r10 :: object + r11 :: tuple[int, int, int] L0: r0 = (4, 6) r1 = __main__.globals :: static @@ -1714,10 +1710,9 @@ L0: r7 = box(tuple[int, int], r0) r8 = CPyList_Extend(r4, r7) r9 = PyList_AsTuple(r4) - r10 = PyDict_New() - r11 = PyObject_Call(r3, r9, r10) - r12 = unbox(tuple[int, int, int], r11) - return r12 + r10 = PyObject_CallObject(r3, r9) + r11 = unbox(tuple[int, int, int], r10) + return r11 [case testStar2Args] from typing import Tuple @@ -3562,15 +3557,12 @@ def wrapper_deco_obj.__call__(__mypyc_self__, args): __mypyc_self__ :: __main__.wrapper_deco_obj args :: tuple r0 :: __main__.deco_env - r1 :: object - r2 :: dict - r3 :: object + r1, r2 :: object L0: r0 = __mypyc_self__.__mypyc_env__ r1 = r0.fn - r2 = PyDict_New() - r3 = PyObject_Call(r1, args, r2) - return r3 + r2 = PyObject_CallObject(r1, args) + return r2 def deco(fn): fn :: object r0 :: __main__.deco_env @@ -3613,15 +3605,13 @@ def wrapper_deco_obj.__call__(__mypyc_self__, args): r0 :: __main__.deco_env r1 :: object r2 :: tuple - r3 :: dict - r4 :: object + r3 :: object L0: r0 = __mypyc_self__.__mypyc_env__ r1 = r0.fn r2 = PyList_AsTuple(args) - r3 = PyDict_New() - r4 = PyObject_Call(r1, r2, r3) - return r4 + r3 = PyObject_CallObject(r1, r2) + return r3 def deco(fn): fn :: object r0 :: __main__.deco_env @@ -3716,15 +3706,13 @@ def wrapper_deco_obj.__call__(__mypyc_self__, args): r0 :: __main__.deco_env r1 :: object r2 :: tuple - r3 :: dict - r4 :: object + r3 :: object L0: r0 = __mypyc_self__.__mypyc_env__ r1 = r0.fn r2 = PySequence_Tuple(args) - r3 = PyDict_New() - r4 = PyObject_Call(r1, r2, r3) - return r4 + r3 = PyObject_CallObject(r1, r2) + return r3 def deco(fn): fn :: object r0 :: __main__.deco_env From dcb4d695e0b3ea6fedd90572bc249ec07731b3bc Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 26 Aug 2025 21:10:20 +0100 Subject: [PATCH 203/424] Use 1 byte per type/symbol tag (#19735) This is a small incremental improvement for fixed format cache. I am adding a dedicated write/read functions for tags (i.e. integers in 0-255 range). I propose to exclusively use these functions for type tags (hence the name), and still use regular `write_int()`/`read_int()` for integers that are "accidentally small" (like argument kinds etc). In a separate PR I will change regular `int` format to be more progressive (e.g. only use 1 byte if an integer happens to be small). I also change the terminology from "marker" to "tag", as this is a more common name for this concept. Note we can probably use `mypy_extensions.u8` for type tags. If there is a desire for this, I can switch to it (either in this or a separate PR). --- mypy/cache.py | 32 +++-- mypy/nodes.py | 85 ++++++------ mypy/types.py | 125 +++++++++--------- .../stubs/mypy-native/native_internal.pyi | 2 + mypyc/lib-rt/native_internal.c | 73 +++++++++- mypyc/lib-rt/native_internal.h | 6 +- mypyc/primitives/misc_ops.py | 16 +++ mypyc/test-data/irbuild-classes.test | 44 +++--- mypyc/test-data/run-classes.test | 15 ++- test-data/unit/lib-stub/native_internal.pyi | 2 + 10 files changed, 262 insertions(+), 138 deletions(-) diff --git a/mypy/cache.py b/mypy/cache.py index 49f568c1f3c19..a16a36900c7ac 100644 --- a/mypy/cache.py +++ b/mypy/cache.py @@ -10,10 +10,12 @@ read_float as read_float, read_int as read_int, read_str as read_str, + read_tag as read_tag, write_bool as write_bool, write_float as write_float, write_int as write_int, write_str as write_str, + write_tag as write_tag, ) except ImportError: # TODO: temporary, remove this after we publish mypy-native on PyPI. @@ -32,6 +34,12 @@ def read_int(data: Buffer) -> int: def write_int(data: Buffer, value: int) -> None: raise NotImplementedError + def read_tag(data: Buffer) -> int: + raise NotImplementedError + + def write_tag(data: Buffer, value: int) -> None: + raise NotImplementedError + def read_str(data: Buffer) -> str: raise NotImplementedError @@ -59,37 +67,37 @@ def write_float(data: Buffer, value: float) -> None: LITERAL_NONE: Final = 6 -def read_literal(data: Buffer, marker: int) -> int | str | bool | float: - if marker == LITERAL_INT: +def read_literal(data: Buffer, tag: int) -> int | str | bool | float: + if tag == LITERAL_INT: return read_int(data) - elif marker == LITERAL_STR: + elif tag == LITERAL_STR: return read_str(data) - elif marker == LITERAL_BOOL: + elif tag == LITERAL_BOOL: return read_bool(data) - elif marker == LITERAL_FLOAT: + elif tag == LITERAL_FLOAT: return read_float(data) - assert False, f"Unknown literal marker {marker}" + assert False, f"Unknown literal tag {tag}" def write_literal(data: Buffer, value: int | str | bool | float | complex | None) -> None: if isinstance(value, bool): - write_int(data, LITERAL_BOOL) + write_tag(data, LITERAL_BOOL) write_bool(data, value) elif isinstance(value, int): - write_int(data, LITERAL_INT) + write_tag(data, LITERAL_INT) write_int(data, value) elif isinstance(value, str): - write_int(data, LITERAL_STR) + write_tag(data, LITERAL_STR) write_str(data, value) elif isinstance(value, float): - write_int(data, LITERAL_FLOAT) + write_tag(data, LITERAL_FLOAT) write_float(data, value) elif isinstance(value, complex): - write_int(data, LITERAL_COMPLEX) + write_tag(data, LITERAL_COMPLEX) write_float(data, value.real) write_float(data, value.imag) else: - write_int(data, LITERAL_NONE) + write_tag(data, LITERAL_NONE) def read_int_opt(data: Buffer) -> int | None: diff --git a/mypy/nodes.py b/mypy/nodes.py index b9c08f02f3168..45e2b60c3e782 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -28,6 +28,7 @@ read_str_list, read_str_opt, read_str_opt_list, + read_tag, write_bool, write_int, write_int_list, @@ -37,6 +38,7 @@ write_str_list, write_str_opt, write_str_opt_list, + write_tag, ) from mypy.options import Options from mypy.util import is_sunder, is_typeshed_file, short_type @@ -417,7 +419,7 @@ def deserialize(cls, data: JsonDict) -> MypyFile: return tree def write(self, data: Buffer) -> None: - write_int(data, MYPY_FILE) + write_tag(data, MYPY_FILE) write_str(data, self._fullname) self.names.write(data, self._fullname) write_bool(data, self.is_stub) @@ -427,7 +429,7 @@ def write(self, data: Buffer) -> None: @classmethod def read(cls, data: Buffer) -> MypyFile: - assert read_int(data) == MYPY_FILE + assert read_tag(data) == MYPY_FILE tree = MypyFile([], []) tree._fullname = read_str(data) tree.names = SymbolTable.read(data) @@ -711,7 +713,7 @@ def deserialize(cls, data: JsonDict) -> OverloadedFuncDef: return res def write(self, data: Buffer) -> None: - write_int(data, OVERLOADED_FUNC_DEF) + write_tag(data, OVERLOADED_FUNC_DEF) write_int(data, len(self.items)) for item in self.items: item.write(data) @@ -1022,7 +1024,7 @@ def deserialize(cls, data: JsonDict) -> FuncDef: return ret def write(self, data: Buffer) -> None: - write_int(data, FUNC_DEF) + write_tag(data, FUNC_DEF) write_str(data, self._name) mypy.types.write_type_opt(data, self.type) write_str(data, self._fullname) @@ -1134,16 +1136,16 @@ def deserialize(cls, data: JsonDict) -> Decorator: return dec def write(self, data: Buffer) -> None: - write_int(data, DECORATOR) + write_tag(data, DECORATOR) self.func.write(data) self.var.write(data) write_bool(data, self.is_overload) @classmethod def read(cls, data: Buffer) -> Decorator: - assert read_int(data) == FUNC_DEF + assert read_tag(data) == FUNC_DEF func = FuncDef.read(data) - assert read_int(data) == VAR + assert read_tag(data) == VAR var = Var.read(data) dec = Decorator(func, [], var) dec.is_overload = read_bool(data) @@ -1326,7 +1328,7 @@ def deserialize(cls, data: JsonDict) -> Var: return v def write(self, data: Buffer) -> None: - write_int(data, VAR) + write_tag(data, VAR) write_str(data, self._name) mypy.types.write_type_opt(data, self.type) mypy.types.write_type_opt(data, self.setter_type) @@ -1341,13 +1343,13 @@ def read(cls, data: Buffer) -> Var: v = Var(name, typ) setter_type: mypy.types.CallableType | None = None if read_bool(data): - assert read_int(data) == mypy.types.CALLABLE_TYPE + assert read_tag(data) == mypy.types.CALLABLE_TYPE setter_type = mypy.types.CallableType.read(data) v.setter_type = setter_type v.is_ready = False # Override True default set in __init__ v._fullname = read_str(data) read_flags(data, v, VAR_FLAGS) - marker = read_int(data) + marker = read_tag(data) if marker == LITERAL_COMPLEX: v.final_value = complex(read_float(data), read_float(data)) elif marker != LITERAL_NONE: @@ -1465,7 +1467,7 @@ def deserialize(cls, data: JsonDict) -> ClassDef: return res def write(self, data: Buffer) -> None: - write_int(data, CLASS_DEF) + write_tag(data, CLASS_DEF) write_str(data, self.name) mypy.types.write_type_list(data, self.type_vars) write_str(data, self.fullname) @@ -2898,7 +2900,7 @@ def deserialize(cls, data: JsonDict) -> TypeVarExpr: ) def write(self, data: Buffer) -> None: - write_int(data, TYPE_VAR_EXPR) + write_tag(data, TYPE_VAR_EXPR) write_str(data, self._name) write_str(data, self._fullname) mypy.types.write_type_list(data, self.values) @@ -2948,7 +2950,7 @@ def deserialize(cls, data: JsonDict) -> ParamSpecExpr: ) def write(self, data: Buffer) -> None: - write_int(data, PARAM_SPEC_EXPR) + write_tag(data, PARAM_SPEC_EXPR) write_str(data, self._name) write_str(data, self._fullname) self.upper_bound.write(data) @@ -3016,7 +3018,7 @@ def deserialize(cls, data: JsonDict) -> TypeVarTupleExpr: ) def write(self, data: Buffer) -> None: - write_int(data, TYPE_VAR_TUPLE_EXPR) + write_tag(data, TYPE_VAR_TUPLE_EXPR) self.tuple_fallback.write(data) write_str(data, self._name) write_str(data, self._fullname) @@ -3026,7 +3028,7 @@ def write(self, data: Buffer) -> None: @classmethod def read(cls, data: Buffer) -> TypeVarTupleExpr: - assert read_int(data) == mypy.types.INSTANCE + assert read_tag(data) == mypy.types.INSTANCE fallback = mypy.types.Instance.read(data) return TypeVarTupleExpr( read_str(data), @@ -3908,7 +3910,7 @@ def deserialize(cls, data: JsonDict) -> TypeInfo: return ti def write(self, data: Buffer) -> None: - write_int(data, TYPE_INFO) + write_tag(data, TYPE_INFO) self.names.write(data, self.fullname) self.defn.write(data) write_str(data, self.module_name) @@ -3944,7 +3946,7 @@ def write(self, data: Buffer) -> None: @classmethod def read(cls, data: Buffer) -> TypeInfo: names = SymbolTable.read(data) - assert read_int(data) == CLASS_DEF + assert read_tag(data) == CLASS_DEF defn = ClassDef.read(data) module_name = read_str(data) ti = TypeInfo(names, defn, module_name) @@ -3954,10 +3956,9 @@ def read(cls, data: Buffer) -> TypeInfo: ti.abstract_attributes = list(zip(attrs, statuses)) ti.type_vars = read_str_list(data) ti.has_param_spec_type = read_bool(data) - num_bases = read_int(data) ti.bases = [] - for _ in range(num_bases): - assert read_int(data) == mypy.types.INSTANCE + for _ in range(read_int(data)): + assert read_tag(data) == mypy.types.INSTANCE ti.bases.append(mypy.types.Instance.read(data)) # NOTE: ti.mro will be set in the fixup phase based on these # names. The reason we need to store the mro instead of just @@ -3972,19 +3973,19 @@ def read(cls, data: Buffer) -> TypeInfo: ti._mro_refs = read_str_list(data) ti._promote = cast(list[mypy.types.ProperType], mypy.types.read_type_list(data)) if read_bool(data): - assert read_int(data) == mypy.types.INSTANCE + assert read_tag(data) == mypy.types.INSTANCE ti.alt_promote = mypy.types.Instance.read(data) if read_bool(data): - assert read_int(data) == mypy.types.INSTANCE + assert read_tag(data) == mypy.types.INSTANCE ti.declared_metaclass = mypy.types.Instance.read(data) if read_bool(data): - assert read_int(data) == mypy.types.INSTANCE + assert read_tag(data) == mypy.types.INSTANCE ti.metaclass_type = mypy.types.Instance.read(data) if read_bool(data): - assert read_int(data) == mypy.types.TUPLE_TYPE + assert read_tag(data) == mypy.types.TUPLE_TYPE ti.tuple_type = mypy.types.TupleType.read(data) if read_bool(data): - assert read_int(data) == mypy.types.TYPED_DICT_TYPE + assert read_tag(data) == mypy.types.TYPED_DICT_TYPE ti.typeddict_type = mypy.types.TypedDictType.read(data) read_flags(data, ti, TypeInfo.FLAGS) metadata = read_str(data) @@ -3994,7 +3995,7 @@ def read(cls, data: Buffer) -> TypeInfo: ti.slots = set(read_str_list(data)) ti.deletable_attributes = read_str_list(data) if read_bool(data): - assert read_int(data) == mypy.types.TYPE_VAR_TYPE + assert read_tag(data) == mypy.types.TYPE_VAR_TYPE ti.self_type = mypy.types.TypeVarType.read(data) if read_bool(data): ti.dataclass_transform_spec = DataclassTransformSpec.read(data) @@ -4270,7 +4271,7 @@ def deserialize(cls, data: JsonDict) -> TypeAlias: ) def write(self, data: Buffer) -> None: - write_int(data, TYPE_ALIAS) + write_tag(data, TYPE_ALIAS) write_str(data, self._fullname) self.target.write(data) mypy.types.write_type_list(data, self.alias_tvars) @@ -4890,33 +4891,33 @@ def local_definitions( def read_symbol(data: Buffer) -> mypy.nodes.SymbolNode: - marker = read_int(data) + tag = read_tag(data) # The branches here are ordered manually by type "popularity". - if marker == VAR: + if tag == VAR: return mypy.nodes.Var.read(data) - if marker == FUNC_DEF: + if tag == FUNC_DEF: return mypy.nodes.FuncDef.read(data) - if marker == DECORATOR: + if tag == DECORATOR: return mypy.nodes.Decorator.read(data) - if marker == TYPE_INFO: + if tag == TYPE_INFO: return mypy.nodes.TypeInfo.read(data) - if marker == OVERLOADED_FUNC_DEF: + if tag == OVERLOADED_FUNC_DEF: return mypy.nodes.OverloadedFuncDef.read(data) - if marker == TYPE_VAR_EXPR: + if tag == TYPE_VAR_EXPR: return mypy.nodes.TypeVarExpr.read(data) - if marker == TYPE_ALIAS: + if tag == TYPE_ALIAS: return mypy.nodes.TypeAlias.read(data) - if marker == PARAM_SPEC_EXPR: + if tag == PARAM_SPEC_EXPR: return mypy.nodes.ParamSpecExpr.read(data) - if marker == TYPE_VAR_TUPLE_EXPR: + if tag == TYPE_VAR_TUPLE_EXPR: return mypy.nodes.TypeVarTupleExpr.read(data) - assert False, f"Unknown symbol marker {marker}" + assert False, f"Unknown symbol tag {tag}" def read_overload_part(data: Buffer) -> OverloadPart: - marker = read_int(data) - if marker == DECORATOR: + tag = read_tag(data) + if tag == DECORATOR: return Decorator.read(data) - if marker == FUNC_DEF: + if tag == FUNC_DEF: return FuncDef.read(data) - assert False, f"Invalid marker for an OverloadPart {marker}" + assert False, f"Invalid tag for an OverloadPart {tag}" diff --git a/mypy/types.py b/mypy/types.py index b48e0ef4d9855..43e6dafe298e0 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -20,6 +20,7 @@ read_str_list, read_str_opt, read_str_opt_list, + read_tag, write_bool, write_int, write_int_list, @@ -28,6 +29,7 @@ write_str_list, write_str_opt, write_str_opt_list, + write_tag, ) from mypy.nodes import ARG_KINDS, ARG_POS, ARG_STAR, ARG_STAR2, INVARIANT, ArgKind, SymbolNode from mypy.options import Options @@ -456,7 +458,7 @@ def deserialize(cls, data: JsonDict) -> TypeAliasType: return alias def write(self, data: Buffer) -> None: - write_int(data, TYPE_ALIAS_TYPE) + write_tag(data, TYPE_ALIAS_TYPE) write_type_list(data, self.args) assert self.alias is not None write_str(data, self.alias.fullname) @@ -735,7 +737,7 @@ def deserialize(cls, data: JsonDict) -> TypeVarType: ) def write(self, data: Buffer) -> None: - write_int(data, TYPE_VAR_TYPE) + write_tag(data, TYPE_VAR_TYPE) write_str(data, self.name) write_str(data, self.fullname) write_int(data, self.id.raw_id) @@ -887,7 +889,7 @@ def deserialize(cls, data: JsonDict) -> ParamSpecType: ) def write(self, data: Buffer) -> None: - write_int(data, PARAM_SPEC_TYPE) + write_tag(data, PARAM_SPEC_TYPE) self.prefix.write(data) write_str(data, self.name) write_str(data, self.fullname) @@ -899,7 +901,7 @@ def write(self, data: Buffer) -> None: @classmethod def read(cls, data: Buffer) -> ParamSpecType: - assert read_int(data) == PARAMETERS + assert read_tag(data) == PARAMETERS prefix = Parameters.read(data) return ParamSpecType( read_str(data), @@ -967,7 +969,7 @@ def deserialize(cls, data: JsonDict) -> TypeVarTupleType: ) def write(self, data: Buffer) -> None: - write_int(data, TYPE_VAR_TUPLE_TYPE) + write_tag(data, TYPE_VAR_TUPLE_TYPE) self.tuple_fallback.write(data) write_str(data, self.name) write_str(data, self.fullname) @@ -979,7 +981,7 @@ def write(self, data: Buffer) -> None: @classmethod def read(cls, data: Buffer) -> TypeVarTupleType: - assert read_int(data) == INSTANCE + assert read_tag(data) == INSTANCE fallback = Instance.read(data) return TypeVarTupleType( read_str(data), @@ -1123,7 +1125,7 @@ def deserialize(cls, data: JsonDict) -> UnboundType: ) def write(self, data: Buffer) -> None: - write_int(data, UNBOUND_TYPE) + write_tag(data, UNBOUND_TYPE) write_str(data, self.name) write_type_list(data, self.args) write_str_opt(data, self.original_str_expr) @@ -1233,7 +1235,7 @@ def serialize(self) -> JsonDict: return {".class": "UnpackType", "type": self.type.serialize()} def write(self, data: Buffer) -> None: - write_int(data, UNPACK_TYPE) + write_tag(data, UNPACK_TYPE) self.type.write(data) @classmethod @@ -1342,7 +1344,7 @@ def deserialize(cls, data: JsonDict) -> AnyType: ) def write(self, data: Buffer) -> None: - write_int(data, ANY_TYPE) + write_tag(data, ANY_TYPE) write_type_opt(data, self.source_any) write_int(data, self.type_of_any) write_str_opt(data, self.missing_import_name) @@ -1350,7 +1352,7 @@ def write(self, data: Buffer) -> None: @classmethod def read(cls, data: Buffer) -> AnyType: if read_bool(data): - assert read_int(data) == ANY_TYPE + assert read_tag(data) == ANY_TYPE source_any = AnyType.read(data) else: source_any = None @@ -1403,7 +1405,7 @@ def deserialize(cls, data: JsonDict) -> UninhabitedType: return UninhabitedType() def write(self, data: Buffer) -> None: - write_int(data, UNINHABITED_TYPE) + write_tag(data, UNINHABITED_TYPE) @classmethod def read(cls, data: Buffer) -> UninhabitedType: @@ -1442,7 +1444,7 @@ def deserialize(cls, data: JsonDict) -> NoneType: return NoneType() def write(self, data: Buffer) -> None: - write_int(data, NONE_TYPE) + write_tag(data, NONE_TYPE) @classmethod def read(cls, data: Buffer) -> NoneType: @@ -1496,7 +1498,7 @@ def deserialize(cls, data: JsonDict) -> DeletedType: return DeletedType(data["source"]) def write(self, data: Buffer) -> None: - write_int(data, DELETED_TYPE) + write_tag(data, DELETED_TYPE) write_str_opt(data, self.source) @classmethod @@ -1704,7 +1706,7 @@ def deserialize(cls, data: JsonDict | str) -> Instance: return inst def write(self, data: Buffer) -> None: - write_int(data, INSTANCE) + write_tag(data, INSTANCE) write_str(data, self.type.fullname) write_type_list(data, self.args) write_type_opt(data, self.last_known_value) @@ -1720,7 +1722,7 @@ def read(cls, data: Buffer) -> Instance: inst = Instance(NOT_READY, read_type_list(data)) inst.type_ref = type_ref if read_bool(data): - assert read_int(data) == LITERAL_TYPE + assert read_tag(data) == LITERAL_TYPE inst.last_known_value = LiteralType.read(data) if read_bool(data): inst.extra_attrs = ExtraAttrs.read(data) @@ -2003,7 +2005,7 @@ def deserialize(cls, data: JsonDict) -> Parameters: ) def write(self, data: Buffer) -> None: - write_int(data, PARAMETERS) + write_tag(data, PARAMETERS) write_type_list(data, self.arg_types) write_int_list(data, [int(x.value) for x in self.arg_kinds]) write_str_opt_list(data, self.arg_names) @@ -2525,7 +2527,7 @@ def deserialize(cls, data: JsonDict) -> CallableType: ) def write(self, data: Buffer) -> None: - write_int(data, CALLABLE_TYPE) + write_tag(data, CALLABLE_TYPE) self.fallback.write(data) write_type_list(data, self.arg_types) write_int_list(data, [int(x.value) for x in self.arg_kinds]) @@ -2544,7 +2546,7 @@ def write(self, data: Buffer) -> None: @classmethod def read(cls, data: Buffer) -> CallableType: - assert read_int(data) == INSTANCE + assert read_tag(data) == INSTANCE fallback = Instance.read(data) return CallableType( read_type_list(data), @@ -2640,15 +2642,14 @@ def deserialize(cls, data: JsonDict) -> Overloaded: return Overloaded([CallableType.deserialize(t) for t in data["items"]]) def write(self, data: Buffer) -> None: - write_int(data, OVERLOADED) + write_tag(data, OVERLOADED) write_type_list(data, self.items) @classmethod def read(cls, data: Buffer) -> Overloaded: items = [] - num_overloads = read_int(data) - for _ in range(num_overloads): - assert read_int(data) == CALLABLE_TYPE + for _ in range(read_int(data)): + assert read_tag(data) == CALLABLE_TYPE items.append(CallableType.read(data)) return Overloaded(items) @@ -2749,14 +2750,14 @@ def deserialize(cls, data: JsonDict) -> TupleType: ) def write(self, data: Buffer) -> None: - write_int(data, TUPLE_TYPE) + write_tag(data, TUPLE_TYPE) self.partial_fallback.write(data) write_type_list(data, self.items) write_bool(data, self.implicit) @classmethod def read(cls, data: Buffer) -> TupleType: - assert read_int(data) == INSTANCE + assert read_tag(data) == INSTANCE fallback = Instance.read(data) return TupleType(read_type_list(data), fallback, implicit=read_bool(data)) @@ -2931,7 +2932,7 @@ def deserialize(cls, data: JsonDict) -> TypedDictType: ) def write(self, data: Buffer) -> None: - write_int(data, TYPED_DICT_TYPE) + write_tag(data, TYPED_DICT_TYPE) self.fallback.write(data) write_type_map(data, self.items) write_str_list(data, sorted(self.required_keys)) @@ -2939,7 +2940,7 @@ def write(self, data: Buffer) -> None: @classmethod def read(cls, data: Buffer) -> TypedDictType: - assert read_int(data) == INSTANCE + assert read_tag(data) == INSTANCE fallback = Instance.read(data) return TypedDictType( read_type_map(data), set(read_str_list(data)), set(read_str_list(data)), fallback @@ -3194,16 +3195,16 @@ def deserialize(cls, data: JsonDict) -> LiteralType: return LiteralType(value=data["value"], fallback=Instance.deserialize(data["fallback"])) def write(self, data: Buffer) -> None: - write_int(data, LITERAL_TYPE) + write_tag(data, LITERAL_TYPE) self.fallback.write(data) write_literal(data, self.value) @classmethod def read(cls, data: Buffer) -> LiteralType: - assert read_int(data) == INSTANCE + assert read_tag(data) == INSTANCE fallback = Instance.read(data) - marker = read_int(data) - return LiteralType(read_literal(data, marker), fallback) + tag = read_tag(data) + return LiteralType(read_literal(data, tag), fallback) def is_singleton_type(self) -> bool: return self.is_enum_literal() or isinstance(self.value, bool) @@ -3307,7 +3308,7 @@ def deserialize(cls, data: JsonDict) -> UnionType: ) def write(self, data: Buffer) -> None: - write_int(data, UNION_TYPE) + write_tag(data, UNION_TYPE) write_type_list(data, self.items) write_bool(data, self.uses_pep604_syntax) @@ -3452,7 +3453,7 @@ def deserialize(cls, data: JsonDict) -> Type: return TypeType.make_normalized(deserialize_type(data["item"])) def write(self, data: Buffer) -> None: - write_int(data, TYPE_TYPE) + write_tag(data, TYPE_TYPE) self.item.write(data) @classmethod @@ -4141,67 +4142,67 @@ def type_vars_as_args(type_vars: Sequence[TypeVarLikeType]) -> tuple[Type, ...]: def read_type(data: Buffer) -> Type: - marker = read_int(data) + tag = read_tag(data) # The branches here are ordered manually by type "popularity". - if marker == INSTANCE: + if tag == INSTANCE: return Instance.read(data) - if marker == ANY_TYPE: + if tag == ANY_TYPE: return AnyType.read(data) - if marker == TYPE_VAR_TYPE: + if tag == TYPE_VAR_TYPE: return TypeVarType.read(data) - if marker == CALLABLE_TYPE: + if tag == CALLABLE_TYPE: return CallableType.read(data) - if marker == NONE_TYPE: + if tag == NONE_TYPE: return NoneType.read(data) - if marker == UNION_TYPE: + if tag == UNION_TYPE: return UnionType.read(data) - if marker == LITERAL_TYPE: + if tag == LITERAL_TYPE: return LiteralType.read(data) - if marker == TYPE_ALIAS_TYPE: + if tag == TYPE_ALIAS_TYPE: return TypeAliasType.read(data) - if marker == TUPLE_TYPE: + if tag == TUPLE_TYPE: return TupleType.read(data) - if marker == TYPED_DICT_TYPE: + if tag == TYPED_DICT_TYPE: return TypedDictType.read(data) - if marker == TYPE_TYPE: + if tag == TYPE_TYPE: return TypeType.read(data) - if marker == OVERLOADED: + if tag == OVERLOADED: return Overloaded.read(data) - if marker == PARAM_SPEC_TYPE: + if tag == PARAM_SPEC_TYPE: return ParamSpecType.read(data) - if marker == TYPE_VAR_TUPLE_TYPE: + if tag == TYPE_VAR_TUPLE_TYPE: return TypeVarTupleType.read(data) - if marker == UNPACK_TYPE: + if tag == UNPACK_TYPE: return UnpackType.read(data) - if marker == PARAMETERS: + if tag == PARAMETERS: return Parameters.read(data) - if marker == UNINHABITED_TYPE: + if tag == UNINHABITED_TYPE: return UninhabitedType.read(data) - if marker == UNBOUND_TYPE: + if tag == UNBOUND_TYPE: return UnboundType.read(data) - if marker == DELETED_TYPE: + if tag == DELETED_TYPE: return DeletedType.read(data) - assert False, f"Unknown type marker {marker}" + assert False, f"Unknown type tag {tag}" def read_function_like(data: Buffer) -> FunctionLike: - marker = read_int(data) - if marker == CALLABLE_TYPE: + tag = read_tag(data) + if tag == CALLABLE_TYPE: return CallableType.read(data) - if marker == OVERLOADED: + if tag == OVERLOADED: return Overloaded.read(data) - assert False, f"Invalid type marker for FunctionLike {marker}" + assert False, f"Invalid type tag for FunctionLike {tag}" def read_type_var_like(data: Buffer) -> TypeVarLikeType: - marker = read_int(data) - if marker == TYPE_VAR_TYPE: + tag = read_tag(data) + if tag == TYPE_VAR_TYPE: return TypeVarType.read(data) - if marker == PARAM_SPEC_TYPE: + if tag == PARAM_SPEC_TYPE: return ParamSpecType.read(data) - if marker == TYPE_VAR_TUPLE_TYPE: + if tag == TYPE_VAR_TUPLE_TYPE: return TypeVarTupleType.read(data) - assert False, f"Invalid type marker for TypeVarLikeType {marker}" + assert False, f"Invalid type tag for TypeVarLikeType {tag}" def read_type_opt(data: Buffer) -> Type | None: diff --git a/mypy/typeshed/stubs/mypy-native/native_internal.pyi b/mypy/typeshed/stubs/mypy-native/native_internal.pyi index bc1f570a8e9c1..3c6a22c938e3c 100644 --- a/mypy/typeshed/stubs/mypy-native/native_internal.pyi +++ b/mypy/typeshed/stubs/mypy-native/native_internal.pyi @@ -10,3 +10,5 @@ def write_float(data: Buffer, value: float) -> None: ... def read_float(data: Buffer) -> float: ... def write_int(data: Buffer, value: int) -> None: ... def read_int(data: Buffer) -> int: ... +def write_tag(data: Buffer, value: int) -> None: ... +def read_tag(data: Buffer) -> int: ... diff --git a/mypyc/lib-rt/native_internal.c b/mypyc/lib-rt/native_internal.c index 11a3fafee56f7..1c35eab946f8c 100644 --- a/mypyc/lib-rt/native_internal.c +++ b/mypyc/lib-rt/native_internal.c @@ -1,10 +1,12 @@ #define PY_SSIZE_T_CLEAN #include +#include #include "CPy.h" #define NATIVE_INTERNAL_MODULE #include "native_internal.h" #define START_SIZE 512 +#define MAX_SHORT_INT_TAGGED (255 << 1) typedef struct { PyObject_HEAD @@ -436,6 +438,71 @@ write_int(PyObject *self, PyObject *args, PyObject *kwds) { return Py_None; } +static CPyTagged +read_tag_internal(PyObject *data) { + if (_check_buffer(data) == 2) + return CPY_INT_TAG; + + if (_check_read((BufferObject *)data, 1) == 2) + return CPY_INT_TAG; + char *buf = ((BufferObject *)data)->buf; + + uint8_t ret = *(uint8_t *)(buf + ((BufferObject *)data)->pos); + ((BufferObject *)data)->pos += 1; + return ((CPyTagged)ret) << 1; +} + +static PyObject* +read_tag(PyObject *self, PyObject *args, PyObject *kwds) { + static char *kwlist[] = {"data", NULL}; + PyObject *data = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O", kwlist, &data)) + return NULL; + CPyTagged retval = read_tag_internal(data); + if (retval == CPY_INT_TAG) { + return NULL; + } + return CPyTagged_StealAsObject(retval); +} + +static char +write_tag_internal(PyObject *data, CPyTagged value) { + if (_check_buffer(data) == 2) + return 2; + + if (value > MAX_SHORT_INT_TAGGED) { + PyErr_SetString(PyExc_OverflowError, "value must fit in single byte"); + return 2; + } + + if (_check_size((BufferObject *)data, 1) == 2) + return 2; + uint8_t *buf = (uint8_t *)((BufferObject *)data)->buf; + *(buf + ((BufferObject *)data)->pos) = (uint8_t)(value >> 1); + ((BufferObject *)data)->pos += 1; + ((BufferObject *)data)->end += 1; + return 1; +} + +static PyObject* +write_tag(PyObject *self, PyObject *args, PyObject *kwds) { + static char *kwlist[] = {"data", "value", NULL}; + PyObject *data = NULL; + PyObject *value = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO", kwlist, &data, &value)) + return NULL; + if (!PyLong_Check(value)) { + PyErr_SetString(PyExc_TypeError, "value must be an int"); + return NULL; + } + CPyTagged tagged_value = CPyTagged_BorrowFromObject(value); + if (write_tag_internal(data, tagged_value) == 2) { + return NULL; + } + Py_INCREF(Py_None); + return Py_None; +} + static PyMethodDef native_internal_module_methods[] = { // TODO: switch public wrappers to METH_FASTCALL. {"write_bool", (PyCFunction)write_bool, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("write a bool")}, @@ -446,6 +513,8 @@ static PyMethodDef native_internal_module_methods[] = { {"read_float", (PyCFunction)read_float, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("read a float")}, {"write_int", (PyCFunction)write_int, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("write an int")}, {"read_int", (PyCFunction)read_int, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("read an int")}, + {"write_tag", (PyCFunction)write_tag, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("write a short int")}, + {"read_tag", (PyCFunction)read_tag, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("read a short int")}, {NULL, NULL, 0, NULL} }; @@ -465,7 +534,7 @@ native_internal_module_exec(PyObject *m) } // Export mypy internal C API, be careful with the order! - static void *NativeInternal_API[12] = { + static void *NativeInternal_API[14] = { (void *)Buffer_internal, (void *)Buffer_internal_empty, (void *)Buffer_getvalue_internal, @@ -477,6 +546,8 @@ native_internal_module_exec(PyObject *m) (void *)read_float_internal, (void *)write_int_internal, (void *)read_int_internal, + (void *)write_tag_internal, + (void *)read_tag_internal, (void *)NativeInternal_ABI_Version, }; PyObject *c_api_object = PyCapsule_New((void *)NativeInternal_API, "native_internal._C_API", NULL); diff --git a/mypyc/lib-rt/native_internal.h b/mypyc/lib-rt/native_internal.h index 3bd3dd1bbb337..5a8905f0e6f0e 100644 --- a/mypyc/lib-rt/native_internal.h +++ b/mypyc/lib-rt/native_internal.h @@ -16,6 +16,8 @@ static char write_float_internal(PyObject *data, double value); static double read_float_internal(PyObject *data); static char write_int_internal(PyObject *data, CPyTagged value); static CPyTagged read_int_internal(PyObject *data); +static char write_tag_internal(PyObject *data, CPyTagged value); +static CPyTagged read_tag_internal(PyObject *data); static int NativeInternal_ABI_Version(void); #else @@ -33,7 +35,9 @@ static void **NativeInternal_API; #define read_float_internal (*(double (*)(PyObject *source)) NativeInternal_API[8]) #define write_int_internal (*(char (*)(PyObject *source, CPyTagged value)) NativeInternal_API[9]) #define read_int_internal (*(CPyTagged (*)(PyObject *source)) NativeInternal_API[10]) -#define NativeInternal_ABI_Version (*(int (*)(void)) NativeInternal_API[11]) +#define write_tag_internal (*(char (*)(PyObject *source, CPyTagged value)) NativeInternal_API[11]) +#define read_tag_internal (*(CPyTagged (*)(PyObject *source)) NativeInternal_API[12]) +#define NativeInternal_ABI_Version (*(int (*)(void)) NativeInternal_API[13]) static int import_native_internal(void) diff --git a/mypyc/primitives/misc_ops.py b/mypyc/primitives/misc_ops.py index 8738255081e2b..5875d5d65e9b4 100644 --- a/mypyc/primitives/misc_ops.py +++ b/mypyc/primitives/misc_ops.py @@ -423,3 +423,19 @@ c_function_name="read_int_internal", error_kind=ERR_MAGIC, ) + +function_op( + name="native_internal.write_tag", + arg_types=[object_rprimitive, int_rprimitive], + return_type=none_rprimitive, + c_function_name="write_tag_internal", + error_kind=ERR_MAGIC, +) + +function_op( + name="native_internal.read_tag", + arg_types=[object_rprimitive], + return_type=int_rprimitive, + c_function_name="read_tag_internal", + error_kind=ERR_MAGIC, +) diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index 68bc18c7bdeb0..3a9657d49f34f 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -1411,7 +1411,8 @@ class TestOverload: [case testNativeBufferFastPath] from native_internal import ( - Buffer, write_bool, read_bool, write_str, read_str, write_float, read_float, write_int, read_int + Buffer, write_bool, read_bool, write_str, read_str, write_float, read_float, + write_int, read_int, write_tag, read_tag ) def foo() -> None: @@ -1420,23 +1421,25 @@ def foo() -> None: write_bool(b, True) write_float(b, 0.1) write_int(b, 1) + write_tag(b, 1) b = Buffer(b.getvalue()) x = read_str(b) y = read_bool(b) z = read_float(b) t = read_int(b) + u = read_tag(b) [out] def foo(): r0, b :: native_internal.Buffer r1 :: str - r2, r3, r4, r5 :: None - r6 :: bytes - r7 :: native_internal.Buffer - r8, x :: str - r9, y :: bool - r10, z :: float - r11, t :: int + r2, r3, r4, r5, r6 :: None + r7 :: bytes + r8 :: native_internal.Buffer + r9, x :: str + r10, y :: bool + r11, z :: float + r12, t, r13, u :: int L0: r0 = Buffer_internal_empty() b = r0 @@ -1445,17 +1448,20 @@ L0: r3 = write_bool_internal(b, 1) r4 = write_float_internal(b, 0.1) r5 = write_int_internal(b, 2) - r6 = Buffer_getvalue_internal(b) - r7 = Buffer_internal(r6) - b = r7 - r8 = read_str_internal(b) - x = r8 - r9 = read_bool_internal(b) - y = r9 - r10 = read_float_internal(b) - z = r10 - r11 = read_int_internal(b) - t = r11 + r6 = write_tag_internal(b, 2) + r7 = Buffer_getvalue_internal(b) + r8 = Buffer_internal(r7) + b = r8 + r9 = read_str_internal(b) + x = r9 + r10 = read_bool_internal(b) + y = r10 + r11 = read_float_internal(b) + z = r11 + r12 = read_int_internal(b) + t = r12 + r13 = read_tag_internal(b) + u = r13 return 1 [case testEnumFastPath] diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 6f1217bd36e68..dc64680f67c1a 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -2712,7 +2712,8 @@ Player.MIN = [case testBufferRoundTrip_native_libs] from native_internal import ( - Buffer, write_bool, read_bool, write_str, read_str, write_float, read_float, write_int, read_int + Buffer, write_bool, read_bool, write_str, read_str, write_float, read_float, + write_int, read_int, write_tag, read_tag ) def test_buffer_basic() -> None: @@ -2728,8 +2729,11 @@ def test_buffer_roundtrip() -> None: write_float(b, 0.1) write_int(b, 0) write_int(b, 1) + write_tag(b, 33) + write_tag(b, 255) write_int(b, 2) write_int(b, 2 ** 85) + write_int(b, -1) b = Buffer(b.getvalue()) assert read_str(b) == "foo" @@ -2739,8 +2743,11 @@ def test_buffer_roundtrip() -> None: assert read_float(b) == 0.1 assert read_int(b) == 0 assert read_int(b) == 1 + assert read_tag(b) == 33 + assert read_tag(b) == 255 assert read_int(b) == 2 assert read_int(b) == 2 ** 85 + assert read_int(b) == -1 [file driver.py] from native import * @@ -2761,8 +2768,11 @@ def test_buffer_roundtrip_interpreted() -> None: write_float(b, 0.1) write_int(b, 0) write_int(b, 1) + write_tag(b, 33) + write_tag(b, 255) write_int(b, 2) write_int(b, 2 ** 85) + write_int(b, -1) b = Buffer(b.getvalue()) assert read_str(b) == "foo" @@ -2772,8 +2782,11 @@ def test_buffer_roundtrip_interpreted() -> None: assert read_float(b) == 0.1 assert read_int(b) == 0 assert read_int(b) == 1 + assert read_tag(b) == 33 + assert read_tag(b) == 255 assert read_int(b) == 2 assert read_int(b) == 2 ** 85 + assert read_int(b) == -1 test_buffer_basic_interpreted() test_buffer_roundtrip_interpreted() diff --git a/test-data/unit/lib-stub/native_internal.pyi b/test-data/unit/lib-stub/native_internal.pyi index bc1f570a8e9c1..3c6a22c938e3c 100644 --- a/test-data/unit/lib-stub/native_internal.pyi +++ b/test-data/unit/lib-stub/native_internal.pyi @@ -10,3 +10,5 @@ def write_float(data: Buffer, value: float) -> None: ... def read_float(data: Buffer) -> float: ... def write_int(data: Buffer, value: int) -> None: ... def read_int(data: Buffer) -> int: ... +def write_tag(data: Buffer, value: int) -> None: ... +def read_tag(data: Buffer) -> int: ... From de1247d76119c91ab36fa687b98dc9aabb261a6e Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Wed, 27 Aug 2025 02:10:35 +0200 Subject: [PATCH 204/424] Fix unwrapping assignment expressions in match subject (#19742) The `else_map` from guard clauses can only be applied properly if the subject expression itself can be put on the binder. Unwrap assignment expressions so we don't need to fall back to a dummy name and loose the guard clause inference. --- mypy/checker.py | 5 ++--- test-data/unit/check-python310.test | 10 ++++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 0fe77e953d066..ae6ae591ed8c4 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -5552,10 +5552,10 @@ def visit_continue_stmt(self, s: ContinueStmt) -> None: return def visit_match_stmt(self, s: MatchStmt) -> None: - named_subject = self._make_named_statement_for_match(s) # In sync with similar actions elsewhere, narrow the target if # we are matching an AssignmentExpr unwrapped_subject = collapse_walrus(s.subject) + named_subject = self._make_named_statement_for_match(s, unwrapped_subject) with self.binder.frame_context(can_skip=False, fall_through=0): subject_type = get_proper_type(self.expr_checker.accept(s.subject)) @@ -5646,9 +5646,8 @@ def visit_match_stmt(self, s: MatchStmt) -> None: with self.binder.frame_context(can_skip=False, fall_through=2): pass - def _make_named_statement_for_match(self, s: MatchStmt) -> Expression: + def _make_named_statement_for_match(self, s: MatchStmt, subject: Expression) -> Expression: """Construct a fake NameExpr for inference if a match clause is complex.""" - subject = s.subject if self.binder.can_put_directly(subject): # Already named - we should infer type of it as given return subject diff --git a/test-data/unit/check-python310.test b/test-data/unit/check-python310.test index 24bf2fdb8fb4e..5c495d2ed863b 100644 --- a/test-data/unit/check-python310.test +++ b/test-data/unit/check-python310.test @@ -1393,6 +1393,16 @@ match m: reveal_type(a) [builtins fixtures/isinstancelist.pyi] +[case testMatchSubjectAssignExprWithGuard] +from typing import Optional +def func() -> Optional[str]: ... + +match m := func(): + case _ if not m: + reveal_type(m) # N: Revealed type is "Union[Literal[''], None]" + case _: + reveal_type(m) # N: Revealed type is "builtins.str" + -- Exhaustiveness -- [case testMatchUnionNegativeNarrowing] From e852829e06aeeef90b12d389eeba775cdfabfd46 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Wed, 27 Aug 2025 02:15:15 +0200 Subject: [PATCH 205/424] Cleanup old ast classes in fastparse (#19743) The ast `ExtSlice` and `Index` classes have been deprecated (and unused) since Python 3.9. https://docs.python.org/3.9/library/ast.html#ast.AST --- mypy/fastparse.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 99d5c48c92d7e..6b2eb532003c9 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -129,9 +129,7 @@ PY_MINOR_VERSION: Final = sys.version_info[1] import ast as ast3 - -# TODO: Index, ExtSlice are deprecated in 3.9. -from ast import AST, Attribute, Call, FunctionType, Index, Name, Starred, UAdd, UnaryOp, USub +from ast import AST, Attribute, Call, FunctionType, Name, Starred, UAdd, UnaryOp, USub def ast3_parse( @@ -1779,18 +1777,6 @@ def visit_Slice(self, n: ast3.Slice) -> SliceExpr: e = SliceExpr(self.visit(n.lower), self.visit(n.upper), self.visit(n.step)) return self.set_line(e, n) - # ExtSlice(slice* dims) - def visit_ExtSlice(self, n: ast3.ExtSlice) -> TupleExpr: - # cast for mypyc's benefit on Python 3.9 - return TupleExpr(self.translate_expr_list(cast(Any, n).dims)) - - # Index(expr value) - def visit_Index(self, n: Index) -> Node: - # cast for mypyc's benefit on Python 3.9 - value = self.visit(cast(Any, n).value) - assert isinstance(value, Node) - return value - # Match(expr subject, match_case* cases) # python 3.10 and later def visit_Match(self, n: Match) -> MatchStmt: node = MatchStmt( From e0ce3e347bb1f2ec0ed885b914858cffe15a77d5 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Wed, 27 Aug 2025 12:53:21 -0400 Subject: [PATCH 206/424] [mypyc] feat: `__mypyc_empty_tuple__` constant (#19654) I realized that any time a user has a kwarg-only call expression like `fn(abc=123, ...)` in their compiled code, and `func` is not a native function, a new empty tuple is created every time This is not really necessary, we can just hold the same empty tuple in memory as a constant and pass it around. It's immutable, and that's already what we're already doing, since `tuple() is tuple()` but our current method involves more steps. This should slightly improve the speed of kwarg-only python func calling. --- mypyc/codegen/emit.py | 26 +++++++++++++++----------- mypyc/irbuild/ll_builder.py | 8 ++++++-- mypyc/lib-rt/CPy.h | 9 +++++++++ mypyc/lib-rt/init.c | 11 +++++++++++ mypyc/primitives/tuple_ops.py | 7 +++++++ mypyc/test-data/irbuild-basic.test | 2 +- mypyc/test-data/irbuild-classes.test | 4 ++-- 7 files changed, 51 insertions(+), 16 deletions(-) diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index 9ca761bd8ac55..4ef53296ef0d1 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -1036,17 +1036,21 @@ def emit_box( self.emit_line(f"{declaration}{dest} = PyFloat_FromDouble({src});") elif isinstance(typ, RTuple): self.declare_tuple_struct(typ) - self.emit_line(f"{declaration}{dest} = PyTuple_New({len(typ.types)});") - self.emit_line(f"if (unlikely({dest} == NULL))") - self.emit_line(" CPyError_OutOfMemory();") - # TODO: Fail if dest is None - for i in range(len(typ.types)): - if not typ.is_unboxed: - self.emit_line(f"PyTuple_SET_ITEM({dest}, {i}, {src}.f{i}") - else: - inner_name = self.temp_name() - self.emit_box(f"{src}.f{i}", inner_name, typ.types[i], declare_dest=True) - self.emit_line(f"PyTuple_SET_ITEM({dest}, {i}, {inner_name});") + if not typ.types: + self.emit_line(f"{declaration}{dest} = CPyTuple_LoadEmptyTupleConstant();") + else: + self.emit_line(f"{declaration}{dest} = PyTuple_New({len(typ.types)});") + self.emit_line(f"if (unlikely({dest} == NULL))") + self.emit_line(" CPyError_OutOfMemory();") + + # TODO: Fail if dest is None + for i in range(len(typ.types)): + if not typ.is_unboxed: + self.emit_line(f"PyTuple_SET_ITEM({dest}, {i}, {src}.f{i}") + else: + inner_name = self.temp_name() + self.emit_box(f"{src}.f{i}", inner_name, typ.types[i], declare_dest=True) + self.emit_line(f"PyTuple_SET_ITEM({dest}, {i}, {inner_name});") else: assert not typ.is_unboxed # Type is boxed -- trivially just assign. diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index ba8ef94b00bdd..112bbdbb50edd 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -189,6 +189,7 @@ ) from mypyc.primitives.tuple_ops import ( list_tuple_op, + load_empty_tuple_constant_op, new_tuple_op, new_tuple_with_length_op, sequence_tuple_op, @@ -2362,8 +2363,11 @@ def builtin_len(self, val: Value, line: int, use_pyssize_t: bool = False) -> Val return self.call_c(generic_len_op, [val], line) def new_tuple(self, items: list[Value], line: int) -> Value: - size: Value = Integer(len(items), c_pyssize_t_rprimitive) - return self.call_c(new_tuple_op, [size] + items, line) + if items: + size: Value = Integer(len(items), c_pyssize_t_rprimitive) + return self.call_c(new_tuple_op, [size] + items, line) + else: + return self.call_c(load_empty_tuple_constant_op, [], line) def new_tuple_with_length(self, length: Value, line: int) -> Value: """This function returns an uninitialized tuple. diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index 8cd141545bbbd..b4d3a0013ae79 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -64,6 +64,15 @@ typedef struct tuple_T4CIOO { } tuple_T4CIOO; #endif +// System-wide empty tuple constant +extern PyObject * __mypyc_empty_tuple__; + +static inline PyObject *CPyTuple_LoadEmptyTupleConstant() { +#if !CPY_3_12_FEATURES + Py_INCREF(__mypyc_empty_tuple__); +#endif + return __mypyc_empty_tuple__; +} // Native object operations diff --git a/mypyc/lib-rt/init.c b/mypyc/lib-rt/init.c index 01b133233489e..9215c2d590194 100644 --- a/mypyc/lib-rt/init.c +++ b/mypyc/lib-rt/init.c @@ -4,10 +4,21 @@ struct ExcDummyStruct _CPy_ExcDummyStruct = { PyObject_HEAD_INIT(NULL) }; PyObject *_CPy_ExcDummy = (PyObject *)&_CPy_ExcDummyStruct; +// System-wide empty tuple constant +PyObject * __mypyc_empty_tuple__ = NULL; + // Because its dynamic linker is more restricted than linux/OS X, // Windows doesn't allow initializing globals with values from // other dynamic libraries. This means we need to initialize // things at load time. void CPy_Init(void) { _CPy_ExcDummyStruct.ob_base.ob_type = &PyBaseObject_Type; + + // Initialize system-wide empty tuple constant + if (__mypyc_empty_tuple__ == NULL) { + __mypyc_empty_tuple__ = PyTuple_New(0); + if (!__mypyc_empty_tuple__) { + CPyError_OutOfMemory(); + } + } } diff --git a/mypyc/primitives/tuple_ops.py b/mypyc/primitives/tuple_ops.py index f262dec8b05ae..ab23f8c441f5b 100644 --- a/mypyc/primitives/tuple_ops.py +++ b/mypyc/primitives/tuple_ops.py @@ -55,6 +55,13 @@ error_kind=ERR_MAGIC, ) +load_empty_tuple_constant_op = custom_op( + arg_types=[], + return_type=tuple_rprimitive, + c_function_name="CPyTuple_LoadEmptyTupleConstant", + error_kind=ERR_NEVER, +) + # PyTuple_SET_ITEM does no error checking, # and should only be used to fill in brand new tuples. new_tuple_set_item_op = custom_op( diff --git a/mypyc/test-data/irbuild-basic.test b/mypyc/test-data/irbuild-basic.test index 4eeeca04719c6..feb7b36a2b52b 100644 --- a/mypyc/test-data/irbuild-basic.test +++ b/mypyc/test-data/irbuild-basic.test @@ -1750,7 +1750,7 @@ L0: r7 = __main__.globals :: static r8 = 'f' r9 = CPyDict_GetItem(r7, r8) - r10 = PyTuple_Pack(0) + r10 = CPyTuple_LoadEmptyTupleConstant() r11 = PyDict_Copy(r6) r12 = PyObject_Call(r9, r10, r11) r13 = unbox(tuple[int, int, int], r12) diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index 3a9657d49f34f..bfd32a523437a 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -297,7 +297,7 @@ L2: r27 = CPyType_FromTemplate(r26, r24, r25) r28 = C_trait_vtable_setup() r29 = '__mypyc_attrs__' - r30 = PyTuple_Pack(0) + r30 = CPyTuple_LoadEmptyTupleConstant() r31 = PyObject_SetAttr(r27, r29, r30) r32 = r31 >= 0 :: signed __main__.C = r27 :: type @@ -310,7 +310,7 @@ L2: r39 = __main__.S_template :: type r40 = CPyType_FromTemplate(r39, r37, r38) r41 = '__mypyc_attrs__' - r42 = PyTuple_Pack(0) + r42 = CPyTuple_LoadEmptyTupleConstant() r43 = PyObject_SetAttr(r40, r41, r42) r44 = r43 >= 0 :: signed __main__.S = r40 :: type From abd9424039fd7b1f7da6b5c143a27c5b682371ee Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 28 Aug 2025 00:28:18 +0100 Subject: [PATCH 207/424] Use u8 for type/symbol tags (#19741) While trying this I found that this is a bit fragile, in the sense that `write_tag(data, 1)` (with a literal `1`) will not get specialized and will go through slow path (btw @JukkaL is this a bug, should type of a literal in fixed int type context be inferred as fixed int?) OTOH this is probably not a big deal since no-one will use `write_tag()` with literals, it will always be something like `write_tag(data, FUNC_DEF)`. --- mypy/cache.py | 23 ++++++----- mypy/nodes.py | 23 +++++------ mypy/types.py | 39 ++++++++++--------- .../stubs/mypy-native/native_internal.pyi | 6 ++- mypyc/lib-rt/native_internal.c | 31 +++++++-------- mypyc/lib-rt/native_internal.h | 8 ++-- mypyc/primitives/misc_ops.py | 11 +++--- mypyc/test-data/irbuild-classes.test | 12 ++++-- mypyc/test-data/run-classes.test | 19 +++++++-- test-data/unit/lib-stub/native_internal.pyi | 6 ++- 10 files changed, 101 insertions(+), 77 deletions(-) diff --git a/mypy/cache.py b/mypy/cache.py index a16a36900c7ac..08e3b05d1a753 100644 --- a/mypy/cache.py +++ b/mypy/cache.py @@ -3,6 +3,8 @@ from collections.abc import Sequence from typing import TYPE_CHECKING, Final +from mypy_extensions import u8 + try: from native_internal import ( Buffer as Buffer, @@ -34,10 +36,10 @@ def read_int(data: Buffer) -> int: def write_int(data: Buffer, value: int) -> None: raise NotImplementedError - def read_tag(data: Buffer) -> int: + def read_tag(data: Buffer) -> u8: raise NotImplementedError - def write_tag(data: Buffer, value: int) -> None: + def write_tag(data: Buffer, value: u8) -> None: raise NotImplementedError def read_str(data: Buffer) -> str: @@ -59,15 +61,18 @@ def write_float(data: Buffer, value: float) -> None: raise NotImplementedError -LITERAL_INT: Final = 1 -LITERAL_STR: Final = 2 -LITERAL_BOOL: Final = 3 -LITERAL_FLOAT: Final = 4 -LITERAL_COMPLEX: Final = 5 -LITERAL_NONE: Final = 6 +# Always use this type alias to refer to type tags. +Tag = u8 + +LITERAL_INT: Final[Tag] = 1 +LITERAL_STR: Final[Tag] = 2 +LITERAL_BOOL: Final[Tag] = 3 +LITERAL_FLOAT: Final[Tag] = 4 +LITERAL_COMPLEX: Final[Tag] = 5 +LITERAL_NONE: Final[Tag] = 6 -def read_literal(data: Buffer, tag: int) -> int | str | bool | float: +def read_literal(data: Buffer, tag: Tag) -> int | str | bool | float: if tag == LITERAL_INT: return read_int(data) elif tag == LITERAL_STR: diff --git a/mypy/nodes.py b/mypy/nodes.py index 45e2b60c3e782..9cfc61c80b3e7 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -18,6 +18,7 @@ LITERAL_COMPLEX, LITERAL_NONE, Buffer, + Tag, read_bool, read_float, read_int, @@ -4877,17 +4878,17 @@ def local_definitions( yield from local_definitions(node.names, fullname, node) -MYPY_FILE: Final = 0 -OVERLOADED_FUNC_DEF: Final = 1 -FUNC_DEF: Final = 2 -DECORATOR: Final = 3 -VAR: Final = 4 -TYPE_VAR_EXPR: Final = 5 -PARAM_SPEC_EXPR: Final = 6 -TYPE_VAR_TUPLE_EXPR: Final = 7 -TYPE_INFO: Final = 8 -TYPE_ALIAS: Final = 9 -CLASS_DEF: Final = 10 +MYPY_FILE: Final[Tag] = 0 +OVERLOADED_FUNC_DEF: Final[Tag] = 1 +FUNC_DEF: Final[Tag] = 2 +DECORATOR: Final[Tag] = 3 +VAR: Final[Tag] = 4 +TYPE_VAR_EXPR: Final[Tag] = 5 +PARAM_SPEC_EXPR: Final[Tag] = 6 +TYPE_VAR_TUPLE_EXPR: Final[Tag] = 7 +TYPE_INFO: Final[Tag] = 8 +TYPE_ALIAS: Final[Tag] = 9 +CLASS_DEF: Final[Tag] = 10 def read_symbol(data: Buffer) -> mypy.nodes.SymbolNode: diff --git a/mypy/types.py b/mypy/types.py index 43e6dafe298e0..8d5648ae0bdac 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -12,6 +12,7 @@ from mypy.bogus_type import Bogus from mypy.cache import ( Buffer, + Tag, read_bool, read_int, read_int_list, @@ -4120,25 +4121,25 @@ def type_vars_as_args(type_vars: Sequence[TypeVarLikeType]) -> tuple[Type, ...]: return tuple(args) -TYPE_ALIAS_TYPE: Final = 1 -TYPE_VAR_TYPE: Final = 2 -PARAM_SPEC_TYPE: Final = 3 -TYPE_VAR_TUPLE_TYPE: Final = 4 -UNBOUND_TYPE: Final = 5 -UNPACK_TYPE: Final = 6 -ANY_TYPE: Final = 7 -UNINHABITED_TYPE: Final = 8 -NONE_TYPE: Final = 9 -DELETED_TYPE: Final = 10 -INSTANCE: Final = 11 -CALLABLE_TYPE: Final = 12 -OVERLOADED: Final = 13 -TUPLE_TYPE: Final = 14 -TYPED_DICT_TYPE: Final = 15 -LITERAL_TYPE: Final = 16 -UNION_TYPE: Final = 17 -TYPE_TYPE: Final = 18 -PARAMETERS: Final = 19 +TYPE_ALIAS_TYPE: Final[Tag] = 1 +TYPE_VAR_TYPE: Final[Tag] = 2 +PARAM_SPEC_TYPE: Final[Tag] = 3 +TYPE_VAR_TUPLE_TYPE: Final[Tag] = 4 +UNBOUND_TYPE: Final[Tag] = 5 +UNPACK_TYPE: Final[Tag] = 6 +ANY_TYPE: Final[Tag] = 7 +UNINHABITED_TYPE: Final[Tag] = 8 +NONE_TYPE: Final[Tag] = 9 +DELETED_TYPE: Final[Tag] = 10 +INSTANCE: Final[Tag] = 11 +CALLABLE_TYPE: Final[Tag] = 12 +OVERLOADED: Final[Tag] = 13 +TUPLE_TYPE: Final[Tag] = 14 +TYPED_DICT_TYPE: Final[Tag] = 15 +LITERAL_TYPE: Final[Tag] = 16 +UNION_TYPE: Final[Tag] = 17 +TYPE_TYPE: Final[Tag] = 18 +PARAMETERS: Final[Tag] = 19 def read_type(data: Buffer) -> Type: diff --git a/mypy/typeshed/stubs/mypy-native/native_internal.pyi b/mypy/typeshed/stubs/mypy-native/native_internal.pyi index 3c6a22c938e3c..a47a4849fe204 100644 --- a/mypy/typeshed/stubs/mypy-native/native_internal.pyi +++ b/mypy/typeshed/stubs/mypy-native/native_internal.pyi @@ -1,3 +1,5 @@ +from mypy_extensions import u8 + class Buffer: def __init__(self, source: bytes = ...) -> None: ... def getvalue(self) -> bytes: ... @@ -10,5 +12,5 @@ def write_float(data: Buffer, value: float) -> None: ... def read_float(data: Buffer) -> float: ... def write_int(data: Buffer, value: int) -> None: ... def read_int(data: Buffer) -> int: ... -def write_tag(data: Buffer, value: int) -> None: ... -def read_tag(data: Buffer) -> int: ... +def write_tag(data: Buffer, value: u8) -> None: ... +def read_tag(data: Buffer) -> u8: ... diff --git a/mypyc/lib-rt/native_internal.c b/mypyc/lib-rt/native_internal.c index 1c35eab946f8c..3228f03307930 100644 --- a/mypyc/lib-rt/native_internal.c +++ b/mypyc/lib-rt/native_internal.c @@ -438,18 +438,18 @@ write_int(PyObject *self, PyObject *args, PyObject *kwds) { return Py_None; } -static CPyTagged +static uint8_t read_tag_internal(PyObject *data) { if (_check_buffer(data) == 2) - return CPY_INT_TAG; + return CPY_LL_UINT_ERROR; if (_check_read((BufferObject *)data, 1) == 2) - return CPY_INT_TAG; + return CPY_LL_UINT_ERROR; char *buf = ((BufferObject *)data)->buf; uint8_t ret = *(uint8_t *)(buf + ((BufferObject *)data)->pos); ((BufferObject *)data)->pos += 1; - return ((CPyTagged)ret) << 1; + return ret; } static PyObject* @@ -458,27 +458,22 @@ read_tag(PyObject *self, PyObject *args, PyObject *kwds) { PyObject *data = NULL; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O", kwlist, &data)) return NULL; - CPyTagged retval = read_tag_internal(data); - if (retval == CPY_INT_TAG) { + uint8_t retval = read_tag_internal(data); + if (retval == CPY_LL_UINT_ERROR && PyErr_Occurred()) { return NULL; } - return CPyTagged_StealAsObject(retval); + return PyLong_FromLong(retval); } static char -write_tag_internal(PyObject *data, CPyTagged value) { +write_tag_internal(PyObject *data, uint8_t value) { if (_check_buffer(data) == 2) return 2; - if (value > MAX_SHORT_INT_TAGGED) { - PyErr_SetString(PyExc_OverflowError, "value must fit in single byte"); - return 2; - } - if (_check_size((BufferObject *)data, 1) == 2) return 2; uint8_t *buf = (uint8_t *)((BufferObject *)data)->buf; - *(buf + ((BufferObject *)data)->pos) = (uint8_t)(value >> 1); + *(buf + ((BufferObject *)data)->pos) = value; ((BufferObject *)data)->pos += 1; ((BufferObject *)data)->end += 1; return 1; @@ -491,12 +486,12 @@ write_tag(PyObject *self, PyObject *args, PyObject *kwds) { PyObject *value = NULL; if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO", kwlist, &data, &value)) return NULL; - if (!PyLong_Check(value)) { - PyErr_SetString(PyExc_TypeError, "value must be an int"); + uint8_t unboxed = CPyLong_AsUInt8(value); + if (unboxed == CPY_LL_UINT_ERROR && PyErr_Occurred()) { + CPy_TypeError("u8", value); return NULL; } - CPyTagged tagged_value = CPyTagged_BorrowFromObject(value); - if (write_tag_internal(data, tagged_value) == 2) { + if (write_tag_internal(data, unboxed) == 2) { return NULL; } Py_INCREF(Py_None); diff --git a/mypyc/lib-rt/native_internal.h b/mypyc/lib-rt/native_internal.h index 5a8905f0e6f0e..63e902a6e1bf2 100644 --- a/mypyc/lib-rt/native_internal.h +++ b/mypyc/lib-rt/native_internal.h @@ -16,8 +16,8 @@ static char write_float_internal(PyObject *data, double value); static double read_float_internal(PyObject *data); static char write_int_internal(PyObject *data, CPyTagged value); static CPyTagged read_int_internal(PyObject *data); -static char write_tag_internal(PyObject *data, CPyTagged value); -static CPyTagged read_tag_internal(PyObject *data); +static char write_tag_internal(PyObject *data, uint8_t value); +static uint8_t read_tag_internal(PyObject *data); static int NativeInternal_ABI_Version(void); #else @@ -35,8 +35,8 @@ static void **NativeInternal_API; #define read_float_internal (*(double (*)(PyObject *source)) NativeInternal_API[8]) #define write_int_internal (*(char (*)(PyObject *source, CPyTagged value)) NativeInternal_API[9]) #define read_int_internal (*(CPyTagged (*)(PyObject *source)) NativeInternal_API[10]) -#define write_tag_internal (*(char (*)(PyObject *source, CPyTagged value)) NativeInternal_API[11]) -#define read_tag_internal (*(CPyTagged (*)(PyObject *source)) NativeInternal_API[12]) +#define write_tag_internal (*(char (*)(PyObject *source, uint8_t value)) NativeInternal_API[11]) +#define read_tag_internal (*(uint8_t (*)(PyObject *source)) NativeInternal_API[12]) #define NativeInternal_ABI_Version (*(int (*)(void)) NativeInternal_API[13]) static int diff --git a/mypyc/primitives/misc_ops.py b/mypyc/primitives/misc_ops.py index 5875d5d65e9b4..943f6fc04b729 100644 --- a/mypyc/primitives/misc_ops.py +++ b/mypyc/primitives/misc_ops.py @@ -2,7 +2,7 @@ from __future__ import annotations -from mypyc.ir.ops import ERR_FALSE, ERR_MAGIC, ERR_NEVER +from mypyc.ir.ops import ERR_FALSE, ERR_MAGIC, ERR_MAGIC_OVERLAPPING, ERR_NEVER from mypyc.ir.rtypes import ( KNOWN_NATIVE_TYPES, bit_rprimitive, @@ -20,6 +20,7 @@ object_rprimitive, pointer_rprimitive, str_rprimitive, + uint8_rprimitive, void_rtype, ) from mypyc.primitives.registry import ( @@ -426,16 +427,16 @@ function_op( name="native_internal.write_tag", - arg_types=[object_rprimitive, int_rprimitive], + arg_types=[object_rprimitive, uint8_rprimitive], return_type=none_rprimitive, c_function_name="write_tag_internal", - error_kind=ERR_MAGIC, + error_kind=ERR_MAGIC_OVERLAPPING, ) function_op( name="native_internal.read_tag", arg_types=[object_rprimitive], - return_type=int_rprimitive, + return_type=uint8_rprimitive, c_function_name="read_tag_internal", - error_kind=ERR_MAGIC, + error_kind=ERR_MAGIC_OVERLAPPING, ) diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index bfd32a523437a..b49b20e13a43b 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -1410,18 +1410,23 @@ class TestOverload: return x [case testNativeBufferFastPath] +from typing import Final +from mypy_extensions import u8 from native_internal import ( Buffer, write_bool, read_bool, write_str, read_str, write_float, read_float, write_int, read_int, write_tag, read_tag ) +Tag = u8 +TAG: Final[Tag] = 1 + def foo() -> None: b = Buffer() write_str(b, "foo") write_bool(b, True) write_float(b, 0.1) write_int(b, 1) - write_tag(b, 1) + write_tag(b, TAG) b = Buffer(b.getvalue()) x = read_str(b) @@ -1439,7 +1444,8 @@ def foo(): r9, x :: str r10, y :: bool r11, z :: float - r12, t, r13, u :: int + r12, t :: int + r13, u :: u8 L0: r0 = Buffer_internal_empty() b = r0 @@ -1448,7 +1454,7 @@ L0: r3 = write_bool_internal(b, 1) r4 = write_float_internal(b, 0.1) r5 = write_int_internal(b, 2) - r6 = write_tag_internal(b, 2) + r6 = write_tag_internal(b, 1) r7 = Buffer_getvalue_internal(b) r8 = Buffer_internal(r7) b = r8 diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index dc64680f67c1a..edc989ea641cb 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -2711,11 +2711,18 @@ from native import Player Player.MIN = [case testBufferRoundTrip_native_libs] +from typing import Final +from mypy_extensions import u8 from native_internal import ( Buffer, write_bool, read_bool, write_str, read_str, write_float, read_float, write_int, read_int, write_tag, read_tag ) +Tag = u8 +TAG_A: Final[Tag] = 33 +TAG_B: Final[Tag] = 255 +TAG_SPECIAL: Final[Tag] = 239 + def test_buffer_basic() -> None: b = Buffer(b"foo") assert b.getvalue() == b"foo" @@ -2729,8 +2736,9 @@ def test_buffer_roundtrip() -> None: write_float(b, 0.1) write_int(b, 0) write_int(b, 1) - write_tag(b, 33) - write_tag(b, 255) + write_tag(b, TAG_A) + write_tag(b, TAG_SPECIAL) + write_tag(b, TAG_B) write_int(b, 2) write_int(b, 2 ** 85) write_int(b, -1) @@ -2743,8 +2751,9 @@ def test_buffer_roundtrip() -> None: assert read_float(b) == 0.1 assert read_int(b) == 0 assert read_int(b) == 1 - assert read_tag(b) == 33 - assert read_tag(b) == 255 + assert read_tag(b) == TAG_A + assert read_tag(b) == TAG_SPECIAL + assert read_tag(b) == TAG_B assert read_int(b) == 2 assert read_int(b) == 2 ** 85 assert read_int(b) == -1 @@ -2769,6 +2778,7 @@ def test_buffer_roundtrip_interpreted() -> None: write_int(b, 0) write_int(b, 1) write_tag(b, 33) + write_tag(b, 239) write_tag(b, 255) write_int(b, 2) write_int(b, 2 ** 85) @@ -2783,6 +2793,7 @@ def test_buffer_roundtrip_interpreted() -> None: assert read_int(b) == 0 assert read_int(b) == 1 assert read_tag(b) == 33 + assert read_tag(b) == 239 assert read_tag(b) == 255 assert read_int(b) == 2 assert read_int(b) == 2 ** 85 diff --git a/test-data/unit/lib-stub/native_internal.pyi b/test-data/unit/lib-stub/native_internal.pyi index 3c6a22c938e3c..a47a4849fe204 100644 --- a/test-data/unit/lib-stub/native_internal.pyi +++ b/test-data/unit/lib-stub/native_internal.pyi @@ -1,3 +1,5 @@ +from mypy_extensions import u8 + class Buffer: def __init__(self, source: bytes = ...) -> None: ... def getvalue(self) -> bytes: ... @@ -10,5 +12,5 @@ def write_float(data: Buffer, value: float) -> None: ... def read_float(data: Buffer) -> float: ... def write_int(data: Buffer, value: int) -> None: ... def read_int(data: Buffer) -> int: ... -def write_tag(data: Buffer, value: int) -> None: ... -def read_tag(data: Buffer) -> int: ... +def write_tag(data: Buffer, value: u8) -> None: ... +def read_tag(data: Buffer) -> u8: ... From e633140ba672d97b9300a44d0cae9fd38db28b2c Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Thu, 28 Aug 2025 14:26:21 +0200 Subject: [PATCH 208/424] Do not use outer context for `or` expr inference if the LHS has Any (#19748) Fixes #19492. In #19695 I tried to add another set of heuristics to `any_constraints`, but can't get that working: trying to join/meet all similar constraints together breaks inference in other cases, and only doing that for Any would be somewhat non-trivial. This PR reverts behaviour introduced in #19492 when LHS of the expression contains `Any`. Outer context is still included in all other cases, and this seems to strike a good balance. --- mypy/checkexpr.py | 4 ++++ test-data/unit/check-inference-context.test | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 88b3005b13763..fd83b6359ddc3 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -6004,6 +6004,10 @@ def analyze_cond_branch( def _combined_context(self, ty: Type | None) -> Type | None: ctx_items = [] if ty is not None: + if has_any_type(ty): + # HACK: Any should be contagious, `dict[str, Any] or ` should still + # infer Any in x. + return ty ctx_items.append(ty) if self.type_context and self.type_context[-1] is not None: ctx_items.append(self.type_context[-1]) diff --git a/test-data/unit/check-inference-context.test b/test-data/unit/check-inference-context.test index 5a674cca09da3..cd44fb5b85cd9 100644 --- a/test-data/unit/check-inference-context.test +++ b/test-data/unit/check-inference-context.test @@ -1530,3 +1530,13 @@ def check3(use: bool, val: str) -> "str | Literal[False]": def check4(use: bool, val: str) -> "str | bool": return use and identity(val) [builtins fixtures/tuple.pyi] + +[case testDictAnyOrLiteralInContext] +from typing import Union, Optional, Any + +def f(x: dict[str, Union[str, None, int]]) -> None: + pass + +def g(x: Optional[dict[str, Any]], s: Optional[str]) -> None: + f(x or {'x': s}) +[builtins fixtures/dict.pyi] From ae87821dd6fe2cf29402a5bb009d72474a97c835 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 28 Aug 2025 14:59:24 +0100 Subject: [PATCH 209/424] Don't write constructor cache without strict optional (#19752) Fixes https://github.com/python/mypy/issues/19751 Fix is straightforward: don't write cache when it is not safe to. --- mypy/typeops.py | 6 ++++-- test-data/unit/check-classes.test | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/mypy/typeops.py b/mypy/typeops.py index 0cb6018d01fd3..87a4d8cefd133 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -208,7 +208,7 @@ def type_object_type(info: TypeInfo, named_type: Callable[[str], Instance]) -> P fallback=named_type("builtins.function"), ) result: FunctionLike = class_callable(sig, info, fallback, None, is_new=False) - if allow_cache: + if allow_cache and state.strict_optional: info.type_object_type = result return result @@ -230,7 +230,9 @@ def type_object_type(info: TypeInfo, named_type: Callable[[str], Instance]) -> P assert isinstance(method.type, FunctionLike) # is_valid_constructor() ensures this t = method.type result = type_object_type_from_function(t, info, method.info, fallback, is_new) - if allow_cache: + # Only write cached result is strict_optional=True, otherwise we may get + # inconsistent behaviour because of union simplification. + if allow_cache and state.strict_optional: info.type_object_type = result return result diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index 5cc4910fb2653..498a2c12b6e8c 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -9273,3 +9273,19 @@ class Bar: [file foo.py] class Bar: ... + +[case testConstructorWithoutStrictOptionalNoCache] +import mod +a = mod.NT(x=None) # OK + +[file typ.py] +from typing import NamedTuple, Optional +NT = NamedTuple("NT", [("x", Optional[str])]) + +[file mod.py] +# mypy: no-strict-optional +from typ import NT + +def f() -> NT: + return NT(x='') +[builtins fixtures/tuple.pyi] From dcc76ea2ef7bc56ab185ab5832583be998ad6a12 Mon Sep 17 00:00:00 2001 From: Randolf Scholz Date: Thu, 28 Aug 2025 23:56:12 +0200 Subject: [PATCH 210/424] Fix crash with variadic tuple arguments to generic type (#19705) Fixes #19704 --- mypy/semanal.py | 5 ++++- mypy/typeanal.py | 2 +- test-data/unit/check-python312.test | 11 ++++++----- test-data/unit/check-typevar-tuple.test | 18 +++++++++--------- 4 files changed, 20 insertions(+), 16 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 77e6b0c005e2a..fa5d9fdc82c44 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -295,6 +295,7 @@ UnboundType, UnionType, UnpackType, + flatten_nested_tuples, get_proper_type, get_proper_types, has_type_vars, @@ -6093,7 +6094,9 @@ def analyze_type_application_args(self, expr: IndexExpr) -> list[Type] | None: types.append(analyzed) if allow_unpack: - types = self.type_analyzer().check_unpacks_in_list(types) + # need to flatten away harmless unpacks like Unpack[tuple[int]] + flattened_items = flatten_nested_tuples(types) + types = self.type_analyzer().check_unpacks_in_list(flattened_items) if has_param_spec and num_args == 1 and types: first_arg = get_proper_type(types[0]) single_any = len(types) == 1 and isinstance(first_arg, AnyType) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 204d3061c7349..d44b13880cbbc 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -1978,7 +1978,7 @@ def check_unpacks_in_list(self, items: list[Type]) -> list[Type]: if num_unpacks > 1: assert final_unpack is not None - self.fail("More than one Unpack in a type is not allowed", final_unpack.type) + self.fail("More than one variadic Unpack in a type is not allowed", final_unpack.type) return new_items def tuple_type(self, items: list[Type], line: int, column: int) -> TupleType: diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 817184dc561cc..01364bdfa32af 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -2038,12 +2038,13 @@ class Z: ... # E: Name "Z" already defined on line 2 # https://github.com/python/mypy/issues/18856 class A[*Ts]: ... -A[*tuple[int, ...], *tuple[int, ...]] # E: More than one Unpack in a type is not allowed -a: A[*tuple[int, ...], *tuple[int, ...]] # E: More than one Unpack in a type is not allowed -def foo(a: A[*tuple[int, ...], *tuple[int, ...]]): ... # E: More than one Unpack in a type is not allowed +A[*tuple[int, ...], *tuple[int, ...]] # E: More than one variadic Unpack in a type is not allowed +A[*tuple[*tuple[int, ...]], *tuple[*tuple[int, ...]]] # E: More than one variadic Unpack in a type is not allowed +a: A[*tuple[int, ...], *tuple[int, ...]] # E: More than one variadic Unpack in a type is not allowed +def foo(a: A[*tuple[int, ...], *tuple[int, ...]]): ... # E: More than one variadic Unpack in a type is not allowed -tuple[*tuple[int, ...], *tuple[int, ...]] # E: More than one Unpack in a type is not allowed -b: tuple[*tuple[int, ...], *tuple[int, ...]] # E: More than one Unpack in a type is not allowed +tuple[*tuple[int, ...], *tuple[int, ...]] # E: More than one variadic Unpack in a type is not allowed +b: tuple[*tuple[int, ...], *tuple[int, ...]] # E: More than one variadic Unpack in a type is not allowed [builtins fixtures/tuple.pyi] [typing fixtures/typing-full.pyi] diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index 2de2e45f0a960..c668f14eaa503 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -123,7 +123,7 @@ reveal_type(empty) # N: Revealed type is "__main__.Variadic[()]" omitted: Variadic reveal_type(omitted) # N: Revealed type is "__main__.Variadic[Unpack[builtins.tuple[Any, ...]]]" -bad: Variadic[Unpack[Tuple[int, ...]], str, Unpack[Tuple[bool, ...]]] # E: More than one Unpack in a type is not allowed +bad: Variadic[Unpack[Tuple[int, ...]], str, Unpack[Tuple[bool, ...]]] # E: More than one variadic Unpack in a type is not allowed reveal_type(bad) # N: Revealed type is "__main__.Variadic[Unpack[builtins.tuple[builtins.int, ...]], builtins.str]" bad2: Unpack[Tuple[int, ...]] # E: Unpack is only valid in a variadic position @@ -353,12 +353,12 @@ expect_variadic_array_2(u) Ts = TypeVarTuple("Ts") Ts2 = TypeVarTuple("Ts2") -def bad(x: Tuple[int, Unpack[Ts], str, Unpack[Ts2]]) -> None: # E: More than one Unpack in a type is not allowed +def bad(x: Tuple[int, Unpack[Ts], str, Unpack[Ts2]]) -> None: # E: More than one variadic Unpack in a type is not allowed ... reveal_type(bad) # N: Revealed type is "def [Ts, Ts2] (x: tuple[builtins.int, Unpack[Ts`-1], builtins.str])" -def bad2(x: Tuple[int, Unpack[Tuple[int, ...]], str, Unpack[Tuple[str, ...]]]) -> None: # E: More than one Unpack in a type is not allowed +def bad2(x: Tuple[int, Unpack[Tuple[int, ...]], str, Unpack[Tuple[str, ...]]]) -> None: # E: More than one variadic Unpack in a type is not allowed ... reveal_type(bad2) # N: Revealed type is "def (x: tuple[builtins.int, Unpack[builtins.tuple[builtins.int, ...]], builtins.str])" [builtins fixtures/tuple.pyi] @@ -571,7 +571,7 @@ from typing_extensions import Unpack, TypeVarTuple Ts = TypeVarTuple("Ts") Us = TypeVarTuple("Us") -a: Callable[[Unpack[Ts], Unpack[Us]], int] # E: More than one Unpack in a type is not allowed +a: Callable[[Unpack[Ts], Unpack[Us]], int] # E: More than one variadic Unpack in a type is not allowed reveal_type(a) # N: Revealed type is "def [Ts, Us] (*Unpack[Ts`-1]) -> builtins.int" b: Callable[[Unpack], int] # E: Unpack[...] requires exactly one type argument reveal_type(b) # N: Revealed type is "def (*Any) -> builtins.int" @@ -725,15 +725,15 @@ Ts = TypeVarTuple("Ts") Us = TypeVarTuple("Us") class G(Generic[Unpack[Ts]]): ... -A = Tuple[Unpack[Ts], Unpack[Us]] # E: More than one Unpack in a type is not allowed +A = Tuple[Unpack[Ts], Unpack[Us]] # E: More than one variadic Unpack in a type is not allowed x: A[int, str] reveal_type(x) # N: Revealed type is "tuple[builtins.int, builtins.str]" -B = Callable[[Unpack[Ts], Unpack[Us]], int] # E: More than one Unpack in a type is not allowed +B = Callable[[Unpack[Ts], Unpack[Us]], int] # E: More than one variadic Unpack in a type is not allowed y: B[int, str] reveal_type(y) # N: Revealed type is "def (builtins.int, builtins.str) -> builtins.int" -C = G[Unpack[Ts], Unpack[Us]] # E: More than one Unpack in a type is not allowed +C = G[Unpack[Ts], Unpack[Us]] # E: More than one variadic Unpack in a type is not allowed z: C[int, str] reveal_type(z) # N: Revealed type is "__main__.G[builtins.int, builtins.str]" [builtins fixtures/tuple.pyi] @@ -2223,12 +2223,12 @@ cb2(1, 2, 3, a="a", b="b") cb2(1, a="a", b="b") # E: Too few arguments cb2(1, 2, 3, a="a") # E: Missing named argument "b" -bad1: Callable[[Unpack[Ints], Unpack[Ints]], None] # E: More than one Unpack in a type is not allowed +bad1: Callable[[Unpack[Ints], Unpack[Ints]], None] # E: More than one variadic Unpack in a type is not allowed reveal_type(bad1) # N: Revealed type is "def (*builtins.int)" bad2: Callable[[Unpack[Keywords], Unpack[Keywords]], None] # E: "Keywords" cannot be unpacked (must be tuple or TypeVarTuple) reveal_type(bad2) # N: Revealed type is "def (*Any, **Unpack[TypedDict('__main__.Keywords', {'a': builtins.str, 'b': builtins.str})])" bad3: Callable[[Unpack[Keywords], Unpack[Ints]], None] # E: "Keywords" cannot be unpacked (must be tuple or TypeVarTuple) \ - # E: More than one Unpack in a type is not allowed + # E: More than one variadic Unpack in a type is not allowed reveal_type(bad3) # N: Revealed type is "def (*Any)" [builtins fixtures/dict.pyi] [typing fixtures/typing-typeddict.pyi] From 6a88c2135cf35875c4bcdc61c0bddd613d72281c Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 29 Aug 2025 10:53:51 +0100 Subject: [PATCH 211/424] Use more compact cache representation for int and str (#19750) After looking more at some real data I found that: * More than 99.9% of all `int`s are between -10 and 117. Values are a bit arbitrary TBH, the idea is that we should include small negative values (for `TypeVarId`s) and still be able to fit them in 1 byte. * More than 99.9% of strings are shorter that 128 bytes (again the idea is to fit the length into a single byte) Note there are very few integers that would fit in two bytes currently. This is because we only store line for type alias nodes, and type aliases are usually defined at the top of a module. We can add special case for two bytes later when needed. We could probably save another byte for long strings and medium integers, but I don't want to have anything fancy that would only affect less than 0.1% cases. Finally you may notice I add a small correctness change I noticed accidentally when working on this, it is not really related, but it is so minor that it doesn't deserve a separate PR. --- mypy/types.py | 3 +- mypyc/lib-rt/native_internal.c | 133 +++++++++++++++++++++++++------ mypyc/primitives/misc_ops.py | 2 +- mypyc/test-data/run-classes.test | 64 +++++++++++++++ 4 files changed, 174 insertions(+), 28 deletions(-) diff --git a/mypy/types.py b/mypy/types.py index 8d5648ae0bdac..e0265e601e0c0 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1674,7 +1674,7 @@ def __eq__(self, other: object) -> bool: def serialize(self) -> JsonDict | str: assert self.type is not None type_ref = self.type.fullname - if not self.args and not self.last_known_value: + if not self.args and not self.last_known_value and not self.extra_attrs: return type_ref data: JsonDict = { ".class": "Instance", @@ -1745,7 +1745,6 @@ def copy_modified( ), extra_attrs=self.extra_attrs, ) - # We intentionally don't copy the extra_attrs here, so they will be erased. new.can_be_true = self.can_be_true new.can_be_false = self.can_be_false return new diff --git a/mypyc/lib-rt/native_internal.c b/mypyc/lib-rt/native_internal.c index 3228f03307930..1c211464b19ca 100644 --- a/mypyc/lib-rt/native_internal.c +++ b/mypyc/lib-rt/native_internal.c @@ -8,6 +8,14 @@ #define START_SIZE 512 #define MAX_SHORT_INT_TAGGED (255 << 1) +#define MAX_SHORT_LEN 127 +#define LONG_STR_TAG 1 + +#define MIN_SHORT_INT -10 +#define MAX_SHORT_INT 117 +#define MEDIUM_INT_TAG 1 +#define LONG_INT_TAG 3 + typedef struct { PyObject_HEAD Py_ssize_t pos; @@ -166,6 +174,12 @@ _check_read(BufferObject *data, Py_ssize_t need) { return 1; } +/* +bool format: single byte + \x00 - False + \x01 - True +*/ + static char read_bool_internal(PyObject *data) { if (_check_buffer(data) == 2) @@ -225,20 +239,34 @@ write_bool(PyObject *self, PyObject *args, PyObject *kwds) { return Py_None; } +/* +str format: size followed by UTF-8 bytes + short strings (len <= 127): single byte for size as `(uint8_t)size << 1` + long strings: \x01 followed by size as Py_ssize_t +*/ + static PyObject* read_str_internal(PyObject *data) { if (_check_buffer(data) == 2) return NULL; - if (_check_read((BufferObject *)data, sizeof(Py_ssize_t)) == 2) - return NULL; + Py_ssize_t size; char *buf = ((BufferObject *)data)->buf; // Read string length. - Py_ssize_t size = *(Py_ssize_t *)(buf + ((BufferObject *)data)->pos); - ((BufferObject *)data)->pos += sizeof(Py_ssize_t); - if (_check_read((BufferObject *)data, size) == 2) + if (_check_read((BufferObject *)data, 1) == 2) return NULL; + uint8_t first = *(uint8_t *)(buf + ((BufferObject *)data)->pos); + ((BufferObject *)data)->pos += 1; + if (first != LONG_STR_TAG) { + // Common case: short string (len <= 127). + size = (Py_ssize_t)(first >> 1); + } else { + size = *(Py_ssize_t *)(buf + ((BufferObject *)data)->pos); + ((BufferObject *)data)->pos += sizeof(Py_ssize_t); + } // Read string content. + if (_check_read((BufferObject *)data, size) == 2) + return NULL; PyObject *res = PyUnicode_FromStringAndSize( buf + ((BufferObject *)data)->pos, (Py_ssize_t)size ); @@ -266,14 +294,28 @@ write_str_internal(PyObject *data, PyObject *value) { const char *chunk = PyUnicode_AsUTF8AndSize(value, &size); if (!chunk) return 2; - Py_ssize_t need = size + sizeof(Py_ssize_t); - if (_check_size((BufferObject *)data, need) == 2) - return 2; - char *buf = ((BufferObject *)data)->buf; + Py_ssize_t need; + char *buf; // Write string length. - *(Py_ssize_t *)(buf + ((BufferObject *)data)->pos) = size; - ((BufferObject *)data)->pos += sizeof(Py_ssize_t); + if (size <= MAX_SHORT_LEN) { + // Common case: short string (len <= 127) store as single byte. + need = size + 1; + if (_check_size((BufferObject *)data, need) == 2) + return 2; + buf = ((BufferObject *)data)->buf; + *(uint8_t *)(buf + ((BufferObject *)data)->pos) = (uint8_t)size << 1; + ((BufferObject *)data)->pos += 1; + } else { + need = size + sizeof(Py_ssize_t) + 1; + if (_check_size((BufferObject *)data, need) == 2) + return 2; + buf = ((BufferObject *)data)->buf; + *(uint8_t *)(buf + ((BufferObject *)data)->pos) = LONG_STR_TAG; + ((BufferObject *)data)->pos += 1; + *(Py_ssize_t *)(buf + ((BufferObject *)data)->pos) = size; + ((BufferObject *)data)->pos += sizeof(Py_ssize_t); + } // Write string content. memcpy(buf + ((BufferObject *)data)->pos, chunk, size); ((BufferObject *)data)->pos += size; @@ -299,6 +341,11 @@ write_str(PyObject *self, PyObject *args, PyObject *kwds) { return Py_None; } +/* +float format: + stored as a C double +*/ + static double read_float_internal(PyObject *data) { if (_check_buffer(data) == 2) @@ -357,19 +404,33 @@ write_float(PyObject *self, PyObject *args, PyObject *kwds) { return Py_None; } +/* +int format: + most common values (-10 <= value <= 117): single byte as `(uint8_t)(value + 10) << 1` + medium values (fit in CPyTagged): \x01 followed by CPyTagged value + long values (very rare): \x03 followed by decimal string (see str format) +*/ + static CPyTagged read_int_internal(PyObject *data) { if (_check_buffer(data) == 2) return CPY_INT_TAG; - if (_check_read((BufferObject *)data, sizeof(CPyTagged)) == 2) - return CPY_INT_TAG; char *buf = ((BufferObject *)data)->buf; + if (_check_read((BufferObject *)data, 1) == 2) + return CPY_INT_TAG; - CPyTagged ret = *(CPyTagged *)(buf + ((BufferObject *)data)->pos); - ((BufferObject *)data)->pos += sizeof(CPyTagged); - if ((ret & CPY_INT_TAG) == 0) + uint8_t first = *(uint8_t *)(buf + ((BufferObject *)data)->pos); + ((BufferObject *)data)->pos += 1; + if ((first & MEDIUM_INT_TAG) == 0) { + // Most common case: int that is small in absolute value. + return ((Py_ssize_t)(first >> 1) + MIN_SHORT_INT) << 1; + } + if (first == MEDIUM_INT_TAG) { + CPyTagged ret = *(CPyTagged *)(buf + ((BufferObject *)data)->pos); + ((BufferObject *)data)->pos += sizeof(CPyTagged); return ret; + } // People who have literal ints not fitting in size_t should be punished :-) PyObject *str_ret = read_str_internal(data); if (str_ret == NULL) @@ -397,17 +458,34 @@ write_int_internal(PyObject *data, CPyTagged value) { if (_check_buffer(data) == 2) return 2; - if (_check_size((BufferObject *)data, sizeof(CPyTagged)) == 2) - return 2; - char *buf = ((BufferObject *)data)->buf; + char *buf; if ((value & CPY_INT_TAG) == 0) { - *(CPyTagged *)(buf + ((BufferObject *)data)->pos) = value; + Py_ssize_t real_value = CPyTagged_ShortAsSsize_t(value); + if (real_value >= MIN_SHORT_INT && real_value <= MAX_SHORT_INT) { + // Most common case: int that is small in absolute value. + if (_check_size((BufferObject *)data, 1) == 2) + return 2; + buf = ((BufferObject *)data)->buf; + *(uint8_t *)(buf + ((BufferObject *)data)->pos) = (uint8_t)(real_value - MIN_SHORT_INT) << 1; + ((BufferObject *)data)->pos += 1; + ((BufferObject *)data)->end += 1; + } else { + if (_check_size((BufferObject *)data, sizeof(CPyTagged) + 1) == 2) + return 2; + buf = ((BufferObject *)data)->buf; + *(uint8_t *)(buf + ((BufferObject *)data)->pos) = MEDIUM_INT_TAG; + ((BufferObject *)data)->pos += 1; + *(CPyTagged *)(buf + ((BufferObject *)data)->pos) = value; + ((BufferObject *)data)->pos += sizeof(CPyTagged); + ((BufferObject *)data)->end += sizeof(CPyTagged) + 1; + } } else { - *(CPyTagged *)(buf + ((BufferObject *)data)->pos) = CPY_INT_TAG; - } - ((BufferObject *)data)->pos += sizeof(CPyTagged); - ((BufferObject *)data)->end += sizeof(CPyTagged); - if ((value & CPY_INT_TAG) != 0) { + if (_check_size((BufferObject *)data, 1) == 2) + return 2; + buf = ((BufferObject *)data)->buf; + *(uint8_t *)(buf + ((BufferObject *)data)->pos) = LONG_INT_TAG; + ((BufferObject *)data)->pos += 1; + ((BufferObject *)data)->end += 1; PyObject *str_value = PyObject_Str(CPyTagged_LongAsObject(value)); if (str_value == NULL) return 2; @@ -438,6 +516,11 @@ write_int(PyObject *self, PyObject *args, PyObject *kwds) { return Py_None; } +/* +integer tag format (0 <= t <= 255): + stored as a uint8_t +*/ + static uint8_t read_tag_internal(PyObject *data) { if (_check_buffer(data) == 2) diff --git a/mypyc/primitives/misc_ops.py b/mypyc/primitives/misc_ops.py index 943f6fc04b729..8e6e450c64dca 100644 --- a/mypyc/primitives/misc_ops.py +++ b/mypyc/primitives/misc_ops.py @@ -430,7 +430,7 @@ arg_types=[object_rprimitive, uint8_rprimitive], return_type=none_rprimitive, c_function_name="write_tag_internal", - error_kind=ERR_MAGIC_OVERLAPPING, + error_kind=ERR_MAGIC, ) function_op( diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index edc989ea641cb..2e55ee70687ea 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -2741,7 +2741,11 @@ def test_buffer_roundtrip() -> None: write_tag(b, TAG_B) write_int(b, 2) write_int(b, 2 ** 85) + write_int(b, 255) write_int(b, -1) + write_int(b, -255) + write_int(b, 1234512344) + write_int(b, 1234512345) b = Buffer(b.getvalue()) assert read_str(b) == "foo" @@ -2756,13 +2760,41 @@ def test_buffer_roundtrip() -> None: assert read_tag(b) == TAG_B assert read_int(b) == 2 assert read_int(b) == 2 ** 85 + assert read_int(b) == 255 assert read_int(b) == -1 + assert read_int(b) == -255 + assert read_int(b) == 1234512344 + assert read_int(b) == 1234512345 + +def test_buffer_int_size() -> None: + for i in (-10, -9, 0, 116, 117): + b = Buffer() + write_int(b, i) + assert len(b.getvalue()) == 1 + b = Buffer(b.getvalue()) + assert read_int(b) == i + for i in (-12345, -12344, -11, 118, 12344, 12345): + b = Buffer() + write_int(b, i) + assert len(b.getvalue()) <= 9 # sizeof(size_t) + 1 + b = Buffer(b.getvalue()) + assert read_int(b) == i + +def test_buffer_str_size() -> None: + for s in ("", "a", "a" * 127): + b = Buffer() + write_str(b, s) + assert len(b.getvalue()) == len(s) + 1 + b = Buffer(b.getvalue()) + assert read_str(b) == s [file driver.py] from native import * test_buffer_basic() test_buffer_roundtrip() +test_buffer_int_size() +test_buffer_str_size() def test_buffer_basic_interpreted() -> None: b = Buffer(b"foo") @@ -2782,7 +2814,11 @@ def test_buffer_roundtrip_interpreted() -> None: write_tag(b, 255) write_int(b, 2) write_int(b, 2 ** 85) + write_int(b, 255) write_int(b, -1) + write_int(b, -255) + write_int(b, 1234512344) + write_int(b, 1234512345) b = Buffer(b.getvalue()) assert read_str(b) == "foo" @@ -2797,10 +2833,38 @@ def test_buffer_roundtrip_interpreted() -> None: assert read_tag(b) == 255 assert read_int(b) == 2 assert read_int(b) == 2 ** 85 + assert read_int(b) == 255 assert read_int(b) == -1 + assert read_int(b) == -255 + assert read_int(b) == 1234512344 + assert read_int(b) == 1234512345 + +def test_buffer_int_size_interpreted() -> None: + for i in (-10, -9, 0, 116, 117): + b = Buffer() + write_int(b, i) + assert len(b.getvalue()) == 1 + b = Buffer(b.getvalue()) + assert read_int(b) == i + for i in (-12345, -12344, -11, 118, 12344, 12345): + b = Buffer() + write_int(b, i) + assert len(b.getvalue()) <= 9 # sizeof(size_t) + 1 + b = Buffer(b.getvalue()) + assert read_int(b) == i + +def test_buffer_str_size_interpreted() -> None: + for s in ("", "a", "a" * 127): + b = Buffer() + write_str(b, s) + assert len(b.getvalue()) == len(s) + 1 + b = Buffer(b.getvalue()) + assert read_str(b) == s test_buffer_basic_interpreted() test_buffer_roundtrip_interpreted() +test_buffer_int_size_interpreted() +test_buffer_str_size_interpreted() [case testEnumMethodCalls] from enum import Enum From d9c77cbe827123be16dd4487bf7cd8248c643100 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 29 Aug 2025 12:56:51 +0100 Subject: [PATCH 212/424] [mypyc] Refactor IR build of equality and unary operators (#19756) Make the code cleaner. This will also make it easier to implement additional optimizations. This probably fixes a few bugs as well. --- mypyc/irbuild/ll_builder.py | 125 ++++++++++++++++++++++-------------- 1 file changed, 76 insertions(+), 49 deletions(-) diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index 112bbdbb50edd..f82be9fe706f0 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -1395,12 +1395,6 @@ def binary_op(self, lreg: Value, rreg: Value, op: str, line: int) -> Value: # Special case various ops if op in ("is", "is not"): return self.translate_is_op(lreg, rreg, op, line) - # TODO: modify 'str' to use same interface as 'compare_bytes' as it avoids - # call to PyErr_Occurred() - if is_str_rprimitive(ltype) and is_str_rprimitive(rtype) and op in ("==", "!="): - return self.compare_strings(lreg, rreg, op, line) - if is_bytes_rprimitive(ltype) and is_bytes_rprimitive(rtype) and op in ("==", "!="): - return self.compare_bytes(lreg, rreg, op, line) if ( is_bool_or_bit_rprimitive(ltype) and is_bool_or_bit_rprimitive(rtype) @@ -1496,6 +1490,7 @@ def binary_op(self, lreg: Value, rreg: Value, op: str, line: int) -> Value: def dunder_op(self, lreg: Value, rreg: Value | None, op: str, line: int) -> Value | None: """ Dispatch a dunder method if applicable. + For example for `a + b` it will use `a.__add__(b)` which can lead to higher performance due to the fact that the method could be already compiled and optimized instead of going all the way through `PyNumber_Add(a, b)` python api (making a jump into the python DL). @@ -1545,6 +1540,10 @@ def compare_strings(self, lhs: Value, rhs: Value, op: str, line: int) -> Value: elif op == "!=": eq = self.primitive_op(str_eq, [lhs, rhs], line) return self.add(ComparisonOp(eq, self.false(), ComparisonOp.EQ, line)) + + # TODO: modify 'str' to use same interface as 'compare_bytes' as it would avoid + # call to PyErr_Occurred() below + compare_result = self.call_c(unicode_compare, [lhs, rhs], line) error_constant = Integer(-1, c_int_rprimitive, line) compare_error_check = self.add( @@ -1648,55 +1647,75 @@ def bool_comparison_op(self, lreg: Value, rreg: Value, op: str, line: int) -> Va op_id = ComparisonOp.signed_ops[op] return self.comparison_op(lreg, rreg, op_id, line) - def unary_not(self, value: Value, line: int) -> Value: - mask = Integer(1, value.type, line) - return self.int_op(value.type, value, mask, IntOp.XOR, line) + def _non_specialized_unary_op(self, value: Value, op: str, line: int) -> Value: + if isinstance(value.type, RInstance): + result = self.dunder_op(value, None, op, line) + if result is not None: + return result + primitive_ops_candidates = unary_ops.get(op, []) + target = self.matching_primitive_op(primitive_ops_candidates, [value], line) + assert target, "Unsupported unary operation: %s" % op + return target - def unary_op(self, value: Value, expr_op: str, line: int) -> Value: + def unary_not(self, value: Value, line: int) -> Value: + """Perform unary 'not'.""" typ = value.type if is_bool_or_bit_rprimitive(typ): - if expr_op == "not": - return self.unary_not(value, line) - if expr_op == "+": - return value - if is_fixed_width_rtype(typ): - if expr_op == "-": - # Translate to '0 - x' - return self.int_op(typ, Integer(0, typ), value, IntOp.SUB, line) - elif expr_op == "~": - if typ.is_signed: - # Translate to 'x ^ -1' - return self.int_op(typ, value, Integer(-1, typ), IntOp.XOR, line) - else: - # Translate to 'x ^ 0xff...' - mask = (1 << (typ.size * 8)) - 1 - return self.int_op(typ, value, Integer(mask, typ), IntOp.XOR, line) - elif expr_op == "+": - return value - if is_float_rprimitive(typ): - if expr_op == "-": - return self.add(FloatNeg(value, line)) - elif expr_op == "+": - return value + mask = Integer(1, typ, line) + return self.int_op(typ, value, mask, IntOp.XOR, line) + return self._non_specialized_unary_op(value, "not", line) + def unary_minus(self, value: Value, line: int) -> Value: + """Perform unary '-'.""" + typ = value.type if isinstance(value, Integer): # TODO: Overflow? Unsigned? - num = value.value - if is_short_int_rprimitive(typ): - num >>= 1 - return Integer(-num, typ, value.line) - if is_tagged(typ) and expr_op == "+": + return Integer(-value.numeric_value(), typ, line) + elif isinstance(value, Float): + return Float(-value.value, line) + elif is_fixed_width_rtype(typ): + # Translate to '0 - x' + return self.int_op(typ, Integer(0, typ), value, IntOp.SUB, line) + elif is_float_rprimitive(typ): + return self.add(FloatNeg(value, line)) + return self._non_specialized_unary_op(value, "-", line) + + def unary_plus(self, value: Value, line: int) -> Value: + """Perform unary '+'.""" + typ = value.type + if ( + is_tagged(typ) + or is_float_rprimitive(typ) + or is_bool_or_bit_rprimitive(typ) + or is_fixed_width_rtype(typ) + ): return value - if isinstance(value, Float): - return Float(-value.value, value.line) - if isinstance(typ, RInstance): - result = self.dunder_op(value, None, expr_op, line) - if result is not None: - return result - primitive_ops_candidates = unary_ops.get(expr_op, []) - target = self.matching_primitive_op(primitive_ops_candidates, [value], line) - assert target, "Unsupported unary operation: %s" % expr_op - return target + return self._non_specialized_unary_op(value, "+", line) + + def unary_invert(self, value: Value, line: int) -> Value: + """Perform unary '~'.""" + typ = value.type + if is_fixed_width_rtype(typ): + if typ.is_signed: + # Translate to 'x ^ -1' + return self.int_op(typ, value, Integer(-1, typ), IntOp.XOR, line) + else: + # Translate to 'x ^ 0xff...' + mask = (1 << (typ.size * 8)) - 1 + return self.int_op(typ, value, Integer(mask, typ), IntOp.XOR, line) + return self._non_specialized_unary_op(value, "~", line) + + def unary_op(self, value: Value, op: str, line: int) -> Value: + """Perform a unary operation.""" + if op == "not": + return self.unary_not(value, line) + elif op == "-": + return self.unary_minus(value, line) + elif op == "+": + return self.unary_plus(value, line) + elif op == "~": + return self.unary_invert(value, line) + raise RuntimeError("Unsupported unary operation: %s" % op) def make_dict(self, key_value_pairs: Sequence[DictEntry], line: int) -> Value: result: Value | None = None @@ -2480,13 +2499,21 @@ def translate_special_method_call( return primitive_op def translate_eq_cmp(self, lreg: Value, rreg: Value, expr_op: str, line: int) -> Value | None: - """Add a equality comparison operation. + """Add an equality comparison operation. + + Note that this doesn't cover all possible types. Args: expr_op: either '==' or '!=' """ ltype = lreg.type rtype = rreg.type + + if is_str_rprimitive(ltype) and is_str_rprimitive(rtype): + return self.compare_strings(lreg, rreg, expr_op, line) + if is_bytes_rprimitive(ltype) and is_bytes_rprimitive(rtype): + return self.compare_bytes(lreg, rreg, expr_op, line) + if not (isinstance(ltype, RInstance) and ltype == rtype): return None From 6a97dc63b4291633ecda87c74f5e8d98d987b8b7 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 29 Aug 2025 14:55:49 +0100 Subject: [PATCH 213/424] [mypyc] Speed up equality with optional str/bytes types (#19758) Specialize most equality (`==` and `!=`) operations when one of the operands is `str | None` or `bytes | None`. First check if the value is `None`, and based on that branch into fast path operations. Previously we used a generic C API primitive for such comparisons, which was quite slow. This could be generalized to other optional types, but let's start with `str | None` since it's a very common type and it's often used in equality tests. `bytes | None` is also covered, since it's very similar to the `str` case. Also add support for unchecked `Cast` operations in the IR. These don't perform a runtime type check -- they can be used to narrow the static type when it can be known statically that the cast is always safe. --- mypyc/codegen/emitfunc.py | 3 + mypyc/ir/ops.py | 10 +++- mypyc/ir/pprint.py | 8 ++- mypyc/ir/rtypes.py | 2 +- mypyc/irbuild/ll_builder.py | 96 +++++++++++++++++++++++++++++- mypyc/test-data/irbuild-bytes.test | 32 ++++++++++ mypyc/test-data/irbuild-str.test | 71 ++++++++++++++++++++++ mypyc/test-data/run-bytes.test | 27 +++++++++ mypyc/test-data/run-strings.test | 31 ++++++++++ 9 files changed, 275 insertions(+), 5 deletions(-) diff --git a/mypyc/codegen/emitfunc.py b/mypyc/codegen/emitfunc.py index f00f2e7002171..7ae49a0d97307 100644 --- a/mypyc/codegen/emitfunc.py +++ b/mypyc/codegen/emitfunc.py @@ -657,6 +657,9 @@ def visit_box(self, op: Box) -> None: self.emitter.emit_box(self.reg(op.src), self.reg(op), op.src.type, can_borrow=True) def visit_cast(self, op: Cast) -> None: + if op.is_unchecked and op.is_borrowed: + self.emit_line(f"{self.reg(op)} = {self.reg(op.src)};") + return branch = self.next_branch() handler = None if branch is not None: diff --git a/mypyc/ir/ops.py b/mypyc/ir/ops.py index 62ac9b8d48e4c..4b3b5eb3c8cae 100644 --- a/mypyc/ir/ops.py +++ b/mypyc/ir/ops.py @@ -1073,11 +1073,19 @@ class Cast(RegisterOp): error_kind = ERR_MAGIC - def __init__(self, src: Value, typ: RType, line: int, *, borrow: bool = False) -> None: + def __init__( + self, src: Value, typ: RType, line: int, *, borrow: bool = False, unchecked: bool = False + ) -> None: super().__init__(line) self.src = src self.type = typ + # If true, don't incref the result. self.is_borrowed = borrow + # If true, don't perform a runtime type check (only changes the static type of + # the operand). Used when we know that the cast will always succeed. + self.is_unchecked = unchecked + if unchecked: + self.error_kind = ERR_NEVER def sources(self) -> list[Value]: return [self.src] diff --git a/mypyc/ir/pprint.py b/mypyc/ir/pprint.py index b0de041e1eaef..efefd76d15f07 100644 --- a/mypyc/ir/pprint.py +++ b/mypyc/ir/pprint.py @@ -196,7 +196,13 @@ def visit_method_call(self, op: MethodCall) -> str: return s def visit_cast(self, op: Cast) -> str: - return self.format("%r = %scast(%s, %r)", op, self.borrow_prefix(op), op.type, op.src) + if op.is_unchecked: + prefix = "unchecked " + else: + prefix = "" + return self.format( + "%r = %s%scast(%s, %r)", op, prefix, self.borrow_prefix(op), op.type, op.src + ) def visit_box(self, op: Box) -> str: return self.format("%r = box(%s, %r)", op, op.src.type, op.src) diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 667ff60b0204a..34824a59cd5c3 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -1027,7 +1027,7 @@ def flatten_nested_unions(types: list[RType]) -> list[RType]: def optional_value_type(rtype: RType) -> RType | None: """If rtype is the union of none_rprimitive and another type X, return X. - Otherwise return None. + Otherwise, return None. """ if isinstance(rtype, RUnion) and len(rtype.items) == 2: if rtype.items[0] == none_rprimitive: diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index f82be9fe706f0..8876003645dd7 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -336,14 +336,20 @@ def box(self, src: Value) -> Value: return src def unbox_or_cast( - self, src: Value, target_type: RType, line: int, *, can_borrow: bool = False + self, + src: Value, + target_type: RType, + line: int, + *, + can_borrow: bool = False, + unchecked: bool = False, ) -> Value: if target_type.is_unboxed: return self.add(Unbox(src, target_type, line)) else: if can_borrow: self.keep_alives.append(src) - return self.add(Cast(src, target_type, line, borrow=can_borrow)) + return self.add(Cast(src, target_type, line, borrow=can_borrow, unchecked=unchecked)) def coerce( self, @@ -2514,6 +2520,22 @@ def translate_eq_cmp(self, lreg: Value, rreg: Value, expr_op: str, line: int) -> if is_bytes_rprimitive(ltype) and is_bytes_rprimitive(rtype): return self.compare_bytes(lreg, rreg, expr_op, line) + lopt = optional_value_type(ltype) + ropt = optional_value_type(rtype) + + # Can we do a quick comparison of two optional types (special case None values)? + fast_opt_eq = False + if lopt is not None: + if ropt is not None and is_same_type(lopt, ropt) and self._never_equal_to_none(lopt): + fast_opt_eq = True + if is_same_type(lopt, rtype) and self._never_equal_to_none(lopt): + fast_opt_eq = True + elif ropt is not None: + if is_same_type(ropt, ltype) and self._never_equal_to_none(ropt): + fast_opt_eq = True + if fast_opt_eq: + return self._translate_fast_optional_eq_cmp(lreg, rreg, expr_op, line) + if not (isinstance(ltype, RInstance) and ltype == rtype): return None @@ -2540,6 +2562,76 @@ def translate_eq_cmp(self, lreg: Value, rreg: Value, expr_op: str, line: int) -> return self.gen_method_call(lreg, op_methods[expr_op], [rreg], ltype, line) + def _never_equal_to_none(self, typ: RType) -> bool: + """Are the values of type never equal to None?""" + # TODO: Support RInstance with no custom __eq__/__ne__ and other primitive types. + return is_str_rprimitive(typ) or is_bytes_rprimitive(typ) + + def _translate_fast_optional_eq_cmp( + self, lreg: Value, rreg: Value, expr_op: str, line: int + ) -> Value: + """Generate eq/ne fast path between 'X | None' and ('X | None' or X). + + Assume 'X' never compares equal to None. + """ + if not isinstance(lreg.type, RUnion): + lreg, rreg = rreg, lreg + value_typ = optional_value_type(lreg.type) + assert value_typ + res = Register(bool_rprimitive) + + # Fast path: left value is None? + cmp = self.add(ComparisonOp(lreg, self.none_object(), ComparisonOp.EQ, line)) + l_none = BasicBlock() + l_not_none = BasicBlock() + out = BasicBlock() + self.add(Branch(cmp, l_none, l_not_none, Branch.BOOL)) + self.activate_block(l_none) + if not isinstance(rreg.type, RUnion): + val = self.false() if expr_op == "==" else self.true() + self.add(Assign(res, val)) + else: + op = ComparisonOp.EQ if expr_op == "==" else ComparisonOp.NEQ + cmp = self.add(ComparisonOp(rreg, self.none_object(), op, line)) + self.add(Assign(res, cmp)) + self.goto(out) + + self.activate_block(l_not_none) + if not isinstance(rreg.type, RUnion): + # Both operands are known to be not None, perform specialized comparison + eq = self.translate_eq_cmp( + self.unbox_or_cast(lreg, value_typ, line, can_borrow=True, unchecked=True), + rreg, + expr_op, + line, + ) + assert eq is not None + self.add(Assign(res, eq)) + else: + r_none = BasicBlock() + r_not_none = BasicBlock() + # Fast path: right value is None? + cmp = self.add(ComparisonOp(rreg, self.none_object(), ComparisonOp.EQ, line)) + self.add(Branch(cmp, r_none, r_not_none, Branch.BOOL)) + self.activate_block(r_none) + # None vs not-None + val = self.false() if expr_op == "==" else self.true() + self.add(Assign(res, val)) + self.goto(out) + self.activate_block(r_not_none) + # Both operands are known to be not None, perform specialized comparison + eq = self.translate_eq_cmp( + self.unbox_or_cast(lreg, value_typ, line, can_borrow=True, unchecked=True), + self.unbox_or_cast(rreg, value_typ, line, can_borrow=True, unchecked=True), + expr_op, + line, + ) + assert eq is not None + self.add(Assign(res, eq)) + self.goto(out) + self.activate_block(out) + return res + def translate_is_op(self, lreg: Value, rreg: Value, expr_op: str, line: int) -> Value: """Create equality comparison operation between object identities diff --git a/mypyc/test-data/irbuild-bytes.test b/mypyc/test-data/irbuild-bytes.test index 476c5ac59f48b..8cfefe03ae22c 100644 --- a/mypyc/test-data/irbuild-bytes.test +++ b/mypyc/test-data/irbuild-bytes.test @@ -185,3 +185,35 @@ L0: r10 = CPyBytes_Build(2, var, r9) b4 = r10 return 1 + +[case testOptionalBytesEquality] +from typing import Optional + +def non_opt_opt(x: bytes, y: Optional[bytes]) -> bool: + return x != y +[out] +def non_opt_opt(x, y): + x :: bytes + y :: union[bytes, None] + r0 :: object + r1 :: bit + r2 :: bool + r3 :: bytes + r4 :: i32 + r5, r6 :: bit +L0: + r0 = load_address _Py_NoneStruct + r1 = y == r0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = 1 + goto L3 +L2: + r3 = unchecked borrow cast(bytes, y) + r4 = CPyBytes_Compare(r3, x) + r5 = r4 >= 0 :: signed + r6 = r4 != 1 + r2 = r6 +L3: + keep_alive y + return r2 diff --git a/mypyc/test-data/irbuild-str.test b/mypyc/test-data/irbuild-str.test index 245acf7402a11..3fa39819498de 100644 --- a/mypyc/test-data/irbuild-str.test +++ b/mypyc/test-data/irbuild-str.test @@ -669,3 +669,74 @@ L0: r2 = 'abc1233.14True' r3 = CPyStr_Build(7, x, r0, x, r1, x, r2, x) return r3 + +[case testOptionalStrEquality1] +from typing import Optional + +def opt_opt(x: Optional[str], y: Optional[str]) -> bool: + return x == y +[out] +def opt_opt(x, y): + x, y :: union[str, None] + r0 :: object + r1 :: bit + r2 :: object + r3 :: bit + r4 :: bool + r5 :: object + r6 :: bit + r7, r8 :: str + r9 :: bool +L0: + r0 = load_address _Py_NoneStruct + r1 = x == r0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = load_address _Py_NoneStruct + r3 = y == r2 + r4 = r3 + goto L5 +L2: + r5 = load_address _Py_NoneStruct + r6 = y == r5 + if r6 goto L3 else goto L4 :: bool +L3: + r4 = 0 + goto L5 +L4: + r7 = unchecked borrow cast(str, x) + r8 = unchecked borrow cast(str, y) + r9 = CPyStr_Equal(r7, r8) + r4 = r9 +L5: + keep_alive x, y + return r4 + +[case testOptionalStrEquality2] +from typing import Optional + +def opt_non_opt(x: Optional[str], y: str) -> bool: + return x == y +[out] +def opt_non_opt(x, y): + x :: union[str, None] + y :: str + r0 :: object + r1 :: bit + r2 :: bool + r3 :: str + r4 :: bool +L0: + r0 = load_address _Py_NoneStruct + r1 = x == r0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = 0 + goto L3 +L2: + r3 = unchecked borrow cast(str, x) + r4 = CPyStr_Equal(r3, y) + r2 = r4 +L3: + keep_alive x + return r2 diff --git a/mypyc/test-data/run-bytes.test b/mypyc/test-data/run-bytes.test index 5a285320c849e..df5cb209b9025 100644 --- a/mypyc/test-data/run-bytes.test +++ b/mypyc/test-data/run-bytes.test @@ -374,3 +374,30 @@ class subbytearray(bytearray): [file userdefinedbytes.py] class bytes: pass + +[case testBytesOptionalEquality] +from __future__ import annotations + +def eq_b_opt_b(x: bytes | None, y: bytes) -> bool: + return x == y + +def ne_b_b_opt(x: bytes, y: bytes | None) -> bool: + return x != y + +def test_optional_eq() -> None: + b = b'x' + assert eq_b_opt_b(b, b) + assert eq_b_opt_b(b + bytes([int()]), b + bytes([int()])) + + assert not eq_b_opt_b(b'x', b'y') + assert not eq_b_opt_b(b'y', b'x') + assert not eq_b_opt_b(None, b'x') + +def test_optional_ne() -> None: + b = b'x' + assert not ne_b_b_opt(b, b) + assert not ne_b_b_opt(b + b'y', b + bytes() + b'y') + + assert ne_b_b_opt(b'x', b'y') + assert ne_b_b_opt(b'y', b'x') + assert ne_b_b_opt(b'x', None) diff --git a/mypyc/test-data/run-strings.test b/mypyc/test-data/run-strings.test index b4f3ebd669104..6960b0a043038 100644 --- a/mypyc/test-data/run-strings.test +++ b/mypyc/test-data/run-strings.test @@ -1063,3 +1063,34 @@ class subc(str): [file userdefinedstr.py] class str: pass + +[case testStrOptionalEquality] +from __future__ import annotations + +def eq_s_opt_s_opt(x: str | None, y: str | None) -> bool: + return x == y + +def ne_s_opt_s_opt(x: str | None, y: str | None) -> bool: + return x != y + +def test_optional_eq() -> None: + s = 'x' + assert eq_s_opt_s_opt(s, s) + assert eq_s_opt_s_opt(s + str(int()), s + str(int())) + assert eq_s_opt_s_opt(None, None) + + assert not eq_s_opt_s_opt('x', 'y') + assert not eq_s_opt_s_opt('y', 'x') + assert not eq_s_opt_s_opt(None, 'x') + assert not eq_s_opt_s_opt('x', None) + +def test_optional_ne() -> None: + s = 'x' + assert not ne_s_opt_s_opt(s, s) + assert not ne_s_opt_s_opt(s + str(int()), s+ str(int())) + assert not ne_s_opt_s_opt(None, None) + + assert ne_s_opt_s_opt('x', 'y') + assert ne_s_opt_s_opt('y', 'x') + assert ne_s_opt_s_opt(None, 'x') + assert ne_s_opt_s_opt('x', None) From f83ec9fffa838978a6ee6411a515a313979c61fa Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 29 Aug 2025 15:53:55 +0100 Subject: [PATCH 214/424] [mypyc] Speed up implicit __ne__ method (#19759) Avoid the use of `PyObject_Not` if `__eq__` returns a boolean (which is common). Also skip incref on the bool/bit result on 3.12+, since the object is immortal. This speeds up a microbenchmark that repeatedly does `!=` operations by 20%. --- mypyc/codegen/emitfunc.py | 4 +- mypyc/irbuild/classdef.py | 4 +- mypyc/irbuild/ll_builder.py | 41 ++++++++++++- mypyc/test-data/irbuild-basic.test | 38 +++++++++--- mypyc/test-data/irbuild-classes.test | 76 ++++++++++++++++++------ mypyc/test-data/run-dunders-special.test | 2 + mypyc/test-data/run-dunders.test | 22 +++++++ 7 files changed, 155 insertions(+), 32 deletions(-) diff --git a/mypyc/codegen/emitfunc.py b/mypyc/codegen/emitfunc.py index 7ae49a0d97307..a1e18353693ee 100644 --- a/mypyc/codegen/emitfunc.py +++ b/mypyc/codegen/emitfunc.py @@ -89,7 +89,7 @@ RStruct, RTuple, RType, - is_bool_rprimitive, + is_bool_or_bit_rprimitive, is_int32_rprimitive, is_int64_rprimitive, is_int_rprimitive, @@ -633,7 +633,7 @@ def emit_method_call(self, dest: str, op_obj: Value, name: str, op_args: list[Va def visit_inc_ref(self, op: IncRef) -> None: if ( isinstance(op.src, Box) - and (is_none_rprimitive(op.src.src.type) or is_bool_rprimitive(op.src.src.type)) + and (is_none_rprimitive(op.src.src.type) or is_bool_or_bit_rprimitive(op.src.src.type)) and HAVE_IMMORTAL ): # On Python 3.12+, None/True/False are immortal, and we can skip inc ref diff --git a/mypyc/irbuild/classdef.py b/mypyc/irbuild/classdef.py index 72482710208af..324b44b95dc40 100644 --- a/mypyc/irbuild/classdef.py +++ b/mypyc/irbuild/classdef.py @@ -845,7 +845,9 @@ def gen_glue_ne_method(builder: IRBuilder, cls: ClassIR, line: int) -> None: ) builder.activate_block(regular_block) rettype = bool_rprimitive if return_bool and strict_typing else object_rprimitive - retval = builder.coerce(builder.unary_op(eqval, "not", line), rettype, line) + retval = builder.coerce( + builder.builder.unary_not(eqval, line, likely_bool=True), rettype, line + ) builder.add(Return(retval)) builder.activate_block(not_implemented_block) builder.add(Return(not_implemented)) diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index 8876003645dd7..475d490a48f2e 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -56,6 +56,7 @@ KeepAlive, LoadAddress, LoadErrorValue, + LoadGlobal, LoadLiteral, LoadMem, LoadStatic, @@ -108,6 +109,7 @@ is_int_rprimitive, is_list_rprimitive, is_none_rprimitive, + is_object_rprimitive, is_set_rprimitive, is_short_int_rprimitive, is_str_rprimitive, @@ -1318,6 +1320,14 @@ def none_object(self) -> Value: """Load Python None value (type: object_rprimitive).""" return self.add(LoadAddress(none_object_op.type, none_object_op.src, line=-1)) + def true_object(self) -> Value: + """Load Python True object (type: object_rprimitive).""" + return self.add(LoadGlobal(object_rprimitive, "Py_True")) + + def false_object(self) -> Value: + """Load Python False object (type: object_rprimitive).""" + return self.add(LoadGlobal(object_rprimitive, "Py_False")) + def load_int(self, value: int) -> Value: """Load a tagged (Python) integer literal value.""" if value > MAX_LITERAL_SHORT_INT or value < MIN_LITERAL_SHORT_INT: @@ -1663,12 +1673,39 @@ def _non_specialized_unary_op(self, value: Value, op: str, line: int) -> Value: assert target, "Unsupported unary operation: %s" % op return target - def unary_not(self, value: Value, line: int) -> Value: - """Perform unary 'not'.""" + def unary_not(self, value: Value, line: int, *, likely_bool: bool = False) -> Value: + """Perform unary 'not'. + + Args: + likely_bool: The operand is likely a bool value, even if the type is something + more general, so specialize for bool values + """ typ = value.type if is_bool_or_bit_rprimitive(typ): mask = Integer(1, typ, line) return self.int_op(typ, value, mask, IntOp.XOR, line) + if likely_bool and is_object_rprimitive(typ): + # First quickly check if it's a bool, and otherwise fall back to generic op. + res = Register(bit_rprimitive) + false, not_false, true, other = BasicBlock(), BasicBlock(), BasicBlock(), BasicBlock() + out = BasicBlock() + cmp = self.add(ComparisonOp(value, self.true_object(), ComparisonOp.EQ, line)) + self.add(Branch(cmp, false, not_false, Branch.BOOL)) + self.activate_block(false) + self.add(Assign(res, self.false())) + self.goto(out) + self.activate_block(not_false) + cmp = self.add(ComparisonOp(value, self.false_object(), ComparisonOp.EQ, line)) + self.add(Branch(cmp, true, other, Branch.BOOL)) + self.activate_block(true) + self.add(Assign(res, self.true())) + self.goto(out) + self.activate_block(other) + val = self._non_specialized_unary_op(value, "not", line) + self.add(Assign(res, val)) + self.goto(out) + self.activate_block(out) + return res return self._non_specialized_unary_op(value, "not", line) def unary_minus(self, value: Value, line: int) -> Value: diff --git a/mypyc/test-data/irbuild-basic.test b/mypyc/test-data/irbuild-basic.test index feb7b36a2b52b..612f3266fd793 100644 --- a/mypyc/test-data/irbuild-basic.test +++ b/mypyc/test-data/irbuild-basic.test @@ -2347,22 +2347,42 @@ def A.__ne__(__mypyc_self__, rhs): __mypyc_self__ :: __main__.A rhs, r0, r1 :: object r2 :: bit - r3 :: i32 - r4 :: bit - r5 :: bool + r3 :: object + r4, r5 :: bit r6 :: object + r7 :: bit + r8 :: i32 + r9 :: bit + r10 :: bool + r11 :: object L0: r0 = __mypyc_self__.__eq__(rhs) r1 = load_address _Py_NotImplementedStruct r2 = r0 == r1 - if r2 goto L2 else goto L1 :: bool + if r2 goto L7 else goto L1 :: bool L1: - r3 = PyObject_Not(r0) - r4 = r3 >= 0 :: signed - r5 = truncate r3: i32 to builtins.bool - r6 = box(bool, r5) - return r6 + r3 = load_global Py_True :: static + r4 = r0 == r3 + if r4 goto L2 else goto L3 :: bool L2: + r5 = 0 + goto L6 +L3: + r6 = load_global Py_False :: static + r7 = r0 == r6 + if r7 goto L4 else goto L5 :: bool +L4: + r5 = 1 + goto L6 +L5: + r8 = PyObject_Not(r0) + r9 = r8 >= 0 :: signed + r10 = truncate r8: i32 to builtins.bool + r5 = r10 +L6: + r11 = box(bit, r5) + return r11 +L7: return r1 [case testDecorators_toplevel] diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index b49b20e13a43b..a573bb8e06685 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -854,22 +854,42 @@ def Base.__ne__(__mypyc_self__, rhs): __mypyc_self__ :: __main__.Base rhs, r0, r1 :: object r2 :: bit - r3 :: i32 - r4 :: bit - r5 :: bool + r3 :: object + r4, r5 :: bit r6 :: object + r7 :: bit + r8 :: i32 + r9 :: bit + r10 :: bool + r11 :: object L0: r0 = __mypyc_self__.__eq__(rhs) r1 = load_address _Py_NotImplementedStruct r2 = r0 == r1 - if r2 goto L2 else goto L1 :: bool + if r2 goto L7 else goto L1 :: bool L1: - r3 = PyObject_Not(r0) - r4 = r3 >= 0 :: signed - r5 = truncate r3: i32 to builtins.bool - r6 = box(bool, r5) - return r6 + r3 = load_global Py_True :: static + r4 = r0 == r3 + if r4 goto L2 else goto L3 :: bool L2: + r5 = 0 + goto L6 +L3: + r6 = load_global Py_False :: static + r7 = r0 == r6 + if r7 goto L4 else goto L5 :: bool +L4: + r5 = 1 + goto L6 +L5: + r8 = PyObject_Not(r0) + r9 = r8 >= 0 :: signed + r10 = truncate r8: i32 to builtins.bool + r5 = r10 +L6: + r11 = box(bit, r5) + return r11 +L7: return r1 def Derived.__eq__(self, other): self :: __main__.Derived @@ -979,22 +999,42 @@ def Derived.__ne__(__mypyc_self__, rhs): __mypyc_self__ :: __main__.Derived rhs, r0, r1 :: object r2 :: bit - r3 :: i32 - r4 :: bit - r5 :: bool + r3 :: object + r4, r5 :: bit r6 :: object + r7 :: bit + r8 :: i32 + r9 :: bit + r10 :: bool + r11 :: object L0: r0 = __mypyc_self__.__eq__(rhs) r1 = load_address _Py_NotImplementedStruct r2 = r0 == r1 - if r2 goto L2 else goto L1 :: bool + if r2 goto L7 else goto L1 :: bool L1: - r3 = PyObject_Not(r0) - r4 = r3 >= 0 :: signed - r5 = truncate r3: i32 to builtins.bool - r6 = box(bool, r5) - return r6 + r3 = load_global Py_True :: static + r4 = r0 == r3 + if r4 goto L2 else goto L3 :: bool L2: + r5 = 0 + goto L6 +L3: + r6 = load_global Py_False :: static + r7 = r0 == r6 + if r7 goto L4 else goto L5 :: bool +L4: + r5 = 1 + goto L6 +L5: + r8 = PyObject_Not(r0) + r9 = r8 >= 0 :: signed + r10 = truncate r8: i32 to builtins.bool + r5 = r10 +L6: + r11 = box(bit, r5) + return r11 +L7: return r1 [case testDefaultVars] diff --git a/mypyc/test-data/run-dunders-special.test b/mypyc/test-data/run-dunders-special.test index 2672434e10ef2..4817435b1e7ce 100644 --- a/mypyc/test-data/run-dunders-special.test +++ b/mypyc/test-data/run-dunders-special.test @@ -8,3 +8,5 @@ class UsesNotImplemented: def test_not_implemented() -> None: assert UsesNotImplemented() != object() + x = UsesNotImplemented() == object() + assert not x diff --git a/mypyc/test-data/run-dunders.test b/mypyc/test-data/run-dunders.test index b8fb13c9dcecb..ec992afbfbd1e 100644 --- a/mypyc/test-data/run-dunders.test +++ b/mypyc/test-data/run-dunders.test @@ -965,3 +965,25 @@ def test_final() -> None: assert b + 3 == 9 assert (a < A(5)) is False assert (b < A(5)) is True + +[case testDundersEq] +class Eq: + def __init__(self, x: int) -> None: + self.x = x + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Eq): + return NotImplemented + return self.x == other.x + +def eq(x: Eq, y: Eq) -> bool: + return x == y + +def ne(x: Eq, y: Eq) -> bool: + return x != y + +def test_equality_with_implicit_ne() -> None: + assert eq(Eq(1), Eq(1)) + assert not eq(Eq(1), Eq(2)) + assert ne(Eq(1), Eq(2)) + assert not ne(Eq(1), Eq(1)) From c32b0a50f2a7329109d049bd439954fdd913e7f2 Mon Sep 17 00:00:00 2001 From: Piotr Sawicki Date: Fri, 29 Aug 2025 18:50:05 +0200 Subject: [PATCH 215/424] [mypyc] Use defined __new__ method in tp_new and constructor (#19739) Fixes #16012 mypyc ignored custom implementations of `__new__` because, even though a C function representing the method was generated, it was called neither in the type constructor nor in the method assigned to the `tp_new` pointer. Now if there's a `__new__` method defined for a type, the corresponding function is called in place of the setup function which is responsible for allocating memory for new objects and initializing their attributes to default values. The setup function is still called when creating instances of the type as calls resolving to `object.__new__()` are transformed to call the setup function. This way, `__new__` can return instances of other types and instances of the type of the class where `__new__` is defined are setup correctly. There are a couple of limitations: - Programs with `super().__new__()` calls in `__new__` methods of non-native classes are rejected because it's more difficult to resolve the setup function for non-native classes but this could probably be supported in the future. - Similarly, programs are rejected when a class inherits from a non-compiled class. In this case calling the `tp_new` method of the parent type results in an error because cpython expects the sub type to use a wrapper for `tp_new` which compiled classes don't. Allowing this would require compiled types to be initialized more closely to the way cpython does it which might need a lot of work. - Lastly, when `__new__` is annotated with `@classmethod`, calling it without the type parameter works in compiled code but raises an error in interpreted. I'm not sure of the reason and it's difficult to make it a compiler error because it's outside of what mypyc sees. --------- Co-authored-by: Brian Schubert Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- mypyc/analysis/attrdefined.py | 1 + mypyc/codegen/emitclass.py | 90 +++-- mypyc/codegen/emitmodule.py | 4 +- mypyc/codegen/emitwrapper.py | 2 +- mypyc/ir/class_ir.py | 14 +- mypyc/irbuild/expression.py | 42 ++- mypyc/irbuild/prepare.py | 50 ++- mypyc/lib-rt/misc_ops.c | 11 + mypyc/test-data/fixtures/ir.py | 4 +- mypyc/test-data/fixtures/typing-full.pyi | 1 + mypyc/test-data/irbuild-classes.test | 75 ++++ mypyc/test-data/run-classes.test | 429 +++++++++++++++++++++++ 12 files changed, 677 insertions(+), 46 deletions(-) diff --git a/mypyc/analysis/attrdefined.py b/mypyc/analysis/attrdefined.py index 4fd0017257a01..5be57d767e355 100644 --- a/mypyc/analysis/attrdefined.py +++ b/mypyc/analysis/attrdefined.py @@ -138,6 +138,7 @@ def analyze_always_defined_attrs_in_class(cl: ClassIR, seen: set[ClassIR]) -> No or cl.builtin_base is not None or cl.children is None or cl.is_serializable() + or cl.has_method("__new__") ): # Give up -- we can't enforce that attributes are always defined. return diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index ecf8c37f83c9f..0931c849131dc 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -5,6 +5,7 @@ from collections.abc import Mapping from typing import Callable +from mypy.nodes import ARG_STAR, ARG_STAR2 from mypyc.codegen.cstring import c_string_initializer from mypyc.codegen.emit import Emitter, HeaderDeclaration, ReturnHandler from mypyc.codegen.emitfunc import native_function_doc_initializer, native_function_header @@ -224,7 +225,7 @@ def generate_class(cl: ClassIR, module: str, emitter: Emitter) -> None: name = cl.name name_prefix = cl.name_prefix(emitter.names) - setup_name = f"{name_prefix}_setup" + setup_name = emitter.native_function_name(cl.setup) new_name = f"{name_prefix}_new" finalize_name = f"{name_prefix}_finalize" members_name = f"{name_prefix}_members" @@ -317,10 +318,8 @@ def emit_line() -> None: fields["tp_basicsize"] = base_size if generate_full: - # Declare setup method that allocates and initializes an object. type is the - # type of the class being initialized, which could be another class if there - # is an interpreted subclass. - emitter.emit_line(f"static PyObject *{setup_name}(PyTypeObject *type);") + assert cl.setup is not None + emitter.emit_line(native_function_header(cl.setup, emitter) + ";") assert cl.ctor is not None emitter.emit_line(native_function_header(cl.ctor, emitter) + ";") @@ -390,9 +389,7 @@ def emit_line() -> None: emitter.emit_line() if generate_full: - generate_setup_for_class( - cl, setup_name, defaults_fn, vtable_name, shadow_vtable_name, emitter - ) + generate_setup_for_class(cl, defaults_fn, vtable_name, shadow_vtable_name, emitter) emitter.emit_line() generate_constructor_for_class(cl, cl.ctor, init_fn, setup_name, vtable_name, emitter) emitter.emit_line() @@ -579,16 +576,16 @@ def generate_vtable( def generate_setup_for_class( cl: ClassIR, - func_name: str, defaults_fn: FuncIR | None, vtable_name: str, shadow_vtable_name: str | None, emitter: Emitter, ) -> None: """Generate a native function that allocates an instance of a class.""" - emitter.emit_line("static PyObject *") - emitter.emit_line(f"{func_name}(PyTypeObject *type)") + emitter.emit_line(native_function_header(cl.setup, emitter)) emitter.emit_line("{") + type_arg_name = REG_PREFIX + cl.setup.sig.args[0].name + emitter.emit_line(f"PyTypeObject *type = (PyTypeObject*){type_arg_name};") struct_name = cl.struct_name(emitter.names) emitter.emit_line(f"{struct_name} *self;") @@ -663,6 +660,35 @@ def emit_attr_defaults_func_call(defaults_fn: FuncIR, self_name: str, emitter: E ) +def emit_setup_or_dunder_new_call( + cl: ClassIR, + setup_name: str, + type_arg: str, + native_prefix: bool, + new_args: str, + emitter: Emitter, +) -> None: + def emit_null_check() -> None: + emitter.emit_line("if (self == NULL)") + emitter.emit_line(" return NULL;") + + new_fn = cl.get_method("__new__") + if not new_fn: + emitter.emit_line(f"PyObject *self = {setup_name}({type_arg});") + emit_null_check() + return + prefix = emitter.get_group_prefix(new_fn.decl) + NATIVE_PREFIX if native_prefix else PREFIX + all_args = type_arg + if new_args != "": + all_args += ", " + new_args + emitter.emit_line(f"PyObject *self = {prefix}{new_fn.cname(emitter.names)}({all_args});") + emit_null_check() + + # skip __init__ if __new__ returns some other type + emitter.emit_line(f"if (Py_TYPE(self) != {emitter.type_struct_name(cl)})") + emitter.emit_line(" return self;") + + def generate_constructor_for_class( cl: ClassIR, fn: FuncDecl, @@ -674,17 +700,30 @@ def generate_constructor_for_class( """Generate a native function that allocates and initializes an instance of a class.""" emitter.emit_line(f"{native_function_header(fn, emitter)}") emitter.emit_line("{") - emitter.emit_line(f"PyObject *self = {setup_name}({emitter.type_struct_name(cl)});") - emitter.emit_line("if (self == NULL)") - emitter.emit_line(" return NULL;") - args = ", ".join(["self"] + [REG_PREFIX + arg.name for arg in fn.sig.args]) + + fn_args = [REG_PREFIX + arg.name for arg in fn.sig.args] + type_arg = "(PyObject *)" + emitter.type_struct_name(cl) + new_args = ", ".join(fn_args) + + use_wrapper = ( + cl.has_method("__new__") + and len(fn.sig.args) == 2 + and fn.sig.args[0].kind == ARG_STAR + and fn.sig.args[1].kind == ARG_STAR2 + ) + emit_setup_or_dunder_new_call(cl, setup_name, type_arg, not use_wrapper, new_args, emitter) + + args = ", ".join(["self"] + fn_args) if init_fn is not None: + prefix = PREFIX if use_wrapper else NATIVE_PREFIX + cast = "!= NULL ? 0 : -1" if use_wrapper else "" emitter.emit_line( - "char res = {}{}{}({});".format( + "char res = {}{}{}({}){};".format( emitter.get_group_prefix(init_fn.decl), - NATIVE_PREFIX, + prefix, init_fn.cname(emitter.names), args, + cast, ) ) emitter.emit_line("if (res == 2) {") @@ -717,7 +756,7 @@ def generate_init_for_class(cl: ClassIR, init_fn: FuncIR, emitter: Emitter) -> s emitter.emit_line("static int") emitter.emit_line(f"{func_name}(PyObject *self, PyObject *args, PyObject *kwds)") emitter.emit_line("{") - if cl.allow_interpreted_subclasses or cl.builtin_base: + if cl.allow_interpreted_subclasses or cl.builtin_base or cl.has_method("__new__"): emitter.emit_line( "return {}{}(self, args, kwds) != NULL ? 0 : -1;".format( PREFIX, init_fn.cname(emitter.names) @@ -750,15 +789,22 @@ def generate_new_for_class( emitter.emit_line("return NULL;") emitter.emit_line("}") - if not init_fn or cl.allow_interpreted_subclasses or cl.builtin_base or cl.is_serializable(): + type_arg = "(PyObject*)type" + new_args = "args, kwds" + emit_setup_or_dunder_new_call(cl, setup_name, type_arg, False, new_args, emitter) + if ( + not init_fn + or cl.allow_interpreted_subclasses + or cl.builtin_base + or cl.is_serializable() + or cl.has_method("__new__") + ): # Match Python semantics -- __new__ doesn't call __init__. - emitter.emit_line(f"return {setup_name}(type);") + emitter.emit_line("return self;") else: # __new__ of a native class implicitly calls __init__ so that we # can enforce that instances are always properly initialized. This # is needed to support always defined attributes. - emitter.emit_line(f"PyObject *self = {setup_name}(type);") - emitter.emit_lines("if (self == NULL)", " return NULL;") emitter.emit_line( f"PyObject *ret = {PREFIX}{init_fn.cname(emitter.names)}(self, args, kwds);" ) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index e31fcf8ea0c9f..ca5db52ab7da3 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -1274,8 +1274,8 @@ def is_fastcall_supported(fn: FuncIR, capi_version: tuple[int, int]) -> bool: if fn.name == "__call__": # We can use vectorcalls (PEP 590) when supported return True - # TODO: Support fastcall for __init__. - return fn.name != "__init__" + # TODO: Support fastcall for __init__ and __new__. + return fn.name != "__init__" and fn.name != "__new__" return True diff --git a/mypyc/codegen/emitwrapper.py b/mypyc/codegen/emitwrapper.py index cd1684255855a..2e5d7efa4e988 100644 --- a/mypyc/codegen/emitwrapper.py +++ b/mypyc/codegen/emitwrapper.py @@ -238,7 +238,7 @@ def generate_legacy_wrapper_function( real_args = list(fn.args) if fn.sig.num_bitmap_args: real_args = real_args[: -fn.sig.num_bitmap_args] - if fn.class_name and fn.decl.kind != FUNC_STATICMETHOD: + if fn.class_name and (fn.decl.name == "__new__" or fn.decl.kind != FUNC_STATICMETHOD): arg = real_args.pop(0) emitter.emit_line(f"PyObject *obj_{arg.name} = self;") diff --git a/mypyc/ir/class_ir.py b/mypyc/ir/class_ir.py index f6015b64dcdd2..0a56aaf5d1011 100644 --- a/mypyc/ir/class_ir.py +++ b/mypyc/ir/class_ir.py @@ -5,9 +5,9 @@ from typing import NamedTuple from mypyc.common import PROPSET_PREFIX, JsonDict -from mypyc.ir.func_ir import FuncDecl, FuncIR, FuncSignature +from mypyc.ir.func_ir import FuncDecl, FuncIR, FuncSignature, RuntimeArg from mypyc.ir.ops import DeserMaps, Value -from mypyc.ir.rtypes import RInstance, RType, deserialize_type +from mypyc.ir.rtypes import RInstance, RType, deserialize_type, object_rprimitive from mypyc.namegen import NameGenerator, exported_name # Some notes on the vtable layout: Each concrete class has a vtable @@ -133,6 +133,16 @@ def __init__( self.builtin_base: str | None = None # Default empty constructor self.ctor = FuncDecl(name, None, module_name, FuncSignature([], RInstance(self))) + # Declare setup method that allocates and initializes an object. type is the + # type of the class being initialized, which could be another class if there + # is an interpreted subclass. + # TODO: Make it a regular method and generate its body in IR + self.setup = FuncDecl( + "__mypyc__" + name + "_setup", + None, + module_name, + FuncSignature([RuntimeArg("type", object_rprimitive)], RInstance(self)), + ) # Attributes defined in the class (not inherited) self.attributes: dict[str, RType] = {} # Deletable attributes diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index e82203021ae31..4409b1acff265 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -57,6 +57,7 @@ from mypyc.ir.ops import ( Assign, BasicBlock, + Call, ComparisonOp, Integer, LoadAddress, @@ -472,23 +473,42 @@ def translate_super_method_call(builder: IRBuilder, expr: CallExpr, callee: Supe if callee.name in base.method_decls: break else: - if ( - ir.is_ext_class - and ir.builtin_base is None - and not ir.inherits_python - and callee.name == "__init__" - and len(expr.args) == 0 - ): - # Call translates to object.__init__(self), which is a - # no-op, so omit the call. - return builder.none() + if ir.is_ext_class and ir.builtin_base is None and not ir.inherits_python: + if callee.name == "__init__" and len(expr.args) == 0: + # Call translates to object.__init__(self), which is a + # no-op, so omit the call. + return builder.none() + elif callee.name == "__new__": + # object.__new__(cls) + assert ( + len(expr.args) == 1 + ), f"Expected object.__new__() call to have exactly 1 argument, got {len(expr.args)}" + typ_arg = expr.args[0] + method_args = builder.fn_info.fitem.arg_names + if ( + isinstance(typ_arg, NameExpr) + and len(method_args) > 0 + and method_args[0] == typ_arg.name + ): + subtype = builder.accept(expr.args[0]) + return builder.add(Call(ir.setup, [subtype], expr.line)) + + if callee.name == "__new__": + call = "super().__new__()" + if not ir.is_ext_class: + builder.error(f"{call} not supported for non-extension classes", expr.line) + if ir.inherits_python: + builder.error( + f"{call} not supported for classes inheriting from non-native classes", + expr.line, + ) return translate_call(builder, expr, callee) decl = base.method_decl(callee.name) arg_values = [builder.accept(arg) for arg in expr.args] arg_kinds, arg_names = expr.arg_kinds.copy(), expr.arg_names.copy() - if decl.kind != FUNC_STATICMETHOD: + if decl.kind != FUNC_STATICMETHOD and decl.name != "__new__": # Grab first argument vself: Value = builder.self() if decl.kind == FUNC_CLASSMETHOD: diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index 83ec3f7c1d382..95c8c448d642c 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -193,9 +193,9 @@ def prepare_func_def( create_generator_class_if_needed(module_name, class_name, fdef, mapper) kind = ( - FUNC_STATICMETHOD - if fdef.is_static - else (FUNC_CLASSMETHOD if fdef.is_class else FUNC_NORMAL) + FUNC_CLASSMETHOD + if fdef.is_class + else (FUNC_STATICMETHOD if fdef.is_static else FUNC_NORMAL) ) sig = mapper.fdef_to_sig(fdef, options.strict_dunders_typing) decl = FuncDecl(fdef.name, class_name, module_name, sig, kind) @@ -555,21 +555,57 @@ def add_setter_declaration( ir.method_decls[setter_name] = decl +def check_matching_args(init_sig: FuncSignature, new_sig: FuncSignature) -> bool: + num_init_args = len(init_sig.args) - init_sig.num_bitmap_args + num_new_args = len(new_sig.args) - new_sig.num_bitmap_args + if num_init_args != num_new_args: + return False + + for idx in range(1, num_init_args): + init_arg = init_sig.args[idx] + new_arg = new_sig.args[idx] + if init_arg.type != new_arg.type: + return False + + if init_arg.kind != new_arg.kind: + return False + + return True + + def prepare_init_method(cdef: ClassDef, ir: ClassIR, module_name: str, mapper: Mapper) -> None: # Set up a constructor decl init_node = cdef.info["__init__"].node + + new_node: SymbolNode | None = None + new_symbol = cdef.info.get("__new__") + # We are only interested in __new__ method defined in a user-defined class, + # so we ignore it if it comes from a builtin type. It's usually builtins.object + # but could also be builtins.type for metaclasses so we detect the prefix which + # matches both. + if new_symbol and new_symbol.fullname and not new_symbol.fullname.startswith("builtins."): + new_node = new_symbol.node + if isinstance(new_node, (Decorator, OverloadedFuncDef)): + new_node = get_func_def(new_node) if not ir.is_trait and not ir.builtin_base and isinstance(init_node, FuncDef): init_sig = mapper.fdef_to_sig(init_node, True) + args_match = True + if isinstance(new_node, FuncDef): + new_sig = mapper.fdef_to_sig(new_node, True) + args_match = check_matching_args(init_sig, new_sig) defining_ir = mapper.type_to_ir.get(init_node.info) # If there is a nontrivial __init__ that wasn't defined in an # extension class, we need to make the constructor take *args, # **kwargs so it can call tp_init. if ( - defining_ir is None - or not defining_ir.is_ext_class - or cdef.info["__init__"].plugin_generated - ) and init_node.info.fullname != "builtins.object": + ( + defining_ir is None + or not defining_ir.is_ext_class + or cdef.info["__init__"].plugin_generated + ) + and init_node.info.fullname != "builtins.object" + ) or not args_match: init_sig = FuncSignature( [ init_sig.args[0], diff --git a/mypyc/lib-rt/misc_ops.c b/mypyc/lib-rt/misc_ops.c index b7593491a6e61..ca09c347b4ffe 100644 --- a/mypyc/lib-rt/misc_ops.c +++ b/mypyc/lib-rt/misc_ops.c @@ -227,6 +227,17 @@ PyObject *CPyType_FromTemplate(PyObject *template, if (!name) goto error; + if (template_->tp_doc) { + // cpython expects tp_doc to be heap-allocated so convert it here to + // avoid segfaults on deallocation. + Py_ssize_t size = strlen(template_->tp_doc) + 1; + char *doc = (char *)PyMem_Malloc(size); + if (!doc) + goto error; + memcpy(doc, template_->tp_doc, size); + template_->tp_doc = doc; + } + // Allocate the type and then copy the main stuff in. t = (PyHeapTypeObject*)PyType_GenericAlloc(&PyType_Type, 0); if (!t) diff --git a/mypyc/test-data/fixtures/ir.py b/mypyc/test-data/fixtures/ir.py index c041c661741c2..fb5512b772793 100644 --- a/mypyc/test-data/fixtures/ir.py +++ b/mypyc/test-data/fixtures/ir.py @@ -3,7 +3,7 @@ import _typeshed from typing import ( - TypeVar, Generic, List, Iterator, Iterable, Dict, Optional, Tuple, Any, Set, + Self, TypeVar, Generic, List, Iterator, Iterable, Dict, Optional, Tuple, Any, Set, overload, Mapping, Union, Callable, Sequence, FrozenSet, Protocol ) @@ -40,9 +40,11 @@ def __pow__(self, other: T_contra, modulo: _M) -> T_co: ... class object: __class__: type + def __new__(cls) -> Self: pass def __init__(self) -> None: pass def __eq__(self, x: object) -> bool: pass def __ne__(self, x: object) -> bool: pass + def __str__(self) -> str: pass class type: def __init__(self, o: object) -> None: ... diff --git a/mypyc/test-data/fixtures/typing-full.pyi b/mypyc/test-data/fixtures/typing-full.pyi index d37129bc2e0ba..8d89e4f93bc9b 100644 --- a/mypyc/test-data/fixtures/typing-full.pyi +++ b/mypyc/test-data/fixtures/typing-full.pyi @@ -32,6 +32,7 @@ Final = 0 TypedDict = 0 NoReturn = 0 NewType = 0 +Self = 0 Callable: _SpecialForm Union: _SpecialForm Literal: _SpecialForm diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index a573bb8e06685..bb55958dc6dcc 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -1659,3 +1659,78 @@ def native_class(x): L0: r0 = CPy_TYPE(x) return r0 + +[case testDunderNew] +from __future__ import annotations + +class Test: + val: int + + def __new__(cls, val: int) -> Test: + obj = super().__new__(cls) + obj.val = val + return obj + +def fn() -> Test: + return Test.__new__(Test, 42) + +class NewClassMethod: + val: int + + @classmethod + def __new__(cls, val: int) -> NewClassMethod: + obj = super().__new__(cls) + obj.val = val + return obj + +def fn2() -> NewClassMethod: + return NewClassMethod.__new__(42) + +[out] +def Test.__new__(cls, val): + cls :: object + val :: int + r0, obj :: __main__.Test + r1 :: bool +L0: + r0 = __mypyc__Test_setup(cls) + obj = r0 + obj.val = val; r1 = is_error + return obj +def fn(): + r0 :: object + r1 :: __main__.Test +L0: + r0 = __main__.Test :: type + r1 = Test.__new__(r0, 84) + return r1 +def NewClassMethod.__new__(cls, val): + cls :: object + val :: int + r0, obj :: __main__.NewClassMethod + r1 :: bool +L0: + r0 = __mypyc__NewClassMethod_setup(cls) + obj = r0 + obj.val = val; r1 = is_error + return obj +def fn2(): + r0 :: object + r1 :: __main__.NewClassMethod +L0: + r0 = __main__.NewClassMethod :: type + r1 = NewClassMethod.__new__(r0, 84) + return r1 + +[case testUnsupportedDunderNew] +from __future__ import annotations +from mypy_extensions import mypyc_attr + +@mypyc_attr(native_class=False) +class NonNative: + def __new__(cls) -> NonNative: + return super().__new__(cls) # E: super().__new__() not supported for non-extension classes + +class InheritsPython(dict): + def __new__(cls) -> InheritsPython: + return super().__new__(cls) # E: super().__new__() not supported for classes inheriting from non-native classes diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 2e55ee70687ea..b25dc9458fd16 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -3365,3 +3365,432 @@ def test_python_class() -> None: [file dynamic.py] class Dyn: pass + +[case testDunderNew] +from __future__ import annotations +from typing import Any, Union + +from testutil import assertRaises + +class Add: + l: IntLike + r: IntLike + + def __new__(cls, l: IntLike, r: IntLike) -> Any: + return ( + l if r == 0 else + r if l == 0 else + super().__new__(cls) + ) + + def __init__(self, l: IntLike, r: IntLike): + self.l = l + self.r = r + +IntLike = Union[int, Add] + +class RaisesException: + def __new__(cls, val: int) -> RaisesException: + if val == 0: + raise RuntimeError("Invalid value!") + return super().__new__(cls) + + def __init__(self, val: int) -> None: + self.val = val + +class ClsArgNotPassed: + def __new__(cls) -> Any: + return super().__new__(str) + +def test_dunder_new() -> None: + add_instance: Any = Add(1, 5) + assert type(add_instance) == Add + assert add_instance.l == 1 + assert add_instance.r == 5 + + # TODO: explicit types should not be needed but mypy does not use + # the return type of __new__ which makes mypyc add casts to Add. + right_int: Any = Add(0, 5) + assert type(right_int) == int + assert right_int == 5 + + left_int: Any = Add(1, 0) + assert type(left_int) == int + assert left_int == 1 + + with assertRaises(RuntimeError, "Invalid value!"): + raised = RaisesException(0) + + not_raised = RaisesException(1) + assert not_raised.val == 1 + + with assertRaises(TypeError, "object.__new__(str) is not safe, use str.__new__()"): + str_as_cls = ClsArgNotPassed() + + +[case testDunderNewInInterpreted] +from __future__ import annotations +from typing import Any, Union + +class Add: + l: IntLike + r: IntLike + + def __new__(cls, l: IntLike, r: IntLike) -> Any: + print(f'running __new__ with {l} and {r}') + + return ( + l if r == 0 else + r if l == 0 else + super().__new__(cls) + ) + + def __init__(self, l: IntLike, r: IntLike): + self.l = l + self.r = r + + def __repr__(self) -> str: + return f'({self.l} + {self.r})' + +IntLike = Union[int, Add] + +class RaisesException: + def __new__(cls, val: int) -> RaisesException: + if val == 0: + raise RuntimeError("Invalid value!") + return super().__new__(cls) + + def __init__(self, val: int) -> None: + self.val = val + +class ClsArgNotPassed: + def __new__(cls) -> Any: + return super().__new__(str) + +[file driver.py] +from native import Add, ClsArgNotPassed, RaisesException + +from testutil import assertRaises + +print(f'{Add(1, 5)=}') +print(f'{Add(0, 5)=}') +print(f'{Add(1, 0)=}') + +with assertRaises(RuntimeError, "Invalid value!"): + raised = RaisesException(0) + +not_raised = RaisesException(1) +assert not_raised.val == 1 + +with assertRaises(TypeError, "object.__new__(str) is not safe, use str.__new__()"): + str_as_cls = ClsArgNotPassed() + +[out] +running __new__ with 1 and 5 +Add(1, 5)=(1 + 5) +running __new__ with 0 and 5 +Add(0, 5)=5 +running __new__ with 1 and 0 +Add(1, 0)=1 + +[case testInheritedDunderNew] +from __future__ import annotations +from mypy_extensions import mypyc_attr +from typing_extensions import Self + +from m import interpreted_subclass + +@mypyc_attr(allow_interpreted_subclasses=True) +class Base: + val: int + + def __new__(cls, val: int) -> Self: + obj = super().__new__(cls) + obj.val = val + 1 + return obj + + def __init__(self, val: int) -> None: + self.init_val = val + +class Sub(Base): + def __new__(cls, val: int) -> Self: + return super().__new__(cls, val + 1) + + def __init__(self, val: int) -> None: + super().__init__(val) + self.init_val = self.init_val * 2 + +class SubWithoutNew(Base): + def __init__(self, val: int) -> None: + super().__init__(val) + self.init_val = self.init_val * 2 + +class BaseWithoutInterpretedSubclasses: + val: int + + def __new__(cls, val: int) -> Self: + obj = super().__new__(cls) + obj.val = val + 1 + return obj + + def __init__(self, val: int) -> None: + self.init_val = val + +class SubNoInterpreted(BaseWithoutInterpretedSubclasses): + def __new__(cls, val: int) -> Self: + return super().__new__(cls, val + 1) + + def __init__(self, val: int) -> None: + super().__init__(val) + self.init_val = self.init_val * 2 + +class SubNoInterpretedWithoutNew(BaseWithoutInterpretedSubclasses): + def __init__(self, val: int) -> None: + super().__init__(val) + self.init_val = self.init_val * 2 + +def test_inherited_dunder_new() -> None: + b = Base(42) + assert type(b) == Base + assert b.val == 43 + assert b.init_val == 42 + + s = Sub(42) + assert type(s) == Sub + assert s.val == 44 + assert s.init_val == 84 + + s2 = SubWithoutNew(42) + assert type(s2) == SubWithoutNew + assert s2.val == 43 + assert s2.init_val == 84 + +def test_inherited_dunder_new_without_interpreted_subclasses() -> None: + b = BaseWithoutInterpretedSubclasses(42) + assert type(b) == BaseWithoutInterpretedSubclasses + assert b.val == 43 + assert b.init_val == 42 + + s = SubNoInterpreted(42) + assert type(s) == SubNoInterpreted + assert s.val == 44 + assert s.init_val == 84 + + s2 = SubNoInterpretedWithoutNew(42) + assert type(s2) == SubNoInterpretedWithoutNew + assert s2.val == 43 + assert s2.init_val == 84 + +def test_interpreted_subclass() -> None: + interpreted_subclass(Base) + +[file m.py] +from __future__ import annotations +from typing_extensions import Self + +def interpreted_subclass(base) -> None: + b = base(42) + assert type(b) == base + assert b.val == 43 + assert b.init_val == 42 + + class InterpretedSub(base): + def __new__(cls, val: int) -> Self: + return super().__new__(cls, val + 1) + + def __init__(self, val: int) -> None: + super().__init__(val) + self.init_val : int = self.init_val * 2 + + s = InterpretedSub(42) + assert type(s) == InterpretedSub + assert s.val == 44 + assert s.init_val == 84 + + class InterpretedSubWithoutNew(base): + def __init__(self, val: int) -> None: + super().__init__(val) + self.init_val : int = self.init_val * 2 + + s2 = InterpretedSubWithoutNew(42) + assert type(s2) == InterpretedSubWithoutNew + assert s2.val == 43 + assert s2.init_val == 84 + +[typing fixtures/typing-full.pyi] + +[case testDunderNewInitArgMismatch] +from __future__ import annotations +from testutil import assertRaises + +class Test0: + @classmethod + def __new__(cls, val: int = 42) -> Test0: + obj = super().__new__(cls) + obj.val = val + return obj + + def __init__(self) -> None: + self.val = 0 + +class Test1: + def __new__(cls, val: int) -> Test1: + obj = super().__new__(cls) + obj.val = val + return obj + + def __init__(self) -> None: + self.val = 0 + +class Test2: + def __new__(cls) -> Test2: + obj = super().__new__(cls) + return obj + + def __init__(self, val: int) -> None: + self.val = val + +def test_arg_mismatch() -> None: + t0 = Test0() + assert t0.val == 0 + t0 = Test0.__new__(1) + assert t0.val == 1 + with assertRaises(TypeError, "__new__() missing required argument 'val'"): + t1 = Test1() + t1 = Test1.__new__(Test1, 2) + assert t1.val == 2 + with assertRaises(TypeError, "__new__() takes at most 0 arguments"): + t2 = Test2(42) + t2 = Test2.__new__(Test2) + with assertRaises(AttributeError, "attribute 'val' of 'Test2' undefined"): + print(t2.val) + +[case testDunderNewInitArgMismatchInInterpreted] +from __future__ import annotations + +class Test0: + # TODO: It should be possible to annotate '@classmethod' here + # but when it's added calling __new__ in interpreted code + # without the explicit type param results in a TypeError. + def __new__(cls, val: int = 42) -> Test0: + obj = super().__new__(cls) + obj.val = val + return obj + + def __init__(self) -> None: + self.val = 0 + +class Test1: + def __new__(cls, val: int) -> Test1: + obj = super().__new__(cls) + obj.val = val + return obj + + def __init__(self) -> None: + self.val = 0 + +class Test2: + def __new__(cls) -> Test2: + obj = super().__new__(cls) + return obj + + def __init__(self, val: int) -> None: + self.val = val + +[file driver.py] +from native import Test0, Test1, Test2 +from testutil import assertRaises + +t0 = Test0() +assert t0.val == 0 +t0 = Test0.__new__(Test0, 1) +assert t0.val == 1 +with assertRaises(TypeError, "__new__() missing required argument 'val'"): + t1 = Test1() +t1 = Test1.__new__(Test1, 2) +assert t1.val == 2 +with assertRaises(TypeError, "__new__() takes at most 0 arguments"): + t2 = Test2(42) +t2 = Test2.__new__(Test2) +with assertRaises(AttributeError, "attribute 'val' of 'Test2' undefined"): + print(t2.val) + +[case testDunderNewAttributeAccess] +from __future__ import annotations + +from mypy_extensions import u8 +from testutil import assertRaises + +class Test: + native: int + generic: object + bitfield: u8 + default: int = 5 + + def __new__(cls, native: int, generic: object, bitfield: u8) -> Test: + obj = super().__new__(cls) + + with assertRaises(AttributeError, "attribute 'native' of 'Test' undefined"): + print(obj.native) + with assertRaises(AttributeError, "attribute 'generic' of 'Test' undefined"): + print(obj.generic) + with assertRaises(AttributeError, "attribute 'bitfield' of 'Test' undefined"): + print(obj.bitfield) + + obj.native = native + obj.generic = generic + obj.bitfield = bitfield + + obj.native = obj.native + 1 + obj.generic = obj.generic.__str__() + obj.bitfield = obj.bitfield & 0x0F + obj.default = obj.default * 2 + return obj + +def test_attribute_access() -> None: + t = Test(42, {}, 0xCC) + assert t.native == 43 + assert t.generic == "{}" + assert t.bitfield == 0x0C + assert t.default == 10 + +[case testDunderNewAttributeAccessInInterpreted] +from __future__ import annotations + +from mypy_extensions import u8 +from testutil import assertRaises + +class Test: + native: int + generic: object + bitfield: u8 + default: int = 5 + + def __new__(cls, native: int, generic: object, bitfield: u8) -> Test: + obj = super().__new__(cls) + + with assertRaises(AttributeError, "attribute 'native' of 'Test' undefined"): + print(obj.native) + with assertRaises(AttributeError, "attribute 'generic' of 'Test' undefined"): + print(obj.generic) + with assertRaises(AttributeError, "attribute 'bitfield' of 'Test' undefined"): + print(obj.bitfield) + + obj.native = native + obj.generic = generic + obj.bitfield = bitfield + + obj.native = obj.native + 1 + obj.generic = obj.generic.__str__() + obj.bitfield = obj.bitfield & 0x0F + obj.default = obj.default * 2 + return obj + +[file driver.py] +from native import Test + +t = Test(42, {}, 0xCC) +assert t.native == 43 +assert t.generic == "{}" +assert t.bitfield == 0x0C +assert t.default == 10 From 4847ec8fcd98b041125a0a08f72b2f02a9e5be71 Mon Sep 17 00:00:00 2001 From: Brian Schubert Date: Fri, 29 Aug 2025 19:38:04 -0400 Subject: [PATCH 216/424] Fix forward references in type parameters of overparameterized PEP 695 type aliases (#19725) Fixes https://github.com/python/mypy/issues/19734 Make semanal defer for placeholders in the upper bounds / constraints, even when the type variable doesn't appear in the target. https://discuss.python.org/t/mypy-raises-exception-for-certain-type-alias-definitions-with-parameters-bounded-by-a-not-yet-defined-type/103305 --- mypy/semanal.py | 3 ++- test-data/unit/check-python312.test | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index fa5d9fdc82c44..213c8d04122d7 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -5598,7 +5598,8 @@ def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None: else: incomplete_target = has_placeholder(res) - if self.found_incomplete_ref(tag) or incomplete_target: + incomplete_tv = any(has_placeholder(tv) for tv in alias_tvars) + if self.found_incomplete_ref(tag) or incomplete_target or incomplete_tv: # Since we have got here, we know this must be a type alias (incomplete refs # may appear in nested positions), therefore use becomes_typeinfo=True. self.mark_incomplete(s.name.name, s.value, becomes_typeinfo=True) diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 01364bdfa32af..382822ced8617 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -2112,3 +2112,12 @@ if T(x) is str: # E: "TypeAliasType" not callable reveal_type(x) # N: Revealed type is "builtins.object" [builtins fixtures/tuple.pyi] [typing fixtures/typing-full.pyi] + +[case testPEP695TypeAliasForwardReferenceInUnusedTypeVar] +# https://discuss.python.org/t/103305 +type Alias1[T: "A"] = int +type Alias2[T: ("A", int)] = int +class A: ... + +x1: Alias1[A] # ok +x2: Alias2[A] # ok From 06f97b256e760d68b067510d6213db7af372ab73 Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Sat, 30 Aug 2025 11:23:25 +0200 Subject: [PATCH 217/424] Do not display import-related errors after module-level always false assert (#19347) Fixes #19346 --- mypy/semanal_pass1.py | 3 +++ test-data/unit/check-incremental.test | 9 +++++++++ test-data/unit/check-unreachable-code.test | 16 ++++++++++++++++ 3 files changed, 28 insertions(+) diff --git a/mypy/semanal_pass1.py b/mypy/semanal_pass1.py index aaa01969217ae..266fd236a01f6 100644 --- a/mypy/semanal_pass1.py +++ b/mypy/semanal_pass1.py @@ -74,6 +74,9 @@ def visit_file(self, file: MypyFile, fnam: str, mod_id: str, options: Options) - if last.end_line is not None: # We are on a Python version recent enough to support end lines. self.skipped_lines |= set(range(next_def.line, last.end_line + 1)) + file.imports = [ + i for i in file.imports if (i.line, i.column) <= (defn.line, defn.column) + ] del file.defs[i + 1 :] break file.skipped_lines = self.skipped_lines diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index c3fe98e69d95f..defe7402730f0 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -6888,3 +6888,12 @@ class A: [out] [out2] main:3: error: Too few arguments + +[case testUnreachableAfterToplevelAssertImportThirdParty] +# flags: --platform unknown +import sys +assert sys.platform == 'linux' +import does_not_exist +[builtins fixtures/ops.pyi] +[out] +[out2] diff --git a/test-data/unit/check-unreachable-code.test b/test-data/unit/check-unreachable-code.test index f425410a97748..645f81e89ca13 100644 --- a/test-data/unit/check-unreachable-code.test +++ b/test-data/unit/check-unreachable-code.test @@ -778,6 +778,22 @@ assert sys.platform == 'lol' def bar() -> None: pass [builtins fixtures/ops.pyi] +[case testUnreachableAfterToplevelAssertImportThirdParty] +# flags: --platform unknown +import sys +assert sys.platform == 'linux' +import does_not_exist +[builtins fixtures/ops.pyi] + +[case testUnreachableAfterToplevelAssertImportThirdParty2] +# flags: --platform unknown +import sys +import bad; assert sys.platform == 'linux'; import does_not_exist +[builtins fixtures/ops.pyi] +[out] +main:3: error: Cannot find implementation or library stub for module named "bad" +main:3: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + [case testUnreachableAfterToplevelAssertNotInsideIf] import sys if sys.version_info[0] >= 2: From 618cf55d4b454dc0b64b06ed387a0f5c70bc0cfe Mon Sep 17 00:00:00 2001 From: Christoph Tyralla Date: Sat, 30 Aug 2025 11:25:16 +0200 Subject: [PATCH 218/424] `--strict-equality` for `None` (#19718) Fixes #18386 (at least partly) (edited) In a first test run, in which I included checks against `None` in `--strict-equality`, the Mypy primer gave hundreds of new `comparison-overlap` reports. Many of them seem really helpful (including those for the Mypy source code itself), because it is often hard to tell if non-overlapping `None` checks are just remnants of incomplete refactorings or can handle cases with corrupted data or similar issues. As it was only a little effort, I decided to add the option `--strict-equality-for-none` to Mypy, which is disabled even in `--strict` mode. Other libraries could adjust to this new behaviour if and when they want. If many of them do so, we could eventually enable `--strict-equality-for-none` in `--strict` mode or even merge it with `--strict-equality` later. The remaining new true positives revealed by the Mypy primer are the result of no longer excluding types with custom `__eq__` methods for identity checks (which, in my opinion, makes sense even in case `--strict-equality-for-none` would be rejected). --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- docs/source/command_line.rst | 15 +++++++++++++- docs/source/config_file.rst | 10 ++++++++- docs/source/error_code_list2.rst | 23 +++++++++++++++++++++ mypy/checkexpr.py | 16 ++++++++++----- mypy/main.py | 11 +++++++++- mypy/options.py | 4 ++++ test-data/unit/check-expressions.test | 29 +++++++++++++++++++++++++++ 7 files changed, 100 insertions(+), 8 deletions(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index db2407e17df8b..c1b757a00ef20 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -728,9 +728,22 @@ of the above sections. if text != b'other bytes': # Error: non-overlapping equality check! ... - assert text is not None # OK, check against None is allowed as a special case. + assert text is not None # OK, check against None is allowed +.. option:: --strict-equality-for-none + + This flag extends :option:`--strict-equality ` for checks + against ``None``: + + .. code-block:: python + + text: str + assert text is not None # Error: non-overlapping identity check! + + Note that :option:`--strict-equality-for-none ` + only works in combination with :option:`--strict-equality `. + .. option:: --strict-bytes By default, mypy treats ``bytearray`` and ``memoryview`` as subtypes of ``bytes`` which diff --git a/docs/source/config_file.rst b/docs/source/config_file.rst index b4f134f26cb14..934e465a7c237 100644 --- a/docs/source/config_file.rst +++ b/docs/source/config_file.rst @@ -834,7 +834,15 @@ section of the command line docs. :default: False Prohibit equality checks, identity checks, and container checks between - non-overlapping types. + non-overlapping types (except ``None``). + +.. confval:: strict_equality_for_none + + :type: boolean + :default: False + + Include ``None`` in strict equality checks (requires :confval:`strict_equality` + to be activated). .. confval:: strict_bytes diff --git a/docs/source/error_code_list2.rst b/docs/source/error_code_list2.rst index 784c2ad728191..125671bc2bef4 100644 --- a/docs/source/error_code_list2.rst +++ b/docs/source/error_code_list2.rst @@ -145,6 +145,29 @@ literal: def is_magic(x: bytes) -> bool: return x == b'magic' # OK +:option:`--strict-equality ` does not include comparisons with +``None``: + +.. code-block:: python + + # mypy: strict-equality + + def is_none(x: str) -> bool: + return x is None # OK + +If you want such checks, you must also activate +:option:`--strict-equality-for-none ` (we might merge +these two options later). + +.. code-block:: python + + # mypy: strict-equality strict-equality-for-none + + def is_none(x: str) -> bool: + # Error: Non-overlapping identity check + # (left operand type: "str", right operand type: "None") + return x is None + .. _code-no-untyped-call: Check that no untyped functions are called [no-untyped-call] diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index fd83b6359ddc3..794af875867a5 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -3721,7 +3721,7 @@ def visit_comparison_expr(self, e: ComparisonExpr) -> Type: elif operator == "is" or operator == "is not": right_type = self.accept(right) # validate the right operand sub_result = self.bool_type() - if self.dangerous_comparison(left_type, right_type): + if self.dangerous_comparison(left_type, right_type, identity_check=True): # Show the most specific literal types possible left_type = try_getting_literal(left_type) right_type = try_getting_literal(right_type) @@ -3763,6 +3763,7 @@ def dangerous_comparison( original_container: Type | None = None, seen_types: set[tuple[Type, Type]] | None = None, prefer_literal: bool = True, + identity_check: bool = False, ) -> bool: """Check for dangerous non-overlapping comparisons like 42 == 'no'. @@ -3790,10 +3791,12 @@ def dangerous_comparison( left, right = get_proper_types((left, right)) - # We suppress the error if there is a custom __eq__() method on either - # side. User defined (or even standard library) classes can define this + # We suppress the error for equality and container checks if there is a custom __eq__() + # method on either side. User defined (or even standard library) classes can define this # to return True for comparisons between non-overlapping types. - if custom_special_method(left, "__eq__") or custom_special_method(right, "__eq__"): + if ( + custom_special_method(left, "__eq__") or custom_special_method(right, "__eq__") + ) and not identity_check: return False if prefer_literal: @@ -3817,7 +3820,10 @@ def dangerous_comparison( # # TODO: find a way of disabling the check only for types resulted from the expansion. return False - if isinstance(left, NoneType) or isinstance(right, NoneType): + if self.chk.options.strict_equality_for_none: + if isinstance(left, NoneType) and isinstance(right, NoneType): + return False + elif isinstance(left, NoneType) or isinstance(right, NoneType): return False if isinstance(left, UnionType) and isinstance(right, UnionType): left = remove_optional(left) diff --git a/mypy/main.py b/mypy/main.py index ad257bab69966..706d1daef6802 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -903,7 +903,16 @@ def add_invertible_flag( "--strict-equality", default=False, strict_flag=True, - help="Prohibit equality, identity, and container checks for non-overlapping types", + help="Prohibit equality, identity, and container checks for non-overlapping types " + "(except `None`)", + group=strictness_group, + ) + + add_invertible_flag( + "--strict-equality-for-none", + default=False, + strict_flag=False, + help="Extend `--strict-equality` for `None` checks", group=strictness_group, ) diff --git a/mypy/options.py b/mypy/options.py index ad4b26cca095f..b3dc9639a41d1 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -55,6 +55,7 @@ class BuildType: "mypyc", "strict_concatenate", "strict_equality", + "strict_equality_for_none", "strict_optional", "warn_no_return", "warn_return_any", @@ -230,6 +231,9 @@ def __init__(self) -> None: # This makes 1 == '1', 1 in ['1'], and 1 is '1' errors. self.strict_equality = False + # Extend the logic of `scrict_equality` for comparisons with `None`. + self.strict_equality_for_none = False + # Disable treating bytearray and memoryview as subtypes of bytes self.strict_bytes = False diff --git a/test-data/unit/check-expressions.test b/test-data/unit/check-expressions.test index 33271a3cc04c1..ea6eac9a39b3a 100644 --- a/test-data/unit/check-expressions.test +++ b/test-data/unit/check-expressions.test @@ -2419,6 +2419,35 @@ assert a == c [builtins fixtures/list.pyi] [typing fixtures/typing-full.pyi] +[case testStrictEqualityForNone] +# flags: --strict-equality --strict-equality-for-none + +class A: ... + +def a1(x: A) -> None: + assert x is None # E: Non-overlapping identity check (left operand type: "A", right operand type: "None") +def a2(x: A) -> None: + x is not None # E: Non-overlapping identity check (left operand type: "A", right operand type: "None") +def a3(x: A) -> None: + None == x # E: Non-overlapping equality check (left operand type: "None", right operand type: "A") +def a4(x: list[A]) -> None: + None in x # E: Non-overlapping container check (element type: "None", container item type: "A") + +class B: + def __eq__(self, x: object) -> bool: ... + +def b1(x: B) -> None: + assert x is None # E: Non-overlapping identity check (left operand type: "B", right operand type: "None") +def b2(x: B) -> None: + x is not None # E: Non-overlapping identity check (left operand type: "B", right operand type: "None") +def b3(x: B) -> None: + x == None +def b4(x: list[B]) -> None: + None in x + +[builtins fixtures/list.pyi] +[typing fixtures/typing-full.pyi] + [case testUnimportedHintAny] def f(x: Any) -> None: # E: Name "Any" is not defined \ # N: Did you forget to import it from "typing"? (Suggestion: "from typing import Any") From 03aa0f857373780f91e1c8b0a7a47a1414a75f28 Mon Sep 17 00:00:00 2001 From: CoolCat467 <52022020+CoolCat467@users.noreply.github.com> Date: Sat, 30 Aug 2025 08:02:13 -0500 Subject: [PATCH 219/424] Add idlemypyextension to IDE integrations section (#18615) In this pull request, we add idlemypyextension (https://github.com/CoolCat467/idlemypyextension) for the IDLE text editor to the Popular IDE integrations section in the README. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 45b71c8a4824b..8040566b18eff 100644 --- a/README.md +++ b/README.md @@ -142,6 +142,7 @@ Mypy can be integrated into popular IDEs: - Emacs: using [Flycheck](https://github.com/flycheck/) - Sublime Text: [SublimeLinter-contrib-mypy](https://github.com/fredcallaway/SublimeLinter-contrib-mypy) - PyCharm: [mypy plugin](https://github.com/dropbox/mypy-PyCharm-plugin) +- IDLE: [idlemypyextension](https://github.com/CoolCat467/idlemypyextension) - pre-commit: use [pre-commit mirrors-mypy](https://github.com/pre-commit/mirrors-mypy), although note by default this will limit mypy's ability to analyse your third party dependencies. From dcaca0e7dca053f3dff5c5c3ad3b1a65df572cce Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 30 Aug 2025 16:24:20 +0100 Subject: [PATCH 220/424] Use some macros in native_internal (#19757) This should be a no-op from semantics/format point if view. Essentially three things here: * Add some macros to hide boilerplate for common steps * Make use of some magic constants more obvious (most notably `2`) * Add some `likely()/unlikely()` directives (this actually seems to have some small positive effect). --- mypyc/lib-rt/native_internal.c | 269 ++++++++++++++------------------- 1 file changed, 113 insertions(+), 156 deletions(-) diff --git a/mypyc/lib-rt/native_internal.c b/mypyc/lib-rt/native_internal.c index 1c211464b19ca..b9bf899588fbc 100644 --- a/mypyc/lib-rt/native_internal.c +++ b/mypyc/lib-rt/native_internal.c @@ -16,6 +16,23 @@ #define MEDIUM_INT_TAG 1 #define LONG_INT_TAG 3 +#define CPY_BOOL_ERROR 2 +#define CPY_NONE_ERROR 2 +#define CPY_NONE 1 + +#define _CHECK_BUFFER(data, err) if (unlikely(_check_buffer(data) == CPY_NONE_ERROR)) \ + return err; +#define _CHECK_SIZE(data, need) if (unlikely(_check_size((BufferObject *)data, need) == CPY_NONE_ERROR)) \ + return CPY_NONE_ERROR; +#define _CHECK_READ(data, size, err) if (unlikely(_check_read((BufferObject *)data, size) == CPY_NONE_ERROR)) \ + return err; + +#define _READ(data, type) *(type *)(((BufferObject *)data)->buf + ((BufferObject *)data)->pos); \ + ((BufferObject *)data)->pos += sizeof(type); + +#define _WRITE(data, type, v) *(type *)(((BufferObject *)data)->buf + ((BufferObject *)data)->pos) = v; \ + ((BufferObject *)data)->pos += sizeof(type); + typedef struct { PyObject_HEAD Py_ssize_t pos; @@ -140,38 +157,38 @@ static PyTypeObject BufferType = { static inline char _check_buffer(PyObject *data) { - if (Py_TYPE(data) != &BufferType) { + if (unlikely(Py_TYPE(data) != &BufferType)) { PyErr_Format( PyExc_TypeError, "data must be a Buffer object, got %s", Py_TYPE(data)->tp_name ); - return 2; + return CPY_NONE_ERROR; } - return 1; + return CPY_NONE; } static inline char _check_size(BufferObject *data, Py_ssize_t need) { Py_ssize_t target = data->pos + need; if (target <= data->size) - return 1; + return CPY_NONE; do data->size *= 2; while (target >= data->size); data->buf = PyMem_Realloc(data->buf, data->size); - if (!data->buf) { + if (unlikely(data->buf == NULL)) { PyErr_NoMemory(); - return 2; + return CPY_NONE_ERROR; } - return 1; + return CPY_NONE; } static inline char _check_read(BufferObject *data, Py_ssize_t need) { - if (data->pos + need > data->end) { + if (unlikely(data->pos + need > data->end)) { PyErr_SetString(PyExc_ValueError, "reading past the buffer end"); - return 2; + return CPY_NONE_ERROR; } - return 1; + return CPY_NONE; } /* @@ -182,14 +199,9 @@ bool format: single byte static char read_bool_internal(PyObject *data) { - if (_check_buffer(data) == 2) - return 2; - - if (_check_read((BufferObject *)data, 1) == 2) - return 2; - char *buf = ((BufferObject *)data)->buf; - char res = buf[((BufferObject *)data)->pos]; - ((BufferObject *)data)->pos += 1; + _CHECK_BUFFER(data, CPY_BOOL_ERROR) + _CHECK_READ(data, 1, CPY_BOOL_ERROR) + char res = _READ(data, char) return res; } @@ -197,10 +209,10 @@ static PyObject* read_bool(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"data", NULL}; PyObject *data = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O", kwlist, &data)) + if (unlikely(!PyArg_ParseTupleAndKeywords(args, kwds, "O", kwlist, &data))) return NULL; char res = read_bool_internal(data); - if (res == 2) + if (unlikely(res == CPY_BOOL_ERROR)) return NULL; PyObject *retval = res ? Py_True : Py_False; Py_INCREF(retval); @@ -209,16 +221,11 @@ read_bool(PyObject *self, PyObject *args, PyObject *kwds) { static char write_bool_internal(PyObject *data, char value) { - if (_check_buffer(data) == 2) - return 2; - - if (_check_size((BufferObject *)data, 1) == 2) - return 2; - char *buf = ((BufferObject *)data)->buf; - buf[((BufferObject *)data)->pos] = value; - ((BufferObject *)data)->pos += 1; + _CHECK_BUFFER(data, CPY_NONE_ERROR) + _CHECK_SIZE(data, 1) + _WRITE(data, char, value) ((BufferObject *)data)->end += 1; - return 1; + return CPY_NONE; } static PyObject* @@ -226,13 +233,13 @@ write_bool(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"data", "value", NULL}; PyObject *data = NULL; PyObject *value = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO", kwlist, &data, &value)) + if (unlikely(!PyArg_ParseTupleAndKeywords(args, kwds, "OO", kwlist, &data, &value))) return NULL; - if (!PyBool_Check(value)) { + if (unlikely(!PyBool_Check(value))) { PyErr_SetString(PyExc_TypeError, "value must be a bool"); return NULL; } - if (write_bool_internal(data, value == Py_True) == 2) { + if (unlikely(write_bool_internal(data, value == Py_True) == CPY_NONE_ERROR)) { return NULL; } Py_INCREF(Py_None); @@ -247,30 +254,26 @@ str format: size followed by UTF-8 bytes static PyObject* read_str_internal(PyObject *data) { - if (_check_buffer(data) == 2) - return NULL; + _CHECK_BUFFER(data, NULL) - Py_ssize_t size; - char *buf = ((BufferObject *)data)->buf; // Read string length. - if (_check_read((BufferObject *)data, 1) == 2) - return NULL; - uint8_t first = *(uint8_t *)(buf + ((BufferObject *)data)->pos); - ((BufferObject *)data)->pos += 1; - if (first != LONG_STR_TAG) { + Py_ssize_t size; + _CHECK_READ(data, 1, NULL) + uint8_t first = _READ(data, uint8_t) + if (likely(first != LONG_STR_TAG)) { // Common case: short string (len <= 127). size = (Py_ssize_t)(first >> 1); } else { - size = *(Py_ssize_t *)(buf + ((BufferObject *)data)->pos); - ((BufferObject *)data)->pos += sizeof(Py_ssize_t); + _CHECK_READ(data, sizeof(CPyTagged), NULL) + size = _READ(data, Py_ssize_t) } // Read string content. - if (_check_read((BufferObject *)data, size) == 2) - return NULL; + char *buf = ((BufferObject *)data)->buf; + _CHECK_READ(data, size, NULL) PyObject *res = PyUnicode_FromStringAndSize( buf + ((BufferObject *)data)->pos, (Py_ssize_t)size ); - if (!res) + if (unlikely(res == NULL)) return NULL; ((BufferObject *)data)->pos += size; return res; @@ -280,47 +283,39 @@ static PyObject* read_str(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"data", NULL}; PyObject *data = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O", kwlist, &data)) + if (unlikely(!PyArg_ParseTupleAndKeywords(args, kwds, "O", kwlist, &data))) return NULL; return read_str_internal(data); } static char write_str_internal(PyObject *data, PyObject *value) { - if (_check_buffer(data) == 2) - return 2; + _CHECK_BUFFER(data, CPY_NONE_ERROR) Py_ssize_t size; const char *chunk = PyUnicode_AsUTF8AndSize(value, &size); - if (!chunk) - return 2; + if (unlikely(chunk == NULL)) + return CPY_NONE_ERROR; Py_ssize_t need; - char *buf; // Write string length. - if (size <= MAX_SHORT_LEN) { + if (likely(size <= MAX_SHORT_LEN)) { // Common case: short string (len <= 127) store as single byte. need = size + 1; - if (_check_size((BufferObject *)data, need) == 2) - return 2; - buf = ((BufferObject *)data)->buf; - *(uint8_t *)(buf + ((BufferObject *)data)->pos) = (uint8_t)size << 1; - ((BufferObject *)data)->pos += 1; + _CHECK_SIZE(data, need) + _WRITE(data, uint8_t, (uint8_t)size << 1) } else { need = size + sizeof(Py_ssize_t) + 1; - if (_check_size((BufferObject *)data, need) == 2) - return 2; - buf = ((BufferObject *)data)->buf; - *(uint8_t *)(buf + ((BufferObject *)data)->pos) = LONG_STR_TAG; - ((BufferObject *)data)->pos += 1; - *(Py_ssize_t *)(buf + ((BufferObject *)data)->pos) = size; - ((BufferObject *)data)->pos += sizeof(Py_ssize_t); + _CHECK_SIZE(data, need) + _WRITE(data, uint8_t, LONG_STR_TAG) + _WRITE(data, Py_ssize_t, size) } // Write string content. + char *buf = ((BufferObject *)data)->buf; memcpy(buf + ((BufferObject *)data)->pos, chunk, size); ((BufferObject *)data)->pos += size; ((BufferObject *)data)->end += need; - return 1; + return CPY_NONE; } static PyObject* @@ -328,13 +323,13 @@ write_str(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"data", "value", NULL}; PyObject *data = NULL; PyObject *value = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO", kwlist, &data, &value)) + if (unlikely(!PyArg_ParseTupleAndKeywords(args, kwds, "OO", kwlist, &data, &value))) return NULL; - if (!PyUnicode_Check(value)) { + if (unlikely(!PyUnicode_Check(value))) { PyErr_SetString(PyExc_TypeError, "value must be a str"); return NULL; } - if (write_str_internal(data, value) == 2) { + if (unlikely(write_str_internal(data, value) == CPY_NONE_ERROR)) { return NULL; } Py_INCREF(Py_None); @@ -348,14 +343,9 @@ float format: static double read_float_internal(PyObject *data) { - if (_check_buffer(data) == 2) - return CPY_FLOAT_ERROR; - - if (_check_read((BufferObject *)data, sizeof(double)) == 2) - return CPY_FLOAT_ERROR; - char *buf = ((BufferObject *)data)->buf; - double res = *(double *)(buf + ((BufferObject *)data)->pos); - ((BufferObject *)data)->pos += sizeof(double); + _CHECK_BUFFER(data, CPY_FLOAT_ERROR) + _CHECK_READ(data, sizeof(double), CPY_FLOAT_ERROR) + double res = _READ(data, double); return res; } @@ -363,10 +353,10 @@ static PyObject* read_float(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"data", NULL}; PyObject *data = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O", kwlist, &data)) + if (unlikely(!PyArg_ParseTupleAndKeywords(args, kwds, "O", kwlist, &data))) return NULL; double retval = read_float_internal(data); - if (retval == CPY_FLOAT_ERROR && PyErr_Occurred()) { + if (unlikely(retval == CPY_FLOAT_ERROR && PyErr_Occurred())) { return NULL; } return PyFloat_FromDouble(retval); @@ -374,16 +364,11 @@ read_float(PyObject *self, PyObject *args, PyObject *kwds) { static char write_float_internal(PyObject *data, double value) { - if (_check_buffer(data) == 2) - return 2; - - if (_check_size((BufferObject *)data, sizeof(double)) == 2) - return 2; - char *buf = ((BufferObject *)data)->buf; - *(double *)(buf + ((BufferObject *)data)->pos) = value; - ((BufferObject *)data)->pos += sizeof(double); + _CHECK_BUFFER(data, CPY_NONE_ERROR) + _CHECK_SIZE(data, sizeof(double)) + _WRITE(data, double, value) ((BufferObject *)data)->end += sizeof(double); - return 1; + return CPY_NONE; } static PyObject* @@ -391,13 +376,13 @@ write_float(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"data", "value", NULL}; PyObject *data = NULL; PyObject *value = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO", kwlist, &data, &value)) + if (unlikely(!PyArg_ParseTupleAndKeywords(args, kwds, "OO", kwlist, &data, &value))) return NULL; - if (!PyFloat_Check(value)) { + if (unlikely(!PyFloat_Check(value))) { PyErr_SetString(PyExc_TypeError, "value must be a float"); return NULL; } - if (write_float_internal(data, PyFloat_AsDouble(value)) == 2) { + if (unlikely(write_float_internal(data, PyFloat_AsDouble(value)) == CPY_NONE_ERROR)) { return NULL; } Py_INCREF(Py_None); @@ -413,27 +398,22 @@ int format: static CPyTagged read_int_internal(PyObject *data) { - if (_check_buffer(data) == 2) - return CPY_INT_TAG; + _CHECK_BUFFER(data, CPY_INT_TAG) + _CHECK_READ(data, 1, CPY_INT_TAG) - char *buf = ((BufferObject *)data)->buf; - if (_check_read((BufferObject *)data, 1) == 2) - return CPY_INT_TAG; - - uint8_t first = *(uint8_t *)(buf + ((BufferObject *)data)->pos); - ((BufferObject *)data)->pos += 1; + uint8_t first = _READ(data, uint8_t) if ((first & MEDIUM_INT_TAG) == 0) { // Most common case: int that is small in absolute value. return ((Py_ssize_t)(first >> 1) + MIN_SHORT_INT) << 1; } if (first == MEDIUM_INT_TAG) { - CPyTagged ret = *(CPyTagged *)(buf + ((BufferObject *)data)->pos); - ((BufferObject *)data)->pos += sizeof(CPyTagged); + _CHECK_READ(data, sizeof(CPyTagged), CPY_INT_TAG) + CPyTagged ret = _READ(data, CPyTagged) return ret; } // People who have literal ints not fitting in size_t should be punished :-) PyObject *str_ret = read_str_internal(data); - if (str_ret == NULL) + if (unlikely(str_ret == NULL)) return CPY_INT_TAG; PyObject* ret_long = PyLong_FromUnicodeObject(str_ret, 10); Py_DECREF(str_ret); @@ -444,10 +424,10 @@ static PyObject* read_int(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"data", NULL}; PyObject *data = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O", kwlist, &data)) + if (unlikely(!PyArg_ParseTupleAndKeywords(args, kwds, "O", kwlist, &data))) return NULL; CPyTagged retval = read_int_internal(data); - if (retval == CPY_INT_TAG) { + if (unlikely(retval == CPY_INT_TAG)) { return NULL; } return CPyTagged_StealAsObject(retval); @@ -455,46 +435,34 @@ read_int(PyObject *self, PyObject *args, PyObject *kwds) { static char write_int_internal(PyObject *data, CPyTagged value) { - if (_check_buffer(data) == 2) - return 2; + _CHECK_BUFFER(data, CPY_NONE_ERROR) - char *buf; - if ((value & CPY_INT_TAG) == 0) { + if (likely((value & CPY_INT_TAG) == 0)) { Py_ssize_t real_value = CPyTagged_ShortAsSsize_t(value); if (real_value >= MIN_SHORT_INT && real_value <= MAX_SHORT_INT) { // Most common case: int that is small in absolute value. - if (_check_size((BufferObject *)data, 1) == 2) - return 2; - buf = ((BufferObject *)data)->buf; - *(uint8_t *)(buf + ((BufferObject *)data)->pos) = (uint8_t)(real_value - MIN_SHORT_INT) << 1; - ((BufferObject *)data)->pos += 1; + _CHECK_SIZE(data, 1) + _WRITE(data, uint8_t, (uint8_t)(real_value - MIN_SHORT_INT) << 1) ((BufferObject *)data)->end += 1; } else { - if (_check_size((BufferObject *)data, sizeof(CPyTagged) + 1) == 2) - return 2; - buf = ((BufferObject *)data)->buf; - *(uint8_t *)(buf + ((BufferObject *)data)->pos) = MEDIUM_INT_TAG; - ((BufferObject *)data)->pos += 1; - *(CPyTagged *)(buf + ((BufferObject *)data)->pos) = value; - ((BufferObject *)data)->pos += sizeof(CPyTagged); + _CHECK_SIZE(data, sizeof(CPyTagged) + 1) + _WRITE(data, uint8_t, MEDIUM_INT_TAG) + _WRITE(data, CPyTagged, value) ((BufferObject *)data)->end += sizeof(CPyTagged) + 1; } } else { - if (_check_size((BufferObject *)data, 1) == 2) - return 2; - buf = ((BufferObject *)data)->buf; - *(uint8_t *)(buf + ((BufferObject *)data)->pos) = LONG_INT_TAG; - ((BufferObject *)data)->pos += 1; + _CHECK_SIZE(data, 1) + _WRITE(data, uint8_t, LONG_INT_TAG) ((BufferObject *)data)->end += 1; PyObject *str_value = PyObject_Str(CPyTagged_LongAsObject(value)); - if (str_value == NULL) - return 2; + if (unlikely(str_value == NULL)) + return CPY_NONE_ERROR; char res = write_str_internal(data, str_value); Py_DECREF(str_value); - if (res == 2) - return 2; + if (unlikely(res == CPY_NONE_ERROR)) + return CPY_NONE_ERROR; } - return 1; + return CPY_NONE; } static PyObject* @@ -502,14 +470,14 @@ write_int(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"data", "value", NULL}; PyObject *data = NULL; PyObject *value = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO", kwlist, &data, &value)) + if (unlikely(!PyArg_ParseTupleAndKeywords(args, kwds, "OO", kwlist, &data, &value))) return NULL; - if (!PyLong_Check(value)) { + if (unlikely(!PyLong_Check(value))) { PyErr_SetString(PyExc_TypeError, "value must be an int"); return NULL; } CPyTagged tagged_value = CPyTagged_BorrowFromObject(value); - if (write_int_internal(data, tagged_value) == 2) { + if (unlikely(write_int_internal(data, tagged_value) == CPY_NONE_ERROR)) { return NULL; } Py_INCREF(Py_None); @@ -523,15 +491,9 @@ integer tag format (0 <= t <= 255): static uint8_t read_tag_internal(PyObject *data) { - if (_check_buffer(data) == 2) - return CPY_LL_UINT_ERROR; - - if (_check_read((BufferObject *)data, 1) == 2) - return CPY_LL_UINT_ERROR; - char *buf = ((BufferObject *)data)->buf; - - uint8_t ret = *(uint8_t *)(buf + ((BufferObject *)data)->pos); - ((BufferObject *)data)->pos += 1; + _CHECK_BUFFER(data, CPY_LL_UINT_ERROR) + _CHECK_READ(data, 1, CPY_LL_UINT_ERROR) + uint8_t ret = _READ(data, uint8_t) return ret; } @@ -539,10 +501,10 @@ static PyObject* read_tag(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"data", NULL}; PyObject *data = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O", kwlist, &data)) + if (unlikely(!PyArg_ParseTupleAndKeywords(args, kwds, "O", kwlist, &data))) return NULL; uint8_t retval = read_tag_internal(data); - if (retval == CPY_LL_UINT_ERROR && PyErr_Occurred()) { + if (unlikely(retval == CPY_LL_UINT_ERROR && PyErr_Occurred())) { return NULL; } return PyLong_FromLong(retval); @@ -550,16 +512,11 @@ read_tag(PyObject *self, PyObject *args, PyObject *kwds) { static char write_tag_internal(PyObject *data, uint8_t value) { - if (_check_buffer(data) == 2) - return 2; - - if (_check_size((BufferObject *)data, 1) == 2) - return 2; - uint8_t *buf = (uint8_t *)((BufferObject *)data)->buf; - *(buf + ((BufferObject *)data)->pos) = value; - ((BufferObject *)data)->pos += 1; + _CHECK_BUFFER(data, CPY_NONE_ERROR) + _CHECK_SIZE(data, 1) + _WRITE(data, uint8_t, value) ((BufferObject *)data)->end += 1; - return 1; + return CPY_NONE; } static PyObject* @@ -567,14 +524,14 @@ write_tag(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"data", "value", NULL}; PyObject *data = NULL; PyObject *value = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO", kwlist, &data, &value)) + if (unlikely(!PyArg_ParseTupleAndKeywords(args, kwds, "OO", kwlist, &data, &value))) return NULL; uint8_t unboxed = CPyLong_AsUInt8(value); - if (unboxed == CPY_LL_UINT_ERROR && PyErr_Occurred()) { + if (unlikely(unboxed == CPY_LL_UINT_ERROR && PyErr_Occurred())) { CPy_TypeError("u8", value); return NULL; } - if (write_tag_internal(data, unboxed) == 2) { + if (unlikely(write_tag_internal(data, unboxed) == CPY_NONE_ERROR)) { return NULL; } Py_INCREF(Py_None); From 1660a392a79a8e36b82e2f572d2af3af468d7874 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 30 Aug 2025 23:44:35 +0100 Subject: [PATCH 221/424] Try fixing test times after GC hack (#19766) I am not sure what happens, but for some reason after GC `freeze()`/`unfreeze()` hack https://github.com/python/mypy/pull/19681 was merged, compiled tests are running twice slower (on GH runner, but I also see much smaller but visible slow-down locally). I have two theories: * The constant overhead we add outweighs the savings when running thousands of tiny builds. * The 8% of extra memory we use goes over the limit in the runner because we were already very close to it. In any case, I propose to try disabling this hack in most tests and see if it helps. --- mypy/build.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 4f22e0703d97d..39199b39b6adc 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -3364,7 +3364,8 @@ def process_graph(graph: Graph, manager: BuildManager) -> None: # TODO: see if it's possible to determine if we need to process only a # _subset_ of the past SCCs instead of having to process them all. if ( - platform.python_implementation() == "CPython" + not manager.options.test_env + and platform.python_implementation() == "CPython" and manager.gc_freeze_cycles < MAX_GC_FREEZE_CYCLES ): # When deserializing cache we create huge amount of new objects, so even @@ -3379,7 +3380,8 @@ def process_graph(graph: Graph, manager: BuildManager) -> None: for prev_scc in fresh_scc_queue: process_fresh_modules(graph, prev_scc, manager) if ( - platform.python_implementation() == "CPython" + not manager.options.test_env + and platform.python_implementation() == "CPython" and manager.gc_freeze_cycles < MAX_GC_FREEZE_CYCLES ): manager.gc_freeze_cycles += 1 From bf70dab2d09b61048a8b720e33efae18281a3313 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 31 Aug 2025 12:06:39 +0100 Subject: [PATCH 222/424] Use fast Python wrappers in native_internal (#19765) This makes fixed format cache significantly faster in _interpreted_ mypy (which is important e.g. for tests). In fact, `load_tree_time` with this PR is 15% faster than when using `orjson`. --- mypyc/lib-rt/native_internal.c | 131 +++++++++++++++++++-------------- mypyc/lib-rt/setup.py | 9 ++- 2 files changed, 83 insertions(+), 57 deletions(-) diff --git a/mypyc/lib-rt/native_internal.c b/mypyc/lib-rt/native_internal.c index b9bf899588fbc..a6511a1caf259 100644 --- a/mypyc/lib-rt/native_internal.c +++ b/mypyc/lib-rt/native_internal.c @@ -206,11 +206,13 @@ read_bool_internal(PyObject *data) { } static PyObject* -read_bool(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"data", NULL}; - PyObject *data = NULL; - if (unlikely(!PyArg_ParseTupleAndKeywords(args, kwds, "O", kwlist, &data))) +read_bool(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames) { + static const char * const kwlist[] = {"data", 0}; + static CPyArg_Parser parser = {"O:read_bool", kwlist, 0}; + PyObject *data; + if (unlikely(!CPyArg_ParseStackAndKeywordsOneArg(args, nargs, kwnames, &parser, &data))) { return NULL; + } char res = read_bool_internal(data); if (unlikely(res == CPY_BOOL_ERROR)) return NULL; @@ -229,12 +231,14 @@ write_bool_internal(PyObject *data, char value) { } static PyObject* -write_bool(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"data", "value", NULL}; - PyObject *data = NULL; - PyObject *value = NULL; - if (unlikely(!PyArg_ParseTupleAndKeywords(args, kwds, "OO", kwlist, &data, &value))) +write_bool(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames) { + static const char * const kwlist[] = {"data", "value", 0}; + static CPyArg_Parser parser = {"OO:write_bool", kwlist, 0}; + PyObject *data; + PyObject *value; + if (unlikely(!CPyArg_ParseStackAndKeywordsSimple(args, nargs, kwnames, &parser, &data, &value))) { return NULL; + } if (unlikely(!PyBool_Check(value))) { PyErr_SetString(PyExc_TypeError, "value must be a bool"); return NULL; @@ -280,11 +284,13 @@ read_str_internal(PyObject *data) { } static PyObject* -read_str(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"data", NULL}; - PyObject *data = NULL; - if (unlikely(!PyArg_ParseTupleAndKeywords(args, kwds, "O", kwlist, &data))) +read_str(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames) { + static const char * const kwlist[] = {"data", 0}; + static CPyArg_Parser parser = {"O:read_str", kwlist, 0}; + PyObject *data; + if (unlikely(!CPyArg_ParseStackAndKeywordsOneArg(args, nargs, kwnames, &parser, &data))) { return NULL; + } return read_str_internal(data); } @@ -319,12 +325,14 @@ write_str_internal(PyObject *data, PyObject *value) { } static PyObject* -write_str(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"data", "value", NULL}; - PyObject *data = NULL; - PyObject *value = NULL; - if (unlikely(!PyArg_ParseTupleAndKeywords(args, kwds, "OO", kwlist, &data, &value))) +write_str(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames) { + static const char * const kwlist[] = {"data", "value", 0}; + static CPyArg_Parser parser = {"OO:write_str", kwlist, 0}; + PyObject *data; + PyObject *value; + if (unlikely(!CPyArg_ParseStackAndKeywordsSimple(args, nargs, kwnames, &parser, &data, &value))) { return NULL; + } if (unlikely(!PyUnicode_Check(value))) { PyErr_SetString(PyExc_TypeError, "value must be a str"); return NULL; @@ -350,11 +358,13 @@ read_float_internal(PyObject *data) { } static PyObject* -read_float(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"data", NULL}; - PyObject *data = NULL; - if (unlikely(!PyArg_ParseTupleAndKeywords(args, kwds, "O", kwlist, &data))) +read_float(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames) { + static const char * const kwlist[] = {"data", 0}; + static CPyArg_Parser parser = {"O:read_float", kwlist, 0}; + PyObject *data; + if (unlikely(!CPyArg_ParseStackAndKeywordsOneArg(args, nargs, kwnames, &parser, &data))) { return NULL; + } double retval = read_float_internal(data); if (unlikely(retval == CPY_FLOAT_ERROR && PyErr_Occurred())) { return NULL; @@ -372,12 +382,14 @@ write_float_internal(PyObject *data, double value) { } static PyObject* -write_float(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"data", "value", NULL}; - PyObject *data = NULL; - PyObject *value = NULL; - if (unlikely(!PyArg_ParseTupleAndKeywords(args, kwds, "OO", kwlist, &data, &value))) +write_float(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames) { + static const char * const kwlist[] = {"data", "value", 0}; + static CPyArg_Parser parser = {"OO:write_float", kwlist, 0}; + PyObject *data; + PyObject *value; + if (unlikely(!CPyArg_ParseStackAndKeywordsSimple(args, nargs, kwnames, &parser, &data, &value))) { return NULL; + } if (unlikely(!PyFloat_Check(value))) { PyErr_SetString(PyExc_TypeError, "value must be a float"); return NULL; @@ -421,11 +433,13 @@ read_int_internal(PyObject *data) { } static PyObject* -read_int(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"data", NULL}; - PyObject *data = NULL; - if (unlikely(!PyArg_ParseTupleAndKeywords(args, kwds, "O", kwlist, &data))) +read_int(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames) { + static const char * const kwlist[] = {"data", 0}; + static CPyArg_Parser parser = {"O:read_int", kwlist, 0}; + PyObject *data; + if (unlikely(!CPyArg_ParseStackAndKeywordsOneArg(args, nargs, kwnames, &parser, &data))) { return NULL; + } CPyTagged retval = read_int_internal(data); if (unlikely(retval == CPY_INT_TAG)) { return NULL; @@ -466,12 +480,14 @@ write_int_internal(PyObject *data, CPyTagged value) { } static PyObject* -write_int(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"data", "value", NULL}; - PyObject *data = NULL; - PyObject *value = NULL; - if (unlikely(!PyArg_ParseTupleAndKeywords(args, kwds, "OO", kwlist, &data, &value))) +write_int(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames) { + static const char * const kwlist[] = {"data", "value", 0}; + static CPyArg_Parser parser = {"OO:write_int", kwlist, 0}; + PyObject *data; + PyObject *value; + if (unlikely(!CPyArg_ParseStackAndKeywordsSimple(args, nargs, kwnames, &parser, &data, &value))) { return NULL; + } if (unlikely(!PyLong_Check(value))) { PyErr_SetString(PyExc_TypeError, "value must be an int"); return NULL; @@ -498,11 +514,13 @@ read_tag_internal(PyObject *data) { } static PyObject* -read_tag(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"data", NULL}; - PyObject *data = NULL; - if (unlikely(!PyArg_ParseTupleAndKeywords(args, kwds, "O", kwlist, &data))) +read_tag(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames) { + static const char * const kwlist[] = {"data", 0}; + static CPyArg_Parser parser = {"O:read_tag", kwlist, 0}; + PyObject *data; + if (unlikely(!CPyArg_ParseStackAndKeywordsOneArg(args, nargs, kwnames, &parser, &data))) { return NULL; + } uint8_t retval = read_tag_internal(data); if (unlikely(retval == CPY_LL_UINT_ERROR && PyErr_Occurred())) { return NULL; @@ -520,12 +538,14 @@ write_tag_internal(PyObject *data, uint8_t value) { } static PyObject* -write_tag(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"data", "value", NULL}; - PyObject *data = NULL; - PyObject *value = NULL; - if (unlikely(!PyArg_ParseTupleAndKeywords(args, kwds, "OO", kwlist, &data, &value))) +write_tag(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames) { + static const char * const kwlist[] = {"data", "value", 0}; + static CPyArg_Parser parser = {"OO:write_tag", kwlist, 0}; + PyObject *data; + PyObject *value; + if (unlikely(!CPyArg_ParseStackAndKeywordsSimple(args, nargs, kwnames, &parser, &data, &value))) { return NULL; + } uint8_t unboxed = CPyLong_AsUInt8(value); if (unlikely(unboxed == CPY_LL_UINT_ERROR && PyErr_Occurred())) { CPy_TypeError("u8", value); @@ -539,17 +559,16 @@ write_tag(PyObject *self, PyObject *args, PyObject *kwds) { } static PyMethodDef native_internal_module_methods[] = { - // TODO: switch public wrappers to METH_FASTCALL. - {"write_bool", (PyCFunction)write_bool, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("write a bool")}, - {"read_bool", (PyCFunction)read_bool, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("read a bool")}, - {"write_str", (PyCFunction)write_str, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("write a string")}, - {"read_str", (PyCFunction)read_str, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("read a string")}, - {"write_float", (PyCFunction)write_float, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("write a float")}, - {"read_float", (PyCFunction)read_float, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("read a float")}, - {"write_int", (PyCFunction)write_int, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("write an int")}, - {"read_int", (PyCFunction)read_int, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("read an int")}, - {"write_tag", (PyCFunction)write_tag, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("write a short int")}, - {"read_tag", (PyCFunction)read_tag, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("read a short int")}, + {"write_bool", (PyCFunction)write_bool, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("write a bool")}, + {"read_bool", (PyCFunction)read_bool, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("read a bool")}, + {"write_str", (PyCFunction)write_str, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("write a string")}, + {"read_str", (PyCFunction)read_str, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("read a string")}, + {"write_float", (PyCFunction)write_float, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("write a float")}, + {"read_float", (PyCFunction)read_float, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("read a float")}, + {"write_int", (PyCFunction)write_int, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("write an int")}, + {"read_int", (PyCFunction)read_int, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("read an int")}, + {"write_tag", (PyCFunction)write_tag, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("write a short int")}, + {"read_tag", (PyCFunction)read_tag, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("read a short int")}, {NULL, NULL, 0, NULL} }; diff --git a/mypyc/lib-rt/setup.py b/mypyc/lib-rt/setup.py index 5b7a2919c0fd2..3a5976cf88b28 100644 --- a/mypyc/lib-rt/setup.py +++ b/mypyc/lib-rt/setup.py @@ -78,7 +78,14 @@ def run(self) -> None: ext_modules=[ Extension( "native_internal", - ["native_internal.c", "init.c", "int_ops.c", "exc_ops.c", "pythonsupport.c"], + [ + "native_internal.c", + "init.c", + "int_ops.c", + "exc_ops.c", + "pythonsupport.c", + "getargsfast.c", + ], include_dirs=["."], ) ], From 87f4dc8e7021a420bc561d491ff756d421a938ab Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 31 Aug 2025 23:36:00 +0100 Subject: [PATCH 223/424] Mypy micro-optimizations (batch 1/3) (#19768) Several mypy micro-optimizations. Together with batches 2 and 3 these improve self check performance by 1.8%. --- mypy/applytype.py | 2 +- mypy/binder.py | 8 +++++--- mypy/plugins/common.py | 2 +- mypy/semanal.py | 2 +- mypy/semanal_namedtuple.py | 2 +- mypy/server/aststrip.py | 2 +- mypy/type_visitor.py | 15 +++++++++------ mypy/typeanal.py | 6 +++--- mypy/types.py | 12 +++++++----- 9 files changed, 29 insertions(+), 22 deletions(-) diff --git a/mypy/applytype.py b/mypy/applytype.py index e87bf939c81ab..dfeaf7752d211 100644 --- a/mypy/applytype.py +++ b/mypy/applytype.py @@ -243,7 +243,7 @@ def visit_callable_type(self, t: CallableType) -> Type: self.bound_tvars -= set(found_vars) assert isinstance(result, ProperType) and isinstance(result, CallableType) - result.variables = list(result.variables) + found_vars + result.variables = result.variables + tuple(found_vars) return result def visit_type_var(self, t: TypeVarType) -> Type: diff --git a/mypy/binder.py b/mypy/binder.py index c95481329a571..a83e65276ff43 100644 --- a/mypy/binder.py +++ b/mypy/binder.py @@ -250,10 +250,12 @@ def update_from_options(self, frames: list[Frame]) -> bool: options are the same. """ all_reachable = all(not f.unreachable for f in frames) - frames = [f for f in frames if not f.unreachable] + if not all_reachable: + frames = [f for f in frames if not f.unreachable] changed = False - keys = {key for f in frames for key in f.types} - + keys = [key for f in frames for key in f.types] + if len(keys) > 1: + keys = list(set(keys)) for key in keys: current_value = self._get(key) resulting_values = [f.types.get(key, current_value) for f in frames] diff --git a/mypy/plugins/common.py b/mypy/plugins/common.py index ac00171a037c0..ed2a91d102f4b 100644 --- a/mypy/plugins/common.py +++ b/mypy/plugins/common.py @@ -360,7 +360,7 @@ def _add_method_by_spec( signature = CallableType(arg_types, arg_kinds, arg_names, return_type, function_type) if tvar_defs: - signature.variables = tvar_defs + signature.variables = tuple(tvar_defs) func = FuncDef(name, args, Block([PassStmt()])) func.info = info diff --git a/mypy/semanal.py b/mypy/semanal.py index 213c8d04122d7..50ee3b5324633 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -995,7 +995,7 @@ def analyze_func_def(self, defn: FuncDef) -> None: if has_self_type and self.type is not None: info = self.type if info.self_type is not None: - result.variables = [info.self_type] + list(result.variables) + result.variables = (info.self_type,) + result.variables defn.type = result self.add_type_alias_deps(analyzer.aliases_used) self.check_function_signature(defn) diff --git a/mypy/semanal_namedtuple.py b/mypy/semanal_namedtuple.py index b67747d16887a..37a650f1b6644 100644 --- a/mypy/semanal_namedtuple.py +++ b/mypy/semanal_namedtuple.py @@ -606,7 +606,7 @@ def add_method( arg_kinds = [arg.kind for arg in args] assert None not in types signature = CallableType(cast(list[Type], types), arg_kinds, items, ret, function_type) - signature.variables = [self_type] + signature.variables = (self_type,) func = FuncDef(funcname, args, Block([])) func.info = info func.is_class = is_classmethod diff --git a/mypy/server/aststrip.py b/mypy/server/aststrip.py index a70dfc30deb55..27c1c4a0eedbb 100644 --- a/mypy/server/aststrip.py +++ b/mypy/server/aststrip.py @@ -165,7 +165,7 @@ def visit_func_def(self, node: FuncDef) -> None: # in order to get the state exactly as it was before semantic analysis. # See also #4814. assert isinstance(node.type, CallableType) - node.type.variables = [] + node.type.variables = () with self.enter_method(node.info) if node.info else nullcontext(): super().visit_func_def(node) diff --git a/mypy/type_visitor.py b/mypy/type_visitor.py index ab1ec8b46fdd0..65051ddbab674 100644 --- a/mypy/type_visitor.py +++ b/mypy/type_visitor.py @@ -228,7 +228,7 @@ def visit_instance(self, t: Instance, /) -> Type: last_known_value = raw_last_known_value return Instance( typ=t.type, - args=self.translate_types(t.args), + args=self.translate_type_tuple(t.args), line=t.line, column=t.column, last_known_value=last_known_value, @@ -242,7 +242,7 @@ def visit_param_spec(self, t: ParamSpecType, /) -> Type: return t def visit_parameters(self, t: Parameters, /) -> Type: - return t.copy_modified(arg_types=self.translate_types(t.arg_types)) + return t.copy_modified(arg_types=self.translate_type_list(t.arg_types)) def visit_type_var_tuple(self, t: TypeVarTupleType, /) -> Type: return t @@ -255,14 +255,14 @@ def visit_unpack_type(self, t: UnpackType, /) -> Type: def visit_callable_type(self, t: CallableType, /) -> Type: return t.copy_modified( - arg_types=self.translate_types(t.arg_types), + arg_types=self.translate_type_list(t.arg_types), ret_type=t.ret_type.accept(self), variables=self.translate_variables(t.variables), ) def visit_tuple_type(self, t: TupleType, /) -> Type: return TupleType( - self.translate_types(t.items), + self.translate_type_list(t.items), # TODO: This appears to be unsafe. cast(Any, t.partial_fallback.accept(self)), t.line, @@ -299,7 +299,7 @@ def visit_union_type(self, t: UnionType, /) -> Type: return cached result = UnionType( - self.translate_types(t.items), + self.translate_type_list(t.items), t.line, t.column, uses_pep604_syntax=t.uses_pep604_syntax, @@ -308,9 +308,12 @@ def visit_union_type(self, t: UnionType, /) -> Type: self.set_cached(t, result) return result - def translate_types(self, types: Iterable[Type]) -> list[Type]: + def translate_type_list(self, types: list[Type]) -> list[Type]: return [t.accept(self) for t in types] + def translate_type_tuple(self, types: tuple[Type, ...]) -> tuple[Type, ...]: + return tuple(t.accept(self) for t in types) + def translate_variables( self, variables: Sequence[TypeVarLikeType] ) -> Sequence[TypeVarLikeType]: diff --git a/mypy/typeanal.py b/mypy/typeanal.py index d44b13880cbbc..af70c52180aa1 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -1820,7 +1820,7 @@ def infer_type_variables( def bind_function_type_variables( self, fun_type: CallableType, defn: Context - ) -> tuple[Sequence[TypeVarLikeType], bool]: + ) -> tuple[tuple[TypeVarLikeType, ...], bool]: """Find the type variables of the function type and bind them in our tvar_scope""" has_self_type = False if fun_type.variables: @@ -1835,7 +1835,7 @@ def bind_function_type_variables( assert isinstance(var_expr, TypeVarLikeExpr) binding = self.tvar_scope.bind_new(var.name, var_expr) defs.append(binding) - return defs, has_self_type + return tuple(defs), has_self_type typevars, has_self_type = self.infer_type_variables(fun_type) # Do not define a new type variable if already defined in scope. typevars = [ @@ -1849,7 +1849,7 @@ def bind_function_type_variables( binding = self.tvar_scope.bind_new(name, tvar) defs.append(binding) - return defs, has_self_type + return tuple(defs), has_self_type def is_defined_type_var(self, tvar: str, context: Context) -> bool: tvar_node = self.lookup_qualified(tvar, context) diff --git a/mypy/types.py b/mypy/types.py index e0265e601e0c0..388a1906285ed 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -2104,14 +2104,12 @@ def __init__( ) -> None: super().__init__(line, column) assert len(arg_types) == len(arg_kinds) == len(arg_names) - for t, k in zip(arg_types, arg_kinds): + self.arg_types = list(arg_types) + for t in self.arg_types: if isinstance(t, ParamSpecType): assert not t.prefix.arg_types # TODO: should we assert that only ARG_STAR contain ParamSpecType? # See testParamSpecJoin, that relies on passing e.g `P.args` as plain argument. - if variables is None: - variables = [] - self.arg_types = list(arg_types) self.arg_kinds = arg_kinds self.arg_names = list(arg_names) self.min_args = arg_kinds.count(ARG_POS) @@ -2123,7 +2121,11 @@ def __init__( # * If it is a non-decorated function, FuncDef is the definition # * If it is a decorated function, enclosing Decorator is the definition self.definition = definition - self.variables = variables + self.variables: tuple[TypeVarLikeType, ...] + if variables is None: + self.variables = () + else: + self.variables = tuple(variables) self.is_ellipsis_args = is_ellipsis_args self.implicit = implicit self.special_sig = special_sig From a009879d540bf38bc46803f6043f564c782a0531 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 31 Aug 2025 23:37:08 +0100 Subject: [PATCH 224/424] Mypy micro-optimizations (batch 2/3) (#19769) Several mypy micro-optimizations. Together with batches 1 and 3 these improve self check performance by 1.8%. --- mypy/argmap.py | 4 +++- mypy/checkexpr.py | 8 +++----- mypy/expandtype.py | 4 +++- mypy/messages.py | 31 +++++++++++++++---------------- mypy/types.py | 4 ++-- 5 files changed, 26 insertions(+), 25 deletions(-) diff --git a/mypy/argmap.py b/mypy/argmap.py index 28fad1f093ddc..a3e8f7fc8c2e1 100644 --- a/mypy/argmap.py +++ b/mypy/argmap.py @@ -167,7 +167,7 @@ def __init__(self, context: ArgumentInferContext) -> None: # Next tuple *args index to use. self.tuple_index = 0 # Keyword arguments in TypedDict **kwargs used. - self.kwargs_used: set[str] = set() + self.kwargs_used: set[str] | None = None # Type context for `*` and `**` arg kinds. self.context = context @@ -241,6 +241,8 @@ def expand_actual_type( from mypy.subtypes import is_subtype if isinstance(actual_type, TypedDictType): + if self.kwargs_used is None: + self.kwargs_used = set() if formal_kind != nodes.ARG_STAR2 and formal_name in actual_type.items: # Lookup type based on keyword argument name. assert formal_name is not None diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 794af875867a5..250decd7567e5 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -248,12 +248,11 @@ def allow_fast_container_literal(t: Type) -> bool: ) -def extract_refexpr_names(expr: RefExpr) -> set[str]: +def extract_refexpr_names(expr: RefExpr, output: set[str]) -> None: """Recursively extracts all module references from a reference expression. Note that currently, the only two subclasses of RefExpr are NameExpr and MemberExpr.""" - output: set[str] = set() while isinstance(expr.node, MypyFile) or expr.fullname: if isinstance(expr.node, MypyFile) and expr.fullname: # If it's None, something's wrong (perhaps due to an @@ -277,7 +276,6 @@ def extract_refexpr_names(expr: RefExpr) -> set[str]: break else: raise AssertionError(f"Unknown RefExpr subclass: {type(expr)}") - return output class Finished(Exception): @@ -372,7 +370,7 @@ def visit_name_expr(self, e: NameExpr) -> Type: It can be of any kind: local, member or global. """ - self.chk.module_refs.update(extract_refexpr_names(e)) + extract_refexpr_names(e, self.chk.module_refs) result = self.analyze_ref_expr(e) narrowed = self.narrow_type_from_binder(e, result) self.chk.check_deprecated(e.node, e) @@ -3345,7 +3343,7 @@ def check_union_call( def visit_member_expr(self, e: MemberExpr, is_lvalue: bool = False) -> Type: """Visit member expression (of form e.id).""" - self.chk.module_refs.update(extract_refexpr_names(e)) + extract_refexpr_names(e, self.chk.module_refs) result = self.analyze_ordinary_member_access(e, is_lvalue) narrowed = self.narrow_type_from_binder(e, result) self.chk.warn_deprecated(e.node, e) diff --git a/mypy/expandtype.py b/mypy/expandtype.py index 8433708eda44a..e2a42317141f4 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -182,7 +182,7 @@ class ExpandTypeVisitor(TrivialSyntheticTypeTranslator): def __init__(self, variables: Mapping[TypeVarId, Type]) -> None: super().__init__() self.variables = variables - self.recursive_tvar_guard: dict[TypeVarId, Type | None] = {} + self.recursive_tvar_guard: dict[TypeVarId, Type | None] | None = None def visit_unbound_type(self, t: UnboundType) -> Type: return t @@ -246,6 +246,8 @@ def visit_type_var(self, t: TypeVarType) -> Type: # If I try to remove this special-casing ~40 tests fail on reveal_type(). return repl.copy_modified(last_known_value=None) if isinstance(repl, TypeVarType) and repl.has_default(): + if self.recursive_tvar_guard is None: + self.recursive_tvar_guard = {} if (tvar_id := repl.id) in self.recursive_tvar_guard: return self.recursive_tvar_guard[tvar_id] or repl self.recursive_tvar_guard[tvar_id] = None diff --git a/mypy/messages.py b/mypy/messages.py index 571cebb1b174c..6329cad687f6d 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -649,26 +649,11 @@ def incompatible_argument( else: base = extract_type(name) - for method, op in op_methods_to_symbols.items(): - for variant in method, "__r" + method[2:]: - # FIX: do not rely on textual formatting - if name.startswith(f'"{variant}" of'): - if op == "in" or variant != method: - # Reversed order of base/argument. - return self.unsupported_operand_types( - op, arg_type, base, context, code=codes.OPERATOR - ) - else: - return self.unsupported_operand_types( - op, base, arg_type, context, code=codes.OPERATOR - ) - if name.startswith('"__getitem__" of'): return self.invalid_index_type( arg_type, callee.arg_types[n - 1], base, context, code=codes.INDEX ) - - if name.startswith('"__setitem__" of'): + elif name.startswith('"__setitem__" of'): if n == 1: return self.invalid_index_type( arg_type, callee.arg_types[n - 1], base, context, code=codes.INDEX @@ -684,6 +669,20 @@ def incompatible_argument( message_registry.INCOMPATIBLE_TYPES_IN_ASSIGNMENT.with_additional_msg(info) ) return self.fail(error_msg.value, context, code=error_msg.code) + elif name.startswith('"__'): + for method, op in op_methods_to_symbols.items(): + for variant in method, "__r" + method[2:]: + # FIX: do not rely on textual formatting + if name.startswith(f'"{variant}" of'): + if op == "in" or variant != method: + # Reversed order of base/argument. + return self.unsupported_operand_types( + op, arg_type, base, context, code=codes.OPERATOR + ) + else: + return self.unsupported_operand_types( + op, base, arg_type, context, code=codes.OPERATOR + ) target = f"to {name} " diff --git a/mypy/types.py b/mypy/types.py index 388a1906285ed..c23997d069d4b 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -543,7 +543,7 @@ class TypeVarId: # function type variables. # Metavariables are allocated unique ids starting from 1. - raw_id: int + raw_id: Final[int] # Level of the variable in type inference. Currently either 0 for # declared types, or 1 for type inference metavariables. @@ -586,7 +586,7 @@ def __ne__(self, other: object) -> bool: return not (self == other) def __hash__(self) -> int: - return hash((self.raw_id, self.meta_level, self.namespace)) + return self.raw_id ^ (self.meta_level << 8) ^ hash(self.namespace) def is_meta_var(self) -> bool: return self.meta_level > 0 From 70d0521a8a61ceaa07a2dadcd916829fde8072b1 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sun, 31 Aug 2025 16:44:44 -0700 Subject: [PATCH 225/424] Update stubinfo for latest typeshed (#19771) --- misc/update-stubinfo.py | 2 +- mypy/stubinfo.py | 39 ++++++++++++++++++++++++++++----------- mypy/test/teststubinfo.py | 3 ++- 3 files changed, 31 insertions(+), 13 deletions(-) diff --git a/misc/update-stubinfo.py b/misc/update-stubinfo.py index beaed34a8a477..4a5b9a40c408c 100644 --- a/misc/update-stubinfo.py +++ b/misc/update-stubinfo.py @@ -58,7 +58,7 @@ def main() -> None: print("Consider removing the following packages no longer in typeshed:") print("=" * 40) for p in sorted(mypy_p - typeshed_p_to_d.keys()): - if p in {"lxml", "pandas"}: # never in typeshed + if p in {"lxml", "pandas", "scipy"}: # never in typeshed continue print(p) diff --git a/mypy/stubinfo.py b/mypy/stubinfo.py index 33064c9d30674..42e53ba21c847 100644 --- a/mypy/stubinfo.py +++ b/mypy/stubinfo.py @@ -45,7 +45,6 @@ def stub_distribution_name(module: str) -> str | None: "first": "types-first", "markdown": "types-Markdown", "mock": "types-mock", - "OpenSSL": "types-pyOpenSSL", "paramiko": "types-paramiko", "polib": "types-polib", "pycurl": "types-pycurl", @@ -75,35 +74,37 @@ def stub_distribution_name(module: str) -> str | None: # but is a non-typeshed stubs package. non_bundled_packages_flat: dict[str, str] = { "_cffi_backend": "types-cffi", + "_jsonnet": "types-jsonnet", "_win32typing": "types-pywin32", "antlr4": "types-antlr4-python3-runtime", "assertpy": "types-assertpy", - "atheris": "types-atheris", + "auth0": "types-auth0-python", "authlib": "types-Authlib", "aws_xray_sdk": "types-aws-xray-sdk", + "binaryornot": "types-binaryornot", "boltons": "types-boltons", "braintree": "types-braintree", - "bs4": "types-beautifulsoup4", "bugbear": "types-flake8-bugbear", - "caldav": "types-caldav", "capturer": "types-capturer", "cffi": "types-cffi", + "channels": "types-channels", "chevron": "types-chevron", "click_default_group": "types-click-default-group", "click_log": "types-click-log", + "click_shell": "types-click-shell", "click_web": "types-click-web", "colorama": "types-colorama", "commctrl": "types-pywin32", - "commonmark": "types-commonmark", "consolemenu": "types-console-menu", - "corus": "types-corus", # codespell:ignore corus + "convertdate": "types-convertdate", "cronlog": "types-python-crontab", "crontab": "types-python-crontab", "crontabs": "types-python-crontab", - "datemath": "types-python-datemath", "dateparser_data": "types-dateparser", "dde": "types-pywin32", "defusedxml": "types-defusedxml", + "dirhash": "types-dirhash", + "django_filters": "types-django-filter", "docker": "types-docker", "dockerfile_parse": "types-dockerfile-parse", "editdistance": "types-editdistance", @@ -122,17 +123,22 @@ def stub_distribution_name(module: str) -> str | None: "flask_socketio": "types-Flask-SocketIO", "fpdf": "types-fpdf2", "gdb": "types-gdb", + "geopandas": "types-geopandas", "gevent": "types-gevent", "greenlet": "types-greenlet", + "grpc_channelz": "types-grpcio-channelz", + "grpc_health": "types-grpcio-health-checking", + "grpc_reflection": "types-grpcio-reflection", + "grpc_status": "types-grpcio-status", + "grpc": "types-grpcio", "hdbcli": "types-hdbcli", + "hnswlib": "types-hnswlib", "html5lib": "types-html5lib", "httplib2": "types-httplib2", - "humanfriendly": "types-humanfriendly", "hvac": "types-hvac", "ibm_db": "types-ibm-db", "icalendar": "types-icalendar", "import_export": "types-django-import-export", - "influxdb_client": "types-influxdb-client", "inifile": "types-inifile", "isapi": "types-pywin32", "jack": "types-JACK-Client", @@ -145,9 +151,11 @@ def stub_distribution_name(module: str) -> str | None: "jwcrypto": "types-jwcrypto", "keyboard": "types-keyboard", "ldap3": "types-ldap3", + "lunardate": "types-lunardate", "lupa": "types-lupa", "lzstring": "types-lzstring", "m3u8": "types-m3u8", + "management": "types-django-import-export", "mmapfile": "types-pywin32", "mmsystem": "types-pywin32", "mypy_extensions": "types-mypy-extensions", @@ -173,6 +181,7 @@ def stub_distribution_name(module: str) -> str | None: "perfmon": "types-pywin32", "pexpect": "types-pexpect", "playhouse": "types-peewee", + "pony": "types-pony", "portpicker": "types-portpicker", "psutil": "types-psutil", "psycopg2": "types-psycopg2", @@ -181,11 +190,13 @@ def stub_distribution_name(module: str) -> str | None: "pyautogui": "types-PyAutoGUI", "pycocotools": "types-pycocotools", "pyflakes": "types-pyflakes", - "pygit2": "types-pygit2", "pygments": "types-Pygments", "pyi_splash": "types-pyinstaller", "PyInstaller": "types-pyinstaller", + "pyluach": "types-pyluach", + "pymeeus": "types-PyMeeus", "pynput": "types-pynput", + "pyperclip": "types-pyperclip", "pyscreeze": "types-PyScreeze", "pysftp": "types-pysftp", "pytest_lazyfixture": "types-pytest-lazy-fixture", @@ -195,10 +206,12 @@ def stub_distribution_name(module: str) -> str | None: "pywintypes": "types-pywin32", "qrbill": "types-qrbill", "qrcode": "types-qrcode", + "ratelimit": "types-ratelimit", "regex": "types-regex", "regutil": "types-pywin32", "reportlab": "types-reportlab", "requests_oauthlib": "types-requests-oauthlib", + "rfc3339_validator": "types-rfc3339-validator", "RPi": "types-RPi.GPIO", "s2clientprotocol": "types-s2clientprotocol", "sass": "types-libsass", @@ -210,6 +223,8 @@ def stub_distribution_name(module: str) -> str | None: "setuptools": "types-setuptools", "shapely": "types-shapely", "slumber": "types-slumber", + "socks": "types-PySocks", + "sockshandler": "types-PySocks", "sspicon": "types-pywin32", "str2bool": "types-str2bool", "tensorflow": "types-tensorflow", @@ -218,7 +233,6 @@ def stub_distribution_name(module: str) -> str | None: "toposort": "types-toposort", "tqdm": "types-tqdm", "translationstring": "types-translationstring", - "tree_sitter_languages": "types-tree-sitter-languages", "ttkthemes": "types-ttkthemes", "unidiff": "types-unidiff", "untangle": "types-untangle", @@ -226,6 +240,7 @@ def stub_distribution_name(module: str) -> str | None: "uwsgi": "types-uWSGI", "uwsgidecorators": "types-uWSGI", "vobject": "types-vobject", + "watchpoints": "types-watchpoints", "webob": "types-WebOb", "whatthepatch": "types-whatthepatch", "win2kras": "types-pywin32", @@ -282,7 +297,9 @@ def stub_distribution_name(module: str) -> str | None: "xdg": "types-pyxdg", "xdgenvpy": "types-xdgenvpy", "Xlib": "types-python-xlib", + "xlrd": "types-xlrd", "xmltodict": "types-xmltodict", + "yt_dlp": "types-yt-dlp", "zstd": "types-zstd", "zxcvbn": "types-zxcvbn", # Stub packages that are not from typeshed diff --git a/mypy/test/teststubinfo.py b/mypy/test/teststubinfo.py index e90c72335bf85..ae34e78f98c67 100644 --- a/mypy/test/teststubinfo.py +++ b/mypy/test/teststubinfo.py @@ -20,7 +20,8 @@ def test_is_legacy_bundled_packages(self) -> None: def test_stub_distribution_name(self) -> None: assert stub_distribution_name("foobar_asdf") is None assert stub_distribution_name("pycurl") == "types-pycurl" - assert stub_distribution_name("bs4") == "types-beautifulsoup4" + assert stub_distribution_name("psutil") == "types-psutil" + assert stub_distribution_name("sassutils") == "types-libsass" assert stub_distribution_name("google.cloud.ndb") == "types-google-cloud-ndb" assert stub_distribution_name("google.cloud.ndb.submodule") == "types-google-cloud-ndb" assert stub_distribution_name("google.cloud.unknown") is None From a856e559756ea0e8b836e94d226a3cecf2cb9166 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 1 Sep 2025 01:19:19 +0100 Subject: [PATCH 226/424] Use empty context as fallback for return statements (#19767) Fixes https://github.com/python/mypy/issues/16924 Fixes https://github.com/python/mypy/issues/15886 Mypy uses external type context first, this can cause bad type inference in return statements (see example in test case added), usually we recommend a workaround to users like replacing: ```python return foo(x) ``` with ```python y = foo(x) return y ``` But this is a bit ugly, and more importantly we can essentially automatically try this workaround. This is what this PR adds. I checked performance impact, and don't see any (but for some reason noise level on my desktop is much higher now). --- mypy/checker.py | 53 +++++++++++++++++++-- mypy/errors.py | 3 +- test-data/unit/check-inference-context.test | 42 ++++++++++++++++ test-data/unit/pythoneval.test | 16 +++++++ 4 files changed, 109 insertions(+), 5 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index ae6ae591ed8c4..12a86fe6fba12 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -90,6 +90,7 @@ ContinueStmt, Decorator, DelStmt, + DictExpr, EllipsisExpr, Expression, ExpressionStmt, @@ -124,6 +125,7 @@ RaiseStmt, RefExpr, ReturnStmt, + SetExpr, StarExpr, Statement, StrExpr, @@ -4859,6 +4861,42 @@ def visit_return_stmt(self, s: ReturnStmt) -> None: self.check_return_stmt(s) self.binder.unreachable() + def infer_context_dependent( + self, expr: Expression, type_ctx: Type, allow_none_func_call: bool + ) -> ProperType: + """Infer type of an expression with fallback to empty type context.""" + with self.msg.filter_errors( + filter_errors=True, filter_deprecated=True, save_filtered_errors=True + ) as msg: + with self.local_type_map as type_map: + typ = get_proper_type( + self.expr_checker.accept( + expr, type_ctx, allow_none_return=allow_none_func_call + ) + ) + if not msg.has_new_errors(): + self.store_types(type_map) + return typ + + # If there are errors with the original type context, try re-inferring in empty context. + original_messages = msg.filtered_errors() + original_type_map = type_map + with self.msg.filter_errors( + filter_errors=True, filter_deprecated=True, save_filtered_errors=True + ) as msg: + with self.local_type_map as type_map: + alt_typ = get_proper_type( + self.expr_checker.accept(expr, None, allow_none_return=allow_none_func_call) + ) + if not msg.has_new_errors() and is_subtype(alt_typ, type_ctx): + self.store_types(type_map) + return alt_typ + + # If empty fallback didn't work, use results from the original type context. + self.msg.add_errors(original_messages) + self.store_types(original_type_map) + return typ + def check_return_stmt(self, s: ReturnStmt) -> None: defn = self.scope.current_function() if defn is not None: @@ -4891,11 +4929,18 @@ def check_return_stmt(self, s: ReturnStmt) -> None: allow_none_func_call = is_lambda or declared_none_return or declared_any_return # Return with a value. - typ = get_proper_type( - self.expr_checker.accept( - s.expr, return_type, allow_none_return=allow_none_func_call + if isinstance(s.expr, (CallExpr, ListExpr, TupleExpr, DictExpr, SetExpr, OpExpr)): + # For expressions that (strongly) depend on type context (i.e. those that + # are handled like a function call), we allow fallback to empty type context + # in case of errors, this improves user experience in some cases, + # see e.g. testReturnFallbackInference. + typ = self.infer_context_dependent(s.expr, return_type, allow_none_func_call) + else: + typ = get_proper_type( + self.expr_checker.accept( + s.expr, return_type, allow_none_return=allow_none_func_call + ) ) - ) # Treat NotImplemented as having type Any, consistent with its # definition in typeshed prior to python/typeshed#4222. if ( diff --git a/mypy/errors.py b/mypy/errors.py index d75c1c62a1edb..f1b2faf67401c 100644 --- a/mypy/errors.py +++ b/mypy/errors.py @@ -206,7 +206,8 @@ def on_error(self, file: str, info: ErrorInfo) -> bool: """ if info.code == codes.DEPRECATED: # Deprecated is not a type error, so it is handled on opt-in basis here. - return self._filter_deprecated + if not self._filter_deprecated: + return False self._has_new_errors = True if isinstance(self._filter, bool): diff --git a/test-data/unit/check-inference-context.test b/test-data/unit/check-inference-context.test index cd44fb5b85cd9..7dbbd68c4215d 100644 --- a/test-data/unit/check-inference-context.test +++ b/test-data/unit/check-inference-context.test @@ -1540,3 +1540,45 @@ def f(x: dict[str, Union[str, None, int]]) -> None: def g(x: Optional[dict[str, Any]], s: Optional[str]) -> None: f(x or {'x': s}) [builtins fixtures/dict.pyi] + +[case testReturnFallbackInferenceTuple] +from typing import TypeVar, Union + +T = TypeVar("T") +def foo(x: list[T]) -> tuple[T, ...]: ... + +def bar(x: list[int]) -> tuple[Union[str, int], ...]: + return foo(x) + +def bar2(x: list[int]) -> tuple[Union[str, int], ...]: + y = foo(x) + return y +[builtins fixtures/tuple.pyi] + +[case testReturnFallbackInferenceUnion] +from typing import Generic, TypeVar, Union + +T = TypeVar("T") + +class Cls(Generic[T]): + pass + +def inner(c: Cls[T]) -> Union[T, int]: + return 1 + +def outer(c: Cls[T]) -> Union[T, int]: + return inner(c) + +[case testReturnFallbackInferenceAsync] +from typing import Generic, TypeVar, Optional + +T = TypeVar("T") + +class Cls(Generic[T]): + pass + +async def inner(c: Cls[T]) -> Optional[T]: + return None + +async def outer(c: Cls[T]) -> Optional[T]: + return await inner(c) diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index 9b5d8a1ac54c0..2910e59b91739 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -2188,3 +2188,19 @@ reveal_type([*map(str, x)]) [out] _testUnpackIteratorBuiltins.py:4: note: Revealed type is "builtins.list[builtins.int]" _testUnpackIteratorBuiltins.py:5: note: Revealed type is "builtins.list[builtins.str]" + +[case testReturnFallbackInferenceDict] +# Requires full dict stubs. +from typing import Dict, Mapping, TypeVar, Union + +K = TypeVar("K") +V = TypeVar("V") +K2 = TypeVar("K2") +V2 = TypeVar("V2") + +def func(one: Dict[K, V], two: Mapping[K2, V2]) -> Dict[Union[K, K2], Union[V, V2]]: + ... + +def caller(arg1: Mapping[K, V], arg2: Mapping[K2, V2]) -> Dict[Union[K, K2], Union[V, V2]]: + _arg1 = arg1 if isinstance(arg1, dict) else dict(arg1) + return func(_arg1, arg2) From b1b8b0ccb1adeab7801bd41e3d688ecdeded6ed8 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Mon, 1 Sep 2025 05:26:36 +0200 Subject: [PATCH 227/424] Sync typeshed (#19772) Source commit: https://github.com/python/typeshed/commit/2480d7e7c74493a024eaf254c5d2c6f452c80ee2 --- ...redundant-inheritances-from-Iterator.patch | 96 ++-- mypy/typeshed/stdlib/_ast.pyi | 10 +- mypy/typeshed/stdlib/_asyncio.pyi | 12 +- mypy/typeshed/stdlib/_blake2.pyi | 133 +++-- mypy/typeshed/stdlib/_collections_abc.pyi | 1 + mypy/typeshed/stdlib/_compat_pickle.pyi | 18 +- mypy/typeshed/stdlib/_csv.pyi | 7 +- mypy/typeshed/stdlib/_ctypes.pyi | 22 +- mypy/typeshed/stdlib/_curses.pyi | 540 +++++++++--------- mypy/typeshed/stdlib/_curses_panel.pyi | 6 +- mypy/typeshed/stdlib/_dbm.pyi | 6 +- mypy/typeshed/stdlib/_decimal.pyi | 16 +- .../stdlib/_frozen_importlib_external.pyi | 23 +- mypy/typeshed/stdlib/_hashlib.pyi | 3 +- mypy/typeshed/stdlib/_interpreters.pyi | 35 +- mypy/typeshed/stdlib/_io.pyi | 122 ++-- mypy/typeshed/stdlib/_lsprof.pyi | 2 + mypy/typeshed/stdlib/_lzma.pyi | 12 +- mypy/typeshed/stdlib/_msi.pyi | 78 +-- mypy/typeshed/stdlib/_multibytecodec.pyi | 5 + mypy/typeshed/stdlib/_pickle.pyi | 4 +- mypy/typeshed/stdlib/_queue.pyi | 2 + mypy/typeshed/stdlib/_random.pyi | 10 +- mypy/typeshed/stdlib/_socket.pyi | 3 +- mypy/typeshed/stdlib/_ssl.pyi | 4 +- mypy/typeshed/stdlib/_struct.pyi | 3 +- mypy/typeshed/stdlib/_thread.pyi | 5 +- mypy/typeshed/stdlib/_threading_local.pyi | 2 + mypy/typeshed/stdlib/_typeshed/__init__.pyi | 15 +- mypy/typeshed/stdlib/_warnings.pyi | 14 +- mypy/typeshed/stdlib/_winapi.pyi | 28 +- mypy/typeshed/stdlib/_zstd.pyi | 10 +- mypy/typeshed/stdlib/abc.pyi | 6 +- mypy/typeshed/stdlib/annotationlib.pyi | 16 +- mypy/typeshed/stdlib/argparse.pyi | 4 +- mypy/typeshed/stdlib/array.pyi | 3 +- mypy/typeshed/stdlib/ast.pyi | 56 +- mypy/typeshed/stdlib/asyncio/base_events.pyi | 26 +- mypy/typeshed/stdlib/asyncio/events.pyi | 42 +- mypy/typeshed/stdlib/asyncio/graph.pyi | 36 +- mypy/typeshed/stdlib/asyncio/protocols.pyi | 6 + mypy/typeshed/stdlib/asyncio/runners.pyi | 2 +- mypy/typeshed/stdlib/asyncio/streams.pyi | 8 +- mypy/typeshed/stdlib/asyncio/subprocess.pyi | 12 +- mypy/typeshed/stdlib/asyncio/tasks.pyi | 8 +- mypy/typeshed/stdlib/asyncio/transports.pyi | 9 +- mypy/typeshed/stdlib/asyncio/trsock.pyi | 30 +- mypy/typeshed/stdlib/asyncio/unix_events.pyi | 12 +- .../stdlib/asyncio/windows_events.pyi | 2 +- .../typeshed/stdlib/asyncio/windows_utils.pyi | 10 +- mypy/typeshed/stdlib/binascii.pyi | 4 +- mypy/typeshed/stdlib/builtins.pyi | 52 +- mypy/typeshed/stdlib/calendar.pyi | 38 +- mypy/typeshed/stdlib/cgi.pyi | 9 +- mypy/typeshed/stdlib/codecs.pyi | 88 ++- mypy/typeshed/stdlib/collections/__init__.pyi | 17 +- mypy/typeshed/stdlib/colorsys.pyi | 8 +- .../stdlib/compression/zstd/__init__.pyi | 1 + .../stdlib/concurrent/futures/_base.pyi | 3 +- .../stdlib/concurrent/futures/process.pyi | 6 +- .../concurrent/interpreters/_crossinterp.pyi | 1 + .../concurrent/interpreters/_queues.pyi | 4 +- mypy/typeshed/stdlib/configparser.pyi | 7 +- mypy/typeshed/stdlib/contextlib.pyi | 2 + mypy/typeshed/stdlib/crypt.pyi | 8 +- mypy/typeshed/stdlib/ctypes/__init__.pyi | 14 +- mypy/typeshed/stdlib/ctypes/_endian.pyi | 8 +- .../stdlib/ctypes/macholib/__init__.pyi | 4 +- mypy/typeshed/stdlib/ctypes/wintypes.pyi | 4 +- mypy/typeshed/stdlib/curses/__init__.pyi | 8 +- mypy/typeshed/stdlib/curses/ascii.pyi | 76 +-- mypy/typeshed/stdlib/dataclasses.pyi | 36 +- mypy/typeshed/stdlib/datetime.pyi | 6 +- mypy/typeshed/stdlib/decimal.pyi | 4 +- mypy/typeshed/stdlib/dis.pyi | 76 +-- mypy/typeshed/stdlib/distutils/file_util.pyi | 4 +- mypy/typeshed/stdlib/doctest.pyi | 34 +- .../stdlib/email/_header_value_parser.pyi | 2 +- mypy/typeshed/stdlib/email/charset.pyi | 13 +- mypy/typeshed/stdlib/enum.pyi | 59 +- mypy/typeshed/stdlib/errno.pyi | 405 ++++++------- mypy/typeshed/stdlib/filecmp.pyi | 2 +- mypy/typeshed/stdlib/fractions.pyi | 1 + mypy/typeshed/stdlib/functools.pyi | 21 +- mypy/typeshed/stdlib/gettext.pyi | 18 +- mypy/typeshed/stdlib/glob.pyi | 8 +- mypy/typeshed/stdlib/hmac.pyi | 1 + mypy/typeshed/stdlib/html/entities.pyi | 10 +- mypy/typeshed/stdlib/http/client.pyi | 138 ++--- mypy/typeshed/stdlib/http/server.pyi | 30 +- mypy/typeshed/stdlib/imp.pyi | 22 +- mypy/typeshed/stdlib/importlib/abc.pyi | 4 +- .../stdlib/importlib/metadata/__init__.pyi | 65 ++- mypy/typeshed/stdlib/inspect.pyi | 88 ++- mypy/typeshed/stdlib/io.pyi | 2 + mypy/typeshed/stdlib/ipaddress.pyi | 8 +- mypy/typeshed/stdlib/itertools.pyi | 22 +- .../stdlib/lib2to3/fixes/fix_tuple_params.pyi | 3 +- mypy/typeshed/stdlib/logging/__init__.pyi | 8 +- mypy/typeshed/stdlib/logging/config.pyi | 27 +- mypy/typeshed/stdlib/logging/handlers.pyi | 12 +- mypy/typeshed/stdlib/mmap.pyi | 5 +- mypy/typeshed/stdlib/msilib/__init__.pyi | 28 +- mypy/typeshed/stdlib/msilib/schema.pyi | 3 +- mypy/typeshed/stdlib/msilib/sequence.pyi | 13 +- mypy/typeshed/stdlib/msilib/text.pyi | 7 +- mypy/typeshed/stdlib/msvcrt.pyi | 8 +- .../stdlib/multiprocessing/managers.pyi | 1 + mypy/typeshed/stdlib/multiprocessing/util.pyi | 2 +- mypy/typeshed/stdlib/nturl2path.pyi | 4 +- mypy/typeshed/stdlib/numbers.pyi | 5 + mypy/typeshed/stdlib/opcode.pyi | 32 +- mypy/typeshed/stdlib/os/__init__.pyi | 373 ++++++------ mypy/typeshed/stdlib/ossaudiodev.pyi | 227 ++++---- mypy/typeshed/stdlib/pathlib/__init__.pyi | 48 +- mypy/typeshed/stdlib/pdb.pyi | 8 +- mypy/typeshed/stdlib/pickle.pyi | 146 ++--- mypy/typeshed/stdlib/pickletools.pyi | 15 +- mypy/typeshed/stdlib/platform.pyi | 24 +- mypy/typeshed/stdlib/plistlib.pyi | 6 +- mypy/typeshed/stdlib/poplib.pyi | 2 +- mypy/typeshed/stdlib/pydoc.pyi | 6 +- mypy/typeshed/stdlib/pydoc_data/topics.pyi | 4 +- mypy/typeshed/stdlib/random.pyi | 5 + mypy/typeshed/stdlib/resource.pyi | 41 +- mypy/typeshed/stdlib/select.pyi | 136 ++--- mypy/typeshed/stdlib/selectors.pyi | 6 +- mypy/typeshed/stdlib/smtplib.pyi | 14 +- mypy/typeshed/stdlib/socket.pyi | 155 ++--- mypy/typeshed/stdlib/sqlite3/__init__.pyi | 5 +- mypy/typeshed/stdlib/sqlite3/dbapi2.pyi | 17 +- mypy/typeshed/stdlib/sre_compile.pyi | 4 +- mypy/typeshed/stdlib/sre_constants.pyi | 15 +- mypy/typeshed/stdlib/sre_parse.pyi | 30 +- mypy/typeshed/stdlib/ssl.pyi | 4 +- mypy/typeshed/stdlib/statistics.pyi | 1 + mypy/typeshed/stdlib/string/__init__.pyi | 20 +- mypy/typeshed/stdlib/stringprep.pyi | 16 +- mypy/typeshed/stdlib/subprocess.pyi | 126 ++-- mypy/typeshed/stdlib/sunau.pyi | 30 +- mypy/typeshed/stdlib/symbol.pyi | 188 +++--- mypy/typeshed/stdlib/symtable.pyi | 7 +- mypy/typeshed/stdlib/sys/__init__.pyi | 2 +- mypy/typeshed/stdlib/sys/_monitoring.pyi | 10 +- mypy/typeshed/stdlib/tarfile.pyi | 30 +- mypy/typeshed/stdlib/telnetlib.pyi | 150 ++--- mypy/typeshed/stdlib/tempfile.pyi | 4 +- mypy/typeshed/stdlib/threading.pyi | 20 +- mypy/typeshed/stdlib/time.pyi | 26 +- mypy/typeshed/stdlib/tkinter/__init__.pyi | 49 +- mypy/typeshed/stdlib/tkinter/messagebox.pyi | 6 +- mypy/typeshed/stdlib/tkinter/ttk.pyi | 165 +++++- mypy/typeshed/stdlib/tokenize.pyi | 89 +-- mypy/typeshed/stdlib/tomllib.pyi | 2 +- mypy/typeshed/stdlib/traceback.pyi | 19 +- mypy/typeshed/stdlib/tracemalloc.pyi | 5 + mypy/typeshed/stdlib/turtle.pyi | 42 +- mypy/typeshed/stdlib/types.pyi | 44 +- mypy/typeshed/stdlib/typing.pyi | 56 +- mypy/typeshed/stdlib/typing_extensions.pyi | 15 +- mypy/typeshed/stdlib/unicodedata.pyi | 4 +- mypy/typeshed/stdlib/unittest/loader.pyi | 53 +- mypy/typeshed/stdlib/unittest/main.pyi | 7 +- mypy/typeshed/stdlib/unittest/mock.pyi | 213 ++++--- mypy/typeshed/stdlib/unittest/util.pyi | 12 +- mypy/typeshed/stdlib/urllib/parse.pyi | 28 +- mypy/typeshed/stdlib/uuid.pyi | 1 + mypy/typeshed/stdlib/venv/__init__.pyi | 3 +- mypy/typeshed/stdlib/wave.pyi | 34 +- mypy/typeshed/stdlib/weakref.pyi | 6 +- mypy/typeshed/stdlib/webbrowser.pyi | 14 +- mypy/typeshed/stdlib/winreg.pyi | 14 +- mypy/typeshed/stdlib/wsgiref/headers.pyi | 4 +- .../typeshed/stdlib/wsgiref/simple_server.pyi | 8 +- mypy/typeshed/stdlib/xml/dom/NodeFilter.pyi | 34 +- mypy/typeshed/stdlib/xml/dom/__init__.pyi | 33 +- mypy/typeshed/stdlib/xml/dom/expatbuilder.pyi | 21 +- mypy/typeshed/stdlib/xml/dom/minicompat.pyi | 2 + mypy/typeshed/stdlib/xml/dom/minidom.pyi | 26 + mypy/typeshed/stdlib/xml/dom/pulldom.pyi | 2 +- mypy/typeshed/stdlib/xml/dom/xmlbuilder.pyi | 24 +- .../stdlib/xml/etree/ElementInclude.pyi | 7 +- .../typeshed/stdlib/xml/etree/ElementPath.pyi | 6 +- .../typeshed/stdlib/xml/etree/ElementTree.pyi | 17 +- mypy/typeshed/stdlib/xml/sax/__init__.pyi | 4 +- mypy/typeshed/stdlib/xml/sax/expatreader.pyi | 4 +- mypy/typeshed/stdlib/xml/sax/handler.pyi | 32 +- mypy/typeshed/stdlib/zipfile/__init__.pyi | 31 +- mypy/typeshed/stdlib/zipfile/_path/glob.pyi | 6 +- mypy/typeshed/stdlib/zoneinfo/__init__.pyi | 3 +- test-data/unit/pythoneval.test | 4 +- 191 files changed, 3640 insertions(+), 2564 deletions(-) diff --git a/misc/typeshed_patches/0001-Revert-Remove-redundant-inheritances-from-Iterator.patch b/misc/typeshed_patches/0001-Revert-Remove-redundant-inheritances-from-Iterator.patch index d3f49a4eef3ef..fdcc14cec3c6a 100644 --- a/misc/typeshed_patches/0001-Revert-Remove-redundant-inheritances-from-Iterator.patch +++ b/misc/typeshed_patches/0001-Revert-Remove-redundant-inheritances-from-Iterator.patch @@ -1,4 +1,4 @@ -From c217544146d36899d50e828d627652a0d8f63bb7 Mon Sep 17 00:00:00 2001 +From 438dbb1300b77331940d7db8f010e97305745116 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Sat, 21 Dec 2024 22:36:38 +0100 Subject: [PATCH] Revert Remove redundant inheritances from Iterator in @@ -15,7 +15,7 @@ Subject: [PATCH] Revert Remove redundant inheritances from Iterator in 7 files changed, 34 insertions(+), 34 deletions(-) diff --git a/mypy/typeshed/stdlib/_asyncio.pyi b/mypy/typeshed/stdlib/_asyncio.pyi -index ed56f33af..5253e967e 100644 +index d663f5d93..f43178e4d 100644 --- a/mypy/typeshed/stdlib/_asyncio.pyi +++ b/mypy/typeshed/stdlib/_asyncio.pyi @@ -1,6 +1,6 @@ @@ -26,59 +26,59 @@ index ed56f33af..5253e967e 100644 from contextvars import Context from types import FrameType, GenericAlias from typing import Any, Literal, TextIO, TypeVar -@@ -10,7 +10,7 @@ _T = TypeVar("_T") - _T_co = TypeVar("_T_co", covariant=True) +@@ -11,7 +11,7 @@ _T_co = TypeVar("_T_co", covariant=True) _TaskYieldType: TypeAlias = Future[object] | None + @disjoint_base -class Future(Awaitable[_T]): +class Future(Awaitable[_T], Iterable[_T]): _state: str @property def _exception(self) -> BaseException | None: ... diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi -index 0575be3c8..d9be595fe 100644 +index f2dd00079..784ee7eac 100644 --- a/mypy/typeshed/stdlib/builtins.pyi +++ b/mypy/typeshed/stdlib/builtins.pyi -@@ -1186,7 +1186,7 @@ class frozenset(AbstractSet[_T_co]): - def __hash__(self) -> int: ... +@@ -1209,7 +1209,7 @@ class frozenset(AbstractSet[_T_co]): def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + @disjoint_base -class enumerate(Generic[_T]): +class enumerate(Iterator[tuple[int, _T]]): def __new__(cls, iterable: Iterable[_T], start: int = 0) -> Self: ... def __iter__(self) -> Self: ... def __next__(self) -> tuple[int, _T]: ... -@@ -1380,7 +1380,7 @@ else: - +@@ -1405,7 +1405,7 @@ else: exit: _sitebuiltins.Quitter + @disjoint_base -class filter(Generic[_T]): +class filter(Iterator[_T]): @overload def __new__(cls, function: None, iterable: Iterable[_T | None], /) -> Self: ... @overload -@@ -1444,7 +1444,7 @@ license: _sitebuiltins._Printer +@@ -1469,7 +1469,7 @@ license: _sitebuiltins._Printer def locals() -> dict[str, Any]: ... - + @disjoint_base -class map(Generic[_S]): +class map(Iterator[_S]): # 3.14 adds `strict` argument. if sys.version_info >= (3, 14): @overload -@@ -1750,7 +1750,7 @@ def pow(base: _SupportsSomeKindOfPow, exp: complex, mod: None = None) -> complex - +@@ -1776,7 +1776,7 @@ def pow(base: _SupportsSomeKindOfPow, exp: complex, mod: None = None) -> complex quit: _sitebuiltins.Quitter + @disjoint_base -class reversed(Generic[_T]): +class reversed(Iterator[_T]): @overload def __new__(cls, sequence: Reversible[_T], /) -> Iterator[_T]: ... # type: ignore[misc] @overload -@@ -1814,7 +1814,7 @@ def vars(object: type, /) -> types.MappingProxyType[str, Any]: ... +@@ -1840,7 +1840,7 @@ def vars(object: type, /) -> types.MappingProxyType[str, Any]: ... @overload def vars(object: Any = ..., /) -> dict[str, Any]: ... - + @disjoint_base -class zip(Generic[_T_co]): +class zip(Iterator[_T_co]): if sys.version_info >= (3, 10): @@ -131,97 +131,102 @@ index 910d63814..eb942bc55 100644 # encoding and errors are added @overload diff --git a/mypy/typeshed/stdlib/itertools.pyi b/mypy/typeshed/stdlib/itertools.pyi -index d0085dd72..7d05b1318 100644 +index fe4ccbdf8..73745fe92 100644 --- a/mypy/typeshed/stdlib/itertools.pyi +++ b/mypy/typeshed/stdlib/itertools.pyi -@@ -27,7 +27,7 @@ _Predicate: TypeAlias = Callable[[_T], object] - +@@ -28,7 +28,7 @@ _Predicate: TypeAlias = Callable[[_T], object] # Technically count can take anything that implements a number protocol and has an add method # but we can't enforce the add method + @disjoint_base -class count(Generic[_N]): +class count(Iterator[_N]): @overload def __new__(cls) -> count[int]: ... @overload -@@ -37,12 +37,12 @@ class count(Generic[_N]): - def __next__(self) -> _N: ... +@@ -39,13 +39,13 @@ class count(Generic[_N]): def __iter__(self) -> Self: ... + @disjoint_base -class cycle(Generic[_T]): +class cycle(Iterator[_T]): def __new__(cls, iterable: Iterable[_T], /) -> Self: ... def __next__(self) -> _T: ... def __iter__(self) -> Self: ... + @disjoint_base -class repeat(Generic[_T]): +class repeat(Iterator[_T]): @overload def __new__(cls, object: _T) -> Self: ... @overload -@@ -51,7 +51,7 @@ class repeat(Generic[_T]): - def __iter__(self) -> Self: ... +@@ -55,7 +55,7 @@ class repeat(Generic[_T]): def __length_hint__(self) -> int: ... + @disjoint_base -class accumulate(Generic[_T]): +class accumulate(Iterator[_T]): @overload def __new__(cls, iterable: Iterable[_T], func: None = None, *, initial: _T | None = ...) -> Self: ... @overload -@@ -59,7 +59,7 @@ class accumulate(Generic[_T]): - def __iter__(self) -> Self: ... +@@ -64,7 +64,7 @@ class accumulate(Generic[_T]): def __next__(self) -> _T: ... + @disjoint_base -class chain(Generic[_T]): +class chain(Iterator[_T]): def __new__(cls, *iterables: Iterable[_T]) -> Self: ... def __next__(self) -> _T: ... def __iter__(self) -> Self: ... -@@ -68,22 +68,22 @@ class chain(Generic[_T]): - def from_iterable(cls: type[Any], iterable: Iterable[Iterable[_S]], /) -> chain[_S]: ... +@@ -74,25 +74,25 @@ class chain(Generic[_T]): def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + @disjoint_base -class compress(Generic[_T]): +class compress(Iterator[_T]): def __new__(cls, data: Iterable[_T], selectors: Iterable[Any]) -> Self: ... def __iter__(self) -> Self: ... def __next__(self) -> _T: ... + @disjoint_base -class dropwhile(Generic[_T]): +class dropwhile(Iterator[_T]): def __new__(cls, predicate: _Predicate[_T], iterable: Iterable[_T], /) -> Self: ... def __iter__(self) -> Self: ... def __next__(self) -> _T: ... + @disjoint_base -class filterfalse(Generic[_T]): +class filterfalse(Iterator[_T]): def __new__(cls, function: _Predicate[_T] | None, iterable: Iterable[_T], /) -> Self: ... def __iter__(self) -> Self: ... def __next__(self) -> _T: ... + @disjoint_base -class groupby(Generic[_T_co, _S_co]): +class groupby(Iterator[tuple[_T_co, Iterator[_S_co]]], Generic[_T_co, _S_co]): @overload def __new__(cls, iterable: Iterable[_T1], key: None = None) -> groupby[_T1, _T1]: ... @overload -@@ -91,7 +91,7 @@ class groupby(Generic[_T_co, _S_co]): - def __iter__(self) -> Self: ... +@@ -101,7 +101,7 @@ class groupby(Generic[_T_co, _S_co]): def __next__(self) -> tuple[_T_co, Iterator[_S_co]]: ... + @disjoint_base -class islice(Generic[_T]): +class islice(Iterator[_T]): @overload def __new__(cls, iterable: Iterable[_T], stop: int | None, /) -> Self: ... @overload -@@ -99,19 +99,19 @@ class islice(Generic[_T]): - def __iter__(self) -> Self: ... +@@ -110,20 +110,20 @@ class islice(Generic[_T]): def __next__(self) -> _T: ... + @disjoint_base -class starmap(Generic[_T_co]): +class starmap(Iterator[_T_co]): def __new__(cls, function: Callable[..., _T], iterable: Iterable[Iterable[Any]], /) -> starmap[_T]: ... def __iter__(self) -> Self: ... def __next__(self) -> _T_co: ... + @disjoint_base -class takewhile(Generic[_T]): +class takewhile(Iterator[_T]): def __new__(cls, predicate: _Predicate[_T], iterable: Iterable[_T], /) -> Self: ... @@ -229,52 +234,52 @@ index d0085dd72..7d05b1318 100644 def __next__(self) -> _T: ... def tee(iterable: Iterable[_T], n: int = 2, /) -> tuple[Iterator[_T], ...]: ... - + @disjoint_base -class zip_longest(Generic[_T_co]): +class zip_longest(Iterator[_T_co]): # one iterable (fillvalue doesn't matter) @overload def __new__(cls, iter1: Iterable[_T1], /, *, fillvalue: object = ...) -> zip_longest[tuple[_T1]]: ... -@@ -189,7 +189,7 @@ class zip_longest(Generic[_T_co]): - def __iter__(self) -> Self: ... +@@ -202,7 +202,7 @@ class zip_longest(Generic[_T_co]): def __next__(self) -> _T_co: ... + @disjoint_base -class product(Generic[_T_co]): +class product(Iterator[_T_co]): @overload def __new__(cls, iter1: Iterable[_T1], /) -> product[tuple[_T1]]: ... @overload -@@ -274,7 +274,7 @@ class product(Generic[_T_co]): - def __iter__(self) -> Self: ... +@@ -288,7 +288,7 @@ class product(Generic[_T_co]): def __next__(self) -> _T_co: ... + @disjoint_base -class permutations(Generic[_T_co]): +class permutations(Iterator[_T_co]): @overload def __new__(cls, iterable: Iterable[_T], r: Literal[2]) -> permutations[tuple[_T, _T]]: ... @overload -@@ -288,7 +288,7 @@ class permutations(Generic[_T_co]): - def __iter__(self) -> Self: ... +@@ -303,7 +303,7 @@ class permutations(Generic[_T_co]): def __next__(self) -> _T_co: ... + @disjoint_base -class combinations(Generic[_T_co]): +class combinations(Iterator[_T_co]): @overload def __new__(cls, iterable: Iterable[_T], r: Literal[2]) -> combinations[tuple[_T, _T]]: ... @overload -@@ -302,7 +302,7 @@ class combinations(Generic[_T_co]): - def __iter__(self) -> Self: ... +@@ -318,7 +318,7 @@ class combinations(Generic[_T_co]): def __next__(self) -> _T_co: ... + @disjoint_base -class combinations_with_replacement(Generic[_T_co]): +class combinations_with_replacement(Iterator[_T_co]): @overload def __new__(cls, iterable: Iterable[_T], r: Literal[2]) -> combinations_with_replacement[tuple[_T, _T]]: ... @overload -@@ -317,13 +317,13 @@ class combinations_with_replacement(Generic[_T_co]): - def __next__(self) -> _T_co: ... +@@ -334,14 +334,14 @@ class combinations_with_replacement(Generic[_T_co]): if sys.version_info >= (3, 10): + @disjoint_base - class pairwise(Generic[_T_co]): + class pairwise(Iterator[_T_co]): def __new__(cls, iterable: Iterable[_T], /) -> pairwise[tuple[_T, _T]]: ... @@ -282,6 +287,7 @@ index d0085dd72..7d05b1318 100644 def __next__(self) -> _T_co: ... if sys.version_info >= (3, 12): + @disjoint_base - class batched(Generic[_T_co]): + class batched(Iterator[tuple[_T_co, ...]], Generic[_T_co]): if sys.version_info >= (3, 13): @@ -307,18 +313,18 @@ index b79f9e773..f276372d0 100644 def __iter__(self) -> Self: ... def next(self, timeout: float | None = None) -> _T: ... diff --git a/mypy/typeshed/stdlib/sqlite3/__init__.pyi b/mypy/typeshed/stdlib/sqlite3/__init__.pyi -index bcfea3a13..5a659deac 100644 +index 6b0f1ba94..882cd143c 100644 --- a/mypy/typeshed/stdlib/sqlite3/__init__.pyi +++ b/mypy/typeshed/stdlib/sqlite3/__init__.pyi -@@ -405,7 +405,7 @@ class Connection: - self, type: type[BaseException] | None, value: BaseException | None, traceback: TracebackType | None, / +@@ -407,7 +407,7 @@ class Connection: ) -> Literal[False]: ... + @disjoint_base -class Cursor: +class Cursor(Iterator[Any]): arraysize: int @property def connection(self) -> Connection: ... -- -2.50.1 +2.51.0 diff --git a/mypy/typeshed/stdlib/_ast.pyi b/mypy/typeshed/stdlib/_ast.pyi index 00c6b357f7d80..d8d5a1829991e 100644 --- a/mypy/typeshed/stdlib/_ast.pyi +++ b/mypy/typeshed/stdlib/_ast.pyi @@ -108,7 +108,7 @@ from ast import ( unaryop as unaryop, withitem as withitem, ) -from typing import Literal +from typing import Final if sys.version_info >= (3, 12): from ast import ( @@ -137,9 +137,9 @@ if sys.version_info >= (3, 10): pattern as pattern, ) -PyCF_ALLOW_TOP_LEVEL_AWAIT: Literal[8192] -PyCF_ONLY_AST: Literal[1024] -PyCF_TYPE_COMMENTS: Literal[4096] +PyCF_ALLOW_TOP_LEVEL_AWAIT: Final = 8192 +PyCF_ONLY_AST: Final = 1024 +PyCF_TYPE_COMMENTS: Final = 4096 if sys.version_info >= (3, 13): - PyCF_OPTIMIZED_AST: Literal[33792] + PyCF_OPTIMIZED_AST: Final = 33792 diff --git a/mypy/typeshed/stdlib/_asyncio.pyi b/mypy/typeshed/stdlib/_asyncio.pyi index 5253e967e5a37..f43178e4d7258 100644 --- a/mypy/typeshed/stdlib/_asyncio.pyi +++ b/mypy/typeshed/stdlib/_asyncio.pyi @@ -4,12 +4,13 @@ from collections.abc import Awaitable, Callable, Coroutine, Generator, Iterable from contextvars import Context from types import FrameType, GenericAlias from typing import Any, Literal, TextIO, TypeVar -from typing_extensions import Self, TypeAlias +from typing_extensions import Self, TypeAlias, disjoint_base _T = TypeVar("_T") _T_co = TypeVar("_T_co", covariant=True) _TaskYieldType: TypeAlias = Future[object] | None +@disjoint_base class Future(Awaitable[_T], Iterable[_T]): _state: str @property @@ -20,7 +21,7 @@ class Future(Awaitable[_T], Iterable[_T]): @_log_traceback.setter def _log_traceback(self, val: Literal[False]) -> None: ... _asyncio_future_blocking: bool # is a part of duck-typing contract for `Future` - def __init__(self, *, loop: AbstractEventLoop | None = ...) -> None: ... + def __init__(self, *, loop: AbstractEventLoop | None = None) -> None: ... def __del__(self) -> None: ... def get_loop(self) -> AbstractEventLoop: ... @property @@ -49,6 +50,7 @@ else: # While this is true in general, here it's sort-of okay to have a covariant subclass, # since the only reason why `asyncio.Future` is invariant is the `set_result()` method, # and `asyncio.Task.set_result()` always raises. +@disjoint_base class Task(Future[_T_co]): # type: ignore[type-var] # pyright: ignore[reportInvalidTypeArguments] if sys.version_info >= (3, 12): def __init__( @@ -56,7 +58,7 @@ class Task(Future[_T_co]): # type: ignore[type-var] # pyright: ignore[reportIn coro: _TaskCompatibleCoro[_T_co], *, loop: AbstractEventLoop | None = None, - name: str | None = ..., + name: str | None = None, context: Context | None = None, eager_start: bool = False, ) -> None: ... @@ -66,12 +68,12 @@ class Task(Future[_T_co]): # type: ignore[type-var] # pyright: ignore[reportIn coro: _TaskCompatibleCoro[_T_co], *, loop: AbstractEventLoop | None = None, - name: str | None = ..., + name: str | None = None, context: Context | None = None, ) -> None: ... else: def __init__( - self, coro: _TaskCompatibleCoro[_T_co], *, loop: AbstractEventLoop | None = None, name: str | None = ... + self, coro: _TaskCompatibleCoro[_T_co], *, loop: AbstractEventLoop | None = None, name: str | None = None ) -> None: ... if sys.version_info >= (3, 12): diff --git a/mypy/typeshed/stdlib/_blake2.pyi b/mypy/typeshed/stdlib/_blake2.pyi index d578df55c2faa..a6c3869fb8513 100644 --- a/mypy/typeshed/stdlib/_blake2.pyi +++ b/mypy/typeshed/stdlib/_blake2.pyi @@ -1,15 +1,16 @@ +import sys from _typeshed import ReadableBuffer -from typing import ClassVar, final +from typing import ClassVar, Final, final from typing_extensions import Self -BLAKE2B_MAX_DIGEST_SIZE: int = 64 -BLAKE2B_MAX_KEY_SIZE: int = 64 -BLAKE2B_PERSON_SIZE: int = 16 -BLAKE2B_SALT_SIZE: int = 16 -BLAKE2S_MAX_DIGEST_SIZE: int = 32 -BLAKE2S_MAX_KEY_SIZE: int = 32 -BLAKE2S_PERSON_SIZE: int = 8 -BLAKE2S_SALT_SIZE: int = 8 +BLAKE2B_MAX_DIGEST_SIZE: Final = 64 +BLAKE2B_MAX_KEY_SIZE: Final = 64 +BLAKE2B_PERSON_SIZE: Final = 16 +BLAKE2B_SALT_SIZE: Final = 16 +BLAKE2S_MAX_DIGEST_SIZE: Final = 32 +BLAKE2S_MAX_KEY_SIZE: Final = 32 +BLAKE2S_PERSON_SIZE: Final = 8 +BLAKE2S_SALT_SIZE: Final = 8 @final class blake2b: @@ -20,24 +21,45 @@ class blake2b: block_size: int digest_size: int name: str - def __new__( - cls, - data: ReadableBuffer = b"", - /, - *, - digest_size: int = 64, - key: ReadableBuffer = b"", - salt: ReadableBuffer = b"", - person: ReadableBuffer = b"", - fanout: int = 1, - depth: int = 1, - leaf_size: int = 0, - node_offset: int = 0, - node_depth: int = 0, - inner_size: int = 0, - last_node: bool = False, - usedforsecurity: bool = True, - ) -> Self: ... + if sys.version_info >= (3, 13): + def __new__( + cls, + data: ReadableBuffer = b"", + *, + digest_size: int = 64, + key: ReadableBuffer = b"", + salt: ReadableBuffer = b"", + person: ReadableBuffer = b"", + fanout: int = 1, + depth: int = 1, + leaf_size: int = 0, + node_offset: int = 0, + node_depth: int = 0, + inner_size: int = 0, + last_node: bool = False, + usedforsecurity: bool = True, + string: ReadableBuffer | None = None, + ) -> Self: ... + else: + def __new__( + cls, + data: ReadableBuffer = b"", + /, + *, + digest_size: int = 64, + key: ReadableBuffer = b"", + salt: ReadableBuffer = b"", + person: ReadableBuffer = b"", + fanout: int = 1, + depth: int = 1, + leaf_size: int = 0, + node_offset: int = 0, + node_depth: int = 0, + inner_size: int = 0, + last_node: bool = False, + usedforsecurity: bool = True, + ) -> Self: ... + def copy(self) -> Self: ... def digest(self) -> bytes: ... def hexdigest(self) -> str: ... @@ -52,24 +74,45 @@ class blake2s: block_size: int digest_size: int name: str - def __new__( - cls, - data: ReadableBuffer = b"", - /, - *, - digest_size: int = 32, - key: ReadableBuffer = b"", - salt: ReadableBuffer = b"", - person: ReadableBuffer = b"", - fanout: int = 1, - depth: int = 1, - leaf_size: int = 0, - node_offset: int = 0, - node_depth: int = 0, - inner_size: int = 0, - last_node: bool = False, - usedforsecurity: bool = True, - ) -> Self: ... + if sys.version_info >= (3, 13): + def __new__( + cls, + data: ReadableBuffer = b"", + *, + digest_size: int = 32, + key: ReadableBuffer = b"", + salt: ReadableBuffer = b"", + person: ReadableBuffer = b"", + fanout: int = 1, + depth: int = 1, + leaf_size: int = 0, + node_offset: int = 0, + node_depth: int = 0, + inner_size: int = 0, + last_node: bool = False, + usedforsecurity: bool = True, + string: ReadableBuffer | None = None, + ) -> Self: ... + else: + def __new__( + cls, + data: ReadableBuffer = b"", + /, + *, + digest_size: int = 32, + key: ReadableBuffer = b"", + salt: ReadableBuffer = b"", + person: ReadableBuffer = b"", + fanout: int = 1, + depth: int = 1, + leaf_size: int = 0, + node_offset: int = 0, + node_depth: int = 0, + inner_size: int = 0, + last_node: bool = False, + usedforsecurity: bool = True, + ) -> Self: ... + def copy(self) -> Self: ... def digest(self) -> bytes: ... def hexdigest(self) -> str: ... diff --git a/mypy/typeshed/stdlib/_collections_abc.pyi b/mypy/typeshed/stdlib/_collections_abc.pyi index b099bdd98f3c4..c63606a13ca99 100644 --- a/mypy/typeshed/stdlib/_collections_abc.pyi +++ b/mypy/typeshed/stdlib/_collections_abc.pyi @@ -103,5 +103,6 @@ class dict_items(ItemsView[_KT_co, _VT_co]): # undocumented if sys.version_info >= (3, 12): @runtime_checkable class Buffer(Protocol): + __slots__ = () @abstractmethod def __buffer__(self, flags: int, /) -> memoryview: ... diff --git a/mypy/typeshed/stdlib/_compat_pickle.pyi b/mypy/typeshed/stdlib/_compat_pickle.pyi index 50fb22442cc92..32c0b542d9913 100644 --- a/mypy/typeshed/stdlib/_compat_pickle.pyi +++ b/mypy/typeshed/stdlib/_compat_pickle.pyi @@ -1,8 +1,10 @@ -IMPORT_MAPPING: dict[str, str] -NAME_MAPPING: dict[tuple[str, str], tuple[str, str]] -PYTHON2_EXCEPTIONS: tuple[str, ...] -MULTIPROCESSING_EXCEPTIONS: tuple[str, ...] -REVERSE_IMPORT_MAPPING: dict[str, str] -REVERSE_NAME_MAPPING: dict[tuple[str, str], tuple[str, str]] -PYTHON3_OSERROR_EXCEPTIONS: tuple[str, ...] -PYTHON3_IMPORTERROR_EXCEPTIONS: tuple[str, ...] +from typing import Final + +IMPORT_MAPPING: Final[dict[str, str]] +NAME_MAPPING: Final[dict[tuple[str, str], tuple[str, str]]] +PYTHON2_EXCEPTIONS: Final[tuple[str, ...]] +MULTIPROCESSING_EXCEPTIONS: Final[tuple[str, ...]] +REVERSE_IMPORT_MAPPING: Final[dict[str, str]] +REVERSE_NAME_MAPPING: Final[dict[tuple[str, str], tuple[str, str]]] +PYTHON3_OSERROR_EXCEPTIONS: Final[tuple[str, ...]] +PYTHON3_IMPORTERROR_EXCEPTIONS: Final[tuple[str, ...]] diff --git a/mypy/typeshed/stdlib/_csv.pyi b/mypy/typeshed/stdlib/_csv.pyi index efe9ad69bd31d..4128178c18b34 100644 --- a/mypy/typeshed/stdlib/_csv.pyi +++ b/mypy/typeshed/stdlib/_csv.pyi @@ -3,7 +3,7 @@ import sys from _typeshed import SupportsWrite from collections.abc import Iterable from typing import Any, Final, Literal, type_check_only -from typing_extensions import Self, TypeAlias +from typing_extensions import Self, TypeAlias, disjoint_base __version__: Final[str] @@ -24,6 +24,7 @@ class Error(Exception): ... _DialectLike: TypeAlias = str | Dialect | csv.Dialect | type[Dialect | csv.Dialect] +@disjoint_base class Dialect: delimiter: str quotechar: str | None @@ -35,7 +36,7 @@ class Dialect: strict: bool def __new__( cls, - dialect: _DialectLike | None = ..., + dialect: _DialectLike | None = None, delimiter: str = ",", doublequote: bool = True, escapechar: str | None = None, @@ -48,6 +49,7 @@ class Dialect: if sys.version_info >= (3, 10): # This class calls itself _csv.reader. + @disjoint_base class Reader: @property def dialect(self) -> Dialect: ... @@ -56,6 +58,7 @@ if sys.version_info >= (3, 10): def __next__(self) -> list[str]: ... # This class calls itself _csv.writer. + @disjoint_base class Writer: @property def dialect(self) -> Dialect: ... diff --git a/mypy/typeshed/stdlib/_ctypes.pyi b/mypy/typeshed/stdlib/_ctypes.pyi index bfd0f910f4822..082a31f705622 100644 --- a/mypy/typeshed/stdlib/_ctypes.pyi +++ b/mypy/typeshed/stdlib/_ctypes.pyi @@ -5,24 +5,24 @@ from abc import abstractmethod from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from ctypes import CDLL, ArgumentError as ArgumentError, c_void_p from types import GenericAlias -from typing import Any, ClassVar, Generic, TypeVar, final, overload, type_check_only +from typing import Any, ClassVar, Final, Generic, TypeVar, final, overload, type_check_only from typing_extensions import Self, TypeAlias _T = TypeVar("_T") _CT = TypeVar("_CT", bound=_CData) -FUNCFLAG_CDECL: int -FUNCFLAG_PYTHONAPI: int -FUNCFLAG_USE_ERRNO: int -FUNCFLAG_USE_LASTERROR: int -RTLD_GLOBAL: int -RTLD_LOCAL: int +FUNCFLAG_CDECL: Final = 0x1 +FUNCFLAG_PYTHONAPI: Final = 0x4 +FUNCFLAG_USE_ERRNO: Final = 0x8 +FUNCFLAG_USE_LASTERROR: Final = 0x10 +RTLD_GLOBAL: Final[int] +RTLD_LOCAL: Final[int] if sys.version_info >= (3, 11): - CTYPES_MAX_ARGCOUNT: int + CTYPES_MAX_ARGCOUNT: Final[int] if sys.version_info >= (3, 12): - SIZEOF_TIME_T: int + SIZEOF_TIME_T: Final[int] if sys.platform == "win32": # Description, Source, HelpFile, HelpContext, scode @@ -37,8 +37,8 @@ if sys.platform == "win32": def CopyComPointer(src: _PointerLike, dst: _PointerLike | _CArgObject) -> int: ... - FUNCFLAG_HRESULT: int - FUNCFLAG_STDCALL: int + FUNCFLAG_HRESULT: Final = 0x2 + FUNCFLAG_STDCALL: Final = 0x0 def FormatError(code: int = ...) -> str: ... def get_last_error() -> int: ... diff --git a/mypy/typeshed/stdlib/_curses.pyi b/mypy/typeshed/stdlib/_curses.pyi index f21a9ca602708..d4e4d48f4e20f 100644 --- a/mypy/typeshed/stdlib/_curses.pyi +++ b/mypy/typeshed/stdlib/_curses.pyi @@ -1,7 +1,7 @@ import sys from _typeshed import ReadOnlyBuffer, SupportsRead, SupportsWrite from curses import _ncurses_version -from typing import Any, final, overload +from typing import Any, Final, final, overload from typing_extensions import TypeAlias # NOTE: This module is ordinarily only available on Unix, but the windows-curses @@ -11,270 +11,270 @@ from typing_extensions import TypeAlias _ChType: TypeAlias = str | bytes | int # ACS codes are only initialized after initscr is called -ACS_BBSS: int -ACS_BLOCK: int -ACS_BOARD: int -ACS_BSBS: int -ACS_BSSB: int -ACS_BSSS: int -ACS_BTEE: int -ACS_BULLET: int -ACS_CKBOARD: int -ACS_DARROW: int -ACS_DEGREE: int -ACS_DIAMOND: int -ACS_GEQUAL: int -ACS_HLINE: int -ACS_LANTERN: int -ACS_LARROW: int -ACS_LEQUAL: int -ACS_LLCORNER: int -ACS_LRCORNER: int -ACS_LTEE: int -ACS_NEQUAL: int -ACS_PI: int -ACS_PLMINUS: int -ACS_PLUS: int -ACS_RARROW: int -ACS_RTEE: int -ACS_S1: int -ACS_S3: int -ACS_S7: int -ACS_S9: int -ACS_SBBS: int -ACS_SBSB: int -ACS_SBSS: int -ACS_SSBB: int -ACS_SSBS: int -ACS_SSSB: int -ACS_SSSS: int -ACS_STERLING: int -ACS_TTEE: int -ACS_UARROW: int -ACS_ULCORNER: int -ACS_URCORNER: int -ACS_VLINE: int -ALL_MOUSE_EVENTS: int -A_ALTCHARSET: int -A_ATTRIBUTES: int -A_BLINK: int -A_BOLD: int -A_CHARTEXT: int -A_COLOR: int -A_DIM: int -A_HORIZONTAL: int -A_INVIS: int -A_ITALIC: int -A_LEFT: int -A_LOW: int -A_NORMAL: int -A_PROTECT: int -A_REVERSE: int -A_RIGHT: int -A_STANDOUT: int -A_TOP: int -A_UNDERLINE: int -A_VERTICAL: int -BUTTON1_CLICKED: int -BUTTON1_DOUBLE_CLICKED: int -BUTTON1_PRESSED: int -BUTTON1_RELEASED: int -BUTTON1_TRIPLE_CLICKED: int -BUTTON2_CLICKED: int -BUTTON2_DOUBLE_CLICKED: int -BUTTON2_PRESSED: int -BUTTON2_RELEASED: int -BUTTON2_TRIPLE_CLICKED: int -BUTTON3_CLICKED: int -BUTTON3_DOUBLE_CLICKED: int -BUTTON3_PRESSED: int -BUTTON3_RELEASED: int -BUTTON3_TRIPLE_CLICKED: int -BUTTON4_CLICKED: int -BUTTON4_DOUBLE_CLICKED: int -BUTTON4_PRESSED: int -BUTTON4_RELEASED: int -BUTTON4_TRIPLE_CLICKED: int +ACS_BBSS: Final[int] +ACS_BLOCK: Final[int] +ACS_BOARD: Final[int] +ACS_BSBS: Final[int] +ACS_BSSB: Final[int] +ACS_BSSS: Final[int] +ACS_BTEE: Final[int] +ACS_BULLET: Final[int] +ACS_CKBOARD: Final[int] +ACS_DARROW: Final[int] +ACS_DEGREE: Final[int] +ACS_DIAMOND: Final[int] +ACS_GEQUAL: Final[int] +ACS_HLINE: Final[int] +ACS_LANTERN: Final[int] +ACS_LARROW: Final[int] +ACS_LEQUAL: Final[int] +ACS_LLCORNER: Final[int] +ACS_LRCORNER: Final[int] +ACS_LTEE: Final[int] +ACS_NEQUAL: Final[int] +ACS_PI: Final[int] +ACS_PLMINUS: Final[int] +ACS_PLUS: Final[int] +ACS_RARROW: Final[int] +ACS_RTEE: Final[int] +ACS_S1: Final[int] +ACS_S3: Final[int] +ACS_S7: Final[int] +ACS_S9: Final[int] +ACS_SBBS: Final[int] +ACS_SBSB: Final[int] +ACS_SBSS: Final[int] +ACS_SSBB: Final[int] +ACS_SSBS: Final[int] +ACS_SSSB: Final[int] +ACS_SSSS: Final[int] +ACS_STERLING: Final[int] +ACS_TTEE: Final[int] +ACS_UARROW: Final[int] +ACS_ULCORNER: Final[int] +ACS_URCORNER: Final[int] +ACS_VLINE: Final[int] +ALL_MOUSE_EVENTS: Final[int] +A_ALTCHARSET: Final[int] +A_ATTRIBUTES: Final[int] +A_BLINK: Final[int] +A_BOLD: Final[int] +A_CHARTEXT: Final[int] +A_COLOR: Final[int] +A_DIM: Final[int] +A_HORIZONTAL: Final[int] +A_INVIS: Final[int] +A_ITALIC: Final[int] +A_LEFT: Final[int] +A_LOW: Final[int] +A_NORMAL: Final[int] +A_PROTECT: Final[int] +A_REVERSE: Final[int] +A_RIGHT: Final[int] +A_STANDOUT: Final[int] +A_TOP: Final[int] +A_UNDERLINE: Final[int] +A_VERTICAL: Final[int] +BUTTON1_CLICKED: Final[int] +BUTTON1_DOUBLE_CLICKED: Final[int] +BUTTON1_PRESSED: Final[int] +BUTTON1_RELEASED: Final[int] +BUTTON1_TRIPLE_CLICKED: Final[int] +BUTTON2_CLICKED: Final[int] +BUTTON2_DOUBLE_CLICKED: Final[int] +BUTTON2_PRESSED: Final[int] +BUTTON2_RELEASED: Final[int] +BUTTON2_TRIPLE_CLICKED: Final[int] +BUTTON3_CLICKED: Final[int] +BUTTON3_DOUBLE_CLICKED: Final[int] +BUTTON3_PRESSED: Final[int] +BUTTON3_RELEASED: Final[int] +BUTTON3_TRIPLE_CLICKED: Final[int] +BUTTON4_CLICKED: Final[int] +BUTTON4_DOUBLE_CLICKED: Final[int] +BUTTON4_PRESSED: Final[int] +BUTTON4_RELEASED: Final[int] +BUTTON4_TRIPLE_CLICKED: Final[int] # Darwin ncurses doesn't provide BUTTON5_* constants prior to 3.12.10 and 3.13.3 if sys.version_info >= (3, 10): if sys.version_info >= (3, 12) or sys.platform != "darwin": - BUTTON5_PRESSED: int - BUTTON5_RELEASED: int - BUTTON5_CLICKED: int - BUTTON5_DOUBLE_CLICKED: int - BUTTON5_TRIPLE_CLICKED: int -BUTTON_ALT: int -BUTTON_CTRL: int -BUTTON_SHIFT: int -COLOR_BLACK: int -COLOR_BLUE: int -COLOR_CYAN: int -COLOR_GREEN: int -COLOR_MAGENTA: int -COLOR_RED: int -COLOR_WHITE: int -COLOR_YELLOW: int -ERR: int -KEY_A1: int -KEY_A3: int -KEY_B2: int -KEY_BACKSPACE: int -KEY_BEG: int -KEY_BREAK: int -KEY_BTAB: int -KEY_C1: int -KEY_C3: int -KEY_CANCEL: int -KEY_CATAB: int -KEY_CLEAR: int -KEY_CLOSE: int -KEY_COMMAND: int -KEY_COPY: int -KEY_CREATE: int -KEY_CTAB: int -KEY_DC: int -KEY_DL: int -KEY_DOWN: int -KEY_EIC: int -KEY_END: int -KEY_ENTER: int -KEY_EOL: int -KEY_EOS: int -KEY_EXIT: int -KEY_F0: int -KEY_F1: int -KEY_F10: int -KEY_F11: int -KEY_F12: int -KEY_F13: int -KEY_F14: int -KEY_F15: int -KEY_F16: int -KEY_F17: int -KEY_F18: int -KEY_F19: int -KEY_F2: int -KEY_F20: int -KEY_F21: int -KEY_F22: int -KEY_F23: int -KEY_F24: int -KEY_F25: int -KEY_F26: int -KEY_F27: int -KEY_F28: int -KEY_F29: int -KEY_F3: int -KEY_F30: int -KEY_F31: int -KEY_F32: int -KEY_F33: int -KEY_F34: int -KEY_F35: int -KEY_F36: int -KEY_F37: int -KEY_F38: int -KEY_F39: int -KEY_F4: int -KEY_F40: int -KEY_F41: int -KEY_F42: int -KEY_F43: int -KEY_F44: int -KEY_F45: int -KEY_F46: int -KEY_F47: int -KEY_F48: int -KEY_F49: int -KEY_F5: int -KEY_F50: int -KEY_F51: int -KEY_F52: int -KEY_F53: int -KEY_F54: int -KEY_F55: int -KEY_F56: int -KEY_F57: int -KEY_F58: int -KEY_F59: int -KEY_F6: int -KEY_F60: int -KEY_F61: int -KEY_F62: int -KEY_F63: int -KEY_F7: int -KEY_F8: int -KEY_F9: int -KEY_FIND: int -KEY_HELP: int -KEY_HOME: int -KEY_IC: int -KEY_IL: int -KEY_LEFT: int -KEY_LL: int -KEY_MARK: int -KEY_MAX: int -KEY_MESSAGE: int -KEY_MIN: int -KEY_MOUSE: int -KEY_MOVE: int -KEY_NEXT: int -KEY_NPAGE: int -KEY_OPEN: int -KEY_OPTIONS: int -KEY_PPAGE: int -KEY_PREVIOUS: int -KEY_PRINT: int -KEY_REDO: int -KEY_REFERENCE: int -KEY_REFRESH: int -KEY_REPLACE: int -KEY_RESET: int -KEY_RESIZE: int -KEY_RESTART: int -KEY_RESUME: int -KEY_RIGHT: int -KEY_SAVE: int -KEY_SBEG: int -KEY_SCANCEL: int -KEY_SCOMMAND: int -KEY_SCOPY: int -KEY_SCREATE: int -KEY_SDC: int -KEY_SDL: int -KEY_SELECT: int -KEY_SEND: int -KEY_SEOL: int -KEY_SEXIT: int -KEY_SF: int -KEY_SFIND: int -KEY_SHELP: int -KEY_SHOME: int -KEY_SIC: int -KEY_SLEFT: int -KEY_SMESSAGE: int -KEY_SMOVE: int -KEY_SNEXT: int -KEY_SOPTIONS: int -KEY_SPREVIOUS: int -KEY_SPRINT: int -KEY_SR: int -KEY_SREDO: int -KEY_SREPLACE: int -KEY_SRESET: int -KEY_SRIGHT: int -KEY_SRSUME: int -KEY_SSAVE: int -KEY_SSUSPEND: int -KEY_STAB: int -KEY_SUNDO: int -KEY_SUSPEND: int -KEY_UNDO: int -KEY_UP: int -OK: int -REPORT_MOUSE_POSITION: int + BUTTON5_PRESSED: Final[int] + BUTTON5_RELEASED: Final[int] + BUTTON5_CLICKED: Final[int] + BUTTON5_DOUBLE_CLICKED: Final[int] + BUTTON5_TRIPLE_CLICKED: Final[int] +BUTTON_ALT: Final[int] +BUTTON_CTRL: Final[int] +BUTTON_SHIFT: Final[int] +COLOR_BLACK: Final[int] +COLOR_BLUE: Final[int] +COLOR_CYAN: Final[int] +COLOR_GREEN: Final[int] +COLOR_MAGENTA: Final[int] +COLOR_RED: Final[int] +COLOR_WHITE: Final[int] +COLOR_YELLOW: Final[int] +ERR: Final[int] +KEY_A1: Final[int] +KEY_A3: Final[int] +KEY_B2: Final[int] +KEY_BACKSPACE: Final[int] +KEY_BEG: Final[int] +KEY_BREAK: Final[int] +KEY_BTAB: Final[int] +KEY_C1: Final[int] +KEY_C3: Final[int] +KEY_CANCEL: Final[int] +KEY_CATAB: Final[int] +KEY_CLEAR: Final[int] +KEY_CLOSE: Final[int] +KEY_COMMAND: Final[int] +KEY_COPY: Final[int] +KEY_CREATE: Final[int] +KEY_CTAB: Final[int] +KEY_DC: Final[int] +KEY_DL: Final[int] +KEY_DOWN: Final[int] +KEY_EIC: Final[int] +KEY_END: Final[int] +KEY_ENTER: Final[int] +KEY_EOL: Final[int] +KEY_EOS: Final[int] +KEY_EXIT: Final[int] +KEY_F0: Final[int] +KEY_F1: Final[int] +KEY_F10: Final[int] +KEY_F11: Final[int] +KEY_F12: Final[int] +KEY_F13: Final[int] +KEY_F14: Final[int] +KEY_F15: Final[int] +KEY_F16: Final[int] +KEY_F17: Final[int] +KEY_F18: Final[int] +KEY_F19: Final[int] +KEY_F2: Final[int] +KEY_F20: Final[int] +KEY_F21: Final[int] +KEY_F22: Final[int] +KEY_F23: Final[int] +KEY_F24: Final[int] +KEY_F25: Final[int] +KEY_F26: Final[int] +KEY_F27: Final[int] +KEY_F28: Final[int] +KEY_F29: Final[int] +KEY_F3: Final[int] +KEY_F30: Final[int] +KEY_F31: Final[int] +KEY_F32: Final[int] +KEY_F33: Final[int] +KEY_F34: Final[int] +KEY_F35: Final[int] +KEY_F36: Final[int] +KEY_F37: Final[int] +KEY_F38: Final[int] +KEY_F39: Final[int] +KEY_F4: Final[int] +KEY_F40: Final[int] +KEY_F41: Final[int] +KEY_F42: Final[int] +KEY_F43: Final[int] +KEY_F44: Final[int] +KEY_F45: Final[int] +KEY_F46: Final[int] +KEY_F47: Final[int] +KEY_F48: Final[int] +KEY_F49: Final[int] +KEY_F5: Final[int] +KEY_F50: Final[int] +KEY_F51: Final[int] +KEY_F52: Final[int] +KEY_F53: Final[int] +KEY_F54: Final[int] +KEY_F55: Final[int] +KEY_F56: Final[int] +KEY_F57: Final[int] +KEY_F58: Final[int] +KEY_F59: Final[int] +KEY_F6: Final[int] +KEY_F60: Final[int] +KEY_F61: Final[int] +KEY_F62: Final[int] +KEY_F63: Final[int] +KEY_F7: Final[int] +KEY_F8: Final[int] +KEY_F9: Final[int] +KEY_FIND: Final[int] +KEY_HELP: Final[int] +KEY_HOME: Final[int] +KEY_IC: Final[int] +KEY_IL: Final[int] +KEY_LEFT: Final[int] +KEY_LL: Final[int] +KEY_MARK: Final[int] +KEY_MAX: Final[int] +KEY_MESSAGE: Final[int] +KEY_MIN: Final[int] +KEY_MOUSE: Final[int] +KEY_MOVE: Final[int] +KEY_NEXT: Final[int] +KEY_NPAGE: Final[int] +KEY_OPEN: Final[int] +KEY_OPTIONS: Final[int] +KEY_PPAGE: Final[int] +KEY_PREVIOUS: Final[int] +KEY_PRINT: Final[int] +KEY_REDO: Final[int] +KEY_REFERENCE: Final[int] +KEY_REFRESH: Final[int] +KEY_REPLACE: Final[int] +KEY_RESET: Final[int] +KEY_RESIZE: Final[int] +KEY_RESTART: Final[int] +KEY_RESUME: Final[int] +KEY_RIGHT: Final[int] +KEY_SAVE: Final[int] +KEY_SBEG: Final[int] +KEY_SCANCEL: Final[int] +KEY_SCOMMAND: Final[int] +KEY_SCOPY: Final[int] +KEY_SCREATE: Final[int] +KEY_SDC: Final[int] +KEY_SDL: Final[int] +KEY_SELECT: Final[int] +KEY_SEND: Final[int] +KEY_SEOL: Final[int] +KEY_SEXIT: Final[int] +KEY_SF: Final[int] +KEY_SFIND: Final[int] +KEY_SHELP: Final[int] +KEY_SHOME: Final[int] +KEY_SIC: Final[int] +KEY_SLEFT: Final[int] +KEY_SMESSAGE: Final[int] +KEY_SMOVE: Final[int] +KEY_SNEXT: Final[int] +KEY_SOPTIONS: Final[int] +KEY_SPREVIOUS: Final[int] +KEY_SPRINT: Final[int] +KEY_SR: Final[int] +KEY_SREDO: Final[int] +KEY_SREPLACE: Final[int] +KEY_SRESET: Final[int] +KEY_SRIGHT: Final[int] +KEY_SRSUME: Final[int] +KEY_SSAVE: Final[int] +KEY_SSUSPEND: Final[int] +KEY_STAB: Final[int] +KEY_SUNDO: Final[int] +KEY_SUSPEND: Final[int] +KEY_UNDO: Final[int] +KEY_UP: Final[int] +OK: Final[int] +REPORT_MOUSE_POSITION: Final[int] _C_API: Any -version: bytes +version: Final[bytes] def baudrate() -> int: ... def beep() -> None: ... @@ -324,7 +324,7 @@ def mouseinterval(interval: int, /) -> None: ... def mousemask(newmask: int, /) -> tuple[int, int]: ... def napms(ms: int, /) -> int: ... def newpad(nlines: int, ncols: int, /) -> window: ... -def newwin(nlines: int, ncols: int, begin_y: int = ..., begin_x: int = ..., /) -> window: ... +def newwin(nlines: int, ncols: int, begin_y: int = 0, begin_x: int = 0, /) -> window: ... def nl(flag: bool = True, /) -> None: ... def nocbreak() -> None: ... def noecho() -> None: ... @@ -394,8 +394,8 @@ class window: # undocumented def attroff(self, attr: int, /) -> None: ... def attron(self, attr: int, /) -> None: ... def attrset(self, attr: int, /) -> None: ... - def bkgd(self, ch: _ChType, attr: int = ..., /) -> None: ... - def bkgdset(self, ch: _ChType, attr: int = ..., /) -> None: ... + def bkgd(self, ch: _ChType, attr: int = 0, /) -> None: ... + def bkgdset(self, ch: _ChType, attr: int = 0, /) -> None: ... def border( self, ls: _ChType = ..., @@ -410,7 +410,7 @@ class window: # undocumented @overload def box(self) -> None: ... @overload - def box(self, vertch: _ChType = ..., horch: _ChType = ...) -> None: ... + def box(self, vertch: _ChType = 0, horch: _ChType = 0) -> None: ... @overload def chgat(self, attr: int) -> None: ... @overload @@ -433,7 +433,7 @@ class window: # undocumented def derwin(self, begin_y: int, begin_x: int) -> window: ... @overload def derwin(self, nlines: int, ncols: int, begin_y: int, begin_x: int) -> window: ... - def echochar(self, ch: _ChType, attr: int = ..., /) -> None: ... + def echochar(self, ch: _ChType, attr: int = 0, /) -> None: ... def enclose(self, y: int, x: int, /) -> bool: ... def erase(self) -> None: ... def getbegyx(self) -> tuple[int, int]: ... @@ -487,9 +487,9 @@ class window: # undocumented @overload def insstr(self, y: int, x: int, str: str, attr: int = ...) -> None: ... @overload - def instr(self, n: int = ...) -> bytes: ... + def instr(self, n: int = 2047) -> bytes: ... @overload - def instr(self, y: int, x: int, n: int = ...) -> bytes: ... + def instr(self, y: int, x: int, n: int = 2047) -> bytes: ... def is_linetouched(self, line: int, /) -> bool: ... def is_wintouched(self) -> bool: ... def keypad(self, yes: bool, /) -> None: ... @@ -523,7 +523,7 @@ class window: # undocumented @overload def refresh(self, pminrow: int, pmincol: int, sminrow: int, smincol: int, smaxrow: int, smaxcol: int) -> None: ... def resize(self, nlines: int, ncols: int) -> None: ... - def scroll(self, lines: int = ...) -> None: ... + def scroll(self, lines: int = 1) -> None: ... def scrollok(self, flag: bool) -> None: ... def setscrreg(self, top: int, bottom: int, /) -> None: ... def standend(self) -> None: ... @@ -540,7 +540,7 @@ class window: # undocumented def syncok(self, flag: bool) -> None: ... def syncup(self) -> None: ... def timeout(self, delay: int) -> None: ... - def touchline(self, start: int, count: int, changed: bool = ...) -> None: ... + def touchline(self, start: int, count: int, changed: bool = True) -> None: ... def touchwin(self) -> None: ... def untouchwin(self) -> None: ... @overload diff --git a/mypy/typeshed/stdlib/_curses_panel.pyi b/mypy/typeshed/stdlib/_curses_panel.pyi index ddec22236b963..a552a151ddf14 100644 --- a/mypy/typeshed/stdlib/_curses_panel.pyi +++ b/mypy/typeshed/stdlib/_curses_panel.pyi @@ -1,8 +1,8 @@ from _curses import window -from typing import final +from typing import Final, final -__version__: str -version: str +__version__: Final[str] +version: Final[str] class error(Exception): ... diff --git a/mypy/typeshed/stdlib/_dbm.pyi b/mypy/typeshed/stdlib/_dbm.pyi index 7e53cca3c704f..222c3ffcb246b 100644 --- a/mypy/typeshed/stdlib/_dbm.pyi +++ b/mypy/typeshed/stdlib/_dbm.pyi @@ -1,7 +1,7 @@ import sys from _typeshed import ReadOnlyBuffer, StrOrBytesPath from types import TracebackType -from typing import TypeVar, final, overload, type_check_only +from typing import Final, TypeVar, final, overload, type_check_only from typing_extensions import Self, TypeAlias if sys.platform != "win32": @@ -10,7 +10,7 @@ if sys.platform != "win32": _ValueType: TypeAlias = str | ReadOnlyBuffer class error(OSError): ... - library: str + library: Final[str] # Actual typename dbm, not exposed by the implementation @final @@ -33,7 +33,7 @@ if sys.platform != "win32": @overload def get(self, k: _KeyType, default: _T, /) -> bytes | _T: ... def keys(self) -> list[bytes]: ... - def setdefault(self, k: _KeyType, default: _ValueType = ..., /) -> bytes: ... + def setdefault(self, k: _KeyType, default: _ValueType = b"", /) -> bytes: ... # This isn't true, but the class can't be instantiated. See #13024 __new__: None # type: ignore[assignment] __init__: None # type: ignore[assignment] diff --git a/mypy/typeshed/stdlib/_decimal.pyi b/mypy/typeshed/stdlib/_decimal.pyi index fd0e6e6ac0914..3cfe8944dfaf4 100644 --- a/mypy/typeshed/stdlib/_decimal.pyi +++ b/mypy/typeshed/stdlib/_decimal.pyi @@ -51,14 +51,14 @@ if sys.version_info >= (3, 11): def localcontext( ctx: Context | None = None, *, - prec: int | None = ..., - rounding: str | None = ..., - Emin: int | None = ..., - Emax: int | None = ..., - capitals: int | None = ..., - clamp: int | None = ..., - traps: dict[_TrapType, bool] | None = ..., - flags: dict[_TrapType, bool] | None = ..., + prec: int | None = None, + rounding: str | None = None, + Emin: int | None = None, + Emax: int | None = None, + capitals: int | None = None, + clamp: int | None = None, + traps: dict[_TrapType, bool] | None = None, + flags: dict[_TrapType, bool] | None = None, ) -> _ContextManager: ... else: diff --git a/mypy/typeshed/stdlib/_frozen_importlib_external.pyi b/mypy/typeshed/stdlib/_frozen_importlib_external.pyi index 80eebe45a7d44..71642c65dc07d 100644 --- a/mypy/typeshed/stdlib/_frozen_importlib_external.pyi +++ b/mypy/typeshed/stdlib/_frozen_importlib_external.pyi @@ -9,7 +9,7 @@ from _typeshed.importlib import LoaderProtocol from collections.abc import Callable, Iterable, Iterator, Mapping, MutableSequence, Sequence from importlib.machinery import ModuleSpec from importlib.metadata import DistributionFinder, PathDistribution -from typing import Any, Literal +from typing import Any, Final, Literal from typing_extensions import Self, deprecated if sys.version_info >= (3, 10): @@ -24,7 +24,7 @@ else: path_sep: Literal["/"] path_sep_tuple: tuple[Literal["/"]] -MAGIC_NUMBER: bytes +MAGIC_NUMBER: Final[bytes] def cache_from_source(path: StrPath, debug_override: bool | None = None, *, optimization: Any | None = None) -> str: ... def source_from_cache(path: StrPath) -> str: ... @@ -37,7 +37,7 @@ def spec_from_file_location( submodule_search_locations: list[str] | None = ..., ) -> importlib.machinery.ModuleSpec | None: ... @deprecated( - "Deprecated as of Python 3.6: Use site configuration instead. " + "Deprecated since Python 3.6. Use site configuration instead. " "Future versions of Python may not enable this finder by default." ) class WindowsRegistryFinder(importlib.abc.MetaPathFinder): @@ -74,11 +74,11 @@ class PathFinder(importlib.abc.MetaPathFinder): @deprecated("Deprecated since Python 3.4; removed in Python 3.12. Use `find_spec()` instead.") def find_module(cls, fullname: str, path: Sequence[str] | None = None) -> importlib.abc.Loader | None: ... -SOURCE_SUFFIXES: list[str] -DEBUG_BYTECODE_SUFFIXES: list[str] -OPTIMIZED_BYTECODE_SUFFIXES: list[str] -BYTECODE_SUFFIXES: list[str] -EXTENSION_SUFFIXES: list[str] +SOURCE_SUFFIXES: Final[list[str]] +DEBUG_BYTECODE_SUFFIXES: Final = [".pyc"] +OPTIMIZED_BYTECODE_SUFFIXES: Final = [".pyc"] +BYTECODE_SUFFIXES: Final = [".pyc"] +EXTENSION_SUFFIXES: Final[list[str]] class FileFinder(importlib.abc.PathEntryFinder): path: str @@ -155,7 +155,7 @@ if sys.version_info >= (3, 11): def get_code(self, fullname: str) -> types.CodeType: ... def create_module(self, spec: ModuleSpec) -> None: ... def exec_module(self, module: types.ModuleType) -> None: ... - @deprecated("load_module() is deprecated; use exec_module() instead") + @deprecated("Deprecated since Python 3.10; will be removed in Python 3.15. Use `exec_module()` instead.") def load_module(self, fullname: str) -> types.ModuleType: ... def get_resource_reader(self, module: types.ModuleType) -> importlib.readers.NamespaceReader: ... if sys.version_info < (3, 12): @@ -177,9 +177,9 @@ else: def get_code(self, fullname: str) -> types.CodeType: ... def create_module(self, spec: ModuleSpec) -> None: ... def exec_module(self, module: types.ModuleType) -> None: ... - @deprecated("load_module() is deprecated; use exec_module() instead") - def load_module(self, fullname: str) -> types.ModuleType: ... if sys.version_info >= (3, 10): + @deprecated("Deprecated since Python 3.10; will be removed in Python 3.15. Use `exec_module()` instead.") + def load_module(self, fullname: str) -> types.ModuleType: ... @staticmethod @deprecated( "Deprecated since Python 3.4; removed in Python 3.12. " @@ -188,6 +188,7 @@ else: def module_repr(module: types.ModuleType) -> str: ... def get_resource_reader(self, module: types.ModuleType) -> importlib.readers.NamespaceReader: ... else: + def load_module(self, fullname: str) -> types.ModuleType: ... @classmethod @deprecated( "Deprecated since Python 3.4; removed in Python 3.12. " diff --git a/mypy/typeshed/stdlib/_hashlib.pyi b/mypy/typeshed/stdlib/_hashlib.pyi index 8b7ef52cdffdf..03c1eef3be3ff 100644 --- a/mypy/typeshed/stdlib/_hashlib.pyi +++ b/mypy/typeshed/stdlib/_hashlib.pyi @@ -3,7 +3,7 @@ from _typeshed import ReadableBuffer from collections.abc import Callable from types import ModuleType from typing import AnyStr, Protocol, final, overload, type_check_only -from typing_extensions import Self, TypeAlias +from typing_extensions import Self, TypeAlias, disjoint_base _DigestMod: TypeAlias = str | Callable[[], _HashObject] | ModuleType | None @@ -22,6 +22,7 @@ class _HashObject(Protocol): def hexdigest(self) -> str: ... def update(self, obj: ReadableBuffer, /) -> None: ... +@disjoint_base class HASH: @property def digest_size(self) -> int: ... diff --git a/mypy/typeshed/stdlib/_interpreters.pyi b/mypy/typeshed/stdlib/_interpreters.pyi index f89a24e7d85c6..8e097efad618a 100644 --- a/mypy/typeshed/stdlib/_interpreters.pyi +++ b/mypy/typeshed/stdlib/_interpreters.pyi @@ -1,7 +1,7 @@ import types from collections.abc import Callable -from typing import Any, Final, Literal, SupportsIndex, TypeVar -from typing_extensions import TypeAlias +from typing import Any, Final, Literal, SupportsIndex, TypeVar, overload +from typing_extensions import TypeAlias, disjoint_base _R = TypeVar("_R") @@ -12,48 +12,45 @@ class InterpreterError(Exception): ... class InterpreterNotFoundError(InterpreterError): ... class NotShareableError(ValueError): ... +@disjoint_base class CrossInterpreterBufferView: def __buffer__(self, flags: int, /) -> memoryview: ... def new_config(name: _Configs = "isolated", /, **overides: object) -> types.SimpleNamespace: ... def create(config: types.SimpleNamespace | _Configs | None = "isolated", *, reqrefs: bool = False) -> int: ... def destroy(id: SupportsIndex, *, restrict: bool = False) -> None: ... -def list_all(*, require_ready: bool) -> list[tuple[int, int]]: ... -def get_current() -> tuple[int, int]: ... -def get_main() -> tuple[int, int]: ... +def list_all(*, require_ready: bool = False) -> list[tuple[int, _Whence]]: ... +def get_current() -> tuple[int, _Whence]: ... +def get_main() -> tuple[int, _Whence]: ... def is_running(id: SupportsIndex, *, restrict: bool = False) -> bool: ... def get_config(id: SupportsIndex, *, restrict: bool = False) -> types.SimpleNamespace: ... def whence(id: SupportsIndex) -> _Whence: ... def exec( - id: SupportsIndex, - code: str | types.CodeType | Callable[[], object], - shared: _SharedDict | None = None, - *, - restrict: bool = False, + id: SupportsIndex, code: str | types.CodeType | Callable[[], object], shared: _SharedDict = {}, *, restrict: bool = False ) -> None | types.SimpleNamespace: ... def call( id: SupportsIndex, callable: Callable[..., _R], - args: tuple[Any, ...] | None = None, - kwargs: dict[str, Any] | None = None, + args: tuple[Any, ...] = (), + kwargs: dict[str, Any] = {}, *, + preserve_exc: bool = False, restrict: bool = False, ) -> tuple[_R, types.SimpleNamespace]: ... def run_string( - id: SupportsIndex, - script: str | types.CodeType | Callable[[], object], - shared: _SharedDict | None = None, - *, - restrict: bool = False, + id: SupportsIndex, script: str | types.CodeType | Callable[[], object], shared: _SharedDict = {}, *, restrict: bool = False ) -> None: ... def run_func( - id: SupportsIndex, func: types.CodeType | Callable[[], object], shared: _SharedDict | None = None, *, restrict: bool = False + id: SupportsIndex, func: types.CodeType | Callable[[], object], shared: _SharedDict = {}, *, restrict: bool = False ) -> None: ... def set___main___attrs(id: SupportsIndex, updates: _SharedDict, *, restrict: bool = False) -> None: ... def incref(id: SupportsIndex, *, implieslink: bool = False, restrict: bool = False) -> None: ... def decref(id: SupportsIndex, *, restrict: bool = False) -> None: ... def is_shareable(obj: object) -> bool: ... -def capture_exception(exc: BaseException | None = None) -> types.SimpleNamespace: ... +@overload +def capture_exception(exc: BaseException) -> types.SimpleNamespace: ... +@overload +def capture_exception(exc: None = None) -> types.SimpleNamespace | None: ... _Whence: TypeAlias = Literal[0, 1, 2, 3, 4, 5] WHENCE_UNKNOWN: Final = 0 diff --git a/mypy/typeshed/stdlib/_io.pyi b/mypy/typeshed/stdlib/_io.pyi index e368ddef7f4e6..2d2a60e4dddf1 100644 --- a/mypy/typeshed/stdlib/_io.pyi +++ b/mypy/typeshed/stdlib/_io.pyi @@ -7,11 +7,14 @@ from io import BufferedIOBase, RawIOBase, TextIOBase, UnsupportedOperation as Un from os import _Opener from types import TracebackType from typing import IO, Any, BinaryIO, Final, Generic, Literal, Protocol, TextIO, TypeVar, overload, type_check_only -from typing_extensions import Self +from typing_extensions import Self, disjoint_base _T = TypeVar("_T") -DEFAULT_BUFFER_SIZE: Final = 8192 +if sys.version_info >= (3, 14): + DEFAULT_BUFFER_SIZE: Final = 131072 +else: + DEFAULT_BUFFER_SIZE: Final = 8192 open = builtins.open @@ -19,32 +22,62 @@ def open_code(path: str) -> IO[bytes]: ... BlockingIOError = builtins.BlockingIOError -class _IOBase: - def __iter__(self) -> Iterator[bytes]: ... - def __next__(self) -> bytes: ... - def __enter__(self) -> Self: ... - def __exit__( - self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None - ) -> None: ... - def close(self) -> None: ... - def fileno(self) -> int: ... - def flush(self) -> None: ... - def isatty(self) -> bool: ... - def readable(self) -> bool: ... - read: Callable[..., Any] - def readlines(self, hint: int = -1, /) -> list[bytes]: ... - def seek(self, offset: int, whence: int = 0, /) -> int: ... - def seekable(self) -> bool: ... - def tell(self) -> int: ... - def truncate(self, size: int | None = None, /) -> int: ... - def writable(self) -> bool: ... - write: Callable[..., Any] - def writelines(self, lines: Iterable[ReadableBuffer], /) -> None: ... - def readline(self, size: int | None = -1, /) -> bytes: ... - def __del__(self) -> None: ... - @property - def closed(self) -> bool: ... - def _checkClosed(self) -> None: ... # undocumented +if sys.version_info >= (3, 12): + @disjoint_base + class _IOBase: + def __iter__(self) -> Iterator[bytes]: ... + def __next__(self) -> bytes: ... + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None + ) -> None: ... + def close(self) -> None: ... + def fileno(self) -> int: ... + def flush(self) -> None: ... + def isatty(self) -> bool: ... + def readable(self) -> bool: ... + read: Callable[..., Any] + def readlines(self, hint: int = -1, /) -> list[bytes]: ... + def seek(self, offset: int, whence: int = 0, /) -> int: ... + def seekable(self) -> bool: ... + def tell(self) -> int: ... + def truncate(self, size: int | None = None, /) -> int: ... + def writable(self) -> bool: ... + write: Callable[..., Any] + def writelines(self, lines: Iterable[ReadableBuffer], /) -> None: ... + def readline(self, size: int | None = -1, /) -> bytes: ... + def __del__(self) -> None: ... + @property + def closed(self) -> bool: ... + def _checkClosed(self) -> None: ... # undocumented + +else: + class _IOBase: + def __iter__(self) -> Iterator[bytes]: ... + def __next__(self) -> bytes: ... + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None + ) -> None: ... + def close(self) -> None: ... + def fileno(self) -> int: ... + def flush(self) -> None: ... + def isatty(self) -> bool: ... + def readable(self) -> bool: ... + read: Callable[..., Any] + def readlines(self, hint: int = -1, /) -> list[bytes]: ... + def seek(self, offset: int, whence: int = 0, /) -> int: ... + def seekable(self) -> bool: ... + def tell(self) -> int: ... + def truncate(self, size: int | None = None, /) -> int: ... + def writable(self) -> bool: ... + write: Callable[..., Any] + def writelines(self, lines: Iterable[ReadableBuffer], /) -> None: ... + def readline(self, size: int | None = -1, /) -> bytes: ... + def __del__(self) -> None: ... + @property + def closed(self) -> bool: ... + def _checkClosed(self) -> None: ... # undocumented class _RawIOBase(_IOBase): def readall(self) -> bytes: ... @@ -62,6 +95,7 @@ class _BufferedIOBase(_IOBase): def read(self, size: int | None = -1, /) -> bytes: ... def read1(self, size: int = -1, /) -> bytes: ... +@disjoint_base class FileIO(RawIOBase, _RawIOBase, BinaryIO): # type: ignore[misc] # incompatible definitions of writelines in the base classes mode: str # The type of "name" equals the argument passed in to the constructor, @@ -76,6 +110,7 @@ class FileIO(RawIOBase, _RawIOBase, BinaryIO): # type: ignore[misc] # incompat def seek(self, pos: int, whence: int = 0, /) -> int: ... def read(self, size: int | None = -1, /) -> bytes | MaybeNone: ... +@disjoint_base class BytesIO(BufferedIOBase, _BufferedIOBase, BinaryIO): # type: ignore[misc] # incompatible definitions of methods in the base classes def __init__(self, initial_bytes: ReadableBuffer = b"") -> None: ... # BytesIO does not contain a "name" field. This workaround is necessary @@ -116,31 +151,51 @@ class _BufferedReaderStream(Protocol): _BufferedReaderStreamT = TypeVar("_BufferedReaderStreamT", bound=_BufferedReaderStream, default=_BufferedReaderStream) +@disjoint_base class BufferedReader(BufferedIOBase, _BufferedIOBase, BinaryIO, Generic[_BufferedReaderStreamT]): # type: ignore[misc] # incompatible definitions of methods in the base classes raw: _BufferedReaderStreamT - def __init__(self, raw: _BufferedReaderStreamT, buffer_size: int = 8192) -> None: ... + if sys.version_info >= (3, 14): + def __init__(self, raw: _BufferedReaderStreamT, buffer_size: int = 131072) -> None: ... + else: + def __init__(self, raw: _BufferedReaderStreamT, buffer_size: int = 8192) -> None: ... + def peek(self, size: int = 0, /) -> bytes: ... def seek(self, target: int, whence: int = 0, /) -> int: ... def truncate(self, pos: int | None = None, /) -> int: ... +@disjoint_base class BufferedWriter(BufferedIOBase, _BufferedIOBase, BinaryIO): # type: ignore[misc] # incompatible definitions of writelines in the base classes raw: RawIOBase - def __init__(self, raw: RawIOBase, buffer_size: int = 8192) -> None: ... + if sys.version_info >= (3, 14): + def __init__(self, raw: RawIOBase, buffer_size: int = 131072) -> None: ... + else: + def __init__(self, raw: RawIOBase, buffer_size: int = 8192) -> None: ... + def write(self, buffer: ReadableBuffer, /) -> int: ... def seek(self, target: int, whence: int = 0, /) -> int: ... def truncate(self, pos: int | None = None, /) -> int: ... +@disjoint_base class BufferedRandom(BufferedIOBase, _BufferedIOBase, BinaryIO): # type: ignore[misc] # incompatible definitions of methods in the base classes mode: str name: Any raw: RawIOBase - def __init__(self, raw: RawIOBase, buffer_size: int = 8192) -> None: ... + if sys.version_info >= (3, 14): + def __init__(self, raw: RawIOBase, buffer_size: int = 131072) -> None: ... + else: + def __init__(self, raw: RawIOBase, buffer_size: int = 8192) -> None: ... + def seek(self, target: int, whence: int = 0, /) -> int: ... # stubtest needs this def peek(self, size: int = 0, /) -> bytes: ... def truncate(self, pos: int | None = None, /) -> int: ... +@disjoint_base class BufferedRWPair(BufferedIOBase, _BufferedIOBase, Generic[_BufferedReaderStreamT]): - def __init__(self, reader: _BufferedReaderStreamT, writer: RawIOBase, buffer_size: int = 8192, /) -> None: ... + if sys.version_info >= (3, 14): + def __init__(self, reader: _BufferedReaderStreamT, writer: RawIOBase, buffer_size: int = 131072, /) -> None: ... + else: + def __init__(self, reader: _BufferedReaderStreamT, writer: RawIOBase, buffer_size: int = 8192, /) -> None: ... + def peek(self, size: int = 0, /) -> bytes: ... class _TextIOBase(_IOBase): @@ -181,6 +236,7 @@ class _WrappedBuffer(Protocol): _BufferT_co = TypeVar("_BufferT_co", bound=_WrappedBuffer, default=_WrappedBuffer, covariant=True) +@disjoint_base class TextIOWrapper(TextIOBase, _TextIOBase, TextIO, Generic[_BufferT_co]): # type: ignore[misc] # incompatible definitions of write in the base classes def __init__( self, @@ -215,6 +271,7 @@ class TextIOWrapper(TextIOBase, _TextIOBase, TextIO, Generic[_BufferT_co]): # t def seek(self, cookie: int, whence: int = 0, /) -> int: ... def truncate(self, pos: int | None = None, /) -> int: ... +@disjoint_base class StringIO(TextIOBase, _TextIOBase, TextIO): # type: ignore[misc] # incompatible definitions of write in the base classes def __init__(self, initial_value: str | None = "", newline: str | None = "\n") -> None: ... # StringIO does not contain a "name" field. This workaround is necessary @@ -227,6 +284,7 @@ class StringIO(TextIOBase, _TextIOBase, TextIO): # type: ignore[misc] # incomp def seek(self, pos: int, whence: int = 0, /) -> int: ... def truncate(self, pos: int | None = None, /) -> int: ... +@disjoint_base class IncrementalNewlineDecoder: def __init__(self, decoder: codecs.IncrementalDecoder | None, translate: bool, errors: str = "strict") -> None: ... def decode(self, input: ReadableBuffer | str, final: bool = False) -> str: ... diff --git a/mypy/typeshed/stdlib/_lsprof.pyi b/mypy/typeshed/stdlib/_lsprof.pyi index 8a6934162c929..4f6d98b8ffb61 100644 --- a/mypy/typeshed/stdlib/_lsprof.pyi +++ b/mypy/typeshed/stdlib/_lsprof.pyi @@ -3,7 +3,9 @@ from _typeshed import structseq from collections.abc import Callable from types import CodeType from typing import Any, Final, final +from typing_extensions import disjoint_base +@disjoint_base class Profiler: def __init__( self, timer: Callable[[], float] | None = None, timeunit: float = 0.0, subcalls: bool = True, builtins: bool = True diff --git a/mypy/typeshed/stdlib/_lzma.pyi b/mypy/typeshed/stdlib/_lzma.pyi index 1a27c7428e8ec..b38dce9fadedf 100644 --- a/mypy/typeshed/stdlib/_lzma.pyi +++ b/mypy/typeshed/stdlib/_lzma.pyi @@ -16,7 +16,7 @@ CHECK_CRC64: Final = 4 CHECK_SHA256: Final = 10 CHECK_ID_MAX: Final = 15 CHECK_UNKNOWN: Final = 16 -FILTER_LZMA1: int # v big number +FILTER_LZMA1: Final[int] # v big number FILTER_LZMA2: Final = 33 FILTER_DELTA: Final = 3 FILTER_X86: Final = 4 @@ -33,14 +33,14 @@ MF_BT4: Final = 20 MODE_FAST: Final = 1 MODE_NORMAL: Final = 2 PRESET_DEFAULT: Final = 6 -PRESET_EXTREME: int # v big number +PRESET_EXTREME: Final[int] # v big number @final class LZMADecompressor: if sys.version_info >= (3, 12): - def __new__(cls, format: int | None = ..., memlimit: int | None = ..., filters: _FilterChain | None = ...) -> Self: ... + def __new__(cls, format: int = 0, memlimit: int | None = None, filters: _FilterChain | None = None) -> Self: ... else: - def __init__(self, format: int | None = ..., memlimit: int | None = ..., filters: _FilterChain | None = ...) -> None: ... + def __init__(self, format: int = 0, memlimit: int | None = None, filters: _FilterChain | None = None) -> None: ... def decompress(self, data: ReadableBuffer, max_length: int = -1) -> bytes: ... @property @@ -56,11 +56,11 @@ class LZMADecompressor: class LZMACompressor: if sys.version_info >= (3, 12): def __new__( - cls, format: int | None = ..., check: int = ..., preset: int | None = ..., filters: _FilterChain | None = ... + cls, format: int = 1, check: int = -1, preset: int | None = None, filters: _FilterChain | None = None ) -> Self: ... else: def __init__( - self, format: int | None = ..., check: int = ..., preset: int | None = ..., filters: _FilterChain | None = ... + self, format: int = 1, check: int = -1, preset: int | None = None, filters: _FilterChain | None = None ) -> None: ... def compress(self, data: ReadableBuffer, /) -> bytes: ... diff --git a/mypy/typeshed/stdlib/_msi.pyi b/mypy/typeshed/stdlib/_msi.pyi index ef45ff6dc3c82..edceed51bf9db 100644 --- a/mypy/typeshed/stdlib/_msi.pyi +++ b/mypy/typeshed/stdlib/_msi.pyi @@ -1,5 +1,5 @@ import sys -from typing import type_check_only +from typing import Final, type_check_only if sys.platform == "win32": class MSIError(Exception): ... @@ -56,42 +56,42 @@ if sys.platform == "win32": def OpenDatabase(path: str, persist: int, /) -> _Database: ... def CreateRecord(count: int, /) -> _Record: ... - MSICOLINFO_NAMES: int - MSICOLINFO_TYPES: int - MSIDBOPEN_CREATE: int - MSIDBOPEN_CREATEDIRECT: int - MSIDBOPEN_DIRECT: int - MSIDBOPEN_PATCHFILE: int - MSIDBOPEN_READONLY: int - MSIDBOPEN_TRANSACT: int - MSIMODIFY_ASSIGN: int - MSIMODIFY_DELETE: int - MSIMODIFY_INSERT: int - MSIMODIFY_INSERT_TEMPORARY: int - MSIMODIFY_MERGE: int - MSIMODIFY_REFRESH: int - MSIMODIFY_REPLACE: int - MSIMODIFY_SEEK: int - MSIMODIFY_UPDATE: int - MSIMODIFY_VALIDATE: int - MSIMODIFY_VALIDATE_DELETE: int - MSIMODIFY_VALIDATE_FIELD: int - MSIMODIFY_VALIDATE_NEW: int + MSICOLINFO_NAMES: Final[int] + MSICOLINFO_TYPES: Final[int] + MSIDBOPEN_CREATE: Final[int] + MSIDBOPEN_CREATEDIRECT: Final[int] + MSIDBOPEN_DIRECT: Final[int] + MSIDBOPEN_PATCHFILE: Final[int] + MSIDBOPEN_READONLY: Final[int] + MSIDBOPEN_TRANSACT: Final[int] + MSIMODIFY_ASSIGN: Final[int] + MSIMODIFY_DELETE: Final[int] + MSIMODIFY_INSERT: Final[int] + MSIMODIFY_INSERT_TEMPORARY: Final[int] + MSIMODIFY_MERGE: Final[int] + MSIMODIFY_REFRESH: Final[int] + MSIMODIFY_REPLACE: Final[int] + MSIMODIFY_SEEK: Final[int] + MSIMODIFY_UPDATE: Final[int] + MSIMODIFY_VALIDATE: Final[int] + MSIMODIFY_VALIDATE_DELETE: Final[int] + MSIMODIFY_VALIDATE_FIELD: Final[int] + MSIMODIFY_VALIDATE_NEW: Final[int] - PID_APPNAME: int - PID_AUTHOR: int - PID_CHARCOUNT: int - PID_CODEPAGE: int - PID_COMMENTS: int - PID_CREATE_DTM: int - PID_KEYWORDS: int - PID_LASTAUTHOR: int - PID_LASTPRINTED: int - PID_LASTSAVE_DTM: int - PID_PAGECOUNT: int - PID_REVNUMBER: int - PID_SECURITY: int - PID_SUBJECT: int - PID_TEMPLATE: int - PID_TITLE: int - PID_WORDCOUNT: int + PID_APPNAME: Final[int] + PID_AUTHOR: Final[int] + PID_CHARCOUNT: Final[int] + PID_CODEPAGE: Final[int] + PID_COMMENTS: Final[int] + PID_CREATE_DTM: Final[int] + PID_KEYWORDS: Final[int] + PID_LASTAUTHOR: Final[int] + PID_LASTPRINTED: Final[int] + PID_LASTSAVE_DTM: Final[int] + PID_PAGECOUNT: Final[int] + PID_REVNUMBER: Final[int] + PID_SECURITY: Final[int] + PID_SUBJECT: Final[int] + PID_TEMPLATE: Final[int] + PID_TITLE: Final[int] + PID_WORDCOUNT: Final[int] diff --git a/mypy/typeshed/stdlib/_multibytecodec.pyi b/mypy/typeshed/stdlib/_multibytecodec.pyi index 7e408f2aa30e1..abe58cb64f319 100644 --- a/mypy/typeshed/stdlib/_multibytecodec.pyi +++ b/mypy/typeshed/stdlib/_multibytecodec.pyi @@ -2,6 +2,7 @@ from _typeshed import ReadableBuffer from codecs import _ReadableStream, _WritableStream from collections.abc import Iterable from typing import final, type_check_only +from typing_extensions import disjoint_base # This class is not exposed. It calls itself _multibytecodec.MultibyteCodec. @final @@ -10,6 +11,7 @@ class _MultibyteCodec: def decode(self, input: ReadableBuffer, errors: str | None = None) -> str: ... def encode(self, input: str, errors: str | None = None) -> bytes: ... +@disjoint_base class MultibyteIncrementalDecoder: errors: str def __init__(self, errors: str = "strict") -> None: ... @@ -18,6 +20,7 @@ class MultibyteIncrementalDecoder: def reset(self) -> None: ... def setstate(self, state: tuple[bytes, int], /) -> None: ... +@disjoint_base class MultibyteIncrementalEncoder: errors: str def __init__(self, errors: str = "strict") -> None: ... @@ -26,6 +29,7 @@ class MultibyteIncrementalEncoder: def reset(self) -> None: ... def setstate(self, state: int, /) -> None: ... +@disjoint_base class MultibyteStreamReader: errors: str stream: _ReadableStream @@ -35,6 +39,7 @@ class MultibyteStreamReader: def readlines(self, sizehintobj: int | None = None, /) -> list[str]: ... def reset(self) -> None: ... +@disjoint_base class MultibyteStreamWriter: errors: str stream: _WritableStream diff --git a/mypy/typeshed/stdlib/_pickle.pyi b/mypy/typeshed/stdlib/_pickle.pyi index 03051bb09d3cf..544f787172d6f 100644 --- a/mypy/typeshed/stdlib/_pickle.pyi +++ b/mypy/typeshed/stdlib/_pickle.pyi @@ -2,7 +2,7 @@ from _typeshed import ReadableBuffer, SupportsWrite from collections.abc import Callable, Iterable, Iterator, Mapping from pickle import PickleBuffer as PickleBuffer from typing import Any, Protocol, type_check_only -from typing_extensions import TypeAlias +from typing_extensions import TypeAlias, disjoint_base @type_check_only class _ReadableFileobj(Protocol): @@ -57,6 +57,7 @@ class PicklerMemoProxy: def clear(self, /) -> None: ... def copy(self, /) -> dict[int, tuple[int, Any]]: ... +@disjoint_base class Pickler: fast: bool dispatch_table: Mapping[type, Callable[[Any], _ReducedType]] @@ -84,6 +85,7 @@ class UnpicklerMemoProxy: def clear(self, /) -> None: ... def copy(self, /) -> dict[int, tuple[int, Any]]: ... +@disjoint_base class Unpickler: def __init__( self, diff --git a/mypy/typeshed/stdlib/_queue.pyi b/mypy/typeshed/stdlib/_queue.pyi index f98397b132aba..edd484a9a71a4 100644 --- a/mypy/typeshed/stdlib/_queue.pyi +++ b/mypy/typeshed/stdlib/_queue.pyi @@ -1,10 +1,12 @@ from types import GenericAlias from typing import Any, Generic, TypeVar +from typing_extensions import disjoint_base _T = TypeVar("_T") class Empty(Exception): ... +@disjoint_base class SimpleQueue(Generic[_T]): def __init__(self) -> None: ... def empty(self) -> bool: ... diff --git a/mypy/typeshed/stdlib/_random.pyi b/mypy/typeshed/stdlib/_random.pyi index 4082344ade8e1..ac00fdfb7272b 100644 --- a/mypy/typeshed/stdlib/_random.pyi +++ b/mypy/typeshed/stdlib/_random.pyi @@ -1,10 +1,16 @@ -from typing_extensions import TypeAlias +import sys +from typing_extensions import Self, TypeAlias, disjoint_base # Actually Tuple[(int,) * 625] _State: TypeAlias = tuple[int, ...] +@disjoint_base class Random: - def __init__(self, seed: object = ...) -> None: ... + if sys.version_info >= (3, 10): + def __init__(self, seed: object = ..., /) -> None: ... + else: + def __new__(self, seed: object = ..., /) -> Self: ... + def seed(self, n: object = None, /) -> None: ... def getstate(self) -> _State: ... def setstate(self, state: _State, /) -> None: ... diff --git a/mypy/typeshed/stdlib/_socket.pyi b/mypy/typeshed/stdlib/_socket.pyi index 9c153a3a6ba0b..cdad886b3415e 100644 --- a/mypy/typeshed/stdlib/_socket.pyi +++ b/mypy/typeshed/stdlib/_socket.pyi @@ -3,7 +3,7 @@ from _typeshed import ReadableBuffer, WriteableBuffer from collections.abc import Iterable from socket import error as error, gaierror as gaierror, herror as herror, timeout as timeout from typing import Any, Final, SupportsIndex, overload -from typing_extensions import CapsuleType, TypeAlias +from typing_extensions import CapsuleType, TypeAlias, disjoint_base _CMSG: TypeAlias = tuple[int, int, bytes] _CMSGArg: TypeAlias = tuple[int, int, ReadableBuffer] @@ -731,6 +731,7 @@ if sys.platform != "win32" and sys.platform != "darwin": # ===== Classes ===== +@disjoint_base class socket: @property def family(self) -> int: ... diff --git a/mypy/typeshed/stdlib/_ssl.pyi b/mypy/typeshed/stdlib/_ssl.pyi index 8afa3e5297bda..73a43f29c8c5f 100644 --- a/mypy/typeshed/stdlib/_ssl.pyi +++ b/mypy/typeshed/stdlib/_ssl.pyi @@ -13,7 +13,7 @@ from ssl import ( SSLZeroReturnError as SSLZeroReturnError, ) from typing import Any, ClassVar, Final, Literal, TypedDict, final, overload, type_check_only -from typing_extensions import NotRequired, Self, TypeAlias, deprecated +from typing_extensions import NotRequired, Self, TypeAlias, deprecated, disjoint_base _PasswordType: TypeAlias = Callable[[], str | bytes | bytearray] | str | bytes | bytearray _PCTRTT: TypeAlias = tuple[tuple[str, str], ...] @@ -67,7 +67,7 @@ if sys.platform == "win32": def txt2obj(txt: str, name: bool = False) -> tuple[int, str, str, str]: ... def nid2obj(nid: int, /) -> tuple[int, str, str, str]: ... - +@disjoint_base class _SSLContext: check_hostname: bool keylog_filename: str | None diff --git a/mypy/typeshed/stdlib/_struct.pyi b/mypy/typeshed/stdlib/_struct.pyi index 662170e869f31..a8fac2aea1b00 100644 --- a/mypy/typeshed/stdlib/_struct.pyi +++ b/mypy/typeshed/stdlib/_struct.pyi @@ -1,6 +1,7 @@ from _typeshed import ReadableBuffer, WriteableBuffer from collections.abc import Iterator from typing import Any +from typing_extensions import disjoint_base def pack(fmt: str | bytes, /, *v: Any) -> bytes: ... def pack_into(fmt: str | bytes, buffer: WriteableBuffer, offset: int, /, *v: Any) -> None: ... @@ -8,7 +9,7 @@ def unpack(format: str | bytes, buffer: ReadableBuffer, /) -> tuple[Any, ...]: . def unpack_from(format: str | bytes, /, buffer: ReadableBuffer, offset: int = 0) -> tuple[Any, ...]: ... def iter_unpack(format: str | bytes, buffer: ReadableBuffer, /) -> Iterator[tuple[Any, ...]]: ... def calcsize(format: str | bytes, /) -> int: ... - +@disjoint_base class Struct: @property def format(self) -> str: ... diff --git a/mypy/typeshed/stdlib/_thread.pyi b/mypy/typeshed/stdlib/_thread.pyi index 970130dfb09c3..6969ae48cae79 100644 --- a/mypy/typeshed/stdlib/_thread.pyi +++ b/mypy/typeshed/stdlib/_thread.pyi @@ -5,7 +5,7 @@ from collections.abc import Callable from threading import Thread from types import TracebackType from typing import Any, Final, NoReturn, final, overload -from typing_extensions import TypeVarTuple, Unpack +from typing_extensions import TypeVarTuple, Unpack, disjoint_base _Ts = TypeVarTuple("_Ts") @@ -85,7 +85,7 @@ def allocate() -> LockType: ... # Obsolete synonym for allocate_lock() def get_ident() -> int: ... def stack_size(size: int = 0, /) -> int: ... -TIMEOUT_MAX: float +TIMEOUT_MAX: Final[float] def get_native_id() -> int: ... # only available on some platforms @final @@ -110,6 +110,7 @@ if sys.version_info >= (3, 12): if sys.version_info >= (3, 14): def set_name(name: str) -> None: ... +@disjoint_base class _local: def __getattribute__(self, name: str, /) -> Any: ... def __setattr__(self, name: str, value: Any, /) -> None: ... diff --git a/mypy/typeshed/stdlib/_threading_local.pyi b/mypy/typeshed/stdlib/_threading_local.pyi index 07a825f0d8168..5f6acaf840aa1 100644 --- a/mypy/typeshed/stdlib/_threading_local.pyi +++ b/mypy/typeshed/stdlib/_threading_local.pyi @@ -7,6 +7,7 @@ __all__ = ["local"] _LocalDict: TypeAlias = dict[Any, Any] class _localimpl: + __slots__ = ("key", "dicts", "localargs", "locallock", "__weakref__") key: str dicts: dict[int, tuple[ReferenceType[Any], _LocalDict]] # Keep localargs in sync with the *args, **kwargs annotation on local.__new__ @@ -16,6 +17,7 @@ class _localimpl: def create_dict(self) -> _LocalDict: ... class local: + __slots__ = ("_local__impl", "__dict__") def __new__(cls, /, *args: Any, **kw: Any) -> Self: ... def __getattribute__(self, name: str) -> Any: ... def __setattr__(self, name: str, value: Any) -> None: ... diff --git a/mypy/typeshed/stdlib/_typeshed/__init__.pyi b/mypy/typeshed/stdlib/_typeshed/__init__.pyi index 98a369dfc5893..25054b601a4f6 100644 --- a/mypy/typeshed/stdlib/_typeshed/__init__.pyi +++ b/mypy/typeshed/stdlib/_typeshed/__init__.pyi @@ -3,7 +3,7 @@ # See the README.md file in this directory for more information. import sys -from collections.abc import Awaitable, Callable, Iterable, Sequence, Set as AbstractSet, Sized +from collections.abc import Awaitable, Callable, Iterable, Iterator, Sequence, Set as AbstractSet, Sized from dataclasses import Field from os import PathLike from types import FrameType, TracebackType @@ -54,7 +54,8 @@ Unused: TypeAlias = object # stable # Marker for return types that include None, but where forcing the user to # check for None can be detrimental. Sometimes called "the Any trick". See -# CONTRIBUTING.md for more information. +# https://typing.python.org/en/latest/guides/writing_stubs.html#the-any-trick +# for more information. MaybeNone: TypeAlias = Any # stable # Used to mark arguments that default to a sentinel value. This prevents @@ -275,6 +276,16 @@ class SupportsWrite(Protocol[_T_contra]): class SupportsFlush(Protocol): def flush(self) -> object: ... +# Suitable for dictionary view objects +class Viewable(Protocol[_T_co]): + def __len__(self) -> int: ... + def __iter__(self) -> Iterator[_T_co]: ... + +class SupportsGetItemViewable(Protocol[_KT, _VT_co]): + def __len__(self) -> int: ... + def __iter__(self) -> Iterator[_KT]: ... + def __getitem__(self, key: _KT, /) -> _VT_co: ... + # Unfortunately PEP 688 does not allow us to distinguish read-only # from writable buffers. We use these aliases for readability for now. # Perhaps a future extension of the buffer protocol will allow us to diff --git a/mypy/typeshed/stdlib/_warnings.pyi b/mypy/typeshed/stdlib/_warnings.pyi index 2e571e676c974..2dbc7b8552813 100644 --- a/mypy/typeshed/stdlib/_warnings.pyi +++ b/mypy/typeshed/stdlib/_warnings.pyi @@ -38,9 +38,9 @@ def warn_explicit( filename: str, lineno: int, module: str | None = ..., - registry: dict[str | tuple[str, type[Warning], int], int] | None = ..., - module_globals: dict[str, Any] | None = ..., - source: Any | None = ..., + registry: dict[str | tuple[str, type[Warning], int], int] | None = None, + module_globals: dict[str, Any] | None = None, + source: Any | None = None, ) -> None: ... @overload def warn_explicit( @@ -48,8 +48,8 @@ def warn_explicit( category: Any, filename: str, lineno: int, - module: str | None = ..., - registry: dict[str | tuple[str, type[Warning], int], int] | None = ..., - module_globals: dict[str, Any] | None = ..., - source: Any | None = ..., + module: str | None = None, + registry: dict[str | tuple[str, type[Warning], int], int] | None = None, + module_globals: dict[str, Any] | None = None, + source: Any | None = None, ) -> None: ... diff --git a/mypy/typeshed/stdlib/_winapi.pyi b/mypy/typeshed/stdlib/_winapi.pyi index 6083ea4ae57a6..d9e2c377b115a 100644 --- a/mypy/typeshed/stdlib/_winapi.pyi +++ b/mypy/typeshed/stdlib/_winapi.pyi @@ -128,21 +128,21 @@ if sys.platform == "win32": WAIT_TIMEOUT: Final = 258 if sys.version_info >= (3, 10): - LOCALE_NAME_INVARIANT: str - LOCALE_NAME_MAX_LENGTH: int - LOCALE_NAME_SYSTEM_DEFAULT: str - LOCALE_NAME_USER_DEFAULT: str | None + LOCALE_NAME_INVARIANT: Final[str] + LOCALE_NAME_MAX_LENGTH: Final[int] + LOCALE_NAME_SYSTEM_DEFAULT: Final[str] + LOCALE_NAME_USER_DEFAULT: Final[str | None] - LCMAP_FULLWIDTH: int - LCMAP_HALFWIDTH: int - LCMAP_HIRAGANA: int - LCMAP_KATAKANA: int - LCMAP_LINGUISTIC_CASING: int - LCMAP_LOWERCASE: int - LCMAP_SIMPLIFIED_CHINESE: int - LCMAP_TITLECASE: int - LCMAP_TRADITIONAL_CHINESE: int - LCMAP_UPPERCASE: int + LCMAP_FULLWIDTH: Final[int] + LCMAP_HALFWIDTH: Final[int] + LCMAP_HIRAGANA: Final[int] + LCMAP_KATAKANA: Final[int] + LCMAP_LINGUISTIC_CASING: Final[int] + LCMAP_LOWERCASE: Final[int] + LCMAP_SIMPLIFIED_CHINESE: Final[int] + LCMAP_TITLECASE: Final[int] + LCMAP_TRADITIONAL_CHINESE: Final[int] + LCMAP_UPPERCASE: Final[int] if sys.version_info >= (3, 12): COPYFILE2_CALLBACK_CHUNK_STARTED: Final = 1 diff --git a/mypy/typeshed/stdlib/_zstd.pyi b/mypy/typeshed/stdlib/_zstd.pyi index 2730232528fc2..f5e98ef88bb9f 100644 --- a/mypy/typeshed/stdlib/_zstd.pyi +++ b/mypy/typeshed/stdlib/_zstd.pyi @@ -45,9 +45,9 @@ class ZstdCompressor: CONTINUE: Final = 0 FLUSH_BLOCK: Final = 1 FLUSH_FRAME: Final = 2 - def __init__( - self, level: int | None = None, options: Mapping[int, int] | None = None, zstd_dict: ZstdDict | None = None - ) -> None: ... + def __new__( + cls, level: int | None = None, options: Mapping[int, int] | None = None, zstd_dict: ZstdDict | None = None + ) -> Self: ... def compress( self, /, data: ReadableBuffer, mode: _ZstdCompressorContinue | _ZstdCompressorFlushBlock | _ZstdCompressorFlushFrame = 0 ) -> bytes: ... @@ -58,7 +58,7 @@ class ZstdCompressor: @final class ZstdDecompressor: - def __init__(self, zstd_dict: ZstdDict | None = None, options: Mapping[int, int] | None = None) -> None: ... + def __new__(cls, zstd_dict: ZstdDict | None = None, options: Mapping[int, int] | None = None) -> Self: ... def decompress(self, /, data: ReadableBuffer, max_length: int = -1) -> bytes: ... @property def eof(self) -> bool: ... @@ -69,7 +69,7 @@ class ZstdDecompressor: @final class ZstdDict: - def __init__(self, dict_content: bytes, /, *, is_raw: bool = False) -> None: ... + def __new__(cls, dict_content: bytes, /, *, is_raw: bool = False) -> Self: ... def __len__(self, /) -> int: ... @property def as_digested_dict(self) -> tuple[Self, int]: ... diff --git a/mypy/typeshed/stdlib/abc.pyi b/mypy/typeshed/stdlib/abc.pyi index fdca48ac7aafe..c8cd549e30eca 100644 --- a/mypy/typeshed/stdlib/abc.pyi +++ b/mypy/typeshed/stdlib/abc.pyi @@ -28,17 +28,17 @@ class ABCMeta(type): def register(cls: ABCMeta, subclass: type[_T]) -> type[_T]: ... def abstractmethod(funcobj: _FuncT) -> _FuncT: ... -@deprecated("Use 'classmethod' with 'abstractmethod' instead") +@deprecated("Deprecated since Python 3.3. Use `@classmethod` stacked on top of `@abstractmethod` instead.") class abstractclassmethod(classmethod[_T, _P, _R_co]): __isabstractmethod__: Literal[True] def __init__(self, callable: Callable[Concatenate[type[_T], _P], _R_co]) -> None: ... -@deprecated("Use 'staticmethod' with 'abstractmethod' instead") +@deprecated("Deprecated since Python 3.3. Use `@staticmethod` stacked on top of `@abstractmethod` instead.") class abstractstaticmethod(staticmethod[_P, _R_co]): __isabstractmethod__: Literal[True] def __init__(self, callable: Callable[_P, _R_co]) -> None: ... -@deprecated("Use 'property' with 'abstractmethod' instead") +@deprecated("Deprecated since Python 3.3. Use `@property` stacked on top of `@abstractmethod` instead.") class abstractproperty(property): __isabstractmethod__: Literal[True] diff --git a/mypy/typeshed/stdlib/annotationlib.pyi b/mypy/typeshed/stdlib/annotationlib.pyi index 7590c632d7856..3679dc29daaa0 100644 --- a/mypy/typeshed/stdlib/annotationlib.pyi +++ b/mypy/typeshed/stdlib/annotationlib.pyi @@ -28,6 +28,20 @@ if sys.version_info >= (3, 14): @final class ForwardRef: + __slots__ = ( + "__forward_is_argument__", + "__forward_is_class__", + "__forward_module__", + "__weakref__", + "__arg__", + "__globals__", + "__extra_names__", + "__code__", + "__ast_node__", + "__cell__", + "__owner__", + "__stringifier_dict__", + ) __forward_is_argument__: bool __forward_is_class__: bool __forward_module__: str | None @@ -64,7 +78,7 @@ if sys.version_info >= (3, 14): owner: object = None, format: Format = Format.VALUE, # noqa: Y011 ) -> AnnotationForm: ... - @deprecated("Use ForwardRef.evaluate() or typing.evaluate_forward_ref() instead.") + @deprecated("Use `ForwardRef.evaluate()` or `typing.evaluate_forward_ref()` instead.") def _evaluate( self, globalns: dict[str, Any] | None, diff --git a/mypy/typeshed/stdlib/argparse.pyi b/mypy/typeshed/stdlib/argparse.pyi index 3c3ba116a692e..f4b3aac09aa96 100644 --- a/mypy/typeshed/stdlib/argparse.pyi +++ b/mypy/typeshed/stdlib/argparse.pyi @@ -156,7 +156,7 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer): exit_on_error: bool = True, *, suggest_on_error: bool = False, - color: bool = False, + color: bool = True, ) -> None: ... else: def __init__( @@ -470,7 +470,7 @@ class Namespace(_AttributeHolder): __hash__: ClassVar[None] # type: ignore[assignment] if sys.version_info >= (3, 14): - @deprecated("Deprecated in Python 3.14; Simply open files after parsing arguments") + @deprecated("Deprecated since Python 3.14. Open files after parsing arguments instead.") class FileType: # undocumented _mode: str diff --git a/mypy/typeshed/stdlib/array.pyi b/mypy/typeshed/stdlib/array.pyi index ccb7f1b98ef33..a6b0344a1e2ea 100644 --- a/mypy/typeshed/stdlib/array.pyi +++ b/mypy/typeshed/stdlib/array.pyi @@ -3,7 +3,7 @@ from _typeshed import ReadableBuffer, SupportsRead, SupportsWrite from collections.abc import Iterable, MutableSequence from types import GenericAlias from typing import Any, ClassVar, Literal, SupportsIndex, TypeVar, overload -from typing_extensions import Self, TypeAlias, deprecated +from typing_extensions import Self, TypeAlias, deprecated, disjoint_base _IntTypeCode: TypeAlias = Literal["b", "B", "h", "H", "i", "I", "l", "L", "q", "Q"] _FloatTypeCode: TypeAlias = Literal["f", "d"] @@ -17,6 +17,7 @@ _T = TypeVar("_T", int, float, str) typecodes: str +@disjoint_base class array(MutableSequence[_T]): @property def typecode(self) -> _TypeCode: ... diff --git a/mypy/typeshed/stdlib/ast.pyi b/mypy/typeshed/stdlib/ast.pyi index 8ee8671163012..d360c2ed60e5c 100644 --- a/mypy/typeshed/stdlib/ast.pyi +++ b/mypy/typeshed/stdlib/ast.pyi @@ -11,7 +11,7 @@ from _ast import ( from _typeshed import ReadableBuffer, Unused from collections.abc import Iterable, Iterator, Sequence from typing import Any, ClassVar, Generic, Literal, TypedDict, TypeVar as _TypeVar, overload, type_check_only -from typing_extensions import Self, Unpack, deprecated +from typing_extensions import Self, Unpack, deprecated, disjoint_base if sys.version_info >= (3, 13): from _ast import PyCF_OPTIMIZED_AST as PyCF_OPTIMIZED_AST @@ -30,16 +30,24 @@ class _Attributes(TypedDict, Generic[_EndPositionT], total=False): # The various AST classes are implemented in C, and imported from _ast at runtime, # but they consider themselves to live in the ast module, # so we'll define the stubs in this file. -class AST: - if sys.version_info >= (3, 10): +if sys.version_info >= (3, 12): + @disjoint_base + class AST: __match_args__ = () - _attributes: ClassVar[tuple[str, ...]] - _fields: ClassVar[tuple[str, ...]] - if sys.version_info >= (3, 13): - _field_types: ClassVar[dict[str, Any]] + _attributes: ClassVar[tuple[str, ...]] + _fields: ClassVar[tuple[str, ...]] + if sys.version_info >= (3, 13): + _field_types: ClassVar[dict[str, Any]] - if sys.version_info >= (3, 14): - def __replace__(self) -> Self: ... + if sys.version_info >= (3, 14): + def __replace__(self) -> Self: ... + +else: + class AST: + if sys.version_info >= (3, 10): + __match_args__ = () + _attributes: ClassVar[tuple[str, ...]] + _fields: ClassVar[tuple[str, ...]] class mod(AST): ... @@ -1098,16 +1106,16 @@ class Constant(expr): if sys.version_info < (3, 14): # Aliases for value, for backwards compatibility @property - @deprecated("Will be removed in Python 3.14. Use `value` instead.") + @deprecated("Removed in Python 3.14. Use `value` instead.") def n(self) -> _ConstantValue: ... @n.setter - @deprecated("Will be removed in Python 3.14. Use `value` instead.") + @deprecated("Removed in Python 3.14. Use `value` instead.") def n(self, value: _ConstantValue) -> None: ... @property - @deprecated("Will be removed in Python 3.14. Use `value` instead.") + @deprecated("Removed in Python 3.14. Use `value` instead.") def s(self) -> _ConstantValue: ... @s.setter - @deprecated("Will be removed in Python 3.14. Use `value` instead.") + @deprecated("Removed in Python 3.14. Use `value` instead.") def s(self, value: _ConstantValue) -> None: ... def __init__(self, value: _ConstantValue, kind: str | None = None, **kwargs: Unpack[_Attributes]) -> None: ... @@ -1206,7 +1214,7 @@ class Slice(expr): self, *, lower: expr | None = ..., upper: expr | None = ..., step: expr | None = ..., **kwargs: Unpack[_Attributes] ) -> Self: ... -@deprecated("Deprecated since Python 3.9. Use ast.Tuple instead.") +@deprecated("Deprecated since Python 3.9. Use `ast.Tuple` instead.") class ExtSlice(slice): def __new__(cls, dims: Iterable[slice] = (), **kwargs: Unpack[_Attributes]) -> Tuple: ... # type: ignore[misc] @@ -1711,23 +1719,23 @@ else: def __init__(cls, *args: Unused) -> None: ... if sys.version_info < (3, 14): - @deprecated("Replaced by ast.Constant; removed in Python 3.14") + @deprecated("Removed in Python 3.14. Use `ast.Constant` instead.") class Num(Constant, metaclass=_ABC): def __new__(cls, n: complex, **kwargs: Unpack[_Attributes]) -> Constant: ... # type: ignore[misc] # pyright: ignore[reportInconsistentConstructor] - @deprecated("Replaced by ast.Constant; removed in Python 3.14") + @deprecated("Removed in Python 3.14. Use `ast.Constant` instead.") class Str(Constant, metaclass=_ABC): def __new__(cls, s: str, **kwargs: Unpack[_Attributes]) -> Constant: ... # type: ignore[misc] # pyright: ignore[reportInconsistentConstructor] - @deprecated("Replaced by ast.Constant; removed in Python 3.14") + @deprecated("Removed in Python 3.14. Use `ast.Constant` instead.") class Bytes(Constant, metaclass=_ABC): def __new__(cls, s: bytes, **kwargs: Unpack[_Attributes]) -> Constant: ... # type: ignore[misc] # pyright: ignore[reportInconsistentConstructor] - @deprecated("Replaced by ast.Constant; removed in Python 3.14") + @deprecated("Removed in Python 3.14. Use `ast.Constant` instead.") class NameConstant(Constant, metaclass=_ABC): def __new__(cls, value: _ConstantValue, kind: str | None, **kwargs: Unpack[_Attributes]) -> Constant: ... # type: ignore[misc] # pyright: ignore[reportInconsistentConstructor] - @deprecated("Replaced by ast.Constant; removed in Python 3.14") + @deprecated("Removed in Python 3.14. Use `ast.Constant` instead.") class Ellipsis(Constant, metaclass=_ABC): def __new__(cls, **kwargs: Unpack[_Attributes]) -> Constant: ... # type: ignore[misc] # pyright: ignore[reportInconsistentConstructor] @@ -2046,15 +2054,15 @@ class NodeVisitor: def visit_Param(self, node: Param) -> Any: ... if sys.version_info < (3, 14): - @deprecated("Replaced by visit_Constant; removed in Python 3.14") + @deprecated("Removed in Python 3.14. Use `visit_Constant` instead.") def visit_Num(self, node: Num) -> Any: ... # type: ignore[deprecated] - @deprecated("Replaced by visit_Constant; removed in Python 3.14") + @deprecated("Removed in Python 3.14. Use `visit_Constant` instead.") def visit_Str(self, node: Str) -> Any: ... # type: ignore[deprecated] - @deprecated("Replaced by visit_Constant; removed in Python 3.14") + @deprecated("Removed in Python 3.14. Use `visit_Constant` instead.") def visit_Bytes(self, node: Bytes) -> Any: ... # type: ignore[deprecated] - @deprecated("Replaced by visit_Constant; removed in Python 3.14") + @deprecated("Removed in Python 3.14. Use `visit_Constant` instead.") def visit_NameConstant(self, node: NameConstant) -> Any: ... # type: ignore[deprecated] - @deprecated("Replaced by visit_Constant; removed in Python 3.14") + @deprecated("Removed in Python 3.14. Use `visit_Constant` instead.") def visit_Ellipsis(self, node: Ellipsis) -> Any: ... # type: ignore[deprecated] class NodeTransformer(NodeVisitor): diff --git a/mypy/typeshed/stdlib/asyncio/base_events.pyi b/mypy/typeshed/stdlib/asyncio/base_events.pyi index cad7dde40b011..1f493210d6655 100644 --- a/mypy/typeshed/stdlib/asyncio/base_events.pyi +++ b/mypy/typeshed/stdlib/asyncio/base_events.pyi @@ -10,7 +10,7 @@ from asyncio.transports import BaseTransport, DatagramTransport, ReadTransport, from collections.abc import Callable, Iterable, Sequence from concurrent.futures import Executor, ThreadPoolExecutor from contextvars import Context -from socket import AddressFamily, SocketKind, _Address, _RetAddress, socket +from socket import AddressFamily, AddressInfo, SocketKind, _Address, _RetAddress, socket from typing import IO, Any, Literal, TypeVar, overload from typing_extensions import TypeAlias, TypeVarTuple, Unpack @@ -235,8 +235,8 @@ class BaseEventLoop(AbstractEventLoop): host: str | Sequence[str] | None = None, port: int = ..., *, - family: int = ..., - flags: int = ..., + family: int = 0, + flags: int = 1, sock: None = None, backlog: int = 100, ssl: _SSLContext = None, @@ -254,8 +254,8 @@ class BaseEventLoop(AbstractEventLoop): host: None = None, port: None = None, *, - family: int = ..., - flags: int = ..., + family: int = 0, + flags: int = 1, sock: socket = ..., backlog: int = 100, ssl: _SSLContext = None, @@ -274,8 +274,8 @@ class BaseEventLoop(AbstractEventLoop): host: str | Sequence[str] | None = None, port: int = ..., *, - family: int = ..., - flags: int = ..., + family: int = AddressFamily.AF_UNSPEC, + flags: int = AddressInfo.AI_PASSIVE, sock: None = None, backlog: int = 100, ssl: _SSLContext = None, @@ -292,8 +292,8 @@ class BaseEventLoop(AbstractEventLoop): host: None = None, port: None = None, *, - family: int = ..., - flags: int = ..., + family: int = AddressFamily.AF_UNSPEC, + flags: int = AddressInfo.AI_PASSIVE, sock: socket = ..., backlog: int = 100, ssl: _SSLContext = None, @@ -311,8 +311,8 @@ class BaseEventLoop(AbstractEventLoop): host: str | Sequence[str] | None = None, port: int = ..., *, - family: int = ..., - flags: int = ..., + family: int = AddressFamily.AF_UNSPEC, + flags: int = AddressInfo.AI_PASSIVE, sock: None = None, backlog: int = 100, ssl: _SSLContext = None, @@ -328,8 +328,8 @@ class BaseEventLoop(AbstractEventLoop): host: None = None, port: None = None, *, - family: int = ..., - flags: int = ..., + family: int = AddressFamily.AF_UNSPEC, + flags: int = AddressInfo.AI_PASSIVE, sock: socket = ..., backlog: int = 100, ssl: _SSLContext = None, diff --git a/mypy/typeshed/stdlib/asyncio/events.pyi b/mypy/typeshed/stdlib/asyncio/events.pyi index a37f6f697b9a1..14c4c0bf3d5ac 100644 --- a/mypy/typeshed/stdlib/asyncio/events.pyi +++ b/mypy/typeshed/stdlib/asyncio/events.pyi @@ -11,7 +11,7 @@ from abc import ABCMeta, abstractmethod from collections.abc import Callable, Sequence from concurrent.futures import Executor from contextvars import Context -from socket import AddressFamily, SocketKind, _Address, _RetAddress, socket +from socket import AddressFamily, AddressInfo, SocketKind, _Address, _RetAddress, socket from typing import IO, Any, Literal, Protocol, TypeVar, overload, type_check_only from typing_extensions import Self, TypeAlias, TypeVarTuple, Unpack, deprecated @@ -73,6 +73,7 @@ class _TaskFactory(Protocol): def __call__(self, loop: AbstractEventLoop, factory: _CoroutineLike[_T], /) -> Future[_T]: ... class Handle: + __slots__ = ("_callback", "_args", "_cancelled", "_loop", "_source_traceback", "_repr", "__weakref__", "_context") _cancelled: bool _args: Sequence[Any] def __init__( @@ -85,6 +86,7 @@ class Handle: def get_context(self) -> Context: ... class TimerHandle(Handle): + __slots__ = ["_scheduled", "_when"] def __init__( self, when: float, @@ -287,8 +289,8 @@ class AbstractEventLoop: host: str | Sequence[str] | None = None, port: int = ..., *, - family: int = ..., - flags: int = ..., + family: int = AddressFamily.AF_UNSPEC, + flags: int = AddressInfo.AI_PASSIVE, sock: None = None, backlog: int = 100, ssl: _SSLContext = None, @@ -307,8 +309,8 @@ class AbstractEventLoop: host: None = None, port: None = None, *, - family: int = ..., - flags: int = ..., + family: int = AddressFamily.AF_UNSPEC, + flags: int = AddressInfo.AI_PASSIVE, sock: socket = ..., backlog: int = 100, ssl: _SSLContext = None, @@ -328,8 +330,8 @@ class AbstractEventLoop: host: str | Sequence[str] | None = None, port: int = ..., *, - family: int = ..., - flags: int = ..., + family: int = AddressFamily.AF_UNSPEC, + flags: int = AddressInfo.AI_PASSIVE, sock: None = None, backlog: int = 100, ssl: _SSLContext = None, @@ -347,8 +349,8 @@ class AbstractEventLoop: host: None = None, port: None = None, *, - family: int = ..., - flags: int = ..., + family: int = AddressFamily.AF_UNSPEC, + flags: int = AddressInfo.AI_PASSIVE, sock: socket = ..., backlog: int = 100, ssl: _SSLContext = None, @@ -367,8 +369,8 @@ class AbstractEventLoop: host: str | Sequence[str] | None = None, port: int = ..., *, - family: int = ..., - flags: int = ..., + family: int = AddressFamily.AF_UNSPEC, + flags: int = AddressInfo.AI_PASSIVE, sock: None = None, backlog: int = 100, ssl: _SSLContext = None, @@ -385,8 +387,8 @@ class AbstractEventLoop: host: None = None, port: None = None, *, - family: int = ..., - flags: int = ..., + family: int = AddressFamily.AF_UNSPEC, + flags: int = AddressInfo.AI_PASSIVE, sock: socket = ..., backlog: int = 100, ssl: _SSLContext = None, @@ -534,7 +536,7 @@ class AbstractEventLoop: bufsize: Literal[0] = 0, encoding: None = None, errors: None = None, - text: Literal[False] | None = ..., + text: Literal[False] | None = None, **kwargs: Any, ) -> tuple[SubprocessTransport, _ProtocolT]: ... @abstractmethod @@ -614,10 +616,10 @@ class _AbstractEventLoopPolicy: if sys.version_info < (3, 14): if sys.version_info >= (3, 12): @abstractmethod - @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + @deprecated("Deprecated since Python 3.12; removed in Python 3.14.") def get_child_watcher(self) -> AbstractChildWatcher: ... @abstractmethod - @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + @deprecated("Deprecated since Python 3.12; removed in Python 3.14.") def set_child_watcher(self, watcher: AbstractChildWatcher) -> None: ... else: @abstractmethod @@ -643,9 +645,9 @@ else: if sys.version_info >= (3, 14): def _get_event_loop_policy() -> _AbstractEventLoopPolicy: ... def _set_event_loop_policy(policy: _AbstractEventLoopPolicy | None) -> None: ... - @deprecated("Deprecated as of Python 3.14; will be removed in Python 3.16") + @deprecated("Deprecated since Python 3.14; will be removed in Python 3.16.") def get_event_loop_policy() -> _AbstractEventLoopPolicy: ... - @deprecated("Deprecated as of Python 3.14; will be removed in Python 3.16") + @deprecated("Deprecated since Python 3.14; will be removed in Python 3.16.") def set_event_loop_policy(policy: _AbstractEventLoopPolicy | None) -> None: ... else: @@ -657,9 +659,9 @@ def new_event_loop() -> AbstractEventLoop: ... if sys.version_info < (3, 14): if sys.version_info >= (3, 12): - @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + @deprecated("Deprecated since Python 3.12; removed in Python 3.14.") def get_child_watcher() -> AbstractChildWatcher: ... - @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + @deprecated("Deprecated since Python 3.12; removed in Python 3.14.") def set_child_watcher(watcher: AbstractChildWatcher) -> None: ... else: diff --git a/mypy/typeshed/stdlib/asyncio/graph.pyi b/mypy/typeshed/stdlib/asyncio/graph.pyi index cb2cf01749955..18a8a6457d757 100644 --- a/mypy/typeshed/stdlib/asyncio/graph.pyi +++ b/mypy/typeshed/stdlib/asyncio/graph.pyi @@ -1,26 +1,28 @@ +import sys from _typeshed import SupportsWrite from asyncio import Future from dataclasses import dataclass from types import FrameType from typing import Any, overload -__all__ = ("capture_call_graph", "format_call_graph", "print_call_graph", "FrameCallGraphEntry", "FutureCallGraph") +if sys.version_info >= (3, 14): + __all__ = ("capture_call_graph", "format_call_graph", "print_call_graph", "FrameCallGraphEntry", "FutureCallGraph") -@dataclass(frozen=True) -class FrameCallGraphEntry: - frame: FrameType + @dataclass(frozen=True, slots=True) + class FrameCallGraphEntry: + frame: FrameType -@dataclass(frozen=True) -class FutureCallGraph: - future: Future[Any] - call_stack: tuple[FrameCallGraphEntry, ...] - awaited_by: tuple[FutureCallGraph, ...] + @dataclass(frozen=True, slots=True) + class FutureCallGraph: + future: Future[Any] + call_stack: tuple[FrameCallGraphEntry, ...] + awaited_by: tuple[FutureCallGraph, ...] -@overload -def capture_call_graph(future: None = None, /, *, depth: int = 1, limit: int | None = None) -> FutureCallGraph | None: ... -@overload -def capture_call_graph(future: Future[Any], /, *, depth: int = 1, limit: int | None = None) -> FutureCallGraph | None: ... -def format_call_graph(future: Future[Any] | None = None, /, *, depth: int = 1, limit: int | None = None) -> str: ... -def print_call_graph( - future: Future[Any] | None = None, /, *, file: SupportsWrite[str] | None = None, depth: int = 1, limit: int | None = None -) -> None: ... + @overload + def capture_call_graph(future: None = None, /, *, depth: int = 1, limit: int | None = None) -> FutureCallGraph | None: ... + @overload + def capture_call_graph(future: Future[Any], /, *, depth: int = 1, limit: int | None = None) -> FutureCallGraph | None: ... + def format_call_graph(future: Future[Any] | None = None, /, *, depth: int = 1, limit: int | None = None) -> str: ... + def print_call_graph( + future: Future[Any] | None = None, /, *, file: SupportsWrite[str] | None = None, depth: int = 1, limit: int | None = None + ) -> None: ... diff --git a/mypy/typeshed/stdlib/asyncio/protocols.pyi b/mypy/typeshed/stdlib/asyncio/protocols.pyi index 5425336c49a82..2c52ad4be4102 100644 --- a/mypy/typeshed/stdlib/asyncio/protocols.pyi +++ b/mypy/typeshed/stdlib/asyncio/protocols.pyi @@ -6,21 +6,26 @@ from typing import Any __all__ = ("BaseProtocol", "Protocol", "DatagramProtocol", "SubprocessProtocol", "BufferedProtocol") class BaseProtocol: + __slots__ = () def connection_made(self, transport: transports.BaseTransport) -> None: ... def connection_lost(self, exc: Exception | None) -> None: ... def pause_writing(self) -> None: ... def resume_writing(self) -> None: ... class Protocol(BaseProtocol): + # Need annotation or mypy will complain about 'Cannot determine type of "__slots__" in base class' + __slots__: tuple[()] = () def data_received(self, data: bytes) -> None: ... def eof_received(self) -> bool | None: ... class BufferedProtocol(BaseProtocol): + __slots__ = () def get_buffer(self, sizehint: int) -> ReadableBuffer: ... def buffer_updated(self, nbytes: int) -> None: ... def eof_received(self) -> bool | None: ... class DatagramProtocol(BaseProtocol): + __slots__ = () def connection_made(self, transport: transports.DatagramTransport) -> None: ... # type: ignore[override] # addr can be a tuple[int, int] for some unusual protocols like socket.AF_NETLINK. # Use tuple[str | Any, int] to not cause typechecking issues on most usual cases. @@ -30,6 +35,7 @@ class DatagramProtocol(BaseProtocol): def error_received(self, exc: Exception) -> None: ... class SubprocessProtocol(BaseProtocol): + __slots__: tuple[()] = () def pipe_data_received(self, fd: int, data: bytes) -> None: ... def pipe_connection_lost(self, fd: int, exc: Exception | None) -> None: ... def process_exited(self) -> None: ... diff --git a/mypy/typeshed/stdlib/asyncio/runners.pyi b/mypy/typeshed/stdlib/asyncio/runners.pyi index caf5e4996cf4b..919e6521f8a15 100644 --- a/mypy/typeshed/stdlib/asyncio/runners.pyi +++ b/mypy/typeshed/stdlib/asyncio/runners.pyi @@ -26,7 +26,7 @@ if sys.version_info >= (3, 11): if sys.version_info >= (3, 12): def run( - main: Coroutine[Any, Any, _T], *, debug: bool | None = ..., loop_factory: Callable[[], AbstractEventLoop] | None = ... + main: Coroutine[Any, Any, _T], *, debug: bool | None = None, loop_factory: Callable[[], AbstractEventLoop] | None = None ) -> _T: ... else: diff --git a/mypy/typeshed/stdlib/asyncio/streams.pyi b/mypy/typeshed/stdlib/asyncio/streams.pyi index bf8db0246ee21..33cffb11ed780 100644 --- a/mypy/typeshed/stdlib/asyncio/streams.pyi +++ b/mypy/typeshed/stdlib/asyncio/streams.pyi @@ -34,7 +34,7 @@ if sys.version_info >= (3, 10): port: int | str | None = None, *, limit: int = 65536, - ssl_handshake_timeout: float | None = ..., + ssl_handshake_timeout: float | None = None, **kwds: Any, ) -> tuple[StreamReader, StreamWriter]: ... async def start_server( @@ -43,7 +43,7 @@ if sys.version_info >= (3, 10): port: int | str | None = None, *, limit: int = 65536, - ssl_handshake_timeout: float | None = ..., + ssl_handshake_timeout: float | None = None, **kwds: Any, ) -> Server: ... @@ -54,7 +54,7 @@ else: *, loop: events.AbstractEventLoop | None = None, limit: int = 65536, - ssl_handshake_timeout: float | None = ..., + ssl_handshake_timeout: float | None = None, **kwds: Any, ) -> tuple[StreamReader, StreamWriter]: ... async def start_server( @@ -64,7 +64,7 @@ else: *, loop: events.AbstractEventLoop | None = None, limit: int = 65536, - ssl_handshake_timeout: float | None = ..., + ssl_handshake_timeout: float | None = None, **kwds: Any, ) -> Server: ... diff --git a/mypy/typeshed/stdlib/asyncio/subprocess.pyi b/mypy/typeshed/stdlib/asyncio/subprocess.pyi index 50d75391f36d6..ceee2b5b90a09 100644 --- a/mypy/typeshed/stdlib/asyncio/subprocess.pyi +++ b/mypy/typeshed/stdlib/asyncio/subprocess.pyi @@ -60,7 +60,7 @@ if sys.version_info >= (3, 11): creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), group: None | str | int = None, extra_groups: None | Collection[str | int] = None, user: None | str | int = None, @@ -92,7 +92,7 @@ if sys.version_info >= (3, 11): creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), group: None | str | int = None, extra_groups: None | Collection[str | int] = None, user: None | str | int = None, @@ -126,7 +126,7 @@ elif sys.version_info >= (3, 10): creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), group: None | str | int = None, extra_groups: None | Collection[str | int] = None, user: None | str | int = None, @@ -157,7 +157,7 @@ elif sys.version_info >= (3, 10): creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), group: None | str | int = None, extra_groups: None | Collection[str | int] = None, user: None | str | int = None, @@ -191,7 +191,7 @@ else: # >= 3.9 creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), group: None | str | int = None, extra_groups: None | Collection[str | int] = None, user: None | str | int = None, @@ -222,7 +222,7 @@ else: # >= 3.9 creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), group: None | str | int = None, extra_groups: None | Collection[str | int] = None, user: None | str | int = None, diff --git a/mypy/typeshed/stdlib/asyncio/tasks.pyi b/mypy/typeshed/stdlib/asyncio/tasks.pyi index 4104b3ecfeee4..1442f7400a9c9 100644 --- a/mypy/typeshed/stdlib/asyncio/tasks.pyi +++ b/mypy/typeshed/stdlib/asyncio/tasks.pyi @@ -8,7 +8,7 @@ from _asyncio import ( _unregister_task as _unregister_task, ) from collections.abc import AsyncIterator, Awaitable, Coroutine, Generator, Iterable, Iterator -from typing import Any, Literal, Protocol, TypeVar, overload, type_check_only +from typing import Any, Final, Literal, Protocol, TypeVar, overload, type_check_only from typing_extensions import TypeAlias from . import _CoroutineLike @@ -82,9 +82,9 @@ else: _TaskYieldType: TypeAlias = Future[object] | None -FIRST_COMPLETED = concurrent.futures.FIRST_COMPLETED -FIRST_EXCEPTION = concurrent.futures.FIRST_EXCEPTION -ALL_COMPLETED = concurrent.futures.ALL_COMPLETED +FIRST_COMPLETED: Final = concurrent.futures.FIRST_COMPLETED +FIRST_EXCEPTION: Final = concurrent.futures.FIRST_EXCEPTION +ALL_COMPLETED: Final = concurrent.futures.ALL_COMPLETED if sys.version_info >= (3, 13): @type_check_only diff --git a/mypy/typeshed/stdlib/asyncio/transports.pyi b/mypy/typeshed/stdlib/asyncio/transports.pyi index bce54897f18f6..cc870d5e0b9ad 100644 --- a/mypy/typeshed/stdlib/asyncio/transports.pyi +++ b/mypy/typeshed/stdlib/asyncio/transports.pyi @@ -8,6 +8,7 @@ from typing import Any __all__ = ("BaseTransport", "ReadTransport", "WriteTransport", "Transport", "DatagramTransport", "SubprocessTransport") class BaseTransport: + __slots__ = ("_extra",) def __init__(self, extra: Mapping[str, Any] | None = None) -> None: ... def get_extra_info(self, name: str, default: Any = None) -> Any: ... def is_closing(self) -> bool: ... @@ -16,11 +17,13 @@ class BaseTransport: def get_protocol(self) -> BaseProtocol: ... class ReadTransport(BaseTransport): + __slots__ = () def is_reading(self) -> bool: ... def pause_reading(self) -> None: ... def resume_reading(self) -> None: ... class WriteTransport(BaseTransport): + __slots__ = () def set_write_buffer_limits(self, high: int | None = None, low: int | None = None) -> None: ... def get_write_buffer_size(self) -> int: ... def get_write_buffer_limits(self) -> tuple[int, int]: ... @@ -32,13 +35,16 @@ class WriteTransport(BaseTransport): def can_write_eof(self) -> bool: ... def abort(self) -> None: ... -class Transport(ReadTransport, WriteTransport): ... +class Transport(ReadTransport, WriteTransport): + __slots__ = () class DatagramTransport(BaseTransport): + __slots__ = () def sendto(self, data: bytes | bytearray | memoryview, addr: _Address | None = None) -> None: ... def abort(self) -> None: ... class SubprocessTransport(BaseTransport): + __slots__ = () def get_pid(self) -> int: ... def get_returncode(self) -> int | None: ... def get_pipe_transport(self, fd: int) -> BaseTransport | None: ... @@ -47,4 +53,5 @@ class SubprocessTransport(BaseTransport): def kill(self) -> None: ... class _FlowControlMixin(Transport): + __slots__ = ("_loop", "_protocol_paused", "_high_water", "_low_water") def __init__(self, extra: Mapping[str, Any] | None = None, loop: AbstractEventLoop | None = None) -> None: ... diff --git a/mypy/typeshed/stdlib/asyncio/trsock.pyi b/mypy/typeshed/stdlib/asyncio/trsock.pyi index 4dacbbd493991..492f1e42adf20 100644 --- a/mypy/typeshed/stdlib/asyncio/trsock.pyi +++ b/mypy/typeshed/stdlib/asyncio/trsock.pyi @@ -14,6 +14,7 @@ _WriteBuffer: TypeAlias = bytearray | memoryview _CMSG: TypeAlias = tuple[int, int, bytes] class TransportSocket: + __slots__ = ("_sock",) def __init__(self, sock: socket.socket) -> None: ... @property def family(self) -> int: ... @@ -62,7 +63,7 @@ class TransportSocket: @deprecated("Removed in Python 3.11") def makefile(self) -> BinaryIO: ... @deprecated("Rmoved in Python 3.11") - def sendfile(self, file: BinaryIO, offset: int = ..., count: int | None = ...) -> int: ... + def sendfile(self, file: BinaryIO, offset: int = 0, count: int | None = None) -> int: ... @deprecated("Removed in Python 3.11") def close(self) -> None: ... @deprecated("Removed in Python 3.11") @@ -70,17 +71,22 @@ class TransportSocket: if sys.platform == "linux": @deprecated("Removed in Python 3.11") def sendmsg_afalg( - self, msg: Iterable[ReadableBuffer] = ..., *, op: int, iv: Any = ..., assoclen: int = ..., flags: int = ... + self, msg: Iterable[ReadableBuffer] = ..., *, op: int, iv: Any = ..., assoclen: int = ..., flags: int = 0 ) -> int: ... else: @deprecated("Removed in Python 3.11.") def sendmsg_afalg( - self, msg: Iterable[ReadableBuffer] = ..., *, op: int, iv: Any = ..., assoclen: int = ..., flags: int = ... + self, msg: Iterable[ReadableBuffer] = ..., *, op: int, iv: Any = ..., assoclen: int = ..., flags: int = 0 ) -> NoReturn: ... @deprecated("Removed in Python 3.11.") def sendmsg( - self, buffers: Iterable[ReadableBuffer], ancdata: Iterable[_CMSG] = ..., flags: int = ..., address: _Address = ..., / + self, + buffers: Iterable[ReadableBuffer], + ancdata: Iterable[_CMSG] = ..., + flags: int = 0, + address: _Address | None = None, + /, ) -> int: ... @overload @deprecated("Removed in Python 3.11.") @@ -89,9 +95,9 @@ class TransportSocket: @deprecated("Removed in Python 3.11.") def sendto(self, data: ReadableBuffer, flags: int, address: _Address) -> int: ... @deprecated("Removed in Python 3.11.") - def send(self, data: ReadableBuffer, flags: int = ...) -> int: ... + def send(self, data: ReadableBuffer, flags: int = 0) -> int: ... @deprecated("Removed in Python 3.11.") - def sendall(self, data: ReadableBuffer, flags: int = ...) -> None: ... + def sendall(self, data: ReadableBuffer, flags: int = 0) -> None: ... @deprecated("Removed in Python 3.11.") def set_inheritable(self, inheritable: bool) -> None: ... if sys.platform == "win32": @@ -102,19 +108,19 @@ class TransportSocket: def share(self, process_id: int) -> NoReturn: ... @deprecated("Removed in Python 3.11.") - def recv_into(self, buffer: _WriteBuffer, nbytes: int = ..., flags: int = ...) -> int: ... + def recv_into(self, buffer: _WriteBuffer, nbytes: int = 0, flags: int = 0) -> int: ... @deprecated("Removed in Python 3.11.") - def recvfrom_into(self, buffer: _WriteBuffer, nbytes: int = ..., flags: int = ...) -> tuple[int, _RetAddress]: ... + def recvfrom_into(self, buffer: _WriteBuffer, nbytes: int = 0, flags: int = 0) -> tuple[int, _RetAddress]: ... @deprecated("Removed in Python 3.11.") def recvmsg_into( - self, buffers: Iterable[_WriteBuffer], ancbufsize: int = ..., flags: int = ..., / + self, buffers: Iterable[_WriteBuffer], ancbufsize: int = 0, flags: int = 0, / ) -> tuple[int, list[_CMSG], int, Any]: ... @deprecated("Removed in Python 3.11.") - def recvmsg(self, bufsize: int, ancbufsize: int = ..., flags: int = ..., /) -> tuple[bytes, list[_CMSG], int, Any]: ... + def recvmsg(self, bufsize: int, ancbufsize: int = 0, flags: int = 0, /) -> tuple[bytes, list[_CMSG], int, Any]: ... @deprecated("Removed in Python 3.11.") - def recvfrom(self, bufsize: int, flags: int = ...) -> tuple[bytes, _RetAddress]: ... + def recvfrom(self, bufsize: int, flags: int = 0) -> tuple[bytes, _RetAddress]: ... @deprecated("Removed in Python 3.11.") - def recv(self, bufsize: int, flags: int = ...) -> bytes: ... + def recv(self, bufsize: int, flags: int = 0) -> bytes: ... @deprecated("Removed in Python 3.11.") def __enter__(self) -> socket.socket: ... @deprecated("Removed in Python 3.11.") diff --git a/mypy/typeshed/stdlib/asyncio/unix_events.pyi b/mypy/typeshed/stdlib/asyncio/unix_events.pyi index b2bf22a27677a..9071ee9a2fa7e 100644 --- a/mypy/typeshed/stdlib/asyncio/unix_events.pyi +++ b/mypy/typeshed/stdlib/asyncio/unix_events.pyi @@ -48,7 +48,7 @@ if sys.platform != "win32": # So, it is special cased. if sys.version_info < (3, 14): if sys.version_info >= (3, 12): - @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + @deprecated("Deprecated since Python 3.12; removed in Python 3.14.") class AbstractChildWatcher: @abstractmethod def add_child_handler( @@ -100,7 +100,7 @@ if sys.platform != "win32": def is_active(self) -> bool: ... def attach_loop(self, loop: events.AbstractEventLoop | None) -> None: ... - @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + @deprecated("Deprecated since Python 3.12; removed in Python 3.14.") class SafeChildWatcher(BaseChildWatcher): def __enter__(self) -> Self: ... def __exit__( @@ -111,7 +111,7 @@ if sys.platform != "win32": ) -> None: ... def remove_child_handler(self, pid: int) -> bool: ... - @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + @deprecated("Deprecated since Python 3.12; removed in Python 3.14.") class FastChildWatcher(BaseChildWatcher): def __enter__(self) -> Self: ... def __exit__( @@ -171,9 +171,9 @@ if sys.platform != "win32": else: class _UnixDefaultEventLoopPolicy(events.BaseDefaultEventLoopPolicy): if sys.version_info >= (3, 12): - @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + @deprecated("Deprecated since Python 3.12; removed in Python 3.14.") def get_child_watcher(self) -> AbstractChildWatcher: ... - @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + @deprecated("Deprecated since Python 3.12; removed in Python 3.14.") def set_child_watcher(self, watcher: AbstractChildWatcher | None) -> None: ... else: def get_child_watcher(self) -> AbstractChildWatcher: ... @@ -191,7 +191,7 @@ if sys.platform != "win32": if sys.version_info < (3, 14): if sys.version_info >= (3, 12): - @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + @deprecated("Deprecated since Python 3.12; removed in Python 3.14.") class MultiLoopChildWatcher(AbstractChildWatcher): def is_active(self) -> bool: ... def close(self) -> None: ... diff --git a/mypy/typeshed/stdlib/asyncio/windows_events.pyi b/mypy/typeshed/stdlib/asyncio/windows_events.pyi index b454aca1f2628..a32381bfb3e63 100644 --- a/mypy/typeshed/stdlib/asyncio/windows_events.pyi +++ b/mypy/typeshed/stdlib/asyncio/windows_events.pyi @@ -116,6 +116,6 @@ if sys.platform == "win32": if sys.version_info >= (3, 14): _DefaultEventLoopPolicy = _WindowsProactorEventLoopPolicy else: - DefaultEventLoopPolicy = WindowsSelectorEventLoopPolicy + DefaultEventLoopPolicy = WindowsProactorEventLoopPolicy if sys.version_info >= (3, 13): EventLoop = ProactorEventLoop diff --git a/mypy/typeshed/stdlib/asyncio/windows_utils.pyi b/mypy/typeshed/stdlib/asyncio/windows_utils.pyi index 4fa0145323762..5cedd61b5f4a3 100644 --- a/mypy/typeshed/stdlib/asyncio/windows_utils.pyi +++ b/mypy/typeshed/stdlib/asyncio/windows_utils.pyi @@ -9,8 +9,8 @@ if sys.platform == "win32": __all__ = ("pipe", "Popen", "PIPE", "PipeHandle") BUFSIZE: Final = 8192 - PIPE = subprocess.PIPE - STDOUT = subprocess.STDOUT + PIPE: Final = subprocess.PIPE + STDOUT: Final = subprocess.STDOUT def pipe(*, duplex: bool = False, overlapped: tuple[bool, bool] = (True, True), bufsize: int = 8192) -> tuple[int, int]: ... class PipeHandle: @@ -34,9 +34,9 @@ if sys.platform == "win32": def __new__( cls, args: subprocess._CMD, - stdin: subprocess._FILE | None = ..., - stdout: subprocess._FILE | None = ..., - stderr: subprocess._FILE | None = ..., + stdin: subprocess._FILE | None = None, + stdout: subprocess._FILE | None = None, + stderr: subprocess._FILE | None = None, **kwds: Any, ) -> Self: ... def __init__( diff --git a/mypy/typeshed/stdlib/binascii.pyi b/mypy/typeshed/stdlib/binascii.pyi index e09d335596fc3..5606d5cdf74d9 100644 --- a/mypy/typeshed/stdlib/binascii.pyi +++ b/mypy/typeshed/stdlib/binascii.pyi @@ -31,8 +31,8 @@ if sys.version_info < (3, 11): def crc_hqx(data: ReadableBuffer, crc: int, /) -> int: ... def crc32(data: ReadableBuffer, crc: int = 0, /) -> int: ... -def b2a_hex(data: ReadableBuffer, sep: str | bytes = ..., bytes_per_sep: int = ...) -> bytes: ... -def hexlify(data: ReadableBuffer, sep: str | bytes = ..., bytes_per_sep: int = ...) -> bytes: ... +def b2a_hex(data: ReadableBuffer, sep: str | bytes = ..., bytes_per_sep: int = 1) -> bytes: ... +def hexlify(data: ReadableBuffer, sep: str | bytes = ..., bytes_per_sep: int = 1) -> bytes: ... def a2b_hex(hexstr: _AsciiBuffer, /) -> bytes: ... def unhexlify(hexstr: _AsciiBuffer, /) -> bytes: ... diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi index d7c0fe27c1ee3..ca8d56cb42970 100644 --- a/mypy/typeshed/stdlib/builtins.pyi +++ b/mypy/typeshed/stdlib/builtins.pyi @@ -70,6 +70,7 @@ from typing_extensions import ( # noqa: Y023 TypeIs, TypeVarTuple, deprecated, + disjoint_base, ) if sys.version_info >= (3, 14): @@ -102,6 +103,7 @@ _StopT_co = TypeVar("_StopT_co", covariant=True, default=_StartT_co) # slice[A # FIXME: https://github.com/python/typing/issues/213 (replace step=start|stop with step=start&stop) _StepT_co = TypeVar("_StepT_co", covariant=True, default=_StartT_co | _StopT_co) # slice[A,B] -> slice[A, B, A|B] +@disjoint_base class object: __doc__: str | None __dict__: dict[str, Any] @@ -137,6 +139,7 @@ class object: @classmethod def __subclasshook__(cls, subclass: type, /) -> bool: ... +@disjoint_base class staticmethod(Generic[_P, _R_co]): @property def __func__(self) -> Callable[_P, _R_co]: ... @@ -157,6 +160,7 @@ class staticmethod(Generic[_P, _R_co]): def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... __annotate__: AnnotateFunc | None +@disjoint_base class classmethod(Generic[_T, _P, _R_co]): @property def __func__(self) -> Callable[Concatenate[type[_T], _P], _R_co]: ... @@ -176,6 +180,7 @@ class classmethod(Generic[_T, _P, _R_co]): def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... __annotate__: AnnotateFunc | None +@disjoint_base class type: # object.__base__ is None. Otherwise, it would be a type. @property @@ -228,6 +233,7 @@ class type: if sys.version_info >= (3, 14): __annotate__: AnnotateFunc | None +@disjoint_base class super: @overload def __init__(self, t: Any, obj: Any, /) -> None: ... @@ -240,6 +246,7 @@ _PositiveInteger: TypeAlias = Literal[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, _NegativeInteger: TypeAlias = Literal[-1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15, -16, -17, -18, -19, -20] _LiteralInteger = _PositiveInteger | _NegativeInteger | Literal[0] # noqa: Y026 # TODO: Use TypeAlias once mypy bugs are fixed +@disjoint_base class int: @overload def __new__(cls, x: ConvertibleToInt = ..., /) -> Self: ... @@ -350,6 +357,7 @@ class int: def __index__(self) -> int: ... def __format__(self, format_spec: str, /) -> str: ... +@disjoint_base class float: def __new__(cls, x: ConvertibleToFloat = ..., /) -> Self: ... def as_integer_ratio(self) -> tuple[int, int]: ... @@ -415,6 +423,7 @@ class float: @classmethod def from_number(cls, number: float | SupportsIndex | SupportsFloat, /) -> Self: ... +@disjoint_base class complex: # Python doesn't currently accept SupportsComplex for the second argument @overload @@ -462,6 +471,7 @@ class _FormatMapMapping(Protocol): class _TranslateTable(Protocol): def __getitem__(self, key: int, /) -> str | int | None: ... +@disjoint_base class str(Sequence[str]): @overload def __new__(cls, object: object = ...) -> Self: ... @@ -549,6 +559,7 @@ class str(Sequence[str]): def __getnewargs__(self) -> tuple[str]: ... def __format__(self, format_spec: str, /) -> str: ... +@disjoint_base class bytes(Sequence[int]): @overload def __new__(cls, o: Iterable[SupportsIndex] | SupportsIndex | SupportsBytes | ReadableBuffer, /) -> Self: ... @@ -647,6 +658,7 @@ class bytes(Sequence[int]): def __buffer__(self, flags: int, /) -> memoryview: ... +@disjoint_base class bytearray(MutableSequence[int]): @overload def __init__(self) -> None: ... @@ -911,6 +923,8 @@ class slice(Generic[_StartT_co, _StopT_co, _StepT_co]): def indices(self, len: SupportsIndex, /) -> tuple[int, int, int]: ... +# Making this a disjoint_base upsets pyright +# @disjoint_base class tuple(Sequence[_T_co]): def __new__(cls, iterable: Iterable[_T_co] = ..., /) -> Self: ... def __len__(self) -> int: ... @@ -987,6 +1001,7 @@ class function: # mypy uses `builtins.function.__get__` to represent methods, properties, and getset_descriptors so we type the return as Any. def __get__(self, instance: object, owner: type | None = None, /) -> Any: ... +@disjoint_base class list(MutableSequence[_T]): @overload def __init__(self) -> None: ... @@ -1041,6 +1056,7 @@ class list(MutableSequence[_T]): def __eq__(self, value: object, /) -> bool: ... def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... +@disjoint_base class dict(MutableMapping[_KT, _VT]): # __init__ should be kept roughly in line with `collections.UserDict.__init__`, which has similar semantics # Also multiprocessing.managers.SyncManager.dict() @@ -1123,6 +1139,7 @@ class dict(MutableMapping[_KT, _VT]): @overload def __ior__(self, value: Iterable[tuple[_KT, _VT]], /) -> Self: ... +@disjoint_base class set(MutableSet[_T]): @overload def __init__(self) -> None: ... @@ -1162,6 +1179,7 @@ class set(MutableSet[_T]): __hash__: ClassVar[None] # type: ignore[assignment] def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... +@disjoint_base class frozenset(AbstractSet[_T_co]): @overload def __new__(cls) -> Self: ... @@ -1190,6 +1208,7 @@ class frozenset(AbstractSet[_T_co]): def __hash__(self) -> int: ... def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... +@disjoint_base class enumerate(Iterator[tuple[int, _T]]): def __new__(cls, iterable: Iterable[_T], start: int = 0) -> Self: ... def __iter__(self) -> Self: ... @@ -1221,6 +1240,7 @@ class range(Sequence[int]): def __getitem__(self, key: slice, /) -> range: ... def __reversed__(self) -> Iterator[int]: ... +@disjoint_base class property: fget: Callable[[Any], Any] | None fset: Callable[[Any, Any], None] | None @@ -1384,6 +1404,7 @@ else: exit: _sitebuiltins.Quitter +@disjoint_base class filter(Iterator[_T]): @overload def __new__(cls, function: None, iterable: Iterable[_T | None], /) -> Self: ... @@ -1447,7 +1468,7 @@ def len(obj: Sized, /) -> int: ... license: _sitebuiltins._Printer def locals() -> dict[str, Any]: ... - +@disjoint_base class map(Iterator[_S]): # 3.14 adds `strict` argument. if sys.version_info >= (3, 14): @@ -1754,6 +1775,7 @@ def pow(base: _SupportsSomeKindOfPow, exp: complex, mod: None = None) -> complex quit: _sitebuiltins.Quitter +@disjoint_base class reversed(Iterator[_T]): @overload def __new__(cls, sequence: Reversible[_T], /) -> Iterator[_T]: ... # type: ignore[misc] @@ -1817,7 +1839,7 @@ def sum(iterable: Iterable[_AddableT1], /, start: _AddableT2) -> _AddableT1 | _A def vars(object: type, /) -> types.MappingProxyType[str, Any]: ... @overload def vars(object: Any = ..., /) -> dict[str, Any]: ... - +@disjoint_base class zip(Iterator[_T_co]): if sys.version_info >= (3, 10): @overload @@ -1921,6 +1943,7 @@ else: Ellipsis: ellipsis +@disjoint_base class BaseException: args: tuple[Any, ...] __cause__: BaseException | None @@ -1939,14 +1962,17 @@ class BaseException: class GeneratorExit(BaseException): ... class KeyboardInterrupt(BaseException): ... +@disjoint_base class SystemExit(BaseException): code: sys._ExitCode class Exception(BaseException): ... +@disjoint_base class StopIteration(Exception): value: Any +@disjoint_base class OSError(Exception): errno: int | None strerror: str | None @@ -1964,15 +1990,20 @@ if sys.platform == "win32": class ArithmeticError(Exception): ... class AssertionError(Exception): ... -class AttributeError(Exception): - if sys.version_info >= (3, 10): +if sys.version_info >= (3, 10): + @disjoint_base + class AttributeError(Exception): def __init__(self, *args: object, name: str | None = ..., obj: object = ...) -> None: ... name: str obj: object +else: + class AttributeError(Exception): ... + class BufferError(Exception): ... class EOFError(Exception): ... +@disjoint_base class ImportError(Exception): def __init__(self, *args: object, name: str | None = ..., path: str | None = ...) -> None: ... name: str | None @@ -1984,15 +2015,20 @@ class ImportError(Exception): class LookupError(Exception): ... class MemoryError(Exception): ... -class NameError(Exception): - if sys.version_info >= (3, 10): +if sys.version_info >= (3, 10): + @disjoint_base + class NameError(Exception): def __init__(self, *args: object, name: str | None = ...) -> None: ... name: str +else: + class NameError(Exception): ... + class ReferenceError(Exception): ... class RuntimeError(Exception): ... class StopAsyncIteration(Exception): ... +@disjoint_base class SyntaxError(Exception): msg: str filename: str | None @@ -2056,6 +2092,7 @@ class IndentationError(SyntaxError): ... class TabError(IndentationError): ... class UnicodeError(ValueError): ... +@disjoint_base class UnicodeDecodeError(UnicodeError): encoding: str object: bytes @@ -2064,6 +2101,7 @@ class UnicodeDecodeError(UnicodeError): reason: str def __init__(self, encoding: str, object: ReadableBuffer, start: int, end: int, reason: str, /) -> None: ... +@disjoint_base class UnicodeEncodeError(UnicodeError): encoding: str object: str @@ -2072,6 +2110,7 @@ class UnicodeEncodeError(UnicodeError): reason: str def __init__(self, encoding: str, object: str, start: int, end: int, reason: str, /) -> None: ... +@disjoint_base class UnicodeTranslateError(UnicodeError): encoding: None object: str @@ -2102,6 +2141,7 @@ if sys.version_info >= (3, 11): _ExceptionT = TypeVar("_ExceptionT", bound=Exception) # See `check_exception_group.py` for use-cases and comments. + @disjoint_base class BaseExceptionGroup(BaseException, Generic[_BaseExceptionT_co]): def __new__(cls, message: str, exceptions: Sequence[_BaseExceptionT_co], /) -> Self: ... def __init__(self, message: str, exceptions: Sequence[_BaseExceptionT_co], /) -> None: ... diff --git a/mypy/typeshed/stdlib/calendar.pyi b/mypy/typeshed/stdlib/calendar.pyi index cabf3b881c30f..d00f0d5d2bce3 100644 --- a/mypy/typeshed/stdlib/calendar.pyi +++ b/mypy/typeshed/stdlib/calendar.pyi @@ -167,18 +167,18 @@ if sys.version_info >= (3, 12): NOVEMBER = 11 DECEMBER = 12 - JANUARY = Month.JANUARY - FEBRUARY = Month.FEBRUARY - MARCH = Month.MARCH - APRIL = Month.APRIL - MAY = Month.MAY - JUNE = Month.JUNE - JULY = Month.JULY - AUGUST = Month.AUGUST - SEPTEMBER = Month.SEPTEMBER - OCTOBER = Month.OCTOBER - NOVEMBER = Month.NOVEMBER - DECEMBER = Month.DECEMBER + JANUARY: Final = Month.JANUARY + FEBRUARY: Final = Month.FEBRUARY + MARCH: Final = Month.MARCH + APRIL: Final = Month.APRIL + MAY: Final = Month.MAY + JUNE: Final = Month.JUNE + JULY: Final = Month.JULY + AUGUST: Final = Month.AUGUST + SEPTEMBER: Final = Month.SEPTEMBER + OCTOBER: Final = Month.OCTOBER + NOVEMBER: Final = Month.NOVEMBER + DECEMBER: Final = Month.DECEMBER class Day(enum.IntEnum): MONDAY = 0 @@ -189,13 +189,13 @@ if sys.version_info >= (3, 12): SATURDAY = 5 SUNDAY = 6 - MONDAY = Day.MONDAY - TUESDAY = Day.TUESDAY - WEDNESDAY = Day.WEDNESDAY - THURSDAY = Day.THURSDAY - FRIDAY = Day.FRIDAY - SATURDAY = Day.SATURDAY - SUNDAY = Day.SUNDAY + MONDAY: Final = Day.MONDAY + TUESDAY: Final = Day.TUESDAY + WEDNESDAY: Final = Day.WEDNESDAY + THURSDAY: Final = Day.THURSDAY + FRIDAY: Final = Day.FRIDAY + SATURDAY: Final = Day.SATURDAY + SUNDAY: Final = Day.SUNDAY else: MONDAY: Final = 0 TUESDAY: Final = 1 diff --git a/mypy/typeshed/stdlib/cgi.pyi b/mypy/typeshed/stdlib/cgi.pyi index a7a95a1393300..0f9d4343b6307 100644 --- a/mypy/typeshed/stdlib/cgi.pyi +++ b/mypy/typeshed/stdlib/cgi.pyi @@ -1,3 +1,4 @@ +import os from _typeshed import SupportsContainsAndGetItem, SupportsGetItem, SupportsItemAccess, Unused from builtins import list as _list, type as _type from collections.abc import Iterable, Iterator, Mapping @@ -23,7 +24,7 @@ __all__ = [ def parse( fp: IO[Any] | None = None, - environ: SupportsItemAccess[str, str] = ..., + environ: SupportsItemAccess[str, str] = os.environ, keep_blank_values: bool = ..., strict_parsing: bool = ..., separator: str = "&", @@ -37,8 +38,8 @@ class _Environ(Protocol): def keys(self) -> Iterable[str]: ... def parse_header(line: str) -> tuple[str, dict[str, str]]: ... -def test(environ: _Environ = ...) -> None: ... -def print_environ(environ: _Environ = ...) -> None: ... +def test(environ: _Environ = os.environ) -> None: ... +def print_environ(environ: _Environ = os.environ) -> None: ... def print_form(form: dict[str, Any]) -> None: ... def print_directory() -> None: ... def print_environ_usage() -> None: ... @@ -85,7 +86,7 @@ class FieldStorage: fp: IO[Any] | None = None, headers: Mapping[str, str] | Message | None = None, outerboundary: bytes = b"", - environ: SupportsContainsAndGetItem[str, str] = ..., + environ: SupportsContainsAndGetItem[str, str] = os.environ, keep_blank_values: int = 0, strict_parsing: int = 0, limit: int | None = None, diff --git a/mypy/typeshed/stdlib/codecs.pyi b/mypy/typeshed/stdlib/codecs.pyi index 15e184fc10388..fa4d4fd4ba928 100644 --- a/mypy/typeshed/stdlib/codecs.pyi +++ b/mypy/typeshed/stdlib/codecs.pyi @@ -1,10 +1,11 @@ +import sys import types from _codecs import * from _typeshed import ReadableBuffer from abc import abstractmethod from collections.abc import Callable, Generator, Iterable from typing import Any, BinaryIO, ClassVar, Final, Literal, Protocol, TextIO, overload, type_check_only -from typing_extensions import Self, TypeAlias +from typing_extensions import Self, TypeAlias, disjoint_base __all__ = [ "register", @@ -122,33 +123,64 @@ class _IncrementalDecoder(Protocol): class _BufferedIncrementalDecoder(Protocol): def __call__(self, errors: str = ...) -> BufferedIncrementalDecoder: ... -class CodecInfo(tuple[_Encoder, _Decoder, _StreamReader, _StreamWriter]): - _is_text_encoding: bool - @property - def encode(self) -> _Encoder: ... - @property - def decode(self) -> _Decoder: ... - @property - def streamreader(self) -> _StreamReader: ... - @property - def streamwriter(self) -> _StreamWriter: ... - @property - def incrementalencoder(self) -> _IncrementalEncoder: ... - @property - def incrementaldecoder(self) -> _IncrementalDecoder: ... - name: str - def __new__( - cls, - encode: _Encoder, - decode: _Decoder, - streamreader: _StreamReader | None = None, - streamwriter: _StreamWriter | None = None, - incrementalencoder: _IncrementalEncoder | None = None, - incrementaldecoder: _IncrementalDecoder | None = None, - name: str | None = None, - *, - _is_text_encoding: bool | None = None, - ) -> Self: ... +if sys.version_info >= (3, 12): + class CodecInfo(tuple[_Encoder, _Decoder, _StreamReader, _StreamWriter]): + _is_text_encoding: bool + @property + def encode(self) -> _Encoder: ... + @property + def decode(self) -> _Decoder: ... + @property + def streamreader(self) -> _StreamReader: ... + @property + def streamwriter(self) -> _StreamWriter: ... + @property + def incrementalencoder(self) -> _IncrementalEncoder: ... + @property + def incrementaldecoder(self) -> _IncrementalDecoder: ... + name: str + def __new__( + cls, + encode: _Encoder, + decode: _Decoder, + streamreader: _StreamReader | None = None, + streamwriter: _StreamWriter | None = None, + incrementalencoder: _IncrementalEncoder | None = None, + incrementaldecoder: _IncrementalDecoder | None = None, + name: str | None = None, + *, + _is_text_encoding: bool | None = None, + ) -> Self: ... + +else: + @disjoint_base + class CodecInfo(tuple[_Encoder, _Decoder, _StreamReader, _StreamWriter]): + _is_text_encoding: bool + @property + def encode(self) -> _Encoder: ... + @property + def decode(self) -> _Decoder: ... + @property + def streamreader(self) -> _StreamReader: ... + @property + def streamwriter(self) -> _StreamWriter: ... + @property + def incrementalencoder(self) -> _IncrementalEncoder: ... + @property + def incrementaldecoder(self) -> _IncrementalDecoder: ... + name: str + def __new__( + cls, + encode: _Encoder, + decode: _Decoder, + streamreader: _StreamReader | None = None, + streamwriter: _StreamWriter | None = None, + incrementalencoder: _IncrementalEncoder | None = None, + incrementaldecoder: _IncrementalDecoder | None = None, + name: str | None = None, + *, + _is_text_encoding: bool | None = None, + ) -> Self: ... def getencoder(encoding: str) -> _Encoder: ... def getdecoder(encoding: str) -> _Decoder: ... diff --git a/mypy/typeshed/stdlib/collections/__init__.pyi b/mypy/typeshed/stdlib/collections/__init__.pyi index df9449ef4c9b0..8636e6cdbdc31 100644 --- a/mypy/typeshed/stdlib/collections/__init__.pyi +++ b/mypy/typeshed/stdlib/collections/__init__.pyi @@ -3,7 +3,7 @@ from _collections_abc import dict_items, dict_keys, dict_values from _typeshed import SupportsItems, SupportsKeysAndGetItem, SupportsRichComparison, SupportsRichComparisonT from types import GenericAlias from typing import Any, ClassVar, Generic, NoReturn, SupportsIndex, TypeVar, final, overload, type_check_only -from typing_extensions import Self +from typing_extensions import Self, disjoint_base if sys.version_info >= (3, 10): from collections.abc import ( @@ -231,6 +231,7 @@ class UserString(Sequence[UserString]): def upper(self) -> Self: ... def zfill(self, width: int) -> Self: ... +@disjoint_base class deque(MutableSequence[_T]): @property def maxlen(self) -> int | None: ... @@ -356,6 +357,7 @@ class _odict_items(dict_items[_KT_co, _VT_co]): # type: ignore[misc] # pyright class _odict_values(dict_values[_KT_co, _VT_co]): # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] def __reversed__(self) -> Iterator[_VT_co]: ... +@disjoint_base class OrderedDict(dict[_KT, _VT]): def popitem(self, last: bool = True) -> tuple[_KT, _VT]: ... def move_to_end(self, key: _KT, last: bool = True) -> None: ... @@ -395,6 +397,7 @@ class OrderedDict(dict[_KT, _VT]): @overload def __ror__(self, value: dict[_T1, _T2], /) -> OrderedDict[_KT | _T1, _VT | _T2]: ... # type: ignore[misc] +@disjoint_base class defaultdict(dict[_KT, _VT]): default_factory: Callable[[], _VT] | None @overload @@ -477,9 +480,15 @@ class ChainMap(MutableMapping[_KT, _VT]): __copy__ = copy # All arguments to `fromkeys` are passed to `dict.fromkeys` at runtime, # so the signature should be kept in line with `dict.fromkeys`. - @classmethod - @overload - def fromkeys(cls, iterable: Iterable[_T]) -> ChainMap[_T, Any | None]: ... + if sys.version_info >= (3, 13): + @classmethod + @overload + def fromkeys(cls, iterable: Iterable[_T], /) -> ChainMap[_T, Any | None]: ... + else: + @classmethod + @overload + def fromkeys(cls, iterable: Iterable[_T]) -> ChainMap[_T, Any | None]: ... + @classmethod @overload # Special-case None: the user probably wants to add non-None values later. diff --git a/mypy/typeshed/stdlib/colorsys.pyi b/mypy/typeshed/stdlib/colorsys.pyi index 7842f80284ef4..4afcb5392b58e 100644 --- a/mypy/typeshed/stdlib/colorsys.pyi +++ b/mypy/typeshed/stdlib/colorsys.pyi @@ -1,3 +1,5 @@ +from typing import Final + __all__ = ["rgb_to_yiq", "yiq_to_rgb", "rgb_to_hls", "hls_to_rgb", "rgb_to_hsv", "hsv_to_rgb"] def rgb_to_yiq(r: float, g: float, b: float) -> tuple[float, float, float]: ... @@ -8,6 +10,6 @@ def rgb_to_hsv(r: float, g: float, b: float) -> tuple[float, float, float]: ... def hsv_to_rgb(h: float, s: float, v: float) -> tuple[float, float, float]: ... # TODO: undocumented -ONE_SIXTH: float -ONE_THIRD: float -TWO_THIRD: float +ONE_SIXTH: Final[float] +ONE_THIRD: Final[float] +TWO_THIRD: Final[float] diff --git a/mypy/typeshed/stdlib/compression/zstd/__init__.pyi b/mypy/typeshed/stdlib/compression/zstd/__init__.pyi index 24a9633c488e6..d5da4be036129 100644 --- a/mypy/typeshed/stdlib/compression/zstd/__init__.pyi +++ b/mypy/typeshed/stdlib/compression/zstd/__init__.pyi @@ -35,6 +35,7 @@ zstd_version_info: Final[tuple[int, int, int]] COMPRESSION_LEVEL_DEFAULT: Final = _zstd.ZSTD_CLEVEL_DEFAULT class FrameInfo: + __slots__ = ("decompressed_size", "dictionary_id") decompressed_size: int dictionary_id: int def __init__(self, decompressed_size: int, dictionary_id: int) -> None: ... diff --git a/mypy/typeshed/stdlib/concurrent/futures/_base.pyi b/mypy/typeshed/stdlib/concurrent/futures/_base.pyi index 4063027f3eed3..be48a6e4289c8 100644 --- a/mypy/typeshed/stdlib/concurrent/futures/_base.pyi +++ b/mypy/typeshed/stdlib/concurrent/futures/_base.pyi @@ -15,8 +15,7 @@ RUNNING: Final = "RUNNING" CANCELLED: Final = "CANCELLED" CANCELLED_AND_NOTIFIED: Final = "CANCELLED_AND_NOTIFIED" FINISHED: Final = "FINISHED" -_FUTURE_STATES: list[str] -_STATE_TO_DESCRIPTION_MAP: dict[str, str] +_STATE_TO_DESCRIPTION_MAP: Final[dict[str, str]] LOGGER: Logger class Error(Exception): ... diff --git a/mypy/typeshed/stdlib/concurrent/futures/process.pyi b/mypy/typeshed/stdlib/concurrent/futures/process.pyi index 607990100369e..071b3aba5d330 100644 --- a/mypy/typeshed/stdlib/concurrent/futures/process.pyi +++ b/mypy/typeshed/stdlib/concurrent/futures/process.pyi @@ -5,7 +5,7 @@ from multiprocessing.context import BaseContext, Process from multiprocessing.queues import Queue, SimpleQueue from threading import Lock, Semaphore, Thread from types import TracebackType -from typing import Any, Generic, TypeVar, overload +from typing import Any, Final, Generic, TypeVar, overload from typing_extensions import TypeVarTuple, Unpack from weakref import ref @@ -28,9 +28,9 @@ class _ThreadWakeup: def _python_exit() -> None: ... -EXTRA_QUEUED_CALLS: int +EXTRA_QUEUED_CALLS: Final = 1 -_MAX_WINDOWS_WORKERS: int +_MAX_WINDOWS_WORKERS: Final = 61 class _RemoteTraceback(Exception): tb: str diff --git a/mypy/typeshed/stdlib/concurrent/interpreters/_crossinterp.pyi b/mypy/typeshed/stdlib/concurrent/interpreters/_crossinterp.pyi index b073aefa7ca7e..7cf1ea34786ed 100644 --- a/mypy/typeshed/stdlib/concurrent/interpreters/_crossinterp.pyi +++ b/mypy/typeshed/stdlib/concurrent/interpreters/_crossinterp.pyi @@ -12,6 +12,7 @@ if sys.version_info >= (3, 13): # needed to satisfy pyright checks for Python < classonly = classmethod class UnboundItem: + __slots__ = () def __new__(cls) -> Never: ... @classonly def singleton(cls, kind: str, module: str, name: str = "UNBOUND") -> Self: ... diff --git a/mypy/typeshed/stdlib/concurrent/interpreters/_queues.pyi b/mypy/typeshed/stdlib/concurrent/interpreters/_queues.pyi index 39a057ee9a7bc..7493f87809c82 100644 --- a/mypy/typeshed/stdlib/concurrent/interpreters/_queues.pyi +++ b/mypy/typeshed/stdlib/concurrent/interpreters/_queues.pyi @@ -51,8 +51,8 @@ if sys.version_info >= (3, 13): # needed to satisfy pyright checks for Python < timeout: SupportsIndex | None = None, *, unbounditems: _AnyUnbound | None = None, - _delay: float = ..., + _delay: float = 0.01, ) -> None: ... def put_nowait(self, obj: object, *, unbounditems: _AnyUnbound | None = None) -> None: ... - def get(self, timeout: SupportsIndex | None = None, *, _delay: float = ...) -> object: ... + def get(self, timeout: SupportsIndex | None = None, *, _delay: float = 0.01) -> object: ... def get_nowait(self) -> object: ... diff --git a/mypy/typeshed/stdlib/configparser.pyi b/mypy/typeshed/stdlib/configparser.pyi index b3a4210030266..764a8a965ea26 100644 --- a/mypy/typeshed/stdlib/configparser.pyi +++ b/mypy/typeshed/stdlib/configparser.pyi @@ -449,10 +449,13 @@ class ParsingError(Error): def __init__(self, source: str) -> None: ... else: @overload - def __init__(self, source: str, filename: None = None) -> None: ... + def __init__(self, source: str) -> None: ... + @overload + @deprecated("The `filename` parameter removed in Python 3.12. Use `source` instead.") + def __init__(self, source: None, filename: str | None) -> None: ... @overload @deprecated("The `filename` parameter removed in Python 3.12. Use `source` instead.") - def __init__(self, source: None = None, filename: str = ...) -> None: ... + def __init__(self, source: None = None, *, filename: str | None) -> None: ... def append(self, lineno: int, line: str) -> None: ... diff --git a/mypy/typeshed/stdlib/contextlib.pyi b/mypy/typeshed/stdlib/contextlib.pyi index c616c1f5bf19f..383a1b7f334b4 100644 --- a/mypy/typeshed/stdlib/contextlib.pyi +++ b/mypy/typeshed/stdlib/contextlib.pyi @@ -47,6 +47,7 @@ _CM_EF = TypeVar("_CM_EF", bound=AbstractContextManager[Any, Any] | _ExitFunc) # allowlist for use as a Protocol. @runtime_checkable class AbstractContextManager(ABC, Protocol[_T_co, _ExitT_co]): # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] + __slots__ = () def __enter__(self) -> _T_co: ... @abstractmethod def __exit__( @@ -58,6 +59,7 @@ class AbstractContextManager(ABC, Protocol[_T_co, _ExitT_co]): # type: ignore[m # allowlist for use as a Protocol. @runtime_checkable class AbstractAsyncContextManager(ABC, Protocol[_T_co, _ExitT_co]): # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] + __slots__ = () async def __aenter__(self) -> _T_co: ... @abstractmethod async def __aexit__( diff --git a/mypy/typeshed/stdlib/crypt.pyi b/mypy/typeshed/stdlib/crypt.pyi index bd22b5f8daba2..f926321969897 100644 --- a/mypy/typeshed/stdlib/crypt.pyi +++ b/mypy/typeshed/stdlib/crypt.pyi @@ -1,5 +1,6 @@ import sys from typing import Final, NamedTuple, type_check_only +from typing_extensions import disjoint_base if sys.platform != "win32": @type_check_only @@ -9,7 +10,12 @@ if sys.platform != "win32": salt_chars: int total_size: int - class _Method(_MethodBase): ... + if sys.version_info >= (3, 12): + class _Method(_MethodBase): ... + else: + @disjoint_base + class _Method(_MethodBase): ... + METHOD_CRYPT: Final[_Method] METHOD_MD5: Final[_Method] METHOD_SHA256: Final[_Method] diff --git a/mypy/typeshed/stdlib/ctypes/__init__.pyi b/mypy/typeshed/stdlib/ctypes/__init__.pyi index 15649da9ff733..9da972240abb7 100644 --- a/mypy/typeshed/stdlib/ctypes/__init__.pyi +++ b/mypy/typeshed/stdlib/ctypes/__init__.pyi @@ -26,7 +26,7 @@ from _ctypes import ( from _typeshed import StrPath from ctypes._endian import BigEndianStructure as BigEndianStructure, LittleEndianStructure as LittleEndianStructure from types import GenericAlias -from typing import Any, ClassVar, Generic, Literal, TypeVar, overload, type_check_only +from typing import Any, ClassVar, Final, Generic, Literal, TypeVar, overload, type_check_only from typing_extensions import Self, TypeAlias, deprecated if sys.platform == "win32": @@ -55,7 +55,7 @@ if sys.version_info >= (3, 14): else: from _ctypes import POINTER as POINTER, pointer as pointer -DEFAULT_MODE: int +DEFAULT_MODE: Final[int] class ArgumentError(Exception): ... @@ -162,8 +162,14 @@ def create_string_buffer(init: int | bytes, size: int | None = None) -> Array[c_ c_buffer = create_string_buffer def create_unicode_buffer(init: int | str, size: int | None = None) -> Array[c_wchar]: ... -@deprecated("Deprecated in Python 3.13; removal scheduled for Python 3.15") -def SetPointerType(pointer: type[_Pointer[Any]], cls: _CTypeBaseType) -> None: ... + +if sys.version_info >= (3, 13): + @deprecated("Deprecated since Python 3.13; will be removed in Python 3.15.") + def SetPointerType(pointer: type[_Pointer[Any]], cls: _CTypeBaseType) -> None: ... + +else: + def SetPointerType(pointer: type[_Pointer[Any]], cls: _CTypeBaseType) -> None: ... + def ARRAY(typ: _CT, len: int) -> Array[_CT]: ... # Soft Deprecated, no plans to remove if sys.platform == "win32": diff --git a/mypy/typeshed/stdlib/ctypes/_endian.pyi b/mypy/typeshed/stdlib/ctypes/_endian.pyi index 144f5ba5dd40f..97852f67aa6eb 100644 --- a/mypy/typeshed/stdlib/ctypes/_endian.pyi +++ b/mypy/typeshed/stdlib/ctypes/_endian.pyi @@ -3,10 +3,14 @@ from ctypes import Structure, Union # At runtime, the native endianness is an alias for Structure, # while the other is a subclass with a metaclass added in. -class BigEndianStructure(Structure): ... +class BigEndianStructure(Structure): + __slots__ = () + class LittleEndianStructure(Structure): ... # Same thing for these: one is an alias of Union at runtime if sys.version_info >= (3, 11): - class BigEndianUnion(Union): ... + class BigEndianUnion(Union): + __slots__ = () + class LittleEndianUnion(Union): ... diff --git a/mypy/typeshed/stdlib/ctypes/macholib/__init__.pyi b/mypy/typeshed/stdlib/ctypes/macholib/__init__.pyi index bda5b5a7f4cc3..c5dd954660638 100644 --- a/mypy/typeshed/stdlib/ctypes/macholib/__init__.pyi +++ b/mypy/typeshed/stdlib/ctypes/macholib/__init__.pyi @@ -1 +1,3 @@ -__version__: str +from typing import Final + +__version__: Final[str] diff --git a/mypy/typeshed/stdlib/ctypes/wintypes.pyi b/mypy/typeshed/stdlib/ctypes/wintypes.pyi index e9ed0df24dd13..0f0d61a396d5f 100644 --- a/mypy/typeshed/stdlib/ctypes/wintypes.pyi +++ b/mypy/typeshed/stdlib/ctypes/wintypes.pyi @@ -21,7 +21,7 @@ from ctypes import ( c_wchar, c_wchar_p, ) -from typing import Any, TypeVar +from typing import Any, Final, TypeVar from typing_extensions import Self, TypeAlias if sys.version_info >= (3, 12): @@ -177,7 +177,7 @@ class MSG(Structure): pt: _CField[POINT, POINT, POINT] tagMSG = MSG -MAX_PATH: int +MAX_PATH: Final = 260 class WIN32_FIND_DATAA(Structure): dwFileAttributes: _CIntLikeField[DWORD] diff --git a/mypy/typeshed/stdlib/curses/__init__.pyi b/mypy/typeshed/stdlib/curses/__init__.pyi index 5c157fd7c2f61..2c0231c13087e 100644 --- a/mypy/typeshed/stdlib/curses/__init__.pyi +++ b/mypy/typeshed/stdlib/curses/__init__.pyi @@ -14,12 +14,12 @@ _T = TypeVar("_T") _P = ParamSpec("_P") # available after calling `curses.initscr()` -LINES: int -COLS: int +LINES: Final[int] +COLS: Final[int] # available after calling `curses.start_color()` -COLORS: int -COLOR_PAIRS: int +COLORS: Final[int] +COLOR_PAIRS: Final[int] def wrapper(func: Callable[Concatenate[window, _P], _T], /, *arg: _P.args, **kwds: _P.kwargs) -> _T: ... diff --git a/mypy/typeshed/stdlib/curses/ascii.pyi b/mypy/typeshed/stdlib/curses/ascii.pyi index 66efbe36a7df2..0234434b8c3de 100644 --- a/mypy/typeshed/stdlib/curses/ascii.pyi +++ b/mypy/typeshed/stdlib/curses/ascii.pyi @@ -1,45 +1,45 @@ -from typing import TypeVar +from typing import Final, TypeVar _CharT = TypeVar("_CharT", str, int) -NUL: int -SOH: int -STX: int -ETX: int -EOT: int -ENQ: int -ACK: int -BEL: int -BS: int -TAB: int -HT: int -LF: int -NL: int -VT: int -FF: int -CR: int -SO: int -SI: int -DLE: int -DC1: int -DC2: int -DC3: int -DC4: int -NAK: int -SYN: int -ETB: int -CAN: int -EM: int -SUB: int -ESC: int -FS: int -GS: int -RS: int -US: int -SP: int -DEL: int +NUL: Final = 0x00 +SOH: Final = 0x01 +STX: Final = 0x02 +ETX: Final = 0x03 +EOT: Final = 0x04 +ENQ: Final = 0x05 +ACK: Final = 0x06 +BEL: Final = 0x07 +BS: Final = 0x08 +TAB: Final = 0x09 +HT: Final = 0x09 +LF: Final = 0x0A +NL: Final = 0x0A +VT: Final = 0x0B +FF: Final = 0x0C +CR: Final = 0x0D +SO: Final = 0x0E +SI: Final = 0x0F +DLE: Final = 0x10 +DC1: Final = 0x11 +DC2: Final = 0x12 +DC3: Final = 0x13 +DC4: Final = 0x14 +NAK: Final = 0x15 +SYN: Final = 0x16 +ETB: Final = 0x17 +CAN: Final = 0x18 +EM: Final = 0x19 +SUB: Final = 0x1A +ESC: Final = 0x1B +FS: Final = 0x1C +GS: Final = 0x1D +RS: Final = 0x1E +US: Final = 0x1F +SP: Final = 0x20 +DEL: Final = 0x7F -controlnames: list[int] +controlnames: Final[list[int]] def isalnum(c: str | int) -> bool: ... def isalpha(c: str | int) -> bool: ... diff --git a/mypy/typeshed/stdlib/dataclasses.pyi b/mypy/typeshed/stdlib/dataclasses.pyi index b3183f57ebd2a..3a1c8cb5d62dd 100644 --- a/mypy/typeshed/stdlib/dataclasses.pyi +++ b/mypy/typeshed/stdlib/dataclasses.pyi @@ -5,7 +5,7 @@ from _typeshed import DataclassInstance from builtins import type as Type # alias to avoid name clashes with fields named "type" from collections.abc import Callable, Iterable, Mapping from types import GenericAlias -from typing import Any, Generic, Literal, Protocol, TypeVar, overload, type_check_only +from typing import Any, Final, Generic, Literal, Protocol, TypeVar, overload, type_check_only from typing_extensions import Never, TypeIs _T = TypeVar("_T") @@ -58,7 +58,7 @@ class _DataclassFactory(Protocol): class _MISSING_TYPE(enum.Enum): MISSING = enum.auto() -MISSING = _MISSING_TYPE.MISSING +MISSING: Final = _MISSING_TYPE.MISSING if sys.version_info >= (3, 10): class KW_ONLY: ... @@ -170,6 +170,37 @@ class _DefaultFactory(Protocol[_T_co]): def __call__(self) -> _T_co: ... class Field(Generic[_T]): + if sys.version_info >= (3, 14): + __slots__ = ( + "name", + "type", + "default", + "default_factory", + "repr", + "hash", + "init", + "compare", + "metadata", + "kw_only", + "doc", + "_field_type", + ) + elif sys.version_info >= (3, 10): + __slots__ = ( + "name", + "type", + "default", + "default_factory", + "repr", + "hash", + "init", + "compare", + "metadata", + "kw_only", + "_field_type", + ) + else: + __slots__ = ("name", "type", "default", "default_factory", "repr", "hash", "init", "compare", "metadata", "_field_type") name: str type: Type[_T] | str | Any default: _T | Literal[_MISSING_TYPE.MISSING] @@ -355,6 +386,7 @@ def is_dataclass(obj: object) -> TypeIs[DataclassInstance | type[DataclassInstan class FrozenInstanceError(AttributeError): ... class InitVar(Generic[_T]): + __slots__ = ("type",) type: Type[_T] def __init__(self, type: Type[_T]) -> None: ... @overload diff --git a/mypy/typeshed/stdlib/datetime.pyi b/mypy/typeshed/stdlib/datetime.pyi index c54de6159b514..8a0536c006d57 100644 --- a/mypy/typeshed/stdlib/datetime.pyi +++ b/mypy/typeshed/stdlib/datetime.pyi @@ -2,7 +2,7 @@ import sys from abc import abstractmethod from time import struct_time from typing import ClassVar, Final, NoReturn, SupportsIndex, final, overload, type_check_only -from typing_extensions import CapsuleType, Self, TypeAlias, deprecated +from typing_extensions import CapsuleType, Self, TypeAlias, deprecated, disjoint_base if sys.version_info >= (3, 11): __all__ = ("date", "datetime", "time", "timedelta", "timezone", "tzinfo", "MINYEAR", "MAXYEAR", "UTC") @@ -51,6 +51,7 @@ class _IsoCalendarDate(tuple[int, int, int]): @property def weekday(self) -> int: ... +@disjoint_base class date: min: ClassVar[date] max: ClassVar[date] @@ -112,6 +113,7 @@ class date: def isoweekday(self) -> int: ... def isocalendar(self) -> _IsoCalendarDate: ... +@disjoint_base class time: min: ClassVar[time] max: ClassVar[time] @@ -191,6 +193,7 @@ class time: _Date: TypeAlias = date _Time: TypeAlias = time +@disjoint_base class timedelta: min: ClassVar[timedelta] max: ClassVar[timedelta] @@ -239,6 +242,7 @@ class timedelta: def __bool__(self) -> bool: ... def __hash__(self) -> int: ... +@disjoint_base class datetime(date): min: ClassVar[datetime] max: ClassVar[datetime] diff --git a/mypy/typeshed/stdlib/decimal.pyi b/mypy/typeshed/stdlib/decimal.pyi index b85c000800920..2e06c2d1b724a 100644 --- a/mypy/typeshed/stdlib/decimal.pyi +++ b/mypy/typeshed/stdlib/decimal.pyi @@ -27,7 +27,7 @@ from _decimal import ( from collections.abc import Container, Sequence from types import TracebackType from typing import Any, ClassVar, Literal, NamedTuple, final, overload, type_check_only -from typing_extensions import Self, TypeAlias +from typing_extensions import Self, TypeAlias, disjoint_base if sys.version_info >= (3, 14): from _decimal import IEEE_CONTEXT_MAX_BITS as IEEE_CONTEXT_MAX_BITS, IEEEContext as IEEEContext @@ -68,6 +68,7 @@ class Overflow(Inexact, Rounded): ... class Underflow(Inexact, Rounded, Subnormal): ... class FloatOperation(DecimalException, TypeError): ... +@disjoint_base class Decimal: def __new__(cls, value: _DecimalNew = "0", context: Context | None = None) -> Self: ... if sys.version_info >= (3, 14): @@ -173,6 +174,7 @@ class Decimal: def __deepcopy__(self, memo: Any, /) -> Self: ... def __format__(self, specifier: str, context: Context | None = None, /) -> str: ... +@disjoint_base class Context: # TODO: Context doesn't allow you to delete *any* attributes from instances of the class at runtime, # even settable attributes like `prec` and `rounding`, diff --git a/mypy/typeshed/stdlib/dis.pyi b/mypy/typeshed/stdlib/dis.pyi index 86b6d01e3120d..896b50fa93847 100644 --- a/mypy/typeshed/stdlib/dis.pyi +++ b/mypy/typeshed/stdlib/dis.pyi @@ -2,8 +2,8 @@ import sys import types from collections.abc import Callable, Iterator from opcode import * # `dis` re-exports it as a part of public API -from typing import IO, Any, NamedTuple -from typing_extensions import Self, TypeAlias +from typing import IO, Any, Final, NamedTuple +from typing_extensions import Self, TypeAlias, disjoint_base __all__ = [ "code_info", @@ -88,39 +88,45 @@ else: starts_line: int | None is_jump_target: bool -class Instruction(_Instruction): - if sys.version_info < (3, 13): +if sys.version_info >= (3, 12): + class Instruction(_Instruction): + if sys.version_info < (3, 13): + def _disassemble(self, lineno_width: int = 3, mark_as_current: bool = False, offset_width: int = 4) -> str: ... + if sys.version_info >= (3, 13): + @property + def oparg(self) -> int: ... + @property + def baseopcode(self) -> int: ... + @property + def baseopname(self) -> str: ... + @property + def cache_offset(self) -> int: ... + @property + def end_offset(self) -> int: ... + @property + def jump_target(self) -> int: ... + @property + def is_jump_target(self) -> bool: ... + if sys.version_info >= (3, 14): + @staticmethod + def make( + opname: str, + arg: int | None, + argval: Any, + argrepr: str, + offset: int, + start_offset: int, + starts_line: bool, + line_number: int | None, + label: int | None = None, + positions: Positions | None = None, + cache_info: list[tuple[str, int, Any]] | None = None, + ) -> Instruction: ... + +else: + @disjoint_base + class Instruction(_Instruction): def _disassemble(self, lineno_width: int = 3, mark_as_current: bool = False, offset_width: int = 4) -> str: ... - if sys.version_info >= (3, 13): - @property - def oparg(self) -> int: ... - @property - def baseopcode(self) -> int: ... - @property - def baseopname(self) -> str: ... - @property - def cache_offset(self) -> int: ... - @property - def end_offset(self) -> int: ... - @property - def jump_target(self) -> int: ... - @property - def is_jump_target(self) -> bool: ... - if sys.version_info >= (3, 14): - @staticmethod - def make( - opname: str, - arg: int | None, - argval: Any, - argrepr: str, - offset: int, - start_offset: int, - starts_line: bool, - line_number: int | None, - label: int | None = None, - positions: Positions | None = None, - cache_info: list[tuple[str, int, Any]] | None = None, - ) -> Instruction: ... class Bytecode: codeobj: types.CodeType @@ -178,7 +184,7 @@ class Bytecode: def info(self) -> str: ... def dis(self) -> str: ... -COMPILER_FLAG_NAMES: dict[int, str] +COMPILER_FLAG_NAMES: Final[dict[int, str]] def findlabels(code: _HaveCodeType) -> list[int]: ... def findlinestarts(code: _HaveCodeType) -> Iterator[tuple[int, int]]: ... diff --git a/mypy/typeshed/stdlib/distutils/file_util.pyi b/mypy/typeshed/stdlib/distutils/file_util.pyi index 873d23ea7e500..c763f91a958d7 100644 --- a/mypy/typeshed/stdlib/distutils/file_util.pyi +++ b/mypy/typeshed/stdlib/distutils/file_util.pyi @@ -29,10 +29,10 @@ def copy_file( ) -> tuple[_BytesPathT | bytes, bool]: ... @overload def move_file( - src: StrPath, dst: _StrPathT, verbose: bool | Literal[0, 1] = 0, dry_run: bool | Literal[0, 1] = 0 + src: StrPath, dst: _StrPathT, verbose: bool | Literal[0, 1] = 1, dry_run: bool | Literal[0, 1] = 0 ) -> _StrPathT | str: ... @overload def move_file( - src: BytesPath, dst: _BytesPathT, verbose: bool | Literal[0, 1] = 0, dry_run: bool | Literal[0, 1] = 0 + src: BytesPath, dst: _BytesPathT, verbose: bool | Literal[0, 1] = 1, dry_run: bool | Literal[0, 1] = 0 ) -> _BytesPathT | bytes: ... def write_file(filename: StrOrBytesPath, contents: Iterable[str]) -> None: ... diff --git a/mypy/typeshed/stdlib/doctest.pyi b/mypy/typeshed/stdlib/doctest.pyi index 562b5a5bdac9e..1bb96e1a77868 100644 --- a/mypy/typeshed/stdlib/doctest.pyi +++ b/mypy/typeshed/stdlib/doctest.pyi @@ -3,7 +3,7 @@ import types import unittest from _typeshed import ExcInfo from collections.abc import Callable -from typing import Any, NamedTuple, type_check_only +from typing import Any, Final, NamedTuple, type_check_only from typing_extensions import Self, TypeAlias __all__ = [ @@ -57,29 +57,29 @@ else: failed: int attempted: int -OPTIONFLAGS_BY_NAME: dict[str, int] +OPTIONFLAGS_BY_NAME: Final[dict[str, int]] def register_optionflag(name: str) -> int: ... -DONT_ACCEPT_TRUE_FOR_1: int -DONT_ACCEPT_BLANKLINE: int -NORMALIZE_WHITESPACE: int -ELLIPSIS: int -SKIP: int -IGNORE_EXCEPTION_DETAIL: int +DONT_ACCEPT_TRUE_FOR_1: Final = 1 +DONT_ACCEPT_BLANKLINE: Final = 2 +NORMALIZE_WHITESPACE: Final = 4 +ELLIPSIS: Final = 8 +SKIP: Final = 16 +IGNORE_EXCEPTION_DETAIL: Final = 32 -COMPARISON_FLAGS: int +COMPARISON_FLAGS: Final = 63 -REPORT_UDIFF: int -REPORT_CDIFF: int -REPORT_NDIFF: int -REPORT_ONLY_FIRST_FAILURE: int -FAIL_FAST: int +REPORT_UDIFF: Final = 64 +REPORT_CDIFF: Final = 128 +REPORT_NDIFF: Final = 256 +REPORT_ONLY_FIRST_FAILURE: Final = 512 +FAIL_FAST: Final = 1024 -REPORTING_FLAGS: int +REPORTING_FLAGS: Final = 1984 -BLANKLINE_MARKER: str -ELLIPSIS_MARKER: str +BLANKLINE_MARKER: Final = "" +ELLIPSIS_MARKER: Final = "..." class Example: source: str diff --git a/mypy/typeshed/stdlib/email/_header_value_parser.pyi b/mypy/typeshed/stdlib/email/_header_value_parser.pyi index 95ada186c4ec8..dededd006e5b5 100644 --- a/mypy/typeshed/stdlib/email/_header_value_parser.pyi +++ b/mypy/typeshed/stdlib/email/_header_value_parser.pyi @@ -25,7 +25,7 @@ SPECIALSNL: Final[set[str]] def make_quoted_pairs(value: Any) -> str: ... def quote_string(value: Any) -> str: ... -rfc2047_matcher: Pattern[str] +rfc2047_matcher: Final[Pattern[str]] class TokenList(list[TokenList | Terminal]): token_type: str | None diff --git a/mypy/typeshed/stdlib/email/charset.pyi b/mypy/typeshed/stdlib/email/charset.pyi index 683daa468cf3b..e1930835bbd11 100644 --- a/mypy/typeshed/stdlib/email/charset.pyi +++ b/mypy/typeshed/stdlib/email/charset.pyi @@ -4,9 +4,16 @@ from typing import ClassVar, Final, overload __all__ = ["Charset", "add_alias", "add_charset", "add_codec"] -QP: Final[int] # undocumented -BASE64: Final[int] # undocumented -SHORTEST: Final[int] # undocumented +QP: Final = 1 # undocumented +BASE64: Final = 2 # undocumented +SHORTEST: Final = 3 # undocumented +RFC2047_CHROME_LEN: Final = 7 # undocumented +DEFAULT_CHARSET: Final = "us-ascii" # undocumented +UNKNOWN8BIT: Final = "unknown-8bit" # undocumented +EMPTYSTRING: Final = "" # undocumented +CHARSETS: Final[dict[str, tuple[int | None, int | None, str | None]]] +ALIASES: Final[dict[str, str]] +CODEC_MAP: Final[dict[str, str | None]] # undocumented class Charset: input_charset: str diff --git a/mypy/typeshed/stdlib/enum.pyi b/mypy/typeshed/stdlib/enum.pyi index eb7d2e3819fd8..4ac860f5e611d 100644 --- a/mypy/typeshed/stdlib/enum.pyi +++ b/mypy/typeshed/stdlib/enum.pyi @@ -4,8 +4,8 @@ import types from _typeshed import SupportsKeysAndGetItem, Unused from builtins import property as _builtins_property from collections.abc import Callable, Iterable, Iterator, Mapping -from typing import Any, Generic, Literal, TypeVar, overload -from typing_extensions import Self, TypeAlias +from typing import Any, Final, Generic, Literal, TypeVar, overload +from typing_extensions import Self, TypeAlias, disjoint_base __all__ = ["EnumMeta", "Enum", "IntEnum", "Flag", "IntFlag", "auto", "unique"] @@ -228,16 +228,25 @@ class Enum(metaclass=EnumMeta): if sys.version_info >= (3, 11): class ReprEnum(Enum): ... -if sys.version_info >= (3, 11): - _IntEnumBase = ReprEnum +if sys.version_info >= (3, 12): + class IntEnum(int, ReprEnum): + _value_: int + @_magic_enum_attr + def value(self) -> int: ... + def __new__(cls, value: int) -> Self: ... + else: - _IntEnumBase = Enum + if sys.version_info >= (3, 11): + _IntEnumBase = ReprEnum + else: + _IntEnumBase = Enum -class IntEnum(int, _IntEnumBase): - _value_: int - @_magic_enum_attr - def value(self) -> int: ... - def __new__(cls, value: int) -> Self: ... + @disjoint_base + class IntEnum(int, _IntEnumBase): + _value_: int + @_magic_enum_attr + def value(self) -> int: ... + def __new__(cls, value: int) -> Self: ... def unique(enumeration: _EnumerationT) -> _EnumerationT: ... @@ -277,9 +286,9 @@ if sys.version_info >= (3, 11): NAMED_FLAGS = "multi-flag aliases may not contain unnamed flags" UNIQUE = "one name per value" - CONTINUOUS = EnumCheck.CONTINUOUS - NAMED_FLAGS = EnumCheck.NAMED_FLAGS - UNIQUE = EnumCheck.UNIQUE + CONTINUOUS: Final = EnumCheck.CONTINUOUS + NAMED_FLAGS: Final = EnumCheck.NAMED_FLAGS + UNIQUE: Final = EnumCheck.UNIQUE class verify: def __init__(self, *checks: EnumCheck) -> None: ... @@ -291,18 +300,31 @@ if sys.version_info >= (3, 11): EJECT = "eject" KEEP = "keep" - STRICT = FlagBoundary.STRICT - CONFORM = FlagBoundary.CONFORM - EJECT = FlagBoundary.EJECT - KEEP = FlagBoundary.KEEP + STRICT: Final = FlagBoundary.STRICT + CONFORM: Final = FlagBoundary.CONFORM + EJECT: Final = FlagBoundary.EJECT + KEEP: Final = FlagBoundary.KEEP def global_str(self: Enum) -> str: ... def global_enum(cls: _EnumerationT, update_str: bool = False) -> _EnumerationT: ... def global_enum_repr(self: Enum) -> str: ... def global_flag_repr(self: Flag) -> str: ... -if sys.version_info >= (3, 11): +if sys.version_info >= (3, 12): + # The body of the class is the same, but the base classes are different. + class IntFlag(int, ReprEnum, Flag, boundary=KEEP): # type: ignore[misc] # complaints about incompatible bases + def __new__(cls, value: int) -> Self: ... + def __or__(self, other: int) -> Self: ... + def __and__(self, other: int) -> Self: ... + def __xor__(self, other: int) -> Self: ... + def __invert__(self) -> Self: ... + __ror__ = __or__ + __rand__ = __and__ + __rxor__ = __xor__ + +elif sys.version_info >= (3, 11): # The body of the class is the same, but the base classes are different. + @disjoint_base class IntFlag(int, ReprEnum, Flag, boundary=KEEP): # type: ignore[misc] # complaints about incompatible bases def __new__(cls, value: int) -> Self: ... def __or__(self, other: int) -> Self: ... @@ -314,6 +336,7 @@ if sys.version_info >= (3, 11): __rxor__ = __xor__ else: + @disjoint_base class IntFlag(int, Flag): # type: ignore[misc] # complaints about incompatible bases def __new__(cls, value: int) -> Self: ... def __or__(self, other: int) -> Self: ... diff --git a/mypy/typeshed/stdlib/errno.pyi b/mypy/typeshed/stdlib/errno.pyi index 3ba8b66d28650..4f19b5aee87e4 100644 --- a/mypy/typeshed/stdlib/errno.pyi +++ b/mypy/typeshed/stdlib/errno.pyi @@ -1,225 +1,226 @@ import sys from collections.abc import Mapping +from typing import Final errorcode: Mapping[int, str] -EPERM: int -ENOENT: int -ESRCH: int -EINTR: int -EIO: int -ENXIO: int -E2BIG: int -ENOEXEC: int -EBADF: int -ECHILD: int -EAGAIN: int -ENOMEM: int -EACCES: int -EFAULT: int -EBUSY: int -EEXIST: int -EXDEV: int -ENODEV: int -ENOTDIR: int -EISDIR: int -EINVAL: int -ENFILE: int -EMFILE: int -ENOTTY: int -ETXTBSY: int -EFBIG: int -ENOSPC: int -ESPIPE: int -EROFS: int -EMLINK: int -EPIPE: int -EDOM: int -ERANGE: int -EDEADLK: int -ENAMETOOLONG: int -ENOLCK: int -ENOSYS: int -ENOTEMPTY: int -ELOOP: int -EWOULDBLOCK: int -ENOMSG: int -EIDRM: int -ENOSTR: int -ENODATA: int -ETIME: int -ENOSR: int -EREMOTE: int -ENOLINK: int -EPROTO: int -EBADMSG: int -EOVERFLOW: int -EILSEQ: int -EUSERS: int -ENOTSOCK: int -EDESTADDRREQ: int -EMSGSIZE: int -EPROTOTYPE: int -ENOPROTOOPT: int -EPROTONOSUPPORT: int -ESOCKTNOSUPPORT: int -ENOTSUP: int -EOPNOTSUPP: int -EPFNOSUPPORT: int -EAFNOSUPPORT: int -EADDRINUSE: int -EADDRNOTAVAIL: int -ENETDOWN: int -ENETUNREACH: int -ENETRESET: int -ECONNABORTED: int -ECONNRESET: int -ENOBUFS: int -EISCONN: int -ENOTCONN: int -ESHUTDOWN: int -ETOOMANYREFS: int -ETIMEDOUT: int -ECONNREFUSED: int -EHOSTDOWN: int -EHOSTUNREACH: int -EALREADY: int -EINPROGRESS: int -ESTALE: int -EDQUOT: int -ECANCELED: int # undocumented -ENOTRECOVERABLE: int # undocumented -EOWNERDEAD: int # undocumented +EPERM: Final[int] +ENOENT: Final[int] +ESRCH: Final[int] +EINTR: Final[int] +EIO: Final[int] +ENXIO: Final[int] +E2BIG: Final[int] +ENOEXEC: Final[int] +EBADF: Final[int] +ECHILD: Final[int] +EAGAIN: Final[int] +ENOMEM: Final[int] +EACCES: Final[int] +EFAULT: Final[int] +EBUSY: Final[int] +EEXIST: Final[int] +EXDEV: Final[int] +ENODEV: Final[int] +ENOTDIR: Final[int] +EISDIR: Final[int] +EINVAL: Final[int] +ENFILE: Final[int] +EMFILE: Final[int] +ENOTTY: Final[int] +ETXTBSY: Final[int] +EFBIG: Final[int] +ENOSPC: Final[int] +ESPIPE: Final[int] +EROFS: Final[int] +EMLINK: Final[int] +EPIPE: Final[int] +EDOM: Final[int] +ERANGE: Final[int] +EDEADLK: Final[int] +ENAMETOOLONG: Final[int] +ENOLCK: Final[int] +ENOSYS: Final[int] +ENOTEMPTY: Final[int] +ELOOP: Final[int] +EWOULDBLOCK: Final[int] +ENOMSG: Final[int] +EIDRM: Final[int] +ENOSTR: Final[int] +ENODATA: Final[int] +ETIME: Final[int] +ENOSR: Final[int] +EREMOTE: Final[int] +ENOLINK: Final[int] +EPROTO: Final[int] +EBADMSG: Final[int] +EOVERFLOW: Final[int] +EILSEQ: Final[int] +EUSERS: Final[int] +ENOTSOCK: Final[int] +EDESTADDRREQ: Final[int] +EMSGSIZE: Final[int] +EPROTOTYPE: Final[int] +ENOPROTOOPT: Final[int] +EPROTONOSUPPORT: Final[int] +ESOCKTNOSUPPORT: Final[int] +ENOTSUP: Final[int] +EOPNOTSUPP: Final[int] +EPFNOSUPPORT: Final[int] +EAFNOSUPPORT: Final[int] +EADDRINUSE: Final[int] +EADDRNOTAVAIL: Final[int] +ENETDOWN: Final[int] +ENETUNREACH: Final[int] +ENETRESET: Final[int] +ECONNABORTED: Final[int] +ECONNRESET: Final[int] +ENOBUFS: Final[int] +EISCONN: Final[int] +ENOTCONN: Final[int] +ESHUTDOWN: Final[int] +ETOOMANYREFS: Final[int] +ETIMEDOUT: Final[int] +ECONNREFUSED: Final[int] +EHOSTDOWN: Final[int] +EHOSTUNREACH: Final[int] +EALREADY: Final[int] +EINPROGRESS: Final[int] +ESTALE: Final[int] +EDQUOT: Final[int] +ECANCELED: Final[int] # undocumented +ENOTRECOVERABLE: Final[int] # undocumented +EOWNERDEAD: Final[int] # undocumented if sys.platform == "sunos5" or sys.platform == "solaris": # noqa: Y008 - ELOCKUNMAPPED: int - ENOTACTIVE: int + ELOCKUNMAPPED: Final[int] + ENOTACTIVE: Final[int] if sys.platform != "win32": - ENOTBLK: int - EMULTIHOP: int + ENOTBLK: Final[int] + EMULTIHOP: Final[int] if sys.platform == "darwin": # All of the below are undocumented - EAUTH: int - EBADARCH: int - EBADEXEC: int - EBADMACHO: int - EBADRPC: int - EDEVERR: int - EFTYPE: int - ENEEDAUTH: int - ENOATTR: int - ENOPOLICY: int - EPROCLIM: int - EPROCUNAVAIL: int - EPROGMISMATCH: int - EPROGUNAVAIL: int - EPWROFF: int - ERPCMISMATCH: int - ESHLIBVERS: int + EAUTH: Final[int] + EBADARCH: Final[int] + EBADEXEC: Final[int] + EBADMACHO: Final[int] + EBADRPC: Final[int] + EDEVERR: Final[int] + EFTYPE: Final[int] + ENEEDAUTH: Final[int] + ENOATTR: Final[int] + ENOPOLICY: Final[int] + EPROCLIM: Final[int] + EPROCUNAVAIL: Final[int] + EPROGMISMATCH: Final[int] + EPROGUNAVAIL: Final[int] + EPWROFF: Final[int] + ERPCMISMATCH: Final[int] + ESHLIBVERS: Final[int] if sys.version_info >= (3, 11): - EQFULL: int + EQFULL: Final[int] if sys.platform != "darwin": - EDEADLOCK: int + EDEADLOCK: Final[int] if sys.platform != "win32" and sys.platform != "darwin": - ECHRNG: int - EL2NSYNC: int - EL3HLT: int - EL3RST: int - ELNRNG: int - EUNATCH: int - ENOCSI: int - EL2HLT: int - EBADE: int - EBADR: int - EXFULL: int - ENOANO: int - EBADRQC: int - EBADSLT: int - EBFONT: int - ENONET: int - ENOPKG: int - EADV: int - ESRMNT: int - ECOMM: int - EDOTDOT: int - ENOTUNIQ: int - EBADFD: int - EREMCHG: int - ELIBACC: int - ELIBBAD: int - ELIBSCN: int - ELIBMAX: int - ELIBEXEC: int - ERESTART: int - ESTRPIPE: int - EUCLEAN: int - ENOTNAM: int - ENAVAIL: int - EISNAM: int - EREMOTEIO: int + ECHRNG: Final[int] + EL2NSYNC: Final[int] + EL3HLT: Final[int] + EL3RST: Final[int] + ELNRNG: Final[int] + EUNATCH: Final[int] + ENOCSI: Final[int] + EL2HLT: Final[int] + EBADE: Final[int] + EBADR: Final[int] + EXFULL: Final[int] + ENOANO: Final[int] + EBADRQC: Final[int] + EBADSLT: Final[int] + EBFONT: Final[int] + ENONET: Final[int] + ENOPKG: Final[int] + EADV: Final[int] + ESRMNT: Final[int] + ECOMM: Final[int] + EDOTDOT: Final[int] + ENOTUNIQ: Final[int] + EBADFD: Final[int] + EREMCHG: Final[int] + ELIBACC: Final[int] + ELIBBAD: Final[int] + ELIBSCN: Final[int] + ELIBMAX: Final[int] + ELIBEXEC: Final[int] + ERESTART: Final[int] + ESTRPIPE: Final[int] + EUCLEAN: Final[int] + ENOTNAM: Final[int] + ENAVAIL: Final[int] + EISNAM: Final[int] + EREMOTEIO: Final[int] # All of the below are undocumented - EKEYEXPIRED: int - EKEYREJECTED: int - EKEYREVOKED: int - EMEDIUMTYPE: int - ENOKEY: int - ENOMEDIUM: int - ERFKILL: int + EKEYEXPIRED: Final[int] + EKEYREJECTED: Final[int] + EKEYREVOKED: Final[int] + EMEDIUMTYPE: Final[int] + ENOKEY: Final[int] + ENOMEDIUM: Final[int] + ERFKILL: Final[int] if sys.version_info >= (3, 14): - EHWPOISON: int + EHWPOISON: Final[int] if sys.platform == "win32": # All of these are undocumented - WSABASEERR: int - WSAEACCES: int - WSAEADDRINUSE: int - WSAEADDRNOTAVAIL: int - WSAEAFNOSUPPORT: int - WSAEALREADY: int - WSAEBADF: int - WSAECONNABORTED: int - WSAECONNREFUSED: int - WSAECONNRESET: int - WSAEDESTADDRREQ: int - WSAEDISCON: int - WSAEDQUOT: int - WSAEFAULT: int - WSAEHOSTDOWN: int - WSAEHOSTUNREACH: int - WSAEINPROGRESS: int - WSAEINTR: int - WSAEINVAL: int - WSAEISCONN: int - WSAELOOP: int - WSAEMFILE: int - WSAEMSGSIZE: int - WSAENAMETOOLONG: int - WSAENETDOWN: int - WSAENETRESET: int - WSAENETUNREACH: int - WSAENOBUFS: int - WSAENOPROTOOPT: int - WSAENOTCONN: int - WSAENOTEMPTY: int - WSAENOTSOCK: int - WSAEOPNOTSUPP: int - WSAEPFNOSUPPORT: int - WSAEPROCLIM: int - WSAEPROTONOSUPPORT: int - WSAEPROTOTYPE: int - WSAEREMOTE: int - WSAESHUTDOWN: int - WSAESOCKTNOSUPPORT: int - WSAESTALE: int - WSAETIMEDOUT: int - WSAETOOMANYREFS: int - WSAEUSERS: int - WSAEWOULDBLOCK: int - WSANOTINITIALISED: int - WSASYSNOTREADY: int - WSAVERNOTSUPPORTED: int + WSABASEERR: Final[int] + WSAEACCES: Final[int] + WSAEADDRINUSE: Final[int] + WSAEADDRNOTAVAIL: Final[int] + WSAEAFNOSUPPORT: Final[int] + WSAEALREADY: Final[int] + WSAEBADF: Final[int] + WSAECONNABORTED: Final[int] + WSAECONNREFUSED: Final[int] + WSAECONNRESET: Final[int] + WSAEDESTADDRREQ: Final[int] + WSAEDISCON: Final[int] + WSAEDQUOT: Final[int] + WSAEFAULT: Final[int] + WSAEHOSTDOWN: Final[int] + WSAEHOSTUNREACH: Final[int] + WSAEINPROGRESS: Final[int] + WSAEINTR: Final[int] + WSAEINVAL: Final[int] + WSAEISCONN: Final[int] + WSAELOOP: Final[int] + WSAEMFILE: Final[int] + WSAEMSGSIZE: Final[int] + WSAENAMETOOLONG: Final[int] + WSAENETDOWN: Final[int] + WSAENETRESET: Final[int] + WSAENETUNREACH: Final[int] + WSAENOBUFS: Final[int] + WSAENOPROTOOPT: Final[int] + WSAENOTCONN: Final[int] + WSAENOTEMPTY: Final[int] + WSAENOTSOCK: Final[int] + WSAEOPNOTSUPP: Final[int] + WSAEPFNOSUPPORT: Final[int] + WSAEPROCLIM: Final[int] + WSAEPROTONOSUPPORT: Final[int] + WSAEPROTOTYPE: Final[int] + WSAEREMOTE: Final[int] + WSAESHUTDOWN: Final[int] + WSAESOCKTNOSUPPORT: Final[int] + WSAESTALE: Final[int] + WSAETIMEDOUT: Final[int] + WSAETOOMANYREFS: Final[int] + WSAEUSERS: Final[int] + WSAEWOULDBLOCK: Final[int] + WSANOTINITIALISED: Final[int] + WSASYSNOTREADY: Final[int] + WSAVERNOTSUPPORTED: Final[int] diff --git a/mypy/typeshed/stdlib/filecmp.pyi b/mypy/typeshed/stdlib/filecmp.pyi index a2a2b235fdadc..620cc177a415a 100644 --- a/mypy/typeshed/stdlib/filecmp.pyi +++ b/mypy/typeshed/stdlib/filecmp.pyi @@ -6,7 +6,7 @@ from typing import Any, AnyStr, Final, Generic, Literal __all__ = ["clear_cache", "cmp", "dircmp", "cmpfiles", "DEFAULT_IGNORES"] -DEFAULT_IGNORES: list[str] +DEFAULT_IGNORES: Final[list[str]] BUFSIZE: Final = 8192 def cmp(f1: StrOrBytesPath, f2: StrOrBytesPath, shallow: bool | Literal[0, 1] = True) -> bool: ... diff --git a/mypy/typeshed/stdlib/fractions.pyi b/mypy/typeshed/stdlib/fractions.pyi index e81fbaf5dad78..ef4066aa65b52 100644 --- a/mypy/typeshed/stdlib/fractions.pyi +++ b/mypy/typeshed/stdlib/fractions.pyi @@ -14,6 +14,7 @@ class _ConvertibleToIntegerRatio(Protocol): def as_integer_ratio(self) -> tuple[int | Rational, int | Rational]: ... class Fraction(Rational): + __slots__ = ("_numerator", "_denominator") @overload def __new__(cls, numerator: int | Rational = 0, denominator: int | Rational | None = None) -> Self: ... @overload diff --git a/mypy/typeshed/stdlib/functools.pyi b/mypy/typeshed/stdlib/functools.pyi index 6e17ba7d35dc7..47baf917294da 100644 --- a/mypy/typeshed/stdlib/functools.pyi +++ b/mypy/typeshed/stdlib/functools.pyi @@ -4,7 +4,7 @@ from _typeshed import SupportsAllComparisons, SupportsItems from collections.abc import Callable, Hashable, Iterable, Sized from types import GenericAlias from typing import Any, Final, Generic, Literal, NamedTuple, TypedDict, TypeVar, final, overload, type_check_only -from typing_extensions import ParamSpec, Self, TypeAlias +from typing_extensions import ParamSpec, Self, TypeAlias, disjoint_base __all__ = [ "update_wrapper", @@ -95,7 +95,7 @@ else: tuple[Literal["__module__"], Literal["__name__"], Literal["__qualname__"], Literal["__doc__"], Literal["__annotations__"]] ] -WRAPPER_UPDATES: tuple[Literal["__dict__"]] +WRAPPER_UPDATES: Final[tuple[Literal["__dict__"]]] @type_check_only class _Wrapped(Generic[_PWrapped, _RWrapped, _PWrapper, _RWrapper]): @@ -150,7 +150,7 @@ else: def total_ordering(cls: type[_T]) -> type[_T]: ... def cmp_to_key(mycmp: Callable[[_T, _T], int]) -> Callable[[_T], SupportsAllComparisons]: ... - +@disjoint_base class partial(Generic[_T]): @property def func(self) -> Callable[..., _T]: ... @@ -169,10 +169,17 @@ class partialmethod(Generic[_T]): func: Callable[..., _T] | _Descriptor args: tuple[Any, ...] keywords: dict[str, Any] - @overload - def __init__(self, func: Callable[..., _T], /, *args: Any, **keywords: Any) -> None: ... - @overload - def __init__(self, func: _Descriptor, /, *args: Any, **keywords: Any) -> None: ... + if sys.version_info >= (3, 14): + @overload + def __new__(self, func: Callable[..., _T], /, *args: Any, **keywords: Any) -> Self: ... + @overload + def __new__(self, func: _Descriptor, /, *args: Any, **keywords: Any) -> Self: ... + else: + @overload + def __init__(self, func: Callable[..., _T], /, *args: Any, **keywords: Any) -> None: ... + @overload + def __init__(self, func: _Descriptor, /, *args: Any, **keywords: Any) -> None: ... + def __get__(self, obj: Any, cls: type[Any] | None = None) -> Callable[..., _T]: ... @property def __isabstractmethod__(self) -> bool: ... diff --git a/mypy/typeshed/stdlib/gettext.pyi b/mypy/typeshed/stdlib/gettext.pyi index 937aece034375..e9ffd7a4a4a42 100644 --- a/mypy/typeshed/stdlib/gettext.pyi +++ b/mypy/typeshed/stdlib/gettext.pyi @@ -120,7 +120,7 @@ else: languages: Iterable[str] | None = None, class_: None = None, fallback: Literal[False] = False, - codeset: str | None = None, + codeset: str | None = ..., ) -> GNUTranslations: ... @overload def translation( @@ -130,7 +130,7 @@ else: *, class_: Callable[[io.BufferedReader], _NullTranslationsT], fallback: Literal[False] = False, - codeset: str | None = None, + codeset: str | None = ..., ) -> _NullTranslationsT: ... @overload def translation( @@ -139,7 +139,7 @@ else: languages: Iterable[str] | None, class_: Callable[[io.BufferedReader], _NullTranslationsT], fallback: Literal[False] = False, - codeset: str | None = None, + codeset: str | None = ..., ) -> _NullTranslationsT: ... @overload def translation( @@ -148,18 +148,18 @@ else: languages: Iterable[str] | None = None, class_: Callable[[io.BufferedReader], NullTranslations] | None = None, fallback: bool = False, - codeset: str | None = None, + codeset: str | None = ..., ) -> NullTranslations: ... @overload - def install( - domain: str, localedir: StrPath | None = None, codeset: None = None, names: Container[str] | None = None - ) -> None: ... + def install(domain: str, localedir: StrPath | None = None, names: Container[str] | None = None) -> None: ... @overload @deprecated("The `codeset` parameter is deprecated since Python 3.8; removed in Python 3.11.") - def install(domain: str, localedir: StrPath | None, codeset: str, /, names: Container[str] | None = None) -> None: ... + def install(domain: str, localedir: StrPath | None, codeset: str | None, /, names: Container[str] | None = None) -> None: ... @overload @deprecated("The `codeset` parameter is deprecated since Python 3.8; removed in Python 3.11.") - def install(domain: str, localedir: StrPath | None = None, *, codeset: str, names: Container[str] | None = None) -> None: ... + def install( + domain: str, localedir: StrPath | None = None, *, codeset: str | None, names: Container[str] | None = None + ) -> None: ... def textdomain(domain: str | None = None) -> str: ... def bindtextdomain(domain: str, localedir: StrPath | None = None) -> str: ... diff --git a/mypy/typeshed/stdlib/glob.pyi b/mypy/typeshed/stdlib/glob.pyi index 63069d8009c8d..942fd73961963 100644 --- a/mypy/typeshed/stdlib/glob.pyi +++ b/mypy/typeshed/stdlib/glob.pyi @@ -10,9 +10,13 @@ if sys.version_info >= (3, 13): __all__ += ["translate"] if sys.version_info >= (3, 10): - @deprecated("Will be removed in Python 3.15; Use `glob.glob` and pass *root_dir* argument instead.") + @deprecated( + "Deprecated since Python 3.10; will be removed in Python 3.15. Use `glob.glob()` with the *root_dir* argument instead." + ) def glob0(dirname: AnyStr, pattern: AnyStr) -> list[AnyStr]: ... - @deprecated("Will be removed in Python 3.15; Use `glob.glob` and pass *root_dir* argument instead.") + @deprecated( + "Deprecated since Python 3.10; will be removed in Python 3.15. Use `glob.glob()` with the *root_dir* argument instead." + ) def glob1(dirname: AnyStr, pattern: AnyStr) -> list[AnyStr]: ... else: diff --git a/mypy/typeshed/stdlib/hmac.pyi b/mypy/typeshed/stdlib/hmac.pyi index 300ed9eb26d85..070c59b1c166d 100644 --- a/mypy/typeshed/stdlib/hmac.pyi +++ b/mypy/typeshed/stdlib/hmac.pyi @@ -20,6 +20,7 @@ def new(key: bytes | bytearray, msg: ReadableBuffer | None, digestmod: _DigestMo def new(key: bytes | bytearray, *, digestmod: _DigestMod) -> HMAC: ... class HMAC: + __slots__ = ("_hmac", "_inner", "_outer", "block_size", "digest_size") digest_size: int block_size: int @property diff --git a/mypy/typeshed/stdlib/html/entities.pyi b/mypy/typeshed/stdlib/html/entities.pyi index be83fd1135be2..e5890d1ecfbd8 100644 --- a/mypy/typeshed/stdlib/html/entities.pyi +++ b/mypy/typeshed/stdlib/html/entities.pyi @@ -1,6 +1,8 @@ +from typing import Final + __all__ = ["html5", "name2codepoint", "codepoint2name", "entitydefs"] -name2codepoint: dict[str, int] -html5: dict[str, str] -codepoint2name: dict[int, str] -entitydefs: dict[str, str] +name2codepoint: Final[dict[str, int]] +html5: Final[dict[str, str]] +codepoint2name: Final[dict[int, str]] +entitydefs: Final[dict[str, str]] diff --git a/mypy/typeshed/stdlib/http/client.pyi b/mypy/typeshed/stdlib/http/client.pyi index 5c35dff28d43a..d259e84e6f2aa 100644 --- a/mypy/typeshed/stdlib/http/client.pyi +++ b/mypy/typeshed/stdlib/http/client.pyi @@ -7,7 +7,7 @@ from _typeshed import MaybeNone, ReadableBuffer, SupportsRead, SupportsReadline, from collections.abc import Callable, Iterable, Iterator, Mapping from email._policybase import _MessageT from socket import socket -from typing import BinaryIO, Literal, TypeVar, overload +from typing import BinaryIO, Final, TypeVar, overload from typing_extensions import Self, TypeAlias __all__ = [ @@ -36,85 +36,85 @@ _DataType: TypeAlias = SupportsRead[bytes] | Iterable[ReadableBuffer] | Readable _T = TypeVar("_T") _HeaderValue: TypeAlias = ReadableBuffer | str | int -HTTP_PORT: int -HTTPS_PORT: int +HTTP_PORT: Final = 80 +HTTPS_PORT: Final = 443 # Keep these global constants in sync with http.HTTPStatus (http/__init__.pyi). # They are present for backward compatibility reasons. -CONTINUE: Literal[100] -SWITCHING_PROTOCOLS: Literal[101] -PROCESSING: Literal[102] -EARLY_HINTS: Literal[103] +CONTINUE: Final = 100 +SWITCHING_PROTOCOLS: Final = 101 +PROCESSING: Final = 102 +EARLY_HINTS: Final = 103 -OK: Literal[200] -CREATED: Literal[201] -ACCEPTED: Literal[202] -NON_AUTHORITATIVE_INFORMATION: Literal[203] -NO_CONTENT: Literal[204] -RESET_CONTENT: Literal[205] -PARTIAL_CONTENT: Literal[206] -MULTI_STATUS: Literal[207] -ALREADY_REPORTED: Literal[208] -IM_USED: Literal[226] +OK: Final = 200 +CREATED: Final = 201 +ACCEPTED: Final = 202 +NON_AUTHORITATIVE_INFORMATION: Final = 203 +NO_CONTENT: Final = 204 +RESET_CONTENT: Final = 205 +PARTIAL_CONTENT: Final = 206 +MULTI_STATUS: Final = 207 +ALREADY_REPORTED: Final = 208 +IM_USED: Final = 226 -MULTIPLE_CHOICES: Literal[300] -MOVED_PERMANENTLY: Literal[301] -FOUND: Literal[302] -SEE_OTHER: Literal[303] -NOT_MODIFIED: Literal[304] -USE_PROXY: Literal[305] -TEMPORARY_REDIRECT: Literal[307] -PERMANENT_REDIRECT: Literal[308] +MULTIPLE_CHOICES: Final = 300 +MOVED_PERMANENTLY: Final = 301 +FOUND: Final = 302 +SEE_OTHER: Final = 303 +NOT_MODIFIED: Final = 304 +USE_PROXY: Final = 305 +TEMPORARY_REDIRECT: Final = 307 +PERMANENT_REDIRECT: Final = 308 -BAD_REQUEST: Literal[400] -UNAUTHORIZED: Literal[401] -PAYMENT_REQUIRED: Literal[402] -FORBIDDEN: Literal[403] -NOT_FOUND: Literal[404] -METHOD_NOT_ALLOWED: Literal[405] -NOT_ACCEPTABLE: Literal[406] -PROXY_AUTHENTICATION_REQUIRED: Literal[407] -REQUEST_TIMEOUT: Literal[408] -CONFLICT: Literal[409] -GONE: Literal[410] -LENGTH_REQUIRED: Literal[411] -PRECONDITION_FAILED: Literal[412] +BAD_REQUEST: Final = 400 +UNAUTHORIZED: Final = 401 +PAYMENT_REQUIRED: Final = 402 +FORBIDDEN: Final = 403 +NOT_FOUND: Final = 404 +METHOD_NOT_ALLOWED: Final = 405 +NOT_ACCEPTABLE: Final = 406 +PROXY_AUTHENTICATION_REQUIRED: Final = 407 +REQUEST_TIMEOUT: Final = 408 +CONFLICT: Final = 409 +GONE: Final = 410 +LENGTH_REQUIRED: Final = 411 +PRECONDITION_FAILED: Final = 412 if sys.version_info >= (3, 13): - CONTENT_TOO_LARGE: Literal[413] -REQUEST_ENTITY_TOO_LARGE: Literal[413] + CONTENT_TOO_LARGE: Final = 413 +REQUEST_ENTITY_TOO_LARGE: Final = 413 if sys.version_info >= (3, 13): - URI_TOO_LONG: Literal[414] -REQUEST_URI_TOO_LONG: Literal[414] -UNSUPPORTED_MEDIA_TYPE: Literal[415] + URI_TOO_LONG: Final = 414 +REQUEST_URI_TOO_LONG: Final = 414 +UNSUPPORTED_MEDIA_TYPE: Final = 415 if sys.version_info >= (3, 13): - RANGE_NOT_SATISFIABLE: Literal[416] -REQUESTED_RANGE_NOT_SATISFIABLE: Literal[416] -EXPECTATION_FAILED: Literal[417] -IM_A_TEAPOT: Literal[418] -MISDIRECTED_REQUEST: Literal[421] + RANGE_NOT_SATISFIABLE: Final = 416 +REQUESTED_RANGE_NOT_SATISFIABLE: Final = 416 +EXPECTATION_FAILED: Final = 417 +IM_A_TEAPOT: Final = 418 +MISDIRECTED_REQUEST: Final = 421 if sys.version_info >= (3, 13): - UNPROCESSABLE_CONTENT: Literal[422] -UNPROCESSABLE_ENTITY: Literal[422] -LOCKED: Literal[423] -FAILED_DEPENDENCY: Literal[424] -TOO_EARLY: Literal[425] -UPGRADE_REQUIRED: Literal[426] -PRECONDITION_REQUIRED: Literal[428] -TOO_MANY_REQUESTS: Literal[429] -REQUEST_HEADER_FIELDS_TOO_LARGE: Literal[431] -UNAVAILABLE_FOR_LEGAL_REASONS: Literal[451] + UNPROCESSABLE_CONTENT: Final = 422 +UNPROCESSABLE_ENTITY: Final = 422 +LOCKED: Final = 423 +FAILED_DEPENDENCY: Final = 424 +TOO_EARLY: Final = 425 +UPGRADE_REQUIRED: Final = 426 +PRECONDITION_REQUIRED: Final = 428 +TOO_MANY_REQUESTS: Final = 429 +REQUEST_HEADER_FIELDS_TOO_LARGE: Final = 431 +UNAVAILABLE_FOR_LEGAL_REASONS: Final = 451 -INTERNAL_SERVER_ERROR: Literal[500] -NOT_IMPLEMENTED: Literal[501] -BAD_GATEWAY: Literal[502] -SERVICE_UNAVAILABLE: Literal[503] -GATEWAY_TIMEOUT: Literal[504] -HTTP_VERSION_NOT_SUPPORTED: Literal[505] -VARIANT_ALSO_NEGOTIATES: Literal[506] -INSUFFICIENT_STORAGE: Literal[507] -LOOP_DETECTED: Literal[508] -NOT_EXTENDED: Literal[510] -NETWORK_AUTHENTICATION_REQUIRED: Literal[511] +INTERNAL_SERVER_ERROR: Final = 500 +NOT_IMPLEMENTED: Final = 501 +BAD_GATEWAY: Final = 502 +SERVICE_UNAVAILABLE: Final = 503 +GATEWAY_TIMEOUT: Final = 504 +HTTP_VERSION_NOT_SUPPORTED: Final = 505 +VARIANT_ALSO_NEGOTIATES: Final = 506 +INSUFFICIENT_STORAGE: Final = 507 +LOOP_DETECTED: Final = 508 +NOT_EXTENDED: Final = 510 +NETWORK_AUTHENTICATION_REQUIRED: Final = 511 responses: dict[int, str] diff --git a/mypy/typeshed/stdlib/http/server.pyi b/mypy/typeshed/stdlib/http/server.pyi index 429bb65bb0efc..2c1a374331bcc 100644 --- a/mypy/typeshed/stdlib/http/server.pyi +++ b/mypy/typeshed/stdlib/http/server.pyi @@ -119,12 +119,24 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler): def guess_type(self, path: StrPath) -> str: ... # undocumented def executable(path: StrPath) -> bool: ... # undocumented -@deprecated("Deprecated in Python 3.13; removal scheduled for Python 3.15") -class CGIHTTPRequestHandler(SimpleHTTPRequestHandler): - cgi_directories: list[str] - have_fork: bool # undocumented - def do_POST(self) -> None: ... - def is_cgi(self) -> bool: ... # undocumented - def is_executable(self, path: StrPath) -> bool: ... # undocumented - def is_python(self, path: StrPath) -> bool: ... # undocumented - def run_cgi(self) -> None: ... # undocumented + +if sys.version_info >= (3, 13): + @deprecated("Deprecated since Python 3.13; will be removed in Python 3.15.") + class CGIHTTPRequestHandler(SimpleHTTPRequestHandler): + cgi_directories: list[str] + have_fork: bool # undocumented + def do_POST(self) -> None: ... + def is_cgi(self) -> bool: ... # undocumented + def is_executable(self, path: StrPath) -> bool: ... # undocumented + def is_python(self, path: StrPath) -> bool: ... # undocumented + def run_cgi(self) -> None: ... # undocumented + +else: + class CGIHTTPRequestHandler(SimpleHTTPRequestHandler): + cgi_directories: list[str] + have_fork: bool # undocumented + def do_POST(self) -> None: ... + def is_cgi(self) -> bool: ... # undocumented + def is_executable(self, path: StrPath) -> bool: ... # undocumented + def is_python(self, path: StrPath) -> bool: ... # undocumented + def run_cgi(self) -> None: ... # undocumented diff --git a/mypy/typeshed/stdlib/imp.pyi b/mypy/typeshed/stdlib/imp.pyi index f045fd969b27d..b5b4223aa58e9 100644 --- a/mypy/typeshed/stdlib/imp.pyi +++ b/mypy/typeshed/stdlib/imp.pyi @@ -13,18 +13,18 @@ from _imp import ( from _typeshed import StrPath from os import PathLike from types import TracebackType -from typing import IO, Any, Protocol, type_check_only +from typing import IO, Any, Final, Protocol, type_check_only -SEARCH_ERROR: int -PY_SOURCE: int -PY_COMPILED: int -C_EXTENSION: int -PY_RESOURCE: int -PKG_DIRECTORY: int -C_BUILTIN: int -PY_FROZEN: int -PY_CODERESOURCE: int -IMP_HOOK: int +SEARCH_ERROR: Final = 0 +PY_SOURCE: Final = 1 +PY_COMPILED: Final = 2 +C_EXTENSION: Final = 3 +PY_RESOURCE: Final = 4 +PKG_DIRECTORY: Final = 5 +C_BUILTIN: Final = 6 +PY_FROZEN: Final = 7 +PY_CODERESOURCE: Final = 8 +IMP_HOOK: Final = 9 def new_module(name: str) -> types.ModuleType: ... def get_magic() -> bytes: ... diff --git a/mypy/typeshed/stdlib/importlib/abc.pyi b/mypy/typeshed/stdlib/importlib/abc.pyi index ef87663cb72dd..72031e0e3bd2e 100644 --- a/mypy/typeshed/stdlib/importlib/abc.pyi +++ b/mypy/typeshed/stdlib/importlib/abc.pyi @@ -40,7 +40,7 @@ if sys.version_info < (3, 12): @deprecated("Deprecated since Python 3.3; removed in Python 3.12. Use `MetaPathFinder` or `PathEntryFinder` instead.") class Finder(metaclass=ABCMeta): ... -@deprecated("Deprecated as of Python 3.7: Use importlib.resources.abc.TraversableResources instead.") +@deprecated("Deprecated since Python 3.7. Use `importlib.resources.abc.TraversableResources` instead.") class ResourceLoader(Loader): @abstractmethod def get_data(self, path: str) -> bytes: ... @@ -61,7 +61,7 @@ class ExecutionLoader(InspectLoader): def get_filename(self, fullname: str) -> str: ... class SourceLoader(_bootstrap_external.SourceLoader, ResourceLoader, ExecutionLoader, metaclass=ABCMeta): # type: ignore[misc] # incompatible definitions of source_to_code in the base classes - @deprecated("Deprecated as of Python 3.3: Use importlib.resources.abc.SourceLoader.path_stats instead.") + @deprecated("Deprecated since Python 3.3. Use `importlib.resources.abc.SourceLoader.path_stats` instead.") def path_mtime(self, path: str) -> float: ... def set_data(self, path: str, data: bytes) -> None: ... def get_source(self, fullname: str) -> str | None: ... diff --git a/mypy/typeshed/stdlib/importlib/metadata/__init__.pyi b/mypy/typeshed/stdlib/importlib/metadata/__init__.pyi index d1315b2eb2f11..9286e92331c82 100644 --- a/mypy/typeshed/stdlib/importlib/metadata/__init__.pyi +++ b/mypy/typeshed/stdlib/importlib/metadata/__init__.pyi @@ -11,7 +11,7 @@ from os import PathLike from pathlib import Path from re import Pattern from typing import Any, ClassVar, Generic, NamedTuple, TypeVar, overload -from typing_extensions import Self, TypeAlias, deprecated +from typing_extensions import Self, TypeAlias, deprecated, disjoint_base _T = TypeVar("_T") _KT = TypeVar("_KT") @@ -59,23 +59,21 @@ else: value: str group: str -class EntryPoint(_EntryPointBase): - pattern: ClassVar[Pattern[str]] - if sys.version_info >= (3, 11): +if sys.version_info >= (3, 11): + class EntryPoint(_EntryPointBase): + pattern: ClassVar[Pattern[str]] name: str value: str group: str def __init__(self, name: str, value: str, group: str) -> None: ... - - def load(self) -> Any: ... # Callable[[], Any] or an importable module - @property - def extras(self) -> list[str]: ... - @property - def module(self) -> str: ... - @property - def attr(self) -> str: ... - if sys.version_info >= (3, 10): + def load(self) -> Any: ... # Callable[[], Any] or an importable module + @property + def extras(self) -> list[str]: ... + @property + def module(self) -> str: ... + @property + def attr(self) -> str: ... dist: ClassVar[Distribution | None] def matches( self, @@ -87,16 +85,43 @@ class EntryPoint(_EntryPointBase): attr: str = ..., extras: list[str] = ..., ) -> bool: ... # undocumented - - def __hash__(self) -> int: ... - if sys.version_info >= (3, 11): + def __hash__(self) -> int: ... def __eq__(self, other: object) -> bool: ... def __lt__(self, other: object) -> bool: ... - if sys.version_info < (3, 12): + if sys.version_info < (3, 12): + def __iter__(self) -> Iterator[Any]: ... # result of iter((str, Self)), really + +else: + @disjoint_base + class EntryPoint(_EntryPointBase): + pattern: ClassVar[Pattern[str]] + + def load(self) -> Any: ... # Callable[[], Any] or an importable module + @property + def extras(self) -> list[str]: ... + @property + def module(self) -> str: ... + @property + def attr(self) -> str: ... + if sys.version_info >= (3, 10): + dist: ClassVar[Distribution | None] + def matches( + self, + *, + name: str = ..., + value: str = ..., + group: str = ..., + module: str = ..., + attr: str = ..., + extras: list[str] = ..., + ) -> bool: ... # undocumented + + def __hash__(self) -> int: ... def __iter__(self) -> Iterator[Any]: ... # result of iter((str, Self)), really if sys.version_info >= (3, 12): class EntryPoints(tuple[EntryPoint, ...]): + __slots__ = () def __getitem__(self, name: str) -> EntryPoint: ... # type: ignore[override] def select( self, @@ -114,10 +139,12 @@ if sys.version_info >= (3, 12): def groups(self) -> set[str]: ... elif sys.version_info >= (3, 10): - class DeprecatedList(list[_T]): ... + class DeprecatedList(list[_T]): + __slots__ = () class EntryPoints(DeprecatedList[EntryPoint]): # use as list is deprecated since 3.10 # int argument is deprecated since 3.10 + __slots__ = () def __getitem__(self, name: int | str) -> EntryPoint: ... # type: ignore[override] def select( self, @@ -230,7 +257,7 @@ class Distribution(_distribution_parent): def name(self) -> str: ... if sys.version_info >= (3, 13): @property - def origin(self) -> types.SimpleNamespace: ... + def origin(self) -> types.SimpleNamespace | None: ... class DistributionFinder(MetaPathFinder): class Context: diff --git a/mypy/typeshed/stdlib/inspect.pyi b/mypy/typeshed/stdlib/inspect.pyi index f8ec6cad01603..55ae61617af7e 100644 --- a/mypy/typeshed/stdlib/inspect.pyi +++ b/mypy/typeshed/stdlib/inspect.pyi @@ -26,7 +26,7 @@ from types import ( WrapperDescriptorType, ) from typing import Any, ClassVar, Final, Literal, NamedTuple, Protocol, TypeVar, overload, type_check_only -from typing_extensions import ParamSpec, Self, TypeAlias, TypeGuard, TypeIs, deprecated +from typing_extensions import ParamSpec, Self, TypeAlias, TypeGuard, TypeIs, deprecated, disjoint_base if sys.version_info >= (3, 14): from annotationlib import Format @@ -336,6 +336,7 @@ class _void: ... class _empty: ... class Signature: + __slots__ = ("_return_annotation", "_parameters") def __init__( self, parameters: Sequence[Parameter] | None = None, *, return_annotation: Any = ..., __validate_parameters__: bool = True ) -> None: ... @@ -416,6 +417,7 @@ if sys.version_info >= (3, 12): def getasyncgenlocals(agen: AsyncGeneratorType[Any, Any]) -> dict[str, Any]: ... class Parameter: + __slots__ = ("_name", "_kind", "_default", "_annotation") def __init__(self, name: str, kind: _ParameterKind, *, default: Any = ..., annotation: Any = ...) -> None: ... empty = _empty @@ -447,6 +449,7 @@ class Parameter: def __hash__(self) -> int: ... class BoundArguments: + __slots__ = ("arguments", "_signature", "__weakref__") arguments: OrderedDict[str, Any] @property def args(self) -> tuple[Any, ...]: ... @@ -567,19 +570,6 @@ if sys.version_info >= (3, 11): code_context: list[str] | None index: int | None # type: ignore[assignment] - class Traceback(_Traceback): - positions: dis.Positions | None - def __new__( - cls, - filename: str, - lineno: int, - function: str, - code_context: list[str] | None, - index: int | None, - *, - positions: dis.Positions | None = None, - ) -> Self: ... - class _FrameInfo(NamedTuple): frame: FrameType filename: str @@ -588,19 +578,63 @@ if sys.version_info >= (3, 11): code_context: list[str] | None index: int | None # type: ignore[assignment] - class FrameInfo(_FrameInfo): - positions: dis.Positions | None - def __new__( - cls, - frame: FrameType, - filename: str, - lineno: int, - function: str, - code_context: list[str] | None, - index: int | None, - *, - positions: dis.Positions | None = None, - ) -> Self: ... + if sys.version_info >= (3, 12): + class Traceback(_Traceback): + positions: dis.Positions | None + def __new__( + cls, + filename: str, + lineno: int, + function: str, + code_context: list[str] | None, + index: int | None, + *, + positions: dis.Positions | None = None, + ) -> Self: ... + + class FrameInfo(_FrameInfo): + positions: dis.Positions | None + def __new__( + cls, + frame: FrameType, + filename: str, + lineno: int, + function: str, + code_context: list[str] | None, + index: int | None, + *, + positions: dis.Positions | None = None, + ) -> Self: ... + + else: + @disjoint_base + class Traceback(_Traceback): + positions: dis.Positions | None + def __new__( + cls, + filename: str, + lineno: int, + function: str, + code_context: list[str] | None, + index: int | None, + *, + positions: dis.Positions | None = None, + ) -> Self: ... + + @disjoint_base + class FrameInfo(_FrameInfo): + positions: dis.Positions | None + def __new__( + cls, + frame: FrameType, + filename: str, + lineno: int, + function: str, + code_context: list[str] | None, + index: int | None, + *, + positions: dis.Positions | None = None, + ) -> Self: ... else: class Traceback(NamedTuple): diff --git a/mypy/typeshed/stdlib/io.pyi b/mypy/typeshed/stdlib/io.pyi index 1313df183d36d..d301d700e9d0f 100644 --- a/mypy/typeshed/stdlib/io.pyi +++ b/mypy/typeshed/stdlib/io.pyi @@ -67,7 +67,9 @@ class TextIOBase(_TextIOBase, IOBase): ... if sys.version_info >= (3, 14): class Reader(Protocol[_T_co]): + __slots__ = () def read(self, size: int = ..., /) -> _T_co: ... class Writer(Protocol[_T_contra]): + __slots__ = () def write(self, data: _T_contra, /) -> int: ... diff --git a/mypy/typeshed/stdlib/ipaddress.pyi b/mypy/typeshed/stdlib/ipaddress.pyi index 6d49eb8bd94ac..e2f3defa2deac 100644 --- a/mypy/typeshed/stdlib/ipaddress.pyi +++ b/mypy/typeshed/stdlib/ipaddress.pyi @@ -22,6 +22,7 @@ def ip_interface( ) -> IPv4Interface | IPv6Interface: ... class _IPAddressBase: + __slots__ = () @property def compressed(self) -> str: ... @property @@ -33,6 +34,7 @@ class _IPAddressBase: def version(self) -> int: ... class _BaseAddress(_IPAddressBase): + __slots__ = () def __add__(self, other: int) -> Self: ... def __hash__(self) -> int: ... def __int__(self) -> int: ... @@ -71,7 +73,7 @@ class _BaseNetwork(_IPAddressBase, Generic[_A]): @property def broadcast_address(self) -> _A: ... def compare_networks(self, other: Self) -> int: ... - def hosts(self) -> Iterator[_A]: ... + def hosts(self) -> Iterator[_A] | list[_A]: ... @property def is_global(self) -> bool: ... @property @@ -105,6 +107,7 @@ class _BaseNetwork(_IPAddressBase, Generic[_A]): def hostmask(self) -> _A: ... class _BaseV4: + __slots__ = () if sys.version_info >= (3, 14): version: Final = 4 max_prefixlen: Final = 32 @@ -115,6 +118,7 @@ class _BaseV4: def max_prefixlen(self) -> Literal[32]: ... class IPv4Address(_BaseV4, _BaseAddress): + __slots__ = ("_ip", "__weakref__") def __init__(self, address: object) -> None: ... @property def is_global(self) -> bool: ... @@ -156,6 +160,7 @@ class IPv4Interface(IPv4Address): def with_prefixlen(self) -> str: ... class _BaseV6: + __slots__ = () if sys.version_info >= (3, 14): version: Final = 6 max_prefixlen: Final = 128 @@ -166,6 +171,7 @@ class _BaseV6: def max_prefixlen(self) -> Literal[128]: ... class IPv6Address(_BaseV6, _BaseAddress): + __slots__ = ("_ip", "_scope_id", "__weakref__") def __init__(self, address: object) -> None: ... @property def is_global(self) -> bool: ... diff --git a/mypy/typeshed/stdlib/itertools.pyi b/mypy/typeshed/stdlib/itertools.pyi index 7d05b1318680b..73745fe92d9eb 100644 --- a/mypy/typeshed/stdlib/itertools.pyi +++ b/mypy/typeshed/stdlib/itertools.pyi @@ -3,7 +3,7 @@ from _typeshed import MaybeNone from collections.abc import Callable, Iterable, Iterator from types import GenericAlias from typing import Any, Generic, Literal, SupportsComplex, SupportsFloat, SupportsIndex, SupportsInt, TypeVar, overload -from typing_extensions import Self, TypeAlias +from typing_extensions import Self, TypeAlias, disjoint_base _T = TypeVar("_T") _S = TypeVar("_S") @@ -27,6 +27,7 @@ _Predicate: TypeAlias = Callable[[_T], object] # Technically count can take anything that implements a number protocol and has an add method # but we can't enforce the add method +@disjoint_base class count(Iterator[_N]): @overload def __new__(cls) -> count[int]: ... @@ -37,11 +38,13 @@ class count(Iterator[_N]): def __next__(self) -> _N: ... def __iter__(self) -> Self: ... +@disjoint_base class cycle(Iterator[_T]): def __new__(cls, iterable: Iterable[_T], /) -> Self: ... def __next__(self) -> _T: ... def __iter__(self) -> Self: ... +@disjoint_base class repeat(Iterator[_T]): @overload def __new__(cls, object: _T) -> Self: ... @@ -51,6 +54,7 @@ class repeat(Iterator[_T]): def __iter__(self) -> Self: ... def __length_hint__(self) -> int: ... +@disjoint_base class accumulate(Iterator[_T]): @overload def __new__(cls, iterable: Iterable[_T], func: None = None, *, initial: _T | None = ...) -> Self: ... @@ -59,6 +63,7 @@ class accumulate(Iterator[_T]): def __iter__(self) -> Self: ... def __next__(self) -> _T: ... +@disjoint_base class chain(Iterator[_T]): def __new__(cls, *iterables: Iterable[_T]) -> Self: ... def __next__(self) -> _T: ... @@ -68,21 +73,25 @@ class chain(Iterator[_T]): def from_iterable(cls: type[Any], iterable: Iterable[Iterable[_S]], /) -> chain[_S]: ... def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... +@disjoint_base class compress(Iterator[_T]): def __new__(cls, data: Iterable[_T], selectors: Iterable[Any]) -> Self: ... def __iter__(self) -> Self: ... def __next__(self) -> _T: ... +@disjoint_base class dropwhile(Iterator[_T]): def __new__(cls, predicate: _Predicate[_T], iterable: Iterable[_T], /) -> Self: ... def __iter__(self) -> Self: ... def __next__(self) -> _T: ... +@disjoint_base class filterfalse(Iterator[_T]): def __new__(cls, function: _Predicate[_T] | None, iterable: Iterable[_T], /) -> Self: ... def __iter__(self) -> Self: ... def __next__(self) -> _T: ... +@disjoint_base class groupby(Iterator[tuple[_T_co, Iterator[_S_co]]], Generic[_T_co, _S_co]): @overload def __new__(cls, iterable: Iterable[_T1], key: None = None) -> groupby[_T1, _T1]: ... @@ -91,6 +100,7 @@ class groupby(Iterator[tuple[_T_co, Iterator[_S_co]]], Generic[_T_co, _S_co]): def __iter__(self) -> Self: ... def __next__(self) -> tuple[_T_co, Iterator[_S_co]]: ... +@disjoint_base class islice(Iterator[_T]): @overload def __new__(cls, iterable: Iterable[_T], stop: int | None, /) -> Self: ... @@ -99,18 +109,20 @@ class islice(Iterator[_T]): def __iter__(self) -> Self: ... def __next__(self) -> _T: ... +@disjoint_base class starmap(Iterator[_T_co]): def __new__(cls, function: Callable[..., _T], iterable: Iterable[Iterable[Any]], /) -> starmap[_T]: ... def __iter__(self) -> Self: ... def __next__(self) -> _T_co: ... +@disjoint_base class takewhile(Iterator[_T]): def __new__(cls, predicate: _Predicate[_T], iterable: Iterable[_T], /) -> Self: ... def __iter__(self) -> Self: ... def __next__(self) -> _T: ... def tee(iterable: Iterable[_T], n: int = 2, /) -> tuple[Iterator[_T], ...]: ... - +@disjoint_base class zip_longest(Iterator[_T_co]): # one iterable (fillvalue doesn't matter) @overload @@ -189,6 +201,7 @@ class zip_longest(Iterator[_T_co]): def __iter__(self) -> Self: ... def __next__(self) -> _T_co: ... +@disjoint_base class product(Iterator[_T_co]): @overload def __new__(cls, iter1: Iterable[_T1], /) -> product[tuple[_T1]]: ... @@ -274,6 +287,7 @@ class product(Iterator[_T_co]): def __iter__(self) -> Self: ... def __next__(self) -> _T_co: ... +@disjoint_base class permutations(Iterator[_T_co]): @overload def __new__(cls, iterable: Iterable[_T], r: Literal[2]) -> permutations[tuple[_T, _T]]: ... @@ -288,6 +302,7 @@ class permutations(Iterator[_T_co]): def __iter__(self) -> Self: ... def __next__(self) -> _T_co: ... +@disjoint_base class combinations(Iterator[_T_co]): @overload def __new__(cls, iterable: Iterable[_T], r: Literal[2]) -> combinations[tuple[_T, _T]]: ... @@ -302,6 +317,7 @@ class combinations(Iterator[_T_co]): def __iter__(self) -> Self: ... def __next__(self) -> _T_co: ... +@disjoint_base class combinations_with_replacement(Iterator[_T_co]): @overload def __new__(cls, iterable: Iterable[_T], r: Literal[2]) -> combinations_with_replacement[tuple[_T, _T]]: ... @@ -317,12 +333,14 @@ class combinations_with_replacement(Iterator[_T_co]): def __next__(self) -> _T_co: ... if sys.version_info >= (3, 10): + @disjoint_base class pairwise(Iterator[_T_co]): def __new__(cls, iterable: Iterable[_T], /) -> pairwise[tuple[_T, _T]]: ... def __iter__(self) -> Self: ... def __next__(self) -> _T_co: ... if sys.version_info >= (3, 12): + @disjoint_base class batched(Iterator[tuple[_T_co, ...]], Generic[_T_co]): if sys.version_info >= (3, 13): def __new__(cls, iterable: Iterable[_T_co], n: int, *, strict: bool = False) -> Self: ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_tuple_params.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_tuple_params.pyi index bfaa9970c996c..7f4f7f4e8656e 100644 --- a/mypy/typeshed/stdlib/lib2to3/fixes/fix_tuple_params.pyi +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_tuple_params.pyi @@ -1,4 +1,3 @@ -from _typeshed import Incomplete from typing import ClassVar, Literal from .. import fixer_base @@ -13,5 +12,5 @@ class FixTupleParams(fixer_base.BaseFix): def simplify_args(node): ... def find_params(node): ... -def map_to_index(param_list, prefix=..., d: Incomplete | None = ...): ... +def map_to_index(param_list, prefix=[], d=None): ... def tuple_name(param_list): ... diff --git a/mypy/typeshed/stdlib/logging/__init__.pyi b/mypy/typeshed/stdlib/logging/__init__.pyi index 03c79cc3e2658..8248f82ea87ac 100644 --- a/mypy/typeshed/stdlib/logging/__init__.pyi +++ b/mypy/typeshed/stdlib/logging/__init__.pyi @@ -155,7 +155,7 @@ class Logger(Filterer): stacklevel: int = 1, extra: Mapping[str, object] | None = None, ) -> None: ... - @deprecated("Deprecated; use warning() instead.") + @deprecated("Deprecated since Python 3.3. Use `Logger.warning()` instead.") def warn( self, msg: object, @@ -409,7 +409,7 @@ class LoggerAdapter(Generic[_L]): extra: Mapping[str, object] | None = None, **kwargs: object, ) -> None: ... - @deprecated("Deprecated; use warning() instead.") + @deprecated("Deprecated since Python 3.3. Use `LoggerAdapter.warning()` instead.") def warn( self, msg: object, @@ -519,7 +519,7 @@ def warning( stacklevel: int = 1, extra: Mapping[str, object] | None = None, ) -> None: ... -@deprecated("Deprecated; use warning() instead.") +@deprecated("Deprecated since Python 3.3. Use `warning()` instead.") def warn( msg: object, *args: object, @@ -659,4 +659,4 @@ class StringTemplateStyle(PercentStyle): # undocumented _STYLES: Final[dict[str, tuple[PercentStyle, str]]] -BASIC_FORMAT: Final[str] +BASIC_FORMAT: Final = "%(levelname)s:%(name)s:%(message)s" diff --git a/mypy/typeshed/stdlib/logging/config.pyi b/mypy/typeshed/stdlib/logging/config.pyi index 000ba1ebb06e0..72412ddc2cea5 100644 --- a/mypy/typeshed/stdlib/logging/config.pyi +++ b/mypy/typeshed/stdlib/logging/config.pyi @@ -5,11 +5,11 @@ from configparser import RawConfigParser from re import Pattern from threading import Thread from typing import IO, Any, Final, Literal, SupportsIndex, TypedDict, overload, type_check_only -from typing_extensions import Required, TypeAlias +from typing_extensions import Required, TypeAlias, disjoint_base from . import Filter, Filterer, Formatter, Handler, Logger, _FilterType, _FormatStyle, _Level -DEFAULT_LOGGING_CONFIG_PORT: int +DEFAULT_LOGGING_CONFIG_PORT: Final = 9030 RESET_ERROR: Final[int] # undocumented IDENTIFIER: Final[Pattern[str]] # undocumented @@ -100,13 +100,22 @@ class ConvertingList(list[Any], ConvertingMixin): # undocumented def __getitem__(self, key: slice) -> Any: ... def pop(self, idx: SupportsIndex = -1) -> Any: ... -class ConvertingTuple(tuple[Any, ...], ConvertingMixin): # undocumented - @overload - def __getitem__(self, key: SupportsIndex) -> Any: ... - @overload - def __getitem__(self, key: slice) -> Any: ... +if sys.version_info >= (3, 12): + class ConvertingTuple(tuple[Any, ...], ConvertingMixin): # undocumented + @overload + def __getitem__(self, key: SupportsIndex) -> Any: ... + @overload + def __getitem__(self, key: slice) -> Any: ... -class BaseConfigurator: # undocumented +else: + @disjoint_base + class ConvertingTuple(tuple[Any, ...], ConvertingMixin): # undocumented + @overload + def __getitem__(self, key: SupportsIndex) -> Any: ... + @overload + def __getitem__(self, key: slice) -> Any: ... + +class BaseConfigurator: CONVERT_PATTERN: Pattern[str] WORD_PATTERN: Pattern[str] DOT_PATTERN: Pattern[str] @@ -115,6 +124,8 @@ class BaseConfigurator: # undocumented value_converters: dict[str, str] importer: Callable[..., Any] + config: dict[str, Any] # undocumented + def __init__(self, config: _DictConfigArgs | dict[str, Any]) -> None: ... def resolve(self, s: str) -> Any: ... def ext_convert(self, value: str) -> Any: ... diff --git a/mypy/typeshed/stdlib/logging/handlers.pyi b/mypy/typeshed/stdlib/logging/handlers.pyi index e231d1de3fb59..535f1c6851831 100644 --- a/mypy/typeshed/stdlib/logging/handlers.pyi +++ b/mypy/typeshed/stdlib/logging/handlers.pyi @@ -14,12 +14,12 @@ from typing_extensions import Self _T = TypeVar("_T") -DEFAULT_TCP_LOGGING_PORT: Final[int] -DEFAULT_UDP_LOGGING_PORT: Final[int] -DEFAULT_HTTP_LOGGING_PORT: Final[int] -DEFAULT_SOAP_LOGGING_PORT: Final[int] -SYSLOG_UDP_PORT: Final[int] -SYSLOG_TCP_PORT: Final[int] +DEFAULT_TCP_LOGGING_PORT: Final = 9020 +DEFAULT_UDP_LOGGING_PORT: Final = 9021 +DEFAULT_HTTP_LOGGING_PORT: Final = 9022 +DEFAULT_SOAP_LOGGING_PORT: Final = 9023 +SYSLOG_UDP_PORT: Final = 514 +SYSLOG_TCP_PORT: Final = 514 class WatchedFileHandler(FileHandler): dev: int # undocumented diff --git a/mypy/typeshed/stdlib/mmap.pyi b/mypy/typeshed/stdlib/mmap.pyi index 261a2bfdfc449..8a5baba629141 100644 --- a/mypy/typeshed/stdlib/mmap.pyi +++ b/mypy/typeshed/stdlib/mmap.pyi @@ -3,7 +3,7 @@ import sys from _typeshed import ReadableBuffer, Unused from collections.abc import Iterator from typing import Final, Literal, NoReturn, overload -from typing_extensions import Self +from typing_extensions import Self, disjoint_base ACCESS_DEFAULT: Final = 0 ACCESS_READ: Final = 1 @@ -31,9 +31,10 @@ if sys.platform != "win32": PAGESIZE: Final[int] +@disjoint_base class mmap: if sys.platform == "win32": - def __init__(self, fileno: int, length: int, tagname: str | None = None, access: int = 0, offset: int = 0) -> None: ... + def __new__(self, fileno: int, length: int, tagname: str | None = None, access: int = 0, offset: int = 0) -> Self: ... else: if sys.version_info >= (3, 13): def __new__( diff --git a/mypy/typeshed/stdlib/msilib/__init__.pyi b/mypy/typeshed/stdlib/msilib/__init__.pyi index 3e43cbc44f520..622f585f5beea 100644 --- a/mypy/typeshed/stdlib/msilib/__init__.pyi +++ b/mypy/typeshed/stdlib/msilib/__init__.pyi @@ -1,26 +1,26 @@ import sys from collections.abc import Container, Iterable, Sequence from types import ModuleType -from typing import Any, Literal +from typing import Any, Final if sys.platform == "win32": from _msi import * from _msi import _Database - AMD64: bool - Win64: bool + AMD64: Final[bool] + Win64: Final[bool] - datasizemask: Literal[0x00FF] - type_valid: Literal[0x0100] - type_localizable: Literal[0x0200] - typemask: Literal[0x0C00] - type_long: Literal[0x0000] - type_short: Literal[0x0400] - type_string: Literal[0x0C00] - type_binary: Literal[0x0800] - type_nullable: Literal[0x1000] - type_key: Literal[0x2000] - knownbits: Literal[0x3FFF] + datasizemask: Final = 0x00FF + type_valid: Final = 0x0100 + type_localizable: Final = 0x0200 + typemask: Final = 0x0C00 + type_long: Final = 0x0000 + type_short: Final = 0x0400 + type_string: Final = 0x0C00 + type_binary: Final = 0x0800 + type_nullable: Final = 0x1000 + type_key: Final = 0x2000 + knownbits: Final = 0x3FFF class Table: name: str diff --git a/mypy/typeshed/stdlib/msilib/schema.pyi b/mypy/typeshed/stdlib/msilib/schema.pyi index 4ad9a1783fcd0..3bbdc41a1e8ec 100644 --- a/mypy/typeshed/stdlib/msilib/schema.pyi +++ b/mypy/typeshed/stdlib/msilib/schema.pyi @@ -1,4 +1,5 @@ import sys +from typing import Final if sys.platform == "win32": from . import Table @@ -89,6 +90,6 @@ if sys.platform == "win32": Upgrade: Table Verb: Table - tables: list[Table] + tables: Final[list[Table]] _Validation_records: list[tuple[str, str, str, int | None, int | None, str | None, int | None, str | None, str | None, str]] diff --git a/mypy/typeshed/stdlib/msilib/sequence.pyi b/mypy/typeshed/stdlib/msilib/sequence.pyi index b8af09f46e65f..a9f5c24717bd3 100644 --- a/mypy/typeshed/stdlib/msilib/sequence.pyi +++ b/mypy/typeshed/stdlib/msilib/sequence.pyi @@ -1,13 +1,14 @@ import sys +from typing import Final from typing_extensions import TypeAlias if sys.platform == "win32": _SequenceType: TypeAlias = list[tuple[str, str | None, int]] - AdminExecuteSequence: _SequenceType - AdminUISequence: _SequenceType - AdvtExecuteSequence: _SequenceType - InstallExecuteSequence: _SequenceType - InstallUISequence: _SequenceType + AdminExecuteSequence: Final[_SequenceType] + AdminUISequence: Final[_SequenceType] + AdvtExecuteSequence: Final[_SequenceType] + InstallExecuteSequence: Final[_SequenceType] + InstallUISequence: Final[_SequenceType] - tables: list[str] + tables: Final[list[str]] diff --git a/mypy/typeshed/stdlib/msilib/text.pyi b/mypy/typeshed/stdlib/msilib/text.pyi index 441c843ca6cfe..da3c5fd0fb7a1 100644 --- a/mypy/typeshed/stdlib/msilib/text.pyi +++ b/mypy/typeshed/stdlib/msilib/text.pyi @@ -1,7 +1,8 @@ import sys +from typing import Final if sys.platform == "win32": - ActionText: list[tuple[str, str, str | None]] - UIText: list[tuple[str, str | None]] + ActionText: Final[list[tuple[str, str, str | None]]] + UIText: Final[list[tuple[str, str | None]]] dirname: str - tables: list[str] + tables: Final[list[str]] diff --git a/mypy/typeshed/stdlib/msvcrt.pyi b/mypy/typeshed/stdlib/msvcrt.pyi index 403a5d9335227..5feca8eab5c1c 100644 --- a/mypy/typeshed/stdlib/msvcrt.pyi +++ b/mypy/typeshed/stdlib/msvcrt.pyi @@ -9,10 +9,10 @@ if sys.platform == "win32": LK_NBLCK: Final = 2 LK_RLCK: Final = 3 LK_NBRLCK: Final = 4 - SEM_FAILCRITICALERRORS: int - SEM_NOALIGNMENTFAULTEXCEPT: int - SEM_NOGPFAULTERRORBOX: int - SEM_NOOPENFILEERRORBOX: int + SEM_FAILCRITICALERRORS: Final = 0x0001 + SEM_NOALIGNMENTFAULTEXCEPT: Final = 0x0004 + SEM_NOGPFAULTERRORBOX: Final = 0x0002 + SEM_NOOPENFILEERRORBOX: Final = 0x8000 def locking(fd: int, mode: int, nbytes: int, /) -> None: ... def setmode(fd: int, mode: int, /) -> int: ... def open_osfhandle(handle: int, flags: int, /) -> int: ... diff --git a/mypy/typeshed/stdlib/multiprocessing/managers.pyi b/mypy/typeshed/stdlib/multiprocessing/managers.pyi index b0ccac41b9253..5efe69a973777 100644 --- a/mypy/typeshed/stdlib/multiprocessing/managers.pyi +++ b/mypy/typeshed/stdlib/multiprocessing/managers.pyi @@ -38,6 +38,7 @@ class Namespace: _Namespace: TypeAlias = Namespace class Token: + __slots__ = ("typeid", "address", "id") typeid: str | bytes | None address: _Address | None id: str | bytes | int | None diff --git a/mypy/typeshed/stdlib/multiprocessing/util.pyi b/mypy/typeshed/stdlib/multiprocessing/util.pyi index ecb4a7ddec7d2..3583194c77e29 100644 --- a/mypy/typeshed/stdlib/multiprocessing/util.pyi +++ b/mypy/typeshed/stdlib/multiprocessing/util.pyi @@ -52,7 +52,7 @@ def get_logger() -> Logger: ... def log_to_stderr(level: _LoggingLevel | None = None) -> Logger: ... def is_abstract_socket_namespace(address: str | bytes | None) -> bool: ... -abstract_sockets_supported: bool +abstract_sockets_supported: Final[bool] def get_temp_dir() -> str: ... def register_after_fork(obj: _T, func: Callable[[_T], object]) -> None: ... diff --git a/mypy/typeshed/stdlib/nturl2path.pyi b/mypy/typeshed/stdlib/nturl2path.pyi index c38a359469d2d..014af8a0fd2ed 100644 --- a/mypy/typeshed/stdlib/nturl2path.pyi +++ b/mypy/typeshed/stdlib/nturl2path.pyi @@ -2,9 +2,9 @@ import sys from typing_extensions import deprecated if sys.version_info >= (3, 14): - @deprecated("nturl2path module was deprecated since Python 3.14") + @deprecated("The `nturl2path` module is deprecated since Python 3.14.") def url2pathname(url: str) -> str: ... - @deprecated("nturl2path module was deprecated since Python 3.14") + @deprecated("The `nturl2path` module is deprecated since Python 3.14.") def pathname2url(p: str) -> str: ... else: diff --git a/mypy/typeshed/stdlib/numbers.pyi b/mypy/typeshed/stdlib/numbers.pyi index b24591719cfff..64fb16581e952 100644 --- a/mypy/typeshed/stdlib/numbers.pyi +++ b/mypy/typeshed/stdlib/numbers.pyi @@ -61,12 +61,14 @@ class _IntegralLike(_RealLike, Protocol): ################# class Number(metaclass=ABCMeta): + __slots__ = () @abstractmethod def __hash__(self) -> int: ... # See comment at the top of the file # for why some of these return types are purposefully vague class Complex(Number, _ComplexLike): + __slots__ = () @abstractmethod def __complex__(self) -> complex: ... def __bool__(self) -> bool: ... @@ -109,6 +111,7 @@ class Complex(Number, _ComplexLike): # See comment at the top of the file # for why some of these return types are purposefully vague class Real(Complex, _RealLike): + __slots__ = () @abstractmethod def __float__(self) -> float: ... @abstractmethod @@ -153,6 +156,7 @@ class Real(Complex, _RealLike): # See comment at the top of the file # for why some of these return types are purposefully vague class Rational(Real): + __slots__ = () @property @abstractmethod def numerator(self) -> _IntegralLike: ... @@ -164,6 +168,7 @@ class Rational(Real): # See comment at the top of the file # for why some of these return types are purposefully vague class Integral(Rational, _IntegralLike): + __slots__ = () @abstractmethod def __int__(self) -> int: ... def __index__(self) -> int: ... diff --git a/mypy/typeshed/stdlib/opcode.pyi b/mypy/typeshed/stdlib/opcode.pyi index a5a3a79c323b0..ed0e96ef1cb9c 100644 --- a/mypy/typeshed/stdlib/opcode.pyi +++ b/mypy/typeshed/stdlib/opcode.pyi @@ -1,5 +1,5 @@ import sys -from typing import Literal +from typing import Final, Literal __all__ = [ "cmp_op", @@ -24,24 +24,24 @@ if sys.version_info >= (3, 13): __all__ += ["hasjump"] cmp_op: tuple[Literal["<"], Literal["<="], Literal["=="], Literal["!="], Literal[">"], Literal[">="]] -hasconst: list[int] -hasname: list[int] -hasjrel: list[int] -hasjabs: list[int] -haslocal: list[int] -hascompare: list[int] -hasfree: list[int] +hasconst: Final[list[int]] +hasname: Final[list[int]] +hasjrel: Final[list[int]] +hasjabs: Final[list[int]] +haslocal: Final[list[int]] +hascompare: Final[list[int]] +hasfree: Final[list[int]] if sys.version_info >= (3, 12): - hasarg: list[int] - hasexc: list[int] + hasarg: Final[list[int]] + hasexc: Final[list[int]] else: - hasnargs: list[int] + hasnargs: Final[list[int]] if sys.version_info >= (3, 13): - hasjump: list[int] -opname: list[str] + hasjump: Final[list[int]] +opname: Final[list[str]] -opmap: dict[str, int] -HAVE_ARGUMENT: int -EXTENDED_ARG: int +opmap: Final[dict[str, int]] +HAVE_ARGUMENT: Final = 43 +EXTENDED_ARG: Final = 69 def stack_effect(opcode: int, oparg: int | None = None, /, *, jump: bool | None = None) -> int: ... diff --git a/mypy/typeshed/stdlib/os/__init__.pyi b/mypy/typeshed/stdlib/os/__init__.pyi index 4047bb0f1c4dc..71c79dfac399f 100644 --- a/mypy/typeshed/stdlib/os/__init__.pyi +++ b/mypy/typeshed/stdlib/os/__init__.pyi @@ -509,22 +509,22 @@ supports_follow_symlinks: set[Callable[..., Any]] if sys.platform != "win32": # Unix only - PRIO_PROCESS: int - PRIO_PGRP: int - PRIO_USER: int + PRIO_PROCESS: Final[int] + PRIO_PGRP: Final[int] + PRIO_USER: Final[int] - F_LOCK: int - F_TLOCK: int - F_ULOCK: int - F_TEST: int + F_LOCK: Final[int] + F_TLOCK: Final[int] + F_ULOCK: Final[int] + F_TEST: Final[int] if sys.platform != "darwin": - POSIX_FADV_NORMAL: int - POSIX_FADV_SEQUENTIAL: int - POSIX_FADV_RANDOM: int - POSIX_FADV_NOREUSE: int - POSIX_FADV_WILLNEED: int - POSIX_FADV_DONTNEED: int + POSIX_FADV_NORMAL: Final[int] + POSIX_FADV_SEQUENTIAL: Final[int] + POSIX_FADV_RANDOM: Final[int] + POSIX_FADV_NOREUSE: Final[int] + POSIX_FADV_WILLNEED: Final[int] + POSIX_FADV_DONTNEED: Final[int] if sys.platform != "linux" and sys.platform != "darwin": # In the os-module docs, these are marked as being available @@ -534,69 +534,69 @@ if sys.platform != "win32": # so the sys-module docs recommend doing `if sys.platform.startswith('freebsd')` # to detect FreeBSD builds. Unfortunately that would be too dynamic # for type checkers, however. - SF_NODISKIO: int - SF_MNOWAIT: int - SF_SYNC: int + SF_NODISKIO: Final[int] + SF_MNOWAIT: Final[int] + SF_SYNC: Final[int] if sys.version_info >= (3, 11): - SF_NOCACHE: int + SF_NOCACHE: Final[int] if sys.platform == "linux": - XATTR_SIZE_MAX: int - XATTR_CREATE: int - XATTR_REPLACE: int + XATTR_SIZE_MAX: Final[int] + XATTR_CREATE: Final[int] + XATTR_REPLACE: Final[int] - P_PID: int - P_PGID: int - P_ALL: int + P_PID: Final[int] + P_PGID: Final[int] + P_ALL: Final[int] if sys.platform == "linux": - P_PIDFD: int - - WEXITED: int - WSTOPPED: int - WNOWAIT: int - - CLD_EXITED: int - CLD_DUMPED: int - CLD_TRAPPED: int - CLD_CONTINUED: int - CLD_KILLED: int - CLD_STOPPED: int - - SCHED_OTHER: int - SCHED_FIFO: int - SCHED_RR: int + P_PIDFD: Final[int] + + WEXITED: Final[int] + WSTOPPED: Final[int] + WNOWAIT: Final[int] + + CLD_EXITED: Final[int] + CLD_DUMPED: Final[int] + CLD_TRAPPED: Final[int] + CLD_CONTINUED: Final[int] + CLD_KILLED: Final[int] + CLD_STOPPED: Final[int] + + SCHED_OTHER: Final[int] + SCHED_FIFO: Final[int] + SCHED_RR: Final[int] if sys.platform != "darwin" and sys.platform != "linux": - SCHED_SPORADIC: int + SCHED_SPORADIC: Final[int] if sys.platform == "linux": - SCHED_BATCH: int - SCHED_IDLE: int - SCHED_RESET_ON_FORK: int + SCHED_BATCH: Final[int] + SCHED_IDLE: Final[int] + SCHED_RESET_ON_FORK: Final[int] if sys.version_info >= (3, 14) and sys.platform == "linux": - SCHED_DEADLINE: int - SCHED_NORMAL: int + SCHED_DEADLINE: Final[int] + SCHED_NORMAL: Final[int] if sys.platform != "win32": - RTLD_LAZY: int - RTLD_NOW: int - RTLD_GLOBAL: int - RTLD_LOCAL: int - RTLD_NODELETE: int - RTLD_NOLOAD: int + RTLD_LAZY: Final[int] + RTLD_NOW: Final[int] + RTLD_GLOBAL: Final[int] + RTLD_LOCAL: Final[int] + RTLD_NODELETE: Final[int] + RTLD_NOLOAD: Final[int] if sys.platform == "linux": - RTLD_DEEPBIND: int - GRND_NONBLOCK: int - GRND_RANDOM: int + RTLD_DEEPBIND: Final[int] + GRND_NONBLOCK: Final[int] + GRND_RANDOM: Final[int] if sys.platform == "darwin" and sys.version_info >= (3, 12): - PRIO_DARWIN_BG: int - PRIO_DARWIN_NONUI: int - PRIO_DARWIN_PROCESS: int - PRIO_DARWIN_THREAD: int + PRIO_DARWIN_BG: Final[int] + PRIO_DARWIN_NONUI: Final[int] + PRIO_DARWIN_PROCESS: Final[int] + PRIO_DARWIN_THREAD: Final[int] SEEK_SET: Final = 0 SEEK_CUR: Final = 1 @@ -605,74 +605,74 @@ if sys.platform != "win32": SEEK_DATA: Final = 3 SEEK_HOLE: Final = 4 -O_RDONLY: int -O_WRONLY: int -O_RDWR: int -O_APPEND: int -O_CREAT: int -O_EXCL: int -O_TRUNC: int +O_RDONLY: Final[int] +O_WRONLY: Final[int] +O_RDWR: Final[int] +O_APPEND: Final[int] +O_CREAT: Final[int] +O_EXCL: Final[int] +O_TRUNC: Final[int] if sys.platform == "win32": - O_BINARY: int - O_NOINHERIT: int - O_SHORT_LIVED: int - O_TEMPORARY: int - O_RANDOM: int - O_SEQUENTIAL: int - O_TEXT: int + O_BINARY: Final[int] + O_NOINHERIT: Final[int] + O_SHORT_LIVED: Final[int] + O_TEMPORARY: Final[int] + O_RANDOM: Final[int] + O_SEQUENTIAL: Final[int] + O_TEXT: Final[int] if sys.platform != "win32": - O_DSYNC: int - O_SYNC: int - O_NDELAY: int - O_NONBLOCK: int - O_NOCTTY: int - O_CLOEXEC: int - O_ASYNC: int # Gnu extension if in C library - O_DIRECTORY: int # Gnu extension if in C library - O_NOFOLLOW: int # Gnu extension if in C library - O_ACCMODE: int # TODO: when does this exist? + O_DSYNC: Final[int] + O_SYNC: Final[int] + O_NDELAY: Final[int] + O_NONBLOCK: Final[int] + O_NOCTTY: Final[int] + O_CLOEXEC: Final[int] + O_ASYNC: Final[int] # Gnu extension if in C library + O_DIRECTORY: Final[int] # Gnu extension if in C library + O_NOFOLLOW: Final[int] # Gnu extension if in C library + O_ACCMODE: Final[int] # TODO: when does this exist? if sys.platform == "linux": - O_RSYNC: int - O_DIRECT: int # Gnu extension if in C library - O_NOATIME: int # Gnu extension if in C library - O_PATH: int # Gnu extension if in C library - O_TMPFILE: int # Gnu extension if in C library - O_LARGEFILE: int # Gnu extension if in C library + O_RSYNC: Final[int] + O_DIRECT: Final[int] # Gnu extension if in C library + O_NOATIME: Final[int] # Gnu extension if in C library + O_PATH: Final[int] # Gnu extension if in C library + O_TMPFILE: Final[int] # Gnu extension if in C library + O_LARGEFILE: Final[int] # Gnu extension if in C library if sys.platform != "linux" and sys.platform != "win32": - O_SHLOCK: int - O_EXLOCK: int + O_SHLOCK: Final[int] + O_EXLOCK: Final[int] if sys.platform == "darwin" and sys.version_info >= (3, 10): - O_EVTONLY: int - O_NOFOLLOW_ANY: int - O_SYMLINK: int + O_EVTONLY: Final[int] + O_NOFOLLOW_ANY: Final[int] + O_SYMLINK: Final[int] if sys.platform != "win32" and sys.version_info >= (3, 10): - O_FSYNC: int + O_FSYNC: Final[int] if sys.platform != "linux" and sys.platform != "win32" and sys.version_info >= (3, 13): - O_EXEC: int - O_SEARCH: int + O_EXEC: Final[int] + O_SEARCH: Final[int] if sys.platform != "win32" and sys.platform != "darwin": # posix, but apparently missing on macos - ST_APPEND: int - ST_MANDLOCK: int - ST_NOATIME: int - ST_NODEV: int - ST_NODIRATIME: int - ST_NOEXEC: int - ST_RELATIME: int - ST_SYNCHRONOUS: int - ST_WRITE: int + ST_APPEND: Final[int] + ST_MANDLOCK: Final[int] + ST_NOATIME: Final[int] + ST_NODEV: Final[int] + ST_NODIRATIME: Final[int] + ST_NOEXEC: Final[int] + ST_RELATIME: Final[int] + ST_SYNCHRONOUS: Final[int] + ST_WRITE: Final[int] if sys.platform != "win32": - NGROUPS_MAX: int - ST_NOSUID: int - ST_RDONLY: int + NGROUPS_MAX: Final[int] + ST_NOSUID: Final[int] + ST_RDONLY: Final[int] curdir: str pardir: str @@ -688,10 +688,10 @@ linesep: Literal["\n", "\r\n"] devnull: str name: str -F_OK: int -R_OK: int -W_OK: int -X_OK: int +F_OK: Final = 0 +R_OK: Final = 4 +W_OK: Final = 2 +X_OK: Final = 1 _EnvironCodeFunc: TypeAlias = Callable[[AnyStr], AnyStr] @@ -730,47 +730,47 @@ if sys.platform != "win32": environb: _Environ[bytes] if sys.version_info >= (3, 11) or sys.platform != "win32": - EX_OK: int + EX_OK: Final[int] if sys.platform != "win32": confstr_names: dict[str, int] pathconf_names: dict[str, int] sysconf_names: dict[str, int] - EX_USAGE: int - EX_DATAERR: int - EX_NOINPUT: int - EX_NOUSER: int - EX_NOHOST: int - EX_UNAVAILABLE: int - EX_SOFTWARE: int - EX_OSERR: int - EX_OSFILE: int - EX_CANTCREAT: int - EX_IOERR: int - EX_TEMPFAIL: int - EX_PROTOCOL: int - EX_NOPERM: int - EX_CONFIG: int + EX_USAGE: Final[int] + EX_DATAERR: Final[int] + EX_NOINPUT: Final[int] + EX_NOUSER: Final[int] + EX_NOHOST: Final[int] + EX_UNAVAILABLE: Final[int] + EX_SOFTWARE: Final[int] + EX_OSERR: Final[int] + EX_OSFILE: Final[int] + EX_CANTCREAT: Final[int] + EX_IOERR: Final[int] + EX_TEMPFAIL: Final[int] + EX_PROTOCOL: Final[int] + EX_NOPERM: Final[int] + EX_CONFIG: Final[int] # Exists on some Unix platforms, e.g. Solaris. if sys.platform != "win32" and sys.platform != "darwin" and sys.platform != "linux": - EX_NOTFOUND: int + EX_NOTFOUND: Final[int] -P_NOWAIT: int -P_NOWAITO: int -P_WAIT: int +P_NOWAIT: Final[int] +P_NOWAITO: Final[int] +P_WAIT: Final[int] if sys.platform == "win32": - P_DETACH: int - P_OVERLAY: int + P_DETACH: Final[int] + P_OVERLAY: Final[int] # wait()/waitpid() options if sys.platform != "win32": - WNOHANG: int # Unix only - WCONTINUED: int # some Unix systems - WUNTRACED: int # Unix only + WNOHANG: Final[int] # Unix only + WCONTINUED: Final[int] # some Unix systems + WUNTRACED: Final[int] # Unix only -TMP_MAX: int # Undocumented, but used by tempfile +TMP_MAX: Final[int] # Undocumented, but used by tempfile # ----- os classes (structures) ----- @final @@ -862,6 +862,7 @@ In the future, this property will contain the last metadata change time.""" # on the allowlist for use as a Protocol starting in 3.14. @runtime_checkable class PathLike(ABC, Protocol[AnyStr_co]): # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] + __slots__ = () @abstractmethod def __fspath__(self) -> AnyStr_co: ... @@ -1136,11 +1137,11 @@ if sys.platform != "win32": def pwritev(fd: int, buffers: SupportsLenAndGetItem[ReadableBuffer], offset: int, flags: int = 0, /) -> int: ... if sys.platform != "darwin": if sys.version_info >= (3, 10): - RWF_APPEND: int # docs say available on 3.7+, stubtest says otherwise - RWF_DSYNC: int - RWF_SYNC: int - RWF_HIPRI: int - RWF_NOWAIT: int + RWF_APPEND: Final[int] # docs say available on 3.7+, stubtest says otherwise + RWF_DSYNC: Final[int] + RWF_SYNC: Final[int] + RWF_HIPRI: Final[int] + RWF_NOWAIT: Final[int] if sys.platform == "linux": def sendfile(out_fd: FileDescriptor, in_fd: FileDescriptor, offset: int | None, count: int) -> int: ... @@ -1150,8 +1151,8 @@ if sys.platform != "win32": in_fd: FileDescriptor, offset: int, count: int, - headers: Sequence[ReadableBuffer] = ..., - trailers: Sequence[ReadableBuffer] = ..., + headers: Sequence[ReadableBuffer] = (), + trailers: Sequence[ReadableBuffer] = (), flags: int = 0, ) -> int: ... # FreeBSD and Mac OS X only @@ -1196,7 +1197,7 @@ if sys.platform != "win32": def getcwd() -> str: ... def getcwdb() -> bytes: ... -def chmod(path: FileDescriptorOrPath, mode: int, *, dir_fd: int | None = None, follow_symlinks: bool = ...) -> None: ... +def chmod(path: FileDescriptorOrPath, mode: int, *, dir_fd: int | None = None, follow_symlinks: bool = True) -> None: ... if sys.platform != "win32" and sys.platform != "linux": def chflags(path: StrOrBytesPath, flags: int, follow_symlinks: bool = True) -> None: ... # some flavors of Unix @@ -1499,9 +1500,9 @@ else: setsigdef: Iterable[int] = ..., scheduler: tuple[Any, sched_param] | None = ..., ) -> int: ... - POSIX_SPAWN_OPEN: int - POSIX_SPAWN_CLOSE: int - POSIX_SPAWN_DUP2: int + POSIX_SPAWN_OPEN: Final = 0 + POSIX_SPAWN_CLOSE: Final = 1 + POSIX_SPAWN_DUP2: Final = 2 if sys.platform != "win32": @final @@ -1565,23 +1566,23 @@ if sys.platform == "win32": def add_dll_directory(path: str) -> _AddedDllDirectory: ... if sys.platform == "linux": - MFD_CLOEXEC: int - MFD_ALLOW_SEALING: int - MFD_HUGETLB: int - MFD_HUGE_SHIFT: int - MFD_HUGE_MASK: int - MFD_HUGE_64KB: int - MFD_HUGE_512KB: int - MFD_HUGE_1MB: int - MFD_HUGE_2MB: int - MFD_HUGE_8MB: int - MFD_HUGE_16MB: int - MFD_HUGE_32MB: int - MFD_HUGE_256MB: int - MFD_HUGE_512MB: int - MFD_HUGE_1GB: int - MFD_HUGE_2GB: int - MFD_HUGE_16GB: int + MFD_CLOEXEC: Final[int] + MFD_ALLOW_SEALING: Final[int] + MFD_HUGETLB: Final[int] + MFD_HUGE_SHIFT: Final[int] + MFD_HUGE_MASK: Final[int] + MFD_HUGE_64KB: Final[int] + MFD_HUGE_512KB: Final[int] + MFD_HUGE_1MB: Final[int] + MFD_HUGE_2MB: Final[int] + MFD_HUGE_8MB: Final[int] + MFD_HUGE_16MB: Final[int] + MFD_HUGE_32MB: Final[int] + MFD_HUGE_256MB: Final[int] + MFD_HUGE_512MB: Final[int] + MFD_HUGE_1GB: Final[int] + MFD_HUGE_2GB: Final[int] + MFD_HUGE_16GB: Final[int] def memfd_create(name: str, flags: int = ...) -> int: ... def copy_file_range(src: int, dst: int, count: int, offset_src: int | None = ..., offset_dst: int | None = ...) -> int: ... @@ -1599,12 +1600,12 @@ if sys.version_info >= (3, 12) and sys.platform == "win32": def listvolumes() -> list[str]: ... if sys.version_info >= (3, 10) and sys.platform == "linux": - EFD_CLOEXEC: int - EFD_NONBLOCK: int - EFD_SEMAPHORE: int - SPLICE_F_MORE: int - SPLICE_F_MOVE: int - SPLICE_F_NONBLOCK: int + EFD_CLOEXEC: Final[int] + EFD_NONBLOCK: Final[int] + EFD_SEMAPHORE: Final[int] + SPLICE_F_MORE: Final[int] + SPLICE_F_MOVE: Final[int] + SPLICE_F_NONBLOCK: Final[int] def eventfd(initval: int, flags: int = 524288) -> FileDescriptor: ... def eventfd_read(fd: FileDescriptor) -> int: ... def eventfd_write(fd: FileDescriptor, value: int) -> None: ... @@ -1618,20 +1619,20 @@ if sys.version_info >= (3, 10) and sys.platform == "linux": ) -> int: ... if sys.version_info >= (3, 12) and sys.platform == "linux": - CLONE_FILES: int - CLONE_FS: int - CLONE_NEWCGROUP: int # Linux 4.6+ - CLONE_NEWIPC: int # Linux 2.6.19+ - CLONE_NEWNET: int # Linux 2.6.24+ - CLONE_NEWNS: int - CLONE_NEWPID: int # Linux 3.8+ - CLONE_NEWTIME: int # Linux 5.6+ - CLONE_NEWUSER: int # Linux 3.8+ - CLONE_NEWUTS: int # Linux 2.6.19+ - CLONE_SIGHAND: int - CLONE_SYSVSEM: int # Linux 2.6.26+ - CLONE_THREAD: int - CLONE_VM: int + CLONE_FILES: Final[int] + CLONE_FS: Final[int] + CLONE_NEWCGROUP: Final[int] # Linux 4.6+ + CLONE_NEWIPC: Final[int] # Linux 2.6.19+ + CLONE_NEWNET: Final[int] # Linux 2.6.24+ + CLONE_NEWNS: Final[int] + CLONE_NEWPID: Final[int] # Linux 3.8+ + CLONE_NEWTIME: Final[int] # Linux 5.6+ + CLONE_NEWUSER: Final[int] # Linux 3.8+ + CLONE_NEWUTS: Final[int] # Linux 2.6.19+ + CLONE_SIGHAND: Final[int] + CLONE_SYSVSEM: Final[int] # Linux 2.6.26+ + CLONE_THREAD: Final[int] + CLONE_VM: Final[int] def unshare(flags: int) -> None: ... def setns(fd: FileDescriptorLike, nstype: int = 0) -> None: ... diff --git a/mypy/typeshed/stdlib/ossaudiodev.pyi b/mypy/typeshed/stdlib/ossaudiodev.pyi index b9ee3edab033e..f8230b4f02123 100644 --- a/mypy/typeshed/stdlib/ossaudiodev.pyi +++ b/mypy/typeshed/stdlib/ossaudiodev.pyi @@ -1,119 +1,120 @@ import sys -from typing import Any, Literal, overload +from typing import Any, Final, Literal, overload if sys.platform != "win32" and sys.platform != "darwin": - AFMT_AC3: int - AFMT_A_LAW: int - AFMT_IMA_ADPCM: int - AFMT_MPEG: int - AFMT_MU_LAW: int - AFMT_QUERY: int - AFMT_S16_BE: int - AFMT_S16_LE: int - AFMT_S16_NE: int - AFMT_S8: int - AFMT_U16_BE: int - AFMT_U16_LE: int - AFMT_U8: int - SNDCTL_COPR_HALT: int - SNDCTL_COPR_LOAD: int - SNDCTL_COPR_RCODE: int - SNDCTL_COPR_RCVMSG: int - SNDCTL_COPR_RDATA: int - SNDCTL_COPR_RESET: int - SNDCTL_COPR_RUN: int - SNDCTL_COPR_SENDMSG: int - SNDCTL_COPR_WCODE: int - SNDCTL_COPR_WDATA: int - SNDCTL_DSP_BIND_CHANNEL: int - SNDCTL_DSP_CHANNELS: int - SNDCTL_DSP_GETBLKSIZE: int - SNDCTL_DSP_GETCAPS: int - SNDCTL_DSP_GETCHANNELMASK: int - SNDCTL_DSP_GETFMTS: int - SNDCTL_DSP_GETIPTR: int - SNDCTL_DSP_GETISPACE: int - SNDCTL_DSP_GETODELAY: int - SNDCTL_DSP_GETOPTR: int - SNDCTL_DSP_GETOSPACE: int - SNDCTL_DSP_GETSPDIF: int - SNDCTL_DSP_GETTRIGGER: int - SNDCTL_DSP_MAPINBUF: int - SNDCTL_DSP_MAPOUTBUF: int - SNDCTL_DSP_NONBLOCK: int - SNDCTL_DSP_POST: int - SNDCTL_DSP_PROFILE: int - SNDCTL_DSP_RESET: int - SNDCTL_DSP_SAMPLESIZE: int - SNDCTL_DSP_SETDUPLEX: int - SNDCTL_DSP_SETFMT: int - SNDCTL_DSP_SETFRAGMENT: int - SNDCTL_DSP_SETSPDIF: int - SNDCTL_DSP_SETSYNCRO: int - SNDCTL_DSP_SETTRIGGER: int - SNDCTL_DSP_SPEED: int - SNDCTL_DSP_STEREO: int - SNDCTL_DSP_SUBDIVIDE: int - SNDCTL_DSP_SYNC: int - SNDCTL_FM_4OP_ENABLE: int - SNDCTL_FM_LOAD_INSTR: int - SNDCTL_MIDI_INFO: int - SNDCTL_MIDI_MPUCMD: int - SNDCTL_MIDI_MPUMODE: int - SNDCTL_MIDI_PRETIME: int - SNDCTL_SEQ_CTRLRATE: int - SNDCTL_SEQ_GETINCOUNT: int - SNDCTL_SEQ_GETOUTCOUNT: int - SNDCTL_SEQ_GETTIME: int - SNDCTL_SEQ_NRMIDIS: int - SNDCTL_SEQ_NRSYNTHS: int - SNDCTL_SEQ_OUTOFBAND: int - SNDCTL_SEQ_PANIC: int - SNDCTL_SEQ_PERCMODE: int - SNDCTL_SEQ_RESET: int - SNDCTL_SEQ_RESETSAMPLES: int - SNDCTL_SEQ_SYNC: int - SNDCTL_SEQ_TESTMIDI: int - SNDCTL_SEQ_THRESHOLD: int - SNDCTL_SYNTH_CONTROL: int - SNDCTL_SYNTH_ID: int - SNDCTL_SYNTH_INFO: int - SNDCTL_SYNTH_MEMAVL: int - SNDCTL_SYNTH_REMOVESAMPLE: int - SNDCTL_TMR_CONTINUE: int - SNDCTL_TMR_METRONOME: int - SNDCTL_TMR_SELECT: int - SNDCTL_TMR_SOURCE: int - SNDCTL_TMR_START: int - SNDCTL_TMR_STOP: int - SNDCTL_TMR_TEMPO: int - SNDCTL_TMR_TIMEBASE: int - SOUND_MIXER_ALTPCM: int - SOUND_MIXER_BASS: int - SOUND_MIXER_CD: int - SOUND_MIXER_DIGITAL1: int - SOUND_MIXER_DIGITAL2: int - SOUND_MIXER_DIGITAL3: int - SOUND_MIXER_IGAIN: int - SOUND_MIXER_IMIX: int - SOUND_MIXER_LINE: int - SOUND_MIXER_LINE1: int - SOUND_MIXER_LINE2: int - SOUND_MIXER_LINE3: int - SOUND_MIXER_MIC: int - SOUND_MIXER_MONITOR: int - SOUND_MIXER_NRDEVICES: int - SOUND_MIXER_OGAIN: int - SOUND_MIXER_PCM: int - SOUND_MIXER_PHONEIN: int - SOUND_MIXER_PHONEOUT: int - SOUND_MIXER_RADIO: int - SOUND_MIXER_RECLEV: int - SOUND_MIXER_SPEAKER: int - SOUND_MIXER_SYNTH: int - SOUND_MIXER_TREBLE: int - SOUND_MIXER_VIDEO: int - SOUND_MIXER_VOLUME: int + # Depends on soundcard.h + AFMT_AC3: Final[int] + AFMT_A_LAW: Final[int] + AFMT_IMA_ADPCM: Final[int] + AFMT_MPEG: Final[int] + AFMT_MU_LAW: Final[int] + AFMT_QUERY: Final[int] + AFMT_S16_BE: Final[int] + AFMT_S16_LE: Final[int] + AFMT_S16_NE: Final[int] + AFMT_S8: Final[int] + AFMT_U16_BE: Final[int] + AFMT_U16_LE: Final[int] + AFMT_U8: Final[int] + SNDCTL_COPR_HALT: Final[int] + SNDCTL_COPR_LOAD: Final[int] + SNDCTL_COPR_RCODE: Final[int] + SNDCTL_COPR_RCVMSG: Final[int] + SNDCTL_COPR_RDATA: Final[int] + SNDCTL_COPR_RESET: Final[int] + SNDCTL_COPR_RUN: Final[int] + SNDCTL_COPR_SENDMSG: Final[int] + SNDCTL_COPR_WCODE: Final[int] + SNDCTL_COPR_WDATA: Final[int] + SNDCTL_DSP_BIND_CHANNEL: Final[int] + SNDCTL_DSP_CHANNELS: Final[int] + SNDCTL_DSP_GETBLKSIZE: Final[int] + SNDCTL_DSP_GETCAPS: Final[int] + SNDCTL_DSP_GETCHANNELMASK: Final[int] + SNDCTL_DSP_GETFMTS: Final[int] + SNDCTL_DSP_GETIPTR: Final[int] + SNDCTL_DSP_GETISPACE: Final[int] + SNDCTL_DSP_GETODELAY: Final[int] + SNDCTL_DSP_GETOPTR: Final[int] + SNDCTL_DSP_GETOSPACE: Final[int] + SNDCTL_DSP_GETSPDIF: Final[int] + SNDCTL_DSP_GETTRIGGER: Final[int] + SNDCTL_DSP_MAPINBUF: Final[int] + SNDCTL_DSP_MAPOUTBUF: Final[int] + SNDCTL_DSP_NONBLOCK: Final[int] + SNDCTL_DSP_POST: Final[int] + SNDCTL_DSP_PROFILE: Final[int] + SNDCTL_DSP_RESET: Final[int] + SNDCTL_DSP_SAMPLESIZE: Final[int] + SNDCTL_DSP_SETDUPLEX: Final[int] + SNDCTL_DSP_SETFMT: Final[int] + SNDCTL_DSP_SETFRAGMENT: Final[int] + SNDCTL_DSP_SETSPDIF: Final[int] + SNDCTL_DSP_SETSYNCRO: Final[int] + SNDCTL_DSP_SETTRIGGER: Final[int] + SNDCTL_DSP_SPEED: Final[int] + SNDCTL_DSP_STEREO: Final[int] + SNDCTL_DSP_SUBDIVIDE: Final[int] + SNDCTL_DSP_SYNC: Final[int] + SNDCTL_FM_4OP_ENABLE: Final[int] + SNDCTL_FM_LOAD_INSTR: Final[int] + SNDCTL_MIDI_INFO: Final[int] + SNDCTL_MIDI_MPUCMD: Final[int] + SNDCTL_MIDI_MPUMODE: Final[int] + SNDCTL_MIDI_PRETIME: Final[int] + SNDCTL_SEQ_CTRLRATE: Final[int] + SNDCTL_SEQ_GETINCOUNT: Final[int] + SNDCTL_SEQ_GETOUTCOUNT: Final[int] + SNDCTL_SEQ_GETTIME: Final[int] + SNDCTL_SEQ_NRMIDIS: Final[int] + SNDCTL_SEQ_NRSYNTHS: Final[int] + SNDCTL_SEQ_OUTOFBAND: Final[int] + SNDCTL_SEQ_PANIC: Final[int] + SNDCTL_SEQ_PERCMODE: Final[int] + SNDCTL_SEQ_RESET: Final[int] + SNDCTL_SEQ_RESETSAMPLES: Final[int] + SNDCTL_SEQ_SYNC: Final[int] + SNDCTL_SEQ_TESTMIDI: Final[int] + SNDCTL_SEQ_THRESHOLD: Final[int] + SNDCTL_SYNTH_CONTROL: Final[int] + SNDCTL_SYNTH_ID: Final[int] + SNDCTL_SYNTH_INFO: Final[int] + SNDCTL_SYNTH_MEMAVL: Final[int] + SNDCTL_SYNTH_REMOVESAMPLE: Final[int] + SNDCTL_TMR_CONTINUE: Final[int] + SNDCTL_TMR_METRONOME: Final[int] + SNDCTL_TMR_SELECT: Final[int] + SNDCTL_TMR_SOURCE: Final[int] + SNDCTL_TMR_START: Final[int] + SNDCTL_TMR_STOP: Final[int] + SNDCTL_TMR_TEMPO: Final[int] + SNDCTL_TMR_TIMEBASE: Final[int] + SOUND_MIXER_ALTPCM: Final[int] + SOUND_MIXER_BASS: Final[int] + SOUND_MIXER_CD: Final[int] + SOUND_MIXER_DIGITAL1: Final[int] + SOUND_MIXER_DIGITAL2: Final[int] + SOUND_MIXER_DIGITAL3: Final[int] + SOUND_MIXER_IGAIN: Final[int] + SOUND_MIXER_IMIX: Final[int] + SOUND_MIXER_LINE: Final[int] + SOUND_MIXER_LINE1: Final[int] + SOUND_MIXER_LINE2: Final[int] + SOUND_MIXER_LINE3: Final[int] + SOUND_MIXER_MIC: Final[int] + SOUND_MIXER_MONITOR: Final[int] + SOUND_MIXER_NRDEVICES: Final[int] + SOUND_MIXER_OGAIN: Final[int] + SOUND_MIXER_PCM: Final[int] + SOUND_MIXER_PHONEIN: Final[int] + SOUND_MIXER_PHONEOUT: Final[int] + SOUND_MIXER_RADIO: Final[int] + SOUND_MIXER_RECLEV: Final[int] + SOUND_MIXER_SPEAKER: Final[int] + SOUND_MIXER_SYNTH: Final[int] + SOUND_MIXER_TREBLE: Final[int] + SOUND_MIXER_VIDEO: Final[int] + SOUND_MIXER_VOLUME: Final[int] control_labels: list[str] control_names: list[str] diff --git a/mypy/typeshed/stdlib/pathlib/__init__.pyi b/mypy/typeshed/stdlib/pathlib/__init__.pyi index 4858f8db1ed09..fa5143f202927 100644 --- a/mypy/typeshed/stdlib/pathlib/__init__.pyi +++ b/mypy/typeshed/stdlib/pathlib/__init__.pyi @@ -29,6 +29,31 @@ if sys.version_info >= (3, 13): __all__ += ["UnsupportedOperation"] class PurePath(PathLike[str]): + if sys.version_info >= (3, 13): + __slots__ = ( + "_raw_paths", + "_drv", + "_root", + "_tail_cached", + "_str", + "_str_normcase_cached", + "_parts_normcase_cached", + "_hash", + ) + elif sys.version_info >= (3, 12): + __slots__ = ( + "_raw_paths", + "_drv", + "_root", + "_tail_cached", + "_str", + "_str_normcase_cached", + "_parts_normcase_cached", + "_lines_cached", + "_hash", + ) + else: + __slots__ = ("_drv", "_root", "_parts", "_str", "_hash", "_pparts", "_cached_cparts") if sys.version_info >= (3, 13): parser: ClassVar[types.ModuleType] def full_match(self, pattern: StrPath, *, case_sensitive: bool | None = None) -> bool: ... @@ -108,10 +133,20 @@ class PurePath(PathLike[str]): if sys.version_info >= (3, 12): def with_segments(self, *args: StrPath) -> Self: ... -class PurePosixPath(PurePath): ... -class PureWindowsPath(PurePath): ... +class PurePosixPath(PurePath): + __slots__ = () + +class PureWindowsPath(PurePath): + __slots__ = () class Path(PurePath): + if sys.version_info >= (3, 14): + __slots__ = ("_info",) + elif sys.version_info >= (3, 10): + __slots__ = () + else: + __slots__ = ("_accessor",) + if sys.version_info >= (3, 12): def __new__(cls, *args: StrPath, **kwargs: Unused) -> Self: ... # pyright: ignore[reportInconsistentConstructor] else: @@ -307,11 +342,14 @@ class Path(PurePath): def link_to(self, target: StrOrBytesPath) -> None: ... if sys.version_info >= (3, 12): def walk( - self, top_down: bool = ..., on_error: Callable[[OSError], object] | None = ..., follow_symlinks: bool = ... + self, top_down: bool = True, on_error: Callable[[OSError], object] | None = None, follow_symlinks: bool = False ) -> Iterator[tuple[Self, list[str], list[str]]]: ... -class PosixPath(Path, PurePosixPath): ... -class WindowsPath(Path, PureWindowsPath): ... +class PosixPath(Path, PurePosixPath): + __slots__ = () + +class WindowsPath(Path, PureWindowsPath): + __slots__ = () if sys.version_info >= (3, 13): class UnsupportedOperation(NotImplementedError): ... diff --git a/mypy/typeshed/stdlib/pdb.pyi b/mypy/typeshed/stdlib/pdb.pyi index ad69fcab16de0..0c16f48e2e220 100644 --- a/mypy/typeshed/stdlib/pdb.pyi +++ b/mypy/typeshed/stdlib/pdb.pyi @@ -17,7 +17,7 @@ _T = TypeVar("_T") _P = ParamSpec("_P") _Mode: TypeAlias = Literal["inline", "cli"] -line_prefix: str # undocumented +line_prefix: Final[str] # undocumented class Restart(Exception): ... @@ -131,7 +131,11 @@ class Pdb(Bdb, Cmd): def completedefault(self, text: str, line: str, begidx: int, endidx: int) -> list[str]: ... def do_commands(self, arg: str) -> bool | None: ... - def do_break(self, arg: str, temporary: bool = ...) -> bool | None: ... + if sys.version_info >= (3, 14): + def do_break(self, arg: str, temporary: bool = False) -> bool | None: ... + else: + def do_break(self, arg: str, temporary: bool | Literal[0, 1] = 0) -> bool | None: ... + def do_tbreak(self, arg: str) -> bool | None: ... def do_enable(self, arg: str) -> bool | None: ... def do_disable(self, arg: str) -> bool | None: ... diff --git a/mypy/typeshed/stdlib/pickle.pyi b/mypy/typeshed/stdlib/pickle.pyi index 2d80d61645e0e..d94fe208f4468 100644 --- a/mypy/typeshed/stdlib/pickle.pyi +++ b/mypy/typeshed/stdlib/pickle.pyi @@ -14,7 +14,7 @@ from _pickle import ( ) from _typeshed import ReadableBuffer, SupportsWrite from collections.abc import Callable, Iterable, Mapping -from typing import Any, ClassVar, SupportsBytes, SupportsIndex, final +from typing import Any, ClassVar, Final, SupportsBytes, SupportsIndex, final from typing_extensions import Self __all__ = [ @@ -102,8 +102,8 @@ __all__ = [ "UNICODE", ] -HIGHEST_PROTOCOL: int -DEFAULT_PROTOCOL: int +HIGHEST_PROTOCOL: Final = 5 +DEFAULT_PROTOCOL: Final = 5 bytes_types: tuple[type[Any], ...] # undocumented @@ -115,85 +115,85 @@ class PickleBuffer: def __buffer__(self, flags: int, /) -> memoryview: ... def __release_buffer__(self, buffer: memoryview, /) -> None: ... -MARK: bytes -STOP: bytes -POP: bytes -POP_MARK: bytes -DUP: bytes -FLOAT: bytes -INT: bytes -BININT: bytes -BININT1: bytes -LONG: bytes -BININT2: bytes -NONE: bytes -PERSID: bytes -BINPERSID: bytes -REDUCE: bytes -STRING: bytes -BINSTRING: bytes -SHORT_BINSTRING: bytes -UNICODE: bytes -BINUNICODE: bytes -APPEND: bytes -BUILD: bytes -GLOBAL: bytes -DICT: bytes -EMPTY_DICT: bytes -APPENDS: bytes -GET: bytes -BINGET: bytes -INST: bytes -LONG_BINGET: bytes -LIST: bytes -EMPTY_LIST: bytes -OBJ: bytes -PUT: bytes -BINPUT: bytes -LONG_BINPUT: bytes -SETITEM: bytes -TUPLE: bytes -EMPTY_TUPLE: bytes -SETITEMS: bytes -BINFLOAT: bytes +MARK: Final = b"(" +STOP: Final = b"." +POP: Final = b"0" +POP_MARK: Final = b"1" +DUP: Final = b"2" +FLOAT: Final = b"F" +INT: Final = b"I" +BININT: Final = b"J" +BININT1: Final = b"K" +LONG: Final = b"L" +BININT2: Final = b"M" +NONE: Final = b"N" +PERSID: Final = b"P" +BINPERSID: Final = b"Q" +REDUCE: Final = b"R" +STRING: Final = b"S" +BINSTRING: Final = b"T" +SHORT_BINSTRING: Final = b"U" +UNICODE: Final = b"V" +BINUNICODE: Final = b"X" +APPEND: Final = b"a" +BUILD: Final = b"b" +GLOBAL: Final = b"c" +DICT: Final = b"d" +EMPTY_DICT: Final = b"}" +APPENDS: Final = b"e" +GET: Final = b"g" +BINGET: Final = b"h" +INST: Final = b"i" +LONG_BINGET: Final = b"j" +LIST: Final = b"l" +EMPTY_LIST: Final = b"]" +OBJ: Final = b"o" +PUT: Final = b"p" +BINPUT: Final = b"q" +LONG_BINPUT: Final = b"r" +SETITEM: Final = b"s" +TUPLE: Final = b"t" +EMPTY_TUPLE: Final = b")" +SETITEMS: Final = b"u" +BINFLOAT: Final = b"G" -TRUE: bytes -FALSE: bytes +TRUE: Final = b"I01\n" +FALSE: Final = b"I00\n" # protocol 2 -PROTO: bytes -NEWOBJ: bytes -EXT1: bytes -EXT2: bytes -EXT4: bytes -TUPLE1: bytes -TUPLE2: bytes -TUPLE3: bytes -NEWTRUE: bytes -NEWFALSE: bytes -LONG1: bytes -LONG4: bytes +PROTO: Final = b"\x80" +NEWOBJ: Final = b"\x81" +EXT1: Final = b"\x82" +EXT2: Final = b"\x83" +EXT4: Final = b"\x84" +TUPLE1: Final = b"\x85" +TUPLE2: Final = b"\x86" +TUPLE3: Final = b"\x87" +NEWTRUE: Final = b"\x88" +NEWFALSE: Final = b"\x89" +LONG1: Final = b"\x8a" +LONG4: Final = b"\x8b" # protocol 3 -BINBYTES: bytes -SHORT_BINBYTES: bytes +BINBYTES: Final = b"B" +SHORT_BINBYTES: Final = b"C" # protocol 4 -SHORT_BINUNICODE: bytes -BINUNICODE8: bytes -BINBYTES8: bytes -EMPTY_SET: bytes -ADDITEMS: bytes -FROZENSET: bytes -NEWOBJ_EX: bytes -STACK_GLOBAL: bytes -MEMOIZE: bytes -FRAME: bytes +SHORT_BINUNICODE: Final = b"\x8c" +BINUNICODE8: Final = b"\x8d" +BINBYTES8: Final = b"\x8e" +EMPTY_SET: Final = b"\x8f" +ADDITEMS: Final = b"\x90" +FROZENSET: Final = b"\x91" +NEWOBJ_EX: Final = b"\x92" +STACK_GLOBAL: Final = b"\x93" +MEMOIZE: Final = b"\x94" +FRAME: Final = b"\x95" # protocol 5 -BYTEARRAY8: bytes -NEXT_BUFFER: bytes -READONLY_BUFFER: bytes +BYTEARRAY8: Final = b"\x96" +NEXT_BUFFER: Final = b"\x97" +READONLY_BUFFER: Final = b"\x98" def encode_long(x: int) -> bytes: ... # undocumented def decode_long(data: Iterable[SupportsIndex] | SupportsBytes | ReadableBuffer) -> int: ... # undocumented diff --git a/mypy/typeshed/stdlib/pickletools.pyi b/mypy/typeshed/stdlib/pickletools.pyi index cdade08d39a87..8bbfaba31b671 100644 --- a/mypy/typeshed/stdlib/pickletools.pyi +++ b/mypy/typeshed/stdlib/pickletools.pyi @@ -1,6 +1,6 @@ import sys from collections.abc import Callable, Iterator, MutableMapping -from typing import IO, Any +from typing import IO, Any, Final from typing_extensions import TypeAlias __all__ = ["dis", "genops", "optimize"] @@ -8,13 +8,14 @@ __all__ = ["dis", "genops", "optimize"] _Reader: TypeAlias = Callable[[IO[bytes]], Any] bytes_types: tuple[type[Any], ...] -UP_TO_NEWLINE: int -TAKEN_FROM_ARGUMENT1: int -TAKEN_FROM_ARGUMENT4: int -TAKEN_FROM_ARGUMENT4U: int -TAKEN_FROM_ARGUMENT8U: int +UP_TO_NEWLINE: Final = -1 +TAKEN_FROM_ARGUMENT1: Final = -2 +TAKEN_FROM_ARGUMENT4: Final = -3 +TAKEN_FROM_ARGUMENT4U: Final = -4 +TAKEN_FROM_ARGUMENT8U: Final = -5 class ArgumentDescriptor: + __slots__ = ("name", "n", "reader", "doc") name: str n: int reader: _Reader @@ -118,6 +119,7 @@ def read_long4(f: IO[bytes]) -> int: ... long4: ArgumentDescriptor class StackObject: + __slots__ = ("name", "obtype", "doc") name: str obtype: type[Any] | tuple[type[Any], ...] doc: str @@ -143,6 +145,7 @@ markobject: StackObject stackslice: StackObject class OpcodeInfo: + __slots__ = ("name", "code", "arg", "stack_before", "stack_after", "proto", "doc") name: str code: str arg: ArgumentDescriptor | None diff --git a/mypy/typeshed/stdlib/platform.pyi b/mypy/typeshed/stdlib/platform.pyi index c6125bd3a56fa..69d702bb155cd 100644 --- a/mypy/typeshed/stdlib/platform.pyi +++ b/mypy/typeshed/stdlib/platform.pyi @@ -1,6 +1,6 @@ import sys from typing import NamedTuple, type_check_only -from typing_extensions import Self, deprecated +from typing_extensions import Self, deprecated, disjoint_base def libc_ver(executable: str | None = None, lib: str = "", version: str = "", chunksize: int = 16384) -> tuple[str, str]: ... def win32_ver(release: str = "", version: str = "", csd: str = "", ptype: str = "") -> tuple[str, str, str, str]: ... @@ -46,13 +46,23 @@ class _uname_result_base(NamedTuple): # uname_result emulates a 6-field named tuple, but the processor field # is lazily evaluated rather than being passed in to the constructor. -class uname_result(_uname_result_base): - if sys.version_info >= (3, 10): +if sys.version_info >= (3, 12): + class uname_result(_uname_result_base): __match_args__ = ("system", "node", "release", "version", "machine") # pyright: ignore[reportAssignmentType] - def __new__(_cls, system: str, node: str, release: str, version: str, machine: str) -> Self: ... - @property - def processor(self) -> str: ... + def __new__(_cls, system: str, node: str, release: str, version: str, machine: str) -> Self: ... + @property + def processor(self) -> str: ... + +else: + @disjoint_base + class uname_result(_uname_result_base): + if sys.version_info >= (3, 10): + __match_args__ = ("system", "node", "release", "version", "machine") # pyright: ignore[reportAssignmentType] + + def __new__(_cls, system: str, node: str, release: str, version: str, machine: str) -> Self: ... + @property + def processor(self) -> str: ... def uname() -> uname_result: ... def system() -> str: ... @@ -68,7 +78,7 @@ def python_branch() -> str: ... def python_revision() -> str: ... def python_build() -> tuple[str, str]: ... def python_compiler() -> str: ... -def platform(aliased: bool = ..., terse: bool = ...) -> str: ... +def platform(aliased: bool = False, terse: bool = False) -> str: ... if sys.version_info >= (3, 10): def freedesktop_os_release() -> dict[str, str]: ... diff --git a/mypy/typeshed/stdlib/plistlib.pyi b/mypy/typeshed/stdlib/plistlib.pyi index 8b39b4217eae6..dc3247ee47fb8 100644 --- a/mypy/typeshed/stdlib/plistlib.pyi +++ b/mypy/typeshed/stdlib/plistlib.pyi @@ -3,7 +3,7 @@ from _typeshed import ReadableBuffer from collections.abc import Mapping, MutableMapping from datetime import datetime from enum import Enum -from typing import IO, Any +from typing import IO, Any, Final from typing_extensions import Self __all__ = ["InvalidFileException", "FMT_XML", "FMT_BINARY", "load", "dump", "loads", "dumps", "UID"] @@ -12,8 +12,8 @@ class PlistFormat(Enum): FMT_XML = 1 FMT_BINARY = 2 -FMT_XML = PlistFormat.FMT_XML -FMT_BINARY = PlistFormat.FMT_BINARY +FMT_XML: Final = PlistFormat.FMT_XML +FMT_BINARY: Final = PlistFormat.FMT_BINARY if sys.version_info >= (3, 13): def load( fp: IO[bytes], diff --git a/mypy/typeshed/stdlib/poplib.pyi b/mypy/typeshed/stdlib/poplib.pyi index a1e41be86a7f8..9ff2b764aeb68 100644 --- a/mypy/typeshed/stdlib/poplib.pyi +++ b/mypy/typeshed/stdlib/poplib.pyi @@ -17,7 +17,7 @@ POP3_SSL_PORT: Final = 995 CR: Final = b"\r" LF: Final = b"\n" CRLF: Final = b"\r\n" -HAVE_SSL: bool +HAVE_SSL: Final[bool] class POP3: encoding: str diff --git a/mypy/typeshed/stdlib/pydoc.pyi b/mypy/typeshed/stdlib/pydoc.pyi index 3c78f9d2de8e9..935f9420f88c0 100644 --- a/mypy/typeshed/stdlib/pydoc.pyi +++ b/mypy/typeshed/stdlib/pydoc.pyi @@ -34,11 +34,11 @@ def visiblename(name: str, all: Container[str] | None = None, obj: object = None def classify_class_attrs(object: object) -> list[tuple[str, str, type, str]]: ... if sys.version_info >= (3, 13): - @deprecated("Deprecated in Python 3.13.") - def ispackage(path: str) -> bool: ... + @deprecated("Deprecated since Python 3.13.") + def ispackage(path: str) -> bool: ... # undocumented else: - def ispackage(path: str) -> bool: ... + def ispackage(path: str) -> bool: ... # undocumented def source_synopsis(file: IO[AnyStr]) -> AnyStr | None: ... def synopsis(filename: str, cache: MutableMapping[str, tuple[int, str]] = {}) -> str | None: ... diff --git a/mypy/typeshed/stdlib/pydoc_data/topics.pyi b/mypy/typeshed/stdlib/pydoc_data/topics.pyi index 091d34300106e..ce907a41c0053 100644 --- a/mypy/typeshed/stdlib/pydoc_data/topics.pyi +++ b/mypy/typeshed/stdlib/pydoc_data/topics.pyi @@ -1 +1,3 @@ -topics: dict[str, str] +from typing import Final + +topics: Final[dict[str, str]] diff --git a/mypy/typeshed/stdlib/random.pyi b/mypy/typeshed/stdlib/random.pyi index 83e37113a941b..a797794b8050f 100644 --- a/mypy/typeshed/stdlib/random.pyi +++ b/mypy/typeshed/stdlib/random.pyi @@ -4,6 +4,7 @@ from _typeshed import SupportsLenAndGetItem from collections.abc import Callable, Iterable, MutableSequence, Sequence, Set as AbstractSet from fractions import Fraction from typing import Any, ClassVar, NoReturn, TypeVar +from typing_extensions import Self __all__ = [ "Random", @@ -44,6 +45,10 @@ class Random(_random.Random): # Using other `seed` types is deprecated since 3.9 and removed in 3.11 # Ignore Y041, since random.seed doesn't treat int like a float subtype. Having an explicit # int better documents conventional usage of random.seed. + if sys.version_info < (3, 10): + # this is a workaround for pyright correctly flagging an inconsistent inherited constructor, see #14624 + def __new__(cls, x: int | float | str | bytes | bytearray | None = None) -> Self: ... # noqa: Y041 + def seed(self, a: int | float | str | bytes | bytearray | None = None, version: int = 2) -> None: ... # type: ignore[override] # noqa: Y041 def getstate(self) -> tuple[Any, ...]: ... def setstate(self, state: tuple[Any, ...]) -> None: ... diff --git a/mypy/typeshed/stdlib/resource.pyi b/mypy/typeshed/stdlib/resource.pyi index 5e468c2cead5f..f99cd5b088056 100644 --- a/mypy/typeshed/stdlib/resource.pyi +++ b/mypy/typeshed/stdlib/resource.pyi @@ -3,27 +3,28 @@ from _typeshed import structseq from typing import Final, final if sys.platform != "win32": - RLIMIT_AS: int - RLIMIT_CORE: int - RLIMIT_CPU: int - RLIMIT_DATA: int - RLIMIT_FSIZE: int - RLIMIT_MEMLOCK: int - RLIMIT_NOFILE: int - RLIMIT_NPROC: int - RLIMIT_RSS: int - RLIMIT_STACK: int - RLIM_INFINITY: int - RUSAGE_CHILDREN: int - RUSAGE_SELF: int + # Depends on resource.h + RLIMIT_AS: Final[int] + RLIMIT_CORE: Final[int] + RLIMIT_CPU: Final[int] + RLIMIT_DATA: Final[int] + RLIMIT_FSIZE: Final[int] + RLIMIT_MEMLOCK: Final[int] + RLIMIT_NOFILE: Final[int] + RLIMIT_NPROC: Final[int] + RLIMIT_RSS: Final[int] + RLIMIT_STACK: Final[int] + RLIM_INFINITY: Final[int] + RUSAGE_CHILDREN: Final[int] + RUSAGE_SELF: Final[int] if sys.platform == "linux": - RLIMIT_MSGQUEUE: int - RLIMIT_NICE: int - RLIMIT_OFILE: int - RLIMIT_RTPRIO: int - RLIMIT_RTTIME: int - RLIMIT_SIGPENDING: int - RUSAGE_THREAD: int + RLIMIT_MSGQUEUE: Final[int] + RLIMIT_NICE: Final[int] + RLIMIT_OFILE: Final[int] + RLIMIT_RTPRIO: Final[int] + RLIMIT_RTTIME: Final[int] + RLIMIT_SIGPENDING: Final[int] + RUSAGE_THREAD: Final[int] @final class struct_rusage( diff --git a/mypy/typeshed/stdlib/select.pyi b/mypy/typeshed/stdlib/select.pyi index 0235473902733..587bc75376ef1 100644 --- a/mypy/typeshed/stdlib/select.pyi +++ b/mypy/typeshed/stdlib/select.pyi @@ -2,25 +2,25 @@ import sys from _typeshed import FileDescriptorLike from collections.abc import Iterable from types import TracebackType -from typing import Any, ClassVar, final +from typing import Any, ClassVar, Final, final from typing_extensions import Self if sys.platform != "win32": - PIPE_BUF: int - POLLERR: int - POLLHUP: int - POLLIN: int + PIPE_BUF: Final[int] + POLLERR: Final[int] + POLLHUP: Final[int] + POLLIN: Final[int] if sys.platform == "linux": - POLLMSG: int - POLLNVAL: int - POLLOUT: int - POLLPRI: int - POLLRDBAND: int + POLLMSG: Final[int] + POLLNVAL: Final[int] + POLLOUT: Final[int] + POLLPRI: Final[int] + POLLRDBAND: Final[int] if sys.platform == "linux": - POLLRDHUP: int - POLLRDNORM: int - POLLWRBAND: int - POLLWRNORM: int + POLLRDHUP: Final[int] + POLLRDNORM: Final[int] + POLLWRBAND: Final[int] + POLLWRNORM: Final[int] # This is actually a function that returns an instance of a class. # The class is not accessible directly, and also calls itself select.poll. @@ -71,50 +71,50 @@ if sys.platform != "linux" and sys.platform != "win32": @classmethod def fromfd(cls, fd: FileDescriptorLike, /) -> kqueue: ... - KQ_EV_ADD: int - KQ_EV_CLEAR: int - KQ_EV_DELETE: int - KQ_EV_DISABLE: int - KQ_EV_ENABLE: int - KQ_EV_EOF: int - KQ_EV_ERROR: int - KQ_EV_FLAG1: int - KQ_EV_ONESHOT: int - KQ_EV_SYSFLAGS: int - KQ_FILTER_AIO: int + KQ_EV_ADD: Final[int] + KQ_EV_CLEAR: Final[int] + KQ_EV_DELETE: Final[int] + KQ_EV_DISABLE: Final[int] + KQ_EV_ENABLE: Final[int] + KQ_EV_EOF: Final[int] + KQ_EV_ERROR: Final[int] + KQ_EV_FLAG1: Final[int] + KQ_EV_ONESHOT: Final[int] + KQ_EV_SYSFLAGS: Final[int] + KQ_FILTER_AIO: Final[int] if sys.platform != "darwin": - KQ_FILTER_NETDEV: int - KQ_FILTER_PROC: int - KQ_FILTER_READ: int - KQ_FILTER_SIGNAL: int - KQ_FILTER_TIMER: int - KQ_FILTER_VNODE: int - KQ_FILTER_WRITE: int - KQ_NOTE_ATTRIB: int - KQ_NOTE_CHILD: int - KQ_NOTE_DELETE: int - KQ_NOTE_EXEC: int - KQ_NOTE_EXIT: int - KQ_NOTE_EXTEND: int - KQ_NOTE_FORK: int - KQ_NOTE_LINK: int + KQ_FILTER_NETDEV: Final[int] + KQ_FILTER_PROC: Final[int] + KQ_FILTER_READ: Final[int] + KQ_FILTER_SIGNAL: Final[int] + KQ_FILTER_TIMER: Final[int] + KQ_FILTER_VNODE: Final[int] + KQ_FILTER_WRITE: Final[int] + KQ_NOTE_ATTRIB: Final[int] + KQ_NOTE_CHILD: Final[int] + KQ_NOTE_DELETE: Final[int] + KQ_NOTE_EXEC: Final[int] + KQ_NOTE_EXIT: Final[int] + KQ_NOTE_EXTEND: Final[int] + KQ_NOTE_FORK: Final[int] + KQ_NOTE_LINK: Final[int] if sys.platform != "darwin": - KQ_NOTE_LINKDOWN: int - KQ_NOTE_LINKINV: int - KQ_NOTE_LINKUP: int - KQ_NOTE_LOWAT: int - KQ_NOTE_PCTRLMASK: int - KQ_NOTE_PDATAMASK: int - KQ_NOTE_RENAME: int - KQ_NOTE_REVOKE: int - KQ_NOTE_TRACK: int - KQ_NOTE_TRACKERR: int - KQ_NOTE_WRITE: int + KQ_NOTE_LINKDOWN: Final[int] + KQ_NOTE_LINKINV: Final[int] + KQ_NOTE_LINKUP: Final[int] + KQ_NOTE_LOWAT: Final[int] + KQ_NOTE_PCTRLMASK: Final[int] + KQ_NOTE_PDATAMASK: Final[int] + KQ_NOTE_RENAME: Final[int] + KQ_NOTE_REVOKE: Final[int] + KQ_NOTE_TRACK: Final[int] + KQ_NOTE_TRACKERR: Final[int] + KQ_NOTE_WRITE: Final[int] if sys.platform == "linux": @final class epoll: - def __init__(self, sizehint: int = ..., flags: int = ...) -> None: ... + def __new__(self, sizehint: int = ..., flags: int = ...) -> Self: ... def __enter__(self) -> Self: ... def __exit__( self, @@ -133,23 +133,23 @@ if sys.platform == "linux": @classmethod def fromfd(cls, fd: FileDescriptorLike, /) -> epoll: ... - EPOLLERR: int - EPOLLEXCLUSIVE: int - EPOLLET: int - EPOLLHUP: int - EPOLLIN: int - EPOLLMSG: int - EPOLLONESHOT: int - EPOLLOUT: int - EPOLLPRI: int - EPOLLRDBAND: int - EPOLLRDHUP: int - EPOLLRDNORM: int - EPOLLWRBAND: int - EPOLLWRNORM: int - EPOLL_CLOEXEC: int + EPOLLERR: Final[int] + EPOLLEXCLUSIVE: Final[int] + EPOLLET: Final[int] + EPOLLHUP: Final[int] + EPOLLIN: Final[int] + EPOLLMSG: Final[int] + EPOLLONESHOT: Final[int] + EPOLLOUT: Final[int] + EPOLLPRI: Final[int] + EPOLLRDBAND: Final[int] + EPOLLRDHUP: Final[int] + EPOLLRDNORM: Final[int] + EPOLLWRBAND: Final[int] + EPOLLWRNORM: Final[int] + EPOLL_CLOEXEC: Final[int] if sys.version_info >= (3, 14): - EPOLLWAKEUP: int + EPOLLWAKEUP: Final[int] if sys.platform != "linux" and sys.platform != "darwin" and sys.platform != "win32": # Solaris only diff --git a/mypy/typeshed/stdlib/selectors.pyi b/mypy/typeshed/stdlib/selectors.pyi index 0ba843a403d8a..bcca4e341b9a1 100644 --- a/mypy/typeshed/stdlib/selectors.pyi +++ b/mypy/typeshed/stdlib/selectors.pyi @@ -2,13 +2,13 @@ import sys from _typeshed import FileDescriptor, FileDescriptorLike, Unused from abc import ABCMeta, abstractmethod from collections.abc import Mapping -from typing import Any, NamedTuple +from typing import Any, Final, NamedTuple from typing_extensions import Self, TypeAlias _EventMask: TypeAlias = int -EVENT_READ: _EventMask -EVENT_WRITE: _EventMask +EVENT_READ: Final = 1 +EVENT_WRITE: Final = 2 class SelectorKey(NamedTuple): fileobj: FileDescriptorLike diff --git a/mypy/typeshed/stdlib/smtplib.pyi b/mypy/typeshed/stdlib/smtplib.pyi index 3d392c0479935..6a8467689367a 100644 --- a/mypy/typeshed/stdlib/smtplib.pyi +++ b/mypy/typeshed/stdlib/smtplib.pyi @@ -7,7 +7,7 @@ from re import Pattern from socket import socket from ssl import SSLContext from types import TracebackType -from typing import Any, Protocol, overload, type_check_only +from typing import Any, Final, Protocol, overload, type_check_only from typing_extensions import Self, TypeAlias __all__ = [ @@ -30,12 +30,12 @@ __all__ = [ _Reply: TypeAlias = tuple[int, bytes] _SendErrs: TypeAlias = dict[str, _Reply] -SMTP_PORT: int -SMTP_SSL_PORT: int -CRLF: str -bCRLF: bytes +SMTP_PORT: Final = 25 +SMTP_SSL_PORT: Final = 465 +CRLF: Final[str] +bCRLF: Final[bytes] -OLDSTYLE_AUTH: Pattern[str] +OLDSTYLE_AUTH: Final[Pattern[str]] class SMTPException(OSError): ... class SMTPNotSupportedError(SMTPException): ... @@ -182,7 +182,7 @@ class SMTP_SSL(SMTP): context: SSLContext | None = None, ) -> None: ... -LMTP_PORT: int +LMTP_PORT: Final = 2003 class LMTP(SMTP): def __init__( diff --git a/mypy/typeshed/stdlib/socket.pyi b/mypy/typeshed/stdlib/socket.pyi index d62f4c228151c..b10b3560b91fa 100644 --- a/mypy/typeshed/stdlib/socket.pyi +++ b/mypy/typeshed/stdlib/socket.pyi @@ -136,7 +136,7 @@ from _typeshed import ReadableBuffer, Unused, WriteableBuffer from collections.abc import Iterable from enum import IntEnum, IntFlag from io import BufferedReader, BufferedRWPair, BufferedWriter, IOBase, RawIOBase, TextIOWrapper -from typing import Any, Literal, Protocol, SupportsIndex, overload, type_check_only +from typing import Any, Final, Literal, Protocol, SupportsIndex, overload, type_check_only from typing_extensions import Self __all__ = [ @@ -1059,9 +1059,9 @@ if sys.version_info >= (3, 14): __all__ += ["IP_FREEBIND", "IP_RECVORIGDSTADDR", "VMADDR_CID_LOCAL"] # Re-exported from errno -EBADF: int -EAGAIN: int -EWOULDBLOCK: int +EBADF: Final[int] +EAGAIN: Final[int] +EWOULDBLOCK: Final[int] # These errors are implemented in _socket at runtime # but they consider themselves to live in socket so we'll put them here. @@ -1124,60 +1124,60 @@ class AddressFamily(IntEnum): # FreeBSD >= 14.0 AF_DIVERT = 44 -AF_INET = AddressFamily.AF_INET -AF_INET6 = AddressFamily.AF_INET6 -AF_APPLETALK = AddressFamily.AF_APPLETALK -AF_DECnet: Literal[12] -AF_IPX = AddressFamily.AF_IPX -AF_SNA = AddressFamily.AF_SNA -AF_UNSPEC = AddressFamily.AF_UNSPEC +AF_INET: Final = AddressFamily.AF_INET +AF_INET6: Final = AddressFamily.AF_INET6 +AF_APPLETALK: Final = AddressFamily.AF_APPLETALK +AF_DECnet: Final = 12 +AF_IPX: Final = AddressFamily.AF_IPX +AF_SNA: Final = AddressFamily.AF_SNA +AF_UNSPEC: Final = AddressFamily.AF_UNSPEC if sys.platform != "darwin": - AF_IRDA = AddressFamily.AF_IRDA + AF_IRDA: Final = AddressFamily.AF_IRDA if sys.platform != "win32": - AF_ROUTE = AddressFamily.AF_ROUTE - AF_UNIX = AddressFamily.AF_UNIX + AF_ROUTE: Final = AddressFamily.AF_ROUTE + AF_UNIX: Final = AddressFamily.AF_UNIX if sys.platform == "darwin": - AF_SYSTEM = AddressFamily.AF_SYSTEM + AF_SYSTEM: Final = AddressFamily.AF_SYSTEM if sys.platform != "win32" and sys.platform != "darwin": - AF_ASH = AddressFamily.AF_ASH - AF_ATMPVC = AddressFamily.AF_ATMPVC - AF_ATMSVC = AddressFamily.AF_ATMSVC - AF_AX25 = AddressFamily.AF_AX25 - AF_BRIDGE = AddressFamily.AF_BRIDGE - AF_ECONET = AddressFamily.AF_ECONET - AF_KEY = AddressFamily.AF_KEY - AF_LLC = AddressFamily.AF_LLC - AF_NETBEUI = AddressFamily.AF_NETBEUI - AF_NETROM = AddressFamily.AF_NETROM - AF_PPPOX = AddressFamily.AF_PPPOX - AF_ROSE = AddressFamily.AF_ROSE - AF_SECURITY = AddressFamily.AF_SECURITY - AF_WANPIPE = AddressFamily.AF_WANPIPE - AF_X25 = AddressFamily.AF_X25 + AF_ASH: Final = AddressFamily.AF_ASH + AF_ATMPVC: Final = AddressFamily.AF_ATMPVC + AF_ATMSVC: Final = AddressFamily.AF_ATMSVC + AF_AX25: Final = AddressFamily.AF_AX25 + AF_BRIDGE: Final = AddressFamily.AF_BRIDGE + AF_ECONET: Final = AddressFamily.AF_ECONET + AF_KEY: Final = AddressFamily.AF_KEY + AF_LLC: Final = AddressFamily.AF_LLC + AF_NETBEUI: Final = AddressFamily.AF_NETBEUI + AF_NETROM: Final = AddressFamily.AF_NETROM + AF_PPPOX: Final = AddressFamily.AF_PPPOX + AF_ROSE: Final = AddressFamily.AF_ROSE + AF_SECURITY: Final = AddressFamily.AF_SECURITY + AF_WANPIPE: Final = AddressFamily.AF_WANPIPE + AF_X25: Final = AddressFamily.AF_X25 if sys.platform == "linux": - AF_CAN = AddressFamily.AF_CAN - AF_PACKET = AddressFamily.AF_PACKET - AF_RDS = AddressFamily.AF_RDS - AF_TIPC = AddressFamily.AF_TIPC - AF_ALG = AddressFamily.AF_ALG - AF_NETLINK = AddressFamily.AF_NETLINK - AF_VSOCK = AddressFamily.AF_VSOCK - AF_QIPCRTR = AddressFamily.AF_QIPCRTR + AF_CAN: Final = AddressFamily.AF_CAN + AF_PACKET: Final = AddressFamily.AF_PACKET + AF_RDS: Final = AddressFamily.AF_RDS + AF_TIPC: Final = AddressFamily.AF_TIPC + AF_ALG: Final = AddressFamily.AF_ALG + AF_NETLINK: Final = AddressFamily.AF_NETLINK + AF_VSOCK: Final = AddressFamily.AF_VSOCK + AF_QIPCRTR: Final = AddressFamily.AF_QIPCRTR if sys.platform != "linux": - AF_LINK = AddressFamily.AF_LINK + AF_LINK: Final = AddressFamily.AF_LINK if sys.platform != "darwin" and sys.platform != "linux": - AF_BLUETOOTH = AddressFamily.AF_BLUETOOTH + AF_BLUETOOTH: Final = AddressFamily.AF_BLUETOOTH if sys.platform == "win32" and sys.version_info >= (3, 12): - AF_HYPERV = AddressFamily.AF_HYPERV + AF_HYPERV: Final = AddressFamily.AF_HYPERV if sys.platform != "linux" and sys.platform != "win32" and sys.platform != "darwin" and sys.version_info >= (3, 12): # FreeBSD >= 14.0 - AF_DIVERT = AddressFamily.AF_DIVERT + AF_DIVERT: Final = AddressFamily.AF_DIVERT class SocketKind(IntEnum): SOCK_STREAM = 1 @@ -1189,14 +1189,14 @@ class SocketKind(IntEnum): SOCK_CLOEXEC = 524288 SOCK_NONBLOCK = 2048 -SOCK_STREAM = SocketKind.SOCK_STREAM -SOCK_DGRAM = SocketKind.SOCK_DGRAM -SOCK_RAW = SocketKind.SOCK_RAW -SOCK_RDM = SocketKind.SOCK_RDM -SOCK_SEQPACKET = SocketKind.SOCK_SEQPACKET +SOCK_STREAM: Final = SocketKind.SOCK_STREAM +SOCK_DGRAM: Final = SocketKind.SOCK_DGRAM +SOCK_RAW: Final = SocketKind.SOCK_RAW +SOCK_RDM: Final = SocketKind.SOCK_RDM +SOCK_SEQPACKET: Final = SocketKind.SOCK_SEQPACKET if sys.platform == "linux": - SOCK_CLOEXEC = SocketKind.SOCK_CLOEXEC - SOCK_NONBLOCK = SocketKind.SOCK_NONBLOCK + SOCK_CLOEXEC: Final = SocketKind.SOCK_CLOEXEC + SOCK_NONBLOCK: Final = SocketKind.SOCK_NONBLOCK class MsgFlag(IntFlag): MSG_CTRUNC = 8 @@ -1228,36 +1228,36 @@ class MsgFlag(IntFlag): if sys.platform != "win32" and sys.platform != "linux": MSG_EOF = 256 -MSG_CTRUNC = MsgFlag.MSG_CTRUNC -MSG_DONTROUTE = MsgFlag.MSG_DONTROUTE -MSG_OOB = MsgFlag.MSG_OOB -MSG_PEEK = MsgFlag.MSG_PEEK -MSG_TRUNC = MsgFlag.MSG_TRUNC -MSG_WAITALL = MsgFlag.MSG_WAITALL +MSG_CTRUNC: Final = MsgFlag.MSG_CTRUNC +MSG_DONTROUTE: Final = MsgFlag.MSG_DONTROUTE +MSG_OOB: Final = MsgFlag.MSG_OOB +MSG_PEEK: Final = MsgFlag.MSG_PEEK +MSG_TRUNC: Final = MsgFlag.MSG_TRUNC +MSG_WAITALL: Final = MsgFlag.MSG_WAITALL if sys.platform == "win32": - MSG_BCAST = MsgFlag.MSG_BCAST - MSG_MCAST = MsgFlag.MSG_MCAST + MSG_BCAST: Final = MsgFlag.MSG_BCAST + MSG_MCAST: Final = MsgFlag.MSG_MCAST if sys.platform != "darwin": - MSG_ERRQUEUE = MsgFlag.MSG_ERRQUEUE + MSG_ERRQUEUE: Final = MsgFlag.MSG_ERRQUEUE if sys.platform != "win32": - MSG_DONTWAIT = MsgFlag.MSG_DONTWAIT - MSG_EOR = MsgFlag.MSG_EOR - MSG_NOSIGNAL = MsgFlag.MSG_NOSIGNAL # Sometimes this exists on darwin, sometimes not + MSG_DONTWAIT: Final = MsgFlag.MSG_DONTWAIT + MSG_EOR: Final = MsgFlag.MSG_EOR + MSG_NOSIGNAL: Final = MsgFlag.MSG_NOSIGNAL # Sometimes this exists on darwin, sometimes not if sys.platform != "win32" and sys.platform != "darwin": - MSG_CMSG_CLOEXEC = MsgFlag.MSG_CMSG_CLOEXEC - MSG_CONFIRM = MsgFlag.MSG_CONFIRM - MSG_FASTOPEN = MsgFlag.MSG_FASTOPEN - MSG_MORE = MsgFlag.MSG_MORE + MSG_CMSG_CLOEXEC: Final = MsgFlag.MSG_CMSG_CLOEXEC + MSG_CONFIRM: Final = MsgFlag.MSG_CONFIRM + MSG_FASTOPEN: Final = MsgFlag.MSG_FASTOPEN + MSG_MORE: Final = MsgFlag.MSG_MORE if sys.platform != "win32" and sys.platform != "darwin" and sys.platform != "linux": - MSG_NOTIFICATION = MsgFlag.MSG_NOTIFICATION + MSG_NOTIFICATION: Final = MsgFlag.MSG_NOTIFICATION if sys.platform != "win32" and sys.platform != "linux": - MSG_EOF = MsgFlag.MSG_EOF + MSG_EOF: Final = MsgFlag.MSG_EOF class AddressInfo(IntFlag): AI_ADDRCONFIG = 32 @@ -1272,18 +1272,18 @@ class AddressInfo(IntFlag): AI_MASK = 5127 AI_V4MAPPED_CFG = 512 -AI_ADDRCONFIG = AddressInfo.AI_ADDRCONFIG -AI_ALL = AddressInfo.AI_ALL -AI_CANONNAME = AddressInfo.AI_CANONNAME -AI_NUMERICHOST = AddressInfo.AI_NUMERICHOST -AI_NUMERICSERV = AddressInfo.AI_NUMERICSERV -AI_PASSIVE = AddressInfo.AI_PASSIVE -AI_V4MAPPED = AddressInfo.AI_V4MAPPED +AI_ADDRCONFIG: Final = AddressInfo.AI_ADDRCONFIG +AI_ALL: Final = AddressInfo.AI_ALL +AI_CANONNAME: Final = AddressInfo.AI_CANONNAME +AI_NUMERICHOST: Final = AddressInfo.AI_NUMERICHOST +AI_NUMERICSERV: Final = AddressInfo.AI_NUMERICSERV +AI_PASSIVE: Final = AddressInfo.AI_PASSIVE +AI_V4MAPPED: Final = AddressInfo.AI_V4MAPPED if sys.platform != "win32" and sys.platform != "linux": - AI_DEFAULT = AddressInfo.AI_DEFAULT - AI_MASK = AddressInfo.AI_MASK - AI_V4MAPPED_CFG = AddressInfo.AI_V4MAPPED_CFG + AI_DEFAULT: Final = AddressInfo.AI_DEFAULT + AI_MASK: Final = AddressInfo.AI_MASK + AI_V4MAPPED_CFG: Final = AddressInfo.AI_V4MAPPED_CFG if sys.platform == "win32": errorTab: dict[int, str] # undocumented @@ -1300,6 +1300,7 @@ class _SendableFile(Protocol): # def fileno(self) -> int: ... class socket(_socket.socket): + __slots__ = ["__weakref__", "_io_refs", "_closed"] def __init__( self, family: AddressFamily | int = -1, type: SocketKind | int = -1, proto: int = -1, fileno: int | None = None ) -> None: ... diff --git a/mypy/typeshed/stdlib/sqlite3/__init__.pyi b/mypy/typeshed/stdlib/sqlite3/__init__.pyi index 5a659deaccf68..882cd143c29ce 100644 --- a/mypy/typeshed/stdlib/sqlite3/__init__.pyi +++ b/mypy/typeshed/stdlib/sqlite3/__init__.pyi @@ -63,7 +63,7 @@ from sqlite3.dbapi2 import ( ) from types import TracebackType from typing import Any, Literal, Protocol, SupportsIndex, TypeVar, final, overload, type_check_only -from typing_extensions import Self, TypeAlias +from typing_extensions import Self, TypeAlias, disjoint_base if sys.version_info < (3, 14): from sqlite3.dbapi2 import version_info as version_info @@ -268,6 +268,7 @@ class OperationalError(DatabaseError): ... class ProgrammingError(DatabaseError): ... class Warning(Exception): ... +@disjoint_base class Connection: @property def DataError(self) -> type[DataError]: ... @@ -405,6 +406,7 @@ class Connection: self, type: type[BaseException] | None, value: BaseException | None, traceback: TracebackType | None, / ) -> Literal[False]: ... +@disjoint_base class Cursor(Iterator[Any]): arraysize: int @property @@ -436,6 +438,7 @@ class Cursor(Iterator[Any]): class PrepareProtocol: def __init__(self, *args: object, **kwargs: object) -> None: ... +@disjoint_base class Row(Sequence[Any]): def __new__(cls, cursor: Cursor, data: tuple[Any, ...], /) -> Self: ... def keys(self) -> list[str]: ... diff --git a/mypy/typeshed/stdlib/sqlite3/dbapi2.pyi b/mypy/typeshed/stdlib/sqlite3/dbapi2.pyi index d37a0d391ec6a..9e170a81243d8 100644 --- a/mypy/typeshed/stdlib/sqlite3/dbapi2.pyi +++ b/mypy/typeshed/stdlib/sqlite3/dbapi2.pyi @@ -66,7 +66,8 @@ from sqlite3 import ( Row as Row, Warning as Warning, ) -from typing import Literal +from typing import Final, Literal +from typing_extensions import deprecated if sys.version_info >= (3, 12): from _sqlite3 import ( @@ -211,11 +212,15 @@ if sys.version_info >= (3, 11): if sys.version_info < (3, 14): # Deprecated and removed from _sqlite3 in 3.12, but removed from here in 3.14. - version: str + version: Final[str] if sys.version_info < (3, 12): if sys.version_info >= (3, 10): # deprecation wrapper that has a different name for the argument... + @deprecated( + "Deprecated since Python 3.10; removed in Python 3.12. " + "Open database in URI mode using `cache=shared` parameter instead." + ) def enable_shared_cache(enable: int) -> None: ... else: from _sqlite3 import enable_shared_cache as enable_shared_cache @@ -223,9 +228,9 @@ if sys.version_info < (3, 12): if sys.version_info < (3, 10): from _sqlite3 import OptimizedUnicode as OptimizedUnicode -paramstyle: str +paramstyle: Final = "qmark" threadsafety: Literal[0, 1, 3] -apilevel: str +apilevel: Final[str] Date = date Time = time Timestamp = datetime @@ -236,7 +241,7 @@ def TimestampFromTicks(ticks: float) -> Timestamp: ... if sys.version_info < (3, 14): # Deprecated in 3.12, removed in 3.14. - version_info: tuple[int, int, int] + version_info: Final[tuple[int, int, int]] -sqlite_version_info: tuple[int, int, int] +sqlite_version_info: Final[tuple[int, int, int]] Binary = memoryview diff --git a/mypy/typeshed/stdlib/sre_compile.pyi b/mypy/typeshed/stdlib/sre_compile.pyi index 2d04a886c931b..d8f0b7937e994 100644 --- a/mypy/typeshed/stdlib/sre_compile.pyi +++ b/mypy/typeshed/stdlib/sre_compile.pyi @@ -2,9 +2,9 @@ from re import Pattern from sre_constants import * from sre_constants import _NamedIntConstant from sre_parse import SubPattern -from typing import Any +from typing import Any, Final -MAXCODE: int +MAXCODE: Final[int] def dis(code: list[_NamedIntConstant]) -> None: ... def isstring(obj: Any) -> bool: ... diff --git a/mypy/typeshed/stdlib/sre_constants.pyi b/mypy/typeshed/stdlib/sre_constants.pyi index a3921aa0fc3b8..9a1da4ac89e7e 100644 --- a/mypy/typeshed/stdlib/sre_constants.pyi +++ b/mypy/typeshed/stdlib/sre_constants.pyi @@ -1,15 +1,22 @@ import sys from re import error as error from typing import Final -from typing_extensions import Self +from typing_extensions import Self, disjoint_base MAXGROUPS: Final[int] MAGIC: Final[int] -class _NamedIntConstant(int): - name: str - def __new__(cls, value: int, name: str) -> Self: ... +if sys.version_info >= (3, 12): + class _NamedIntConstant(int): + name: str + def __new__(cls, value: int, name: str) -> Self: ... + +else: + @disjoint_base + class _NamedIntConstant(int): + name: str + def __new__(cls, value: int, name: str) -> Self: ... MAXREPEAT: Final[_NamedIntConstant] OPCODES: list[_NamedIntConstant] diff --git a/mypy/typeshed/stdlib/sre_parse.pyi b/mypy/typeshed/stdlib/sre_parse.pyi index c242bd2a065fb..eaacbff312a92 100644 --- a/mypy/typeshed/stdlib/sre_parse.pyi +++ b/mypy/typeshed/stdlib/sre_parse.pyi @@ -3,24 +3,24 @@ from collections.abc import Iterable from re import Match, Pattern as _Pattern from sre_constants import * from sre_constants import _NamedIntConstant as _NIC, error as _Error -from typing import Any, overload +from typing import Any, Final, overload from typing_extensions import TypeAlias -SPECIAL_CHARS: str -REPEAT_CHARS: str -DIGITS: frozenset[str] -OCTDIGITS: frozenset[str] -HEXDIGITS: frozenset[str] -ASCIILETTERS: frozenset[str] -WHITESPACE: frozenset[str] -ESCAPES: dict[str, tuple[_NIC, int]] -CATEGORIES: dict[str, tuple[_NIC, _NIC] | tuple[_NIC, list[tuple[_NIC, _NIC]]]] -FLAGS: dict[str, int] -TYPE_FLAGS: int -GLOBAL_FLAGS: int +SPECIAL_CHARS: Final = ".\\[{()*+?^$|" +REPEAT_CHARS: Final = "*+?{" +DIGITS: Final[frozenset[str]] +OCTDIGITS: Final[frozenset[str]] +HEXDIGITS: Final[frozenset[str]] +ASCIILETTERS: Final[frozenset[str]] +WHITESPACE: Final[frozenset[str]] +ESCAPES: Final[dict[str, tuple[_NIC, int]]] +CATEGORIES: Final[dict[str, tuple[_NIC, _NIC] | tuple[_NIC, list[tuple[_NIC, _NIC]]]]] +FLAGS: Final[dict[str, int]] +TYPE_FLAGS: Final[int] +GLOBAL_FLAGS: Final[int] if sys.version_info >= (3, 11): - MAXWIDTH: int + MAXWIDTH: Final[int] if sys.version_info < (3, 11): class Verbose(Exception): ... @@ -39,7 +39,7 @@ class State: lookbehindgroups: int | None @property def groups(self) -> int: ... - def opengroup(self, name: str | None = ...) -> int: ... + def opengroup(self, name: str | None = None) -> int: ... def closegroup(self, gid: int, p: SubPattern) -> None: ... def checkgroup(self, gid: int) -> bool: ... def checklookbehindgroup(self, gid: int, source: Tokenizer) -> None: ... diff --git a/mypy/typeshed/stdlib/ssl.pyi b/mypy/typeshed/stdlib/ssl.pyi index f1893ec3194f0..faa98cb399200 100644 --- a/mypy/typeshed/stdlib/ssl.pyi +++ b/mypy/typeshed/stdlib/ssl.pyi @@ -81,7 +81,7 @@ class SSLCertVerificationError(SSLError, ValueError): CertificateError = SSLCertVerificationError if sys.version_info < (3, 12): - @deprecated("Deprecated since Python 3.7. Removed in Python 3.12. Use `SSLContext.wrap_socket()` instead.") + @deprecated("Deprecated since Python 3.7; removed in Python 3.12. Use `SSLContext.wrap_socket()` instead.") def wrap_socket( sock: socket.socket, keyfile: StrOrBytesPath | None = None, @@ -94,7 +94,7 @@ if sys.version_info < (3, 12): suppress_ragged_eofs: bool = True, ciphers: str | None = None, ) -> SSLSocket: ... - @deprecated("Deprecated since Python 3.7. Removed in Python 3.12.") + @deprecated("Deprecated since Python 3.7; removed in Python 3.12.") def match_hostname(cert: _PeerCertRetDictType, hostname: str) -> None: ... def cert_time_to_seconds(cert_time: str) -> int: ... diff --git a/mypy/typeshed/stdlib/statistics.pyi b/mypy/typeshed/stdlib/statistics.pyi index 6d7d3fbb4956a..ba9e5f1b6b71f 100644 --- a/mypy/typeshed/stdlib/statistics.pyi +++ b/mypy/typeshed/stdlib/statistics.pyi @@ -79,6 +79,7 @@ def stdev(data: Iterable[_NumberT], xbar: _NumberT | None = None) -> _NumberT: . def variance(data: Iterable[_NumberT], xbar: _NumberT | None = None) -> _NumberT: ... class NormalDist: + __slots__ = {"_mu": "Arithmetic mean of a normal distribution", "_sigma": "Standard deviation of a normal distribution"} def __init__(self, mu: float = 0.0, sigma: float = 1.0) -> None: ... @property def mean(self) -> float: ... diff --git a/mypy/typeshed/stdlib/string/__init__.pyi b/mypy/typeshed/stdlib/string/__init__.pyi index 29fe27f39b809..c8b32a98e26d7 100644 --- a/mypy/typeshed/stdlib/string/__init__.pyi +++ b/mypy/typeshed/stdlib/string/__init__.pyi @@ -2,7 +2,7 @@ import sys from _typeshed import StrOrLiteralStr from collections.abc import Iterable, Mapping, Sequence from re import Pattern, RegexFlag -from typing import Any, ClassVar, overload +from typing import Any, ClassVar, Final, overload from typing_extensions import LiteralString __all__ = [ @@ -20,15 +20,15 @@ __all__ = [ "Template", ] -ascii_letters: LiteralString -ascii_lowercase: LiteralString -ascii_uppercase: LiteralString -digits: LiteralString -hexdigits: LiteralString -octdigits: LiteralString -punctuation: LiteralString -printable: LiteralString -whitespace: LiteralString +whitespace: Final = " \t\n\r\v\f" +ascii_lowercase: Final = "abcdefghijklmnopqrstuvwxyz" +ascii_uppercase: Final = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" +ascii_letters: Final[LiteralString] # string too long +digits: Final = "0123456789" +hexdigits: Final = "0123456789abcdefABCDEF" +octdigits: Final = "01234567" +punctuation: Final = r"""!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~""" +printable: Final[LiteralString] # string too long def capwords(s: StrOrLiteralStr, sep: StrOrLiteralStr | None = None) -> StrOrLiteralStr: ... diff --git a/mypy/typeshed/stdlib/stringprep.pyi b/mypy/typeshed/stdlib/stringprep.pyi index fc28c027ca9b1..d67955e499c85 100644 --- a/mypy/typeshed/stdlib/stringprep.pyi +++ b/mypy/typeshed/stdlib/stringprep.pyi @@ -1,10 +1,12 @@ -b1_set: set[int] -b3_exceptions: dict[int, str] -c22_specials: set[int] -c6_set: set[int] -c7_set: set[int] -c8_set: set[int] -c9_set: set[int] +from typing import Final + +b1_set: Final[set[int]] +b3_exceptions: Final[dict[int, str]] +c22_specials: Final[set[int]] +c6_set: Final[set[int]] +c7_set: Final[set[int]] +c8_set: Final[set[int]] +c9_set: Final[set[int]] def in_table_a1(code: str) -> bool: ... def in_table_b1(code: str) -> bool: ... diff --git a/mypy/typeshed/stdlib/subprocess.pyi b/mypy/typeshed/stdlib/subprocess.pyi index 8b72e2ec7ae2d..e1e25bcb50cbe 100644 --- a/mypy/typeshed/stdlib/subprocess.pyi +++ b/mypy/typeshed/stdlib/subprocess.pyi @@ -106,7 +106,7 @@ if sys.version_info >= (3, 11): creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), *, capture_output: bool = False, check: bool = False, @@ -140,7 +140,7 @@ if sys.version_info >= (3, 11): creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), *, capture_output: bool = False, check: bool = False, @@ -174,7 +174,7 @@ if sys.version_info >= (3, 11): creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), *, capture_output: bool = False, check: bool = False, @@ -209,7 +209,7 @@ if sys.version_info >= (3, 11): creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), # where the *real* keyword only args start capture_output: bool = False, check: bool = False, @@ -243,7 +243,7 @@ if sys.version_info >= (3, 11): creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), *, capture_output: bool = False, check: bool = False, @@ -277,7 +277,7 @@ if sys.version_info >= (3, 11): creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), *, capture_output: bool = False, check: bool = False, @@ -314,7 +314,7 @@ elif sys.version_info >= (3, 10): creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), *, capture_output: bool = False, check: bool = False, @@ -347,7 +347,7 @@ elif sys.version_info >= (3, 10): creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), *, capture_output: bool = False, check: bool = False, @@ -380,7 +380,7 @@ elif sys.version_info >= (3, 10): creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), *, capture_output: bool = False, check: bool = False, @@ -414,7 +414,7 @@ elif sys.version_info >= (3, 10): creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), # where the *real* keyword only args start capture_output: bool = False, check: bool = False, @@ -447,7 +447,7 @@ elif sys.version_info >= (3, 10): creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), *, capture_output: bool = False, check: bool = False, @@ -480,7 +480,7 @@ elif sys.version_info >= (3, 10): creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), *, capture_output: bool = False, check: bool = False, @@ -516,7 +516,7 @@ else: creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), *, capture_output: bool = False, check: bool = False, @@ -548,7 +548,7 @@ else: creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), *, capture_output: bool = False, check: bool = False, @@ -580,7 +580,7 @@ else: creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), *, capture_output: bool = False, check: bool = False, @@ -613,7 +613,7 @@ else: creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), # where the *real* keyword only args start capture_output: bool = False, check: bool = False, @@ -645,7 +645,7 @@ else: creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), *, capture_output: bool = False, check: bool = False, @@ -677,7 +677,7 @@ else: creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), *, capture_output: bool = False, check: bool = False, @@ -712,7 +712,7 @@ if sys.version_info >= (3, 11): creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), *, encoding: str | None = None, timeout: float | None = None, @@ -744,7 +744,7 @@ elif sys.version_info >= (3, 10): creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), *, encoding: str | None = None, timeout: float | None = None, @@ -774,7 +774,7 @@ else: creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), *, encoding: str | None = None, timeout: float | None = None, @@ -805,8 +805,8 @@ if sys.version_info >= (3, 11): creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., - timeout: float | None = ..., + pass_fds: Collection[int] = (), + timeout: float | None = None, *, encoding: str | None = None, text: bool | None = None, @@ -837,8 +837,8 @@ elif sys.version_info >= (3, 10): creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., - timeout: float | None = ..., + pass_fds: Collection[int] = (), + timeout: float | None = None, *, encoding: str | None = None, text: bool | None = None, @@ -867,8 +867,8 @@ else: creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., - timeout: float | None = ..., + pass_fds: Collection[int] = (), + timeout: float | None = None, *, encoding: str | None = None, text: bool | None = None, @@ -897,10 +897,10 @@ if sys.version_info >= (3, 11): creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), *, timeout: float | None = None, - input: _InputString | None = ..., + input: _InputString | None = None, encoding: str | None = None, errors: str | None = None, text: Literal[True], @@ -928,10 +928,10 @@ if sys.version_info >= (3, 11): creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), *, timeout: float | None = None, - input: _InputString | None = ..., + input: _InputString | None = None, encoding: str, errors: str | None = None, text: bool | None = None, @@ -959,10 +959,10 @@ if sys.version_info >= (3, 11): creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), *, timeout: float | None = None, - input: _InputString | None = ..., + input: _InputString | None = None, encoding: str | None = None, errors: str, text: bool | None = None, @@ -991,10 +991,10 @@ if sys.version_info >= (3, 11): creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), # where the real keyword only ones start timeout: float | None = None, - input: _InputString | None = ..., + input: _InputString | None = None, encoding: str | None = None, errors: str | None = None, text: bool | None = None, @@ -1022,10 +1022,10 @@ if sys.version_info >= (3, 11): creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), *, timeout: float | None = None, - input: _InputString | None = ..., + input: _InputString | None = None, encoding: None = None, errors: None = None, text: Literal[False] | None = None, @@ -1053,10 +1053,10 @@ if sys.version_info >= (3, 11): creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), *, timeout: float | None = None, - input: _InputString | None = ..., + input: _InputString | None = None, encoding: str | None = None, errors: str | None = None, text: bool | None = None, @@ -1087,10 +1087,10 @@ elif sys.version_info >= (3, 10): creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), *, timeout: float | None = None, - input: _InputString | None = ..., + input: _InputString | None = None, encoding: str | None = None, errors: str | None = None, text: Literal[True], @@ -1117,10 +1117,10 @@ elif sys.version_info >= (3, 10): creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), *, timeout: float | None = None, - input: _InputString | None = ..., + input: _InputString | None = None, encoding: str, errors: str | None = None, text: bool | None = None, @@ -1147,10 +1147,10 @@ elif sys.version_info >= (3, 10): creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), *, timeout: float | None = None, - input: _InputString | None = ..., + input: _InputString | None = None, encoding: str | None = None, errors: str, text: bool | None = None, @@ -1178,10 +1178,10 @@ elif sys.version_info >= (3, 10): creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), # where the real keyword only ones start timeout: float | None = None, - input: _InputString | None = ..., + input: _InputString | None = None, encoding: str | None = None, errors: str | None = None, text: bool | None = None, @@ -1208,10 +1208,10 @@ elif sys.version_info >= (3, 10): creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), *, timeout: float | None = None, - input: _InputString | None = ..., + input: _InputString | None = None, encoding: None = None, errors: None = None, text: Literal[False] | None = None, @@ -1238,10 +1238,10 @@ elif sys.version_info >= (3, 10): creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), *, timeout: float | None = None, - input: _InputString | None = ..., + input: _InputString | None = None, encoding: str | None = None, errors: str | None = None, text: bool | None = None, @@ -1270,10 +1270,10 @@ else: creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), *, timeout: float | None = None, - input: _InputString | None = ..., + input: _InputString | None = None, encoding: str | None = None, errors: str | None = None, text: Literal[True], @@ -1299,10 +1299,10 @@ else: creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), *, timeout: float | None = None, - input: _InputString | None = ..., + input: _InputString | None = None, encoding: str, errors: str | None = None, text: bool | None = None, @@ -1328,10 +1328,10 @@ else: creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), *, timeout: float | None = None, - input: _InputString | None = ..., + input: _InputString | None = None, encoding: str | None = None, errors: str, text: bool | None = None, @@ -1358,10 +1358,10 @@ else: creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), # where the real keyword only ones start timeout: float | None = None, - input: _InputString | None = ..., + input: _InputString | None = None, encoding: str | None = None, errors: str | None = None, text: bool | None = None, @@ -1387,10 +1387,10 @@ else: creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), *, timeout: float | None = None, - input: _InputString | None = ..., + input: _InputString | None = None, encoding: None = None, errors: None = None, text: Literal[False] | None = None, @@ -1416,10 +1416,10 @@ else: creationflags: int = 0, restore_signals: bool = True, start_new_session: bool = False, - pass_fds: Collection[int] = ..., + pass_fds: Collection[int] = (), *, timeout: float | None = None, - input: _InputString | None = ..., + input: _InputString | None = None, encoding: str | None = None, errors: str | None = None, text: bool | None = None, diff --git a/mypy/typeshed/stdlib/sunau.pyi b/mypy/typeshed/stdlib/sunau.pyi index d81645cb5687f..f83a0a4c520e7 100644 --- a/mypy/typeshed/stdlib/sunau.pyi +++ b/mypy/typeshed/stdlib/sunau.pyi @@ -1,25 +1,25 @@ from _typeshed import Unused -from typing import IO, Any, Literal, NamedTuple, NoReturn, overload +from typing import IO, Any, Final, Literal, NamedTuple, NoReturn, overload from typing_extensions import Self, TypeAlias _File: TypeAlias = str | IO[bytes] class Error(Exception): ... -AUDIO_FILE_MAGIC: int -AUDIO_FILE_ENCODING_MULAW_8: int -AUDIO_FILE_ENCODING_LINEAR_8: int -AUDIO_FILE_ENCODING_LINEAR_16: int -AUDIO_FILE_ENCODING_LINEAR_24: int -AUDIO_FILE_ENCODING_LINEAR_32: int -AUDIO_FILE_ENCODING_FLOAT: int -AUDIO_FILE_ENCODING_DOUBLE: int -AUDIO_FILE_ENCODING_ADPCM_G721: int -AUDIO_FILE_ENCODING_ADPCM_G722: int -AUDIO_FILE_ENCODING_ADPCM_G723_3: int -AUDIO_FILE_ENCODING_ADPCM_G723_5: int -AUDIO_FILE_ENCODING_ALAW_8: int -AUDIO_UNKNOWN_SIZE: int +AUDIO_FILE_MAGIC: Final = 0x2E736E64 +AUDIO_FILE_ENCODING_MULAW_8: Final = 1 +AUDIO_FILE_ENCODING_LINEAR_8: Final = 2 +AUDIO_FILE_ENCODING_LINEAR_16: Final = 3 +AUDIO_FILE_ENCODING_LINEAR_24: Final = 4 +AUDIO_FILE_ENCODING_LINEAR_32: Final = 5 +AUDIO_FILE_ENCODING_FLOAT: Final = 6 +AUDIO_FILE_ENCODING_DOUBLE: Final = 7 +AUDIO_FILE_ENCODING_ADPCM_G721: Final = 23 +AUDIO_FILE_ENCODING_ADPCM_G722: Final = 24 +AUDIO_FILE_ENCODING_ADPCM_G723_3: Final = 25 +AUDIO_FILE_ENCODING_ADPCM_G723_5: Final = 26 +AUDIO_FILE_ENCODING_ALAW_8: Final = 27 +AUDIO_UNKNOWN_SIZE: Final = 0xFFFFFFFF class _sunau_params(NamedTuple): nchannels: int diff --git a/mypy/typeshed/stdlib/symbol.pyi b/mypy/typeshed/stdlib/symbol.pyi index 48ae3567a1a56..5344ce504c6c7 100644 --- a/mypy/typeshed/stdlib/symbol.pyi +++ b/mypy/typeshed/stdlib/symbol.pyi @@ -1,93 +1,95 @@ -single_input: int -file_input: int -eval_input: int -decorator: int -decorators: int -decorated: int -async_funcdef: int -funcdef: int -parameters: int -typedargslist: int -tfpdef: int -varargslist: int -vfpdef: int -stmt: int -simple_stmt: int -small_stmt: int -expr_stmt: int -annassign: int -testlist_star_expr: int -augassign: int -del_stmt: int -pass_stmt: int -flow_stmt: int -break_stmt: int -continue_stmt: int -return_stmt: int -yield_stmt: int -raise_stmt: int -import_stmt: int -import_name: int -import_from: int -import_as_name: int -dotted_as_name: int -import_as_names: int -dotted_as_names: int -dotted_name: int -global_stmt: int -nonlocal_stmt: int -assert_stmt: int -compound_stmt: int -async_stmt: int -if_stmt: int -while_stmt: int -for_stmt: int -try_stmt: int -with_stmt: int -with_item: int -except_clause: int -suite: int -test: int -test_nocond: int -lambdef: int -lambdef_nocond: int -or_test: int -and_test: int -not_test: int -comparison: int -comp_op: int -star_expr: int -expr: int -xor_expr: int -and_expr: int -shift_expr: int -arith_expr: int -term: int -factor: int -power: int -atom_expr: int -atom: int -testlist_comp: int -trailer: int -subscriptlist: int -subscript: int -sliceop: int -exprlist: int -testlist: int -dictorsetmaker: int -classdef: int -arglist: int -argument: int -comp_iter: int -comp_for: int -comp_if: int -encoding_decl: int -yield_expr: int -yield_arg: int -sync_comp_for: int -func_body_suite: int -func_type: int -func_type_input: int -namedexpr_test: int -typelist: int -sym_name: dict[int, str] +from typing import Final + +single_input: Final[int] +file_input: Final[int] +eval_input: Final[int] +decorator: Final[int] +decorators: Final[int] +decorated: Final[int] +async_funcdef: Final[int] +funcdef: Final[int] +parameters: Final[int] +typedargslist: Final[int] +tfpdef: Final[int] +varargslist: Final[int] +vfpdef: Final[int] +stmt: Final[int] +simple_stmt: Final[int] +small_stmt: Final[int] +expr_stmt: Final[int] +annassign: Final[int] +testlist_star_expr: Final[int] +augassign: Final[int] +del_stmt: Final[int] +pass_stmt: Final[int] +flow_stmt: Final[int] +break_stmt: Final[int] +continue_stmt: Final[int] +return_stmt: Final[int] +yield_stmt: Final[int] +raise_stmt: Final[int] +import_stmt: Final[int] +import_name: Final[int] +import_from: Final[int] +import_as_name: Final[int] +dotted_as_name: Final[int] +import_as_names: Final[int] +dotted_as_names: Final[int] +dotted_name: Final[int] +global_stmt: Final[int] +nonlocal_stmt: Final[int] +assert_stmt: Final[int] +compound_stmt: Final[int] +async_stmt: Final[int] +if_stmt: Final[int] +while_stmt: Final[int] +for_stmt: Final[int] +try_stmt: Final[int] +with_stmt: Final[int] +with_item: Final[int] +except_clause: Final[int] +suite: Final[int] +test: Final[int] +test_nocond: Final[int] +lambdef: Final[int] +lambdef_nocond: Final[int] +or_test: Final[int] +and_test: Final[int] +not_test: Final[int] +comparison: Final[int] +comp_op: Final[int] +star_expr: Final[int] +expr: Final[int] +xor_expr: Final[int] +and_expr: Final[int] +shift_expr: Final[int] +arith_expr: Final[int] +term: Final[int] +factor: Final[int] +power: Final[int] +atom_expr: Final[int] +atom: Final[int] +testlist_comp: Final[int] +trailer: Final[int] +subscriptlist: Final[int] +subscript: Final[int] +sliceop: Final[int] +exprlist: Final[int] +testlist: Final[int] +dictorsetmaker: Final[int] +classdef: Final[int] +arglist: Final[int] +argument: Final[int] +comp_iter: Final[int] +comp_for: Final[int] +comp_if: Final[int] +encoding_decl: Final[int] +yield_expr: Final[int] +yield_arg: Final[int] +sync_comp_for: Final[int] +func_body_suite: Final[int] +func_type: Final[int] +func_type_input: Final[int] +namedexpr_test: Final[int] +typelist: Final[int] +sym_name: Final[dict[int, str]] diff --git a/mypy/typeshed/stdlib/symtable.pyi b/mypy/typeshed/stdlib/symtable.pyi index d5f2be04b6009..a727b878688ed 100644 --- a/mypy/typeshed/stdlib/symtable.pyi +++ b/mypy/typeshed/stdlib/symtable.pyi @@ -49,8 +49,11 @@ class Function(SymbolTable): def get_nonlocals(self) -> tuple[str, ...]: ... class Class(SymbolTable): - @deprecated("deprecated in Python 3.14, will be removed in Python 3.16") - def get_methods(self) -> tuple[str, ...]: ... + if sys.version_info >= (3, 14): + @deprecated("Deprecated since Python 3.14; will be removed in Python 3.16.") + def get_methods(self) -> tuple[str, ...]: ... + else: + def get_methods(self) -> tuple[str, ...]: ... class Symbol: def __init__( diff --git a/mypy/typeshed/stdlib/sys/__init__.pyi b/mypy/typeshed/stdlib/sys/__init__.pyi index b16e7c0abd052..7807b0eab01f6 100644 --- a/mypy/typeshed/stdlib/sys/__init__.pyi +++ b/mypy/typeshed/stdlib/sys/__init__.pyi @@ -345,7 +345,7 @@ version_info: _version_info def call_tracing(func: Callable[..., _T], args: Any, /) -> _T: ... if sys.version_info >= (3, 13): - @deprecated("Deprecated in Python 3.13; use _clear_internal_caches() instead.") + @deprecated("Deprecated since Python 3.13. Use `_clear_internal_caches()` instead.") def _clear_type_cache() -> None: ... else: diff --git a/mypy/typeshed/stdlib/sys/_monitoring.pyi b/mypy/typeshed/stdlib/sys/_monitoring.pyi index 3a8292ea0df4e..5d231c7a93b39 100644 --- a/mypy/typeshed/stdlib/sys/_monitoring.pyi +++ b/mypy/typeshed/stdlib/sys/_monitoring.pyi @@ -11,10 +11,10 @@ from types import CodeType from typing import Any, Final, type_check_only from typing_extensions import deprecated -DEBUGGER_ID: Final[int] -COVERAGE_ID: Final[int] -PROFILER_ID: Final[int] -OPTIMIZER_ID: Final[int] +DEBUGGER_ID: Final = 0 +COVERAGE_ID: Final = 1 +PROFILER_ID: Final = 2 +OPTIMIZER_ID: Final = 5 def use_tool_id(tool_id: int, name: str, /) -> None: ... def free_tool_id(tool_id: int, /) -> None: ... @@ -46,7 +46,7 @@ class _events: BRANCH_TAKEN: Final[int] @property - @deprecated("BRANCH is deprecated; use BRANCH_LEFT or BRANCH_TAKEN instead") + @deprecated("Deprecated since Python 3.14. Use `BRANCH_LEFT` or `BRANCH_TAKEN` instead.") def BRANCH(self) -> int: ... else: diff --git a/mypy/typeshed/stdlib/tarfile.pyi b/mypy/typeshed/stdlib/tarfile.pyi index 4e394409bbe0f..f6623ea9929d4 100644 --- a/mypy/typeshed/stdlib/tarfile.pyi +++ b/mypy/typeshed/stdlib/tarfile.pyi @@ -656,7 +656,7 @@ class TarFile: members: Iterable[TarInfo] | None = None, *, numeric_owner: bool = False, - filter: _TarfileFilter | None = ..., + filter: _TarfileFilter | None = None, ) -> None: ... # Same situation as for `extractall`. def extract( @@ -666,7 +666,7 @@ class TarFile: set_attrs: bool = True, *, numeric_owner: bool = False, - filter: _TarfileFilter | None = ..., + filter: _TarfileFilter | None = None, ) -> None: ... def _extract_member( self, @@ -744,6 +744,28 @@ def tar_filter(member: TarInfo, dest_path: str) -> TarInfo: ... def data_filter(member: TarInfo, dest_path: str) -> TarInfo: ... class TarInfo: + __slots__ = ( + "name", + "mode", + "uid", + "gid", + "size", + "mtime", + "chksum", + "type", + "linkname", + "uname", + "gname", + "devmajor", + "devminor", + "offset", + "offset_data", + "pax_headers", + "sparse", + "_tarfile", + "_sparse_structs", + "_link_target", + ) name: str path: str size: int @@ -765,10 +787,10 @@ class TarInfo: def __init__(self, name: str = "") -> None: ... if sys.version_info >= (3, 13): @property - @deprecated("Deprecated in Python 3.13; removal scheduled for Python 3.16") + @deprecated("Deprecated since Python 3.13; will be removed in Python 3.16.") def tarfile(self) -> TarFile | None: ... @tarfile.setter - @deprecated("Deprecated in Python 3.13; removal scheduled for Python 3.16") + @deprecated("Deprecated since Python 3.13; will be removed in Python 3.16.") def tarfile(self, tarfile: TarFile | None) -> None: ... else: tarfile: TarFile | None diff --git a/mypy/typeshed/stdlib/telnetlib.pyi b/mypy/typeshed/stdlib/telnetlib.pyi index 6b599256d17b6..88aa43d248996 100644 --- a/mypy/typeshed/stdlib/telnetlib.pyi +++ b/mypy/typeshed/stdlib/telnetlib.pyi @@ -2,89 +2,89 @@ import socket from collections.abc import Callable, MutableSequence, Sequence from re import Match, Pattern from types import TracebackType -from typing import Any +from typing import Any, Final from typing_extensions import Self __all__ = ["Telnet"] -DEBUGLEVEL: int -TELNET_PORT: int +DEBUGLEVEL: Final = 0 +TELNET_PORT: Final = 23 -IAC: bytes -DONT: bytes -DO: bytes -WONT: bytes -WILL: bytes -theNULL: bytes +IAC: Final = b"\xff" +DONT: Final = b"\xfe" +DO: Final = b"\xfd" +WONT: Final = b"\xfc" +WILL: Final = b"\xfb" +theNULL: Final = b"\x00" -SE: bytes -NOP: bytes -DM: bytes -BRK: bytes -IP: bytes -AO: bytes -AYT: bytes -EC: bytes -EL: bytes -GA: bytes -SB: bytes +SE: Final = b"\xf0" +NOP: Final = b"\xf1" +DM: Final = b"\xf2" +BRK: Final = b"\xf3" +IP: Final = b"\xf4" +AO: Final = b"\xf5" +AYT: Final = b"\xf6" +EC: Final = b"\xf7" +EL: Final = b"\xf8" +GA: Final = b"\xf9" +SB: Final = b"\xfa" -BINARY: bytes -ECHO: bytes -RCP: bytes -SGA: bytes -NAMS: bytes -STATUS: bytes -TM: bytes -RCTE: bytes -NAOL: bytes -NAOP: bytes -NAOCRD: bytes -NAOHTS: bytes -NAOHTD: bytes -NAOFFD: bytes -NAOVTS: bytes -NAOVTD: bytes -NAOLFD: bytes -XASCII: bytes -LOGOUT: bytes -BM: bytes -DET: bytes -SUPDUP: bytes -SUPDUPOUTPUT: bytes -SNDLOC: bytes -TTYPE: bytes -EOR: bytes -TUID: bytes -OUTMRK: bytes -TTYLOC: bytes -VT3270REGIME: bytes -X3PAD: bytes -NAWS: bytes -TSPEED: bytes -LFLOW: bytes -LINEMODE: bytes -XDISPLOC: bytes -OLD_ENVIRON: bytes -AUTHENTICATION: bytes -ENCRYPT: bytes -NEW_ENVIRON: bytes +BINARY: Final = b"\x00" +ECHO: Final = b"\x01" +RCP: Final = b"\x02" +SGA: Final = b"\x03" +NAMS: Final = b"\x04" +STATUS: Final = b"\x05" +TM: Final = b"\x06" +RCTE: Final = b"\x07" +NAOL: Final = b"\x08" +NAOP: Final = b"\t" +NAOCRD: Final = b"\n" +NAOHTS: Final = b"\x0b" +NAOHTD: Final = b"\x0c" +NAOFFD: Final = b"\r" +NAOVTS: Final = b"\x0e" +NAOVTD: Final = b"\x0f" +NAOLFD: Final = b"\x10" +XASCII: Final = b"\x11" +LOGOUT: Final = b"\x12" +BM: Final = b"\x13" +DET: Final = b"\x14" +SUPDUP: Final = b"\x15" +SUPDUPOUTPUT: Final = b"\x16" +SNDLOC: Final = b"\x17" +TTYPE: Final = b"\x18" +EOR: Final = b"\x19" +TUID: Final = b"\x1a" +OUTMRK: Final = b"\x1b" +TTYLOC: Final = b"\x1c" +VT3270REGIME: Final = b"\x1d" +X3PAD: Final = b"\x1e" +NAWS: Final = b"\x1f" +TSPEED: Final = b" " +LFLOW: Final = b"!" +LINEMODE: Final = b'"' +XDISPLOC: Final = b"#" +OLD_ENVIRON: Final = b"$" +AUTHENTICATION: Final = b"%" +ENCRYPT: Final = b"&" +NEW_ENVIRON: Final = b"'" -TN3270E: bytes -XAUTH: bytes -CHARSET: bytes -RSP: bytes -COM_PORT_OPTION: bytes -SUPPRESS_LOCAL_ECHO: bytes -TLS: bytes -KERMIT: bytes -SEND_URL: bytes -FORWARD_X: bytes -PRAGMA_LOGON: bytes -SSPI_LOGON: bytes -PRAGMA_HEARTBEAT: bytes -EXOPL: bytes -NOOPT: bytes +TN3270E: Final = b"(" +XAUTH: Final = b")" +CHARSET: Final = b"*" +RSP: Final = b"+" +COM_PORT_OPTION: Final = b"," +SUPPRESS_LOCAL_ECHO: Final = b"-" +TLS: Final = b"." +KERMIT: Final = b"/" +SEND_URL: Final = b"0" +FORWARD_X: Final = b"1" +PRAGMA_LOGON: Final = b"\x8a" +SSPI_LOGON: Final = b"\x8b" +PRAGMA_HEARTBEAT: Final = b"\x8c" +EXOPL: Final = b"\xff" +NOOPT: Final = b"\x00" class Telnet: host: str | None # undocumented diff --git a/mypy/typeshed/stdlib/tempfile.pyi b/mypy/typeshed/stdlib/tempfile.pyi index 6b2abe4398d2f..26491074ff71d 100644 --- a/mypy/typeshed/stdlib/tempfile.pyi +++ b/mypy/typeshed/stdlib/tempfile.pyi @@ -14,7 +14,7 @@ from _typeshed import ( ) from collections.abc import Iterable, Iterator from types import GenericAlias, TracebackType -from typing import IO, Any, AnyStr, Generic, Literal, overload +from typing import IO, Any, AnyStr, Final, Generic, Literal, overload from typing_extensions import Self, deprecated __all__ = [ @@ -34,7 +34,7 @@ __all__ = [ ] # global variables -TMP_MAX: int +TMP_MAX: Final[int] tempdir: str | None template: str diff --git a/mypy/typeshed/stdlib/threading.pyi b/mypy/typeshed/stdlib/threading.pyi index 033cad3931f5c..28fa5267a9975 100644 --- a/mypy/typeshed/stdlib/threading.pyi +++ b/mypy/typeshed/stdlib/threading.pyi @@ -5,7 +5,7 @@ from _typeshed import ProfileFunction, TraceFunction from collections.abc import Callable, Iterable, Mapping from contextvars import ContextVar from types import TracebackType -from typing import Any, TypeVar, final +from typing import Any, Final, TypeVar, final from typing_extensions import deprecated _T = TypeVar("_T") @@ -46,10 +46,10 @@ if sys.version_info >= (3, 12): _profile_hook: ProfileFunction | None def active_count() -> int: ... -@deprecated("Use active_count() instead") +@deprecated("Deprecated since Python 3.10. Use `active_count()` instead.") def activeCount() -> int: ... def current_thread() -> Thread: ... -@deprecated("Use current_thread() instead") +@deprecated("Deprecated since Python 3.10. Use `current_thread()` instead.") def currentThread() -> Thread: ... def get_ident() -> int: ... def enumerate() -> list[Thread]: ... @@ -67,7 +67,7 @@ if sys.version_info >= (3, 10): def stack_size(size: int = 0, /) -> int: ... -TIMEOUT_MAX: float +TIMEOUT_MAX: Final[float] ThreadError = _thread.error local = _thread._local @@ -107,13 +107,13 @@ class Thread: @property def native_id(self) -> int | None: ... # only available on some platforms def is_alive(self) -> bool: ... - @deprecated("Get the daemon attribute instead") + @deprecated("Deprecated since Python 3.10. Read the `daemon` attribute instead.") def isDaemon(self) -> bool: ... - @deprecated("Set the daemon attribute instead") + @deprecated("Deprecated since Python 3.10. Set the `daemon` attribute instead.") def setDaemon(self, daemonic: bool) -> None: ... - @deprecated("Use the name attribute instead") + @deprecated("Deprecated since Python 3.10. Read the `name` attribute instead.") def getName(self) -> str: ... - @deprecated("Use the name attribute instead") + @deprecated("Deprecated since Python 3.10. Set the `name` attribute instead.") def setName(self, name: str) -> None: ... class _DummyThread(Thread): @@ -148,7 +148,7 @@ class Condition: def wait_for(self, predicate: Callable[[], _T], timeout: float | None = None) -> _T: ... def notify(self, n: int = 1) -> None: ... def notify_all(self) -> None: ... - @deprecated("Use notify_all() instead") + @deprecated("Deprecated since Python 3.10. Use `notify_all()` instead.") def notifyAll(self) -> None: ... class Semaphore: @@ -163,7 +163,7 @@ class BoundedSemaphore(Semaphore): ... class Event: def is_set(self) -> bool: ... - @deprecated("Use is_set() instead") + @deprecated("Deprecated since Python 3.10. Use `is_set()` instead.") def isSet(self) -> bool: ... def set(self) -> None: ... def clear(self) -> None: ... diff --git a/mypy/typeshed/stdlib/time.pyi b/mypy/typeshed/stdlib/time.pyi index a921722b62c5b..5665efbba69d0 100644 --- a/mypy/typeshed/stdlib/time.pyi +++ b/mypy/typeshed/stdlib/time.pyi @@ -11,28 +11,28 @@ timezone: int tzname: tuple[str, str] if sys.platform == "linux": - CLOCK_BOOTTIME: int + CLOCK_BOOTTIME: Final[int] if sys.platform != "linux" and sys.platform != "win32" and sys.platform != "darwin": - CLOCK_PROF: int # FreeBSD, NetBSD, OpenBSD - CLOCK_UPTIME: int # FreeBSD, OpenBSD + CLOCK_PROF: Final[int] # FreeBSD, NetBSD, OpenBSD + CLOCK_UPTIME: Final[int] # FreeBSD, OpenBSD if sys.platform != "win32": - CLOCK_MONOTONIC: int - CLOCK_MONOTONIC_RAW: int - CLOCK_PROCESS_CPUTIME_ID: int - CLOCK_REALTIME: int - CLOCK_THREAD_CPUTIME_ID: int + CLOCK_MONOTONIC: Final[int] + CLOCK_MONOTONIC_RAW: Final[int] + CLOCK_PROCESS_CPUTIME_ID: Final[int] + CLOCK_REALTIME: Final[int] + CLOCK_THREAD_CPUTIME_ID: Final[int] if sys.platform != "linux" and sys.platform != "darwin": - CLOCK_HIGHRES: int # Solaris only + CLOCK_HIGHRES: Final[int] # Solaris only if sys.platform == "darwin": - CLOCK_UPTIME_RAW: int + CLOCK_UPTIME_RAW: Final[int] if sys.version_info >= (3, 13): - CLOCK_UPTIME_RAW_APPROX: int - CLOCK_MONOTONIC_RAW_APPROX: int + CLOCK_UPTIME_RAW_APPROX: Final[int] + CLOCK_MONOTONIC_RAW_APPROX: Final[int] if sys.platform == "linux": - CLOCK_TAI: int + CLOCK_TAI: Final[int] # Constructor takes an iterable of any type, of length between 9 and 11 elements. # However, it always *behaves* like a tuple of 9 elements, diff --git a/mypy/typeshed/stdlib/tkinter/__init__.pyi b/mypy/typeshed/stdlib/tkinter/__init__.pyi index 76b2ddcf17df1..54dd70baf1996 100644 --- a/mypy/typeshed/stdlib/tkinter/__init__.pyi +++ b/mypy/typeshed/stdlib/tkinter/__init__.pyi @@ -5,8 +5,8 @@ from collections.abc import Callable, Iterable, Mapping, Sequence from tkinter.constants import * from tkinter.font import _FontDescription from types import GenericAlias, TracebackType -from typing import Any, ClassVar, Generic, Literal, NamedTuple, Protocol, TypedDict, TypeVar, overload, type_check_only -from typing_extensions import TypeAlias, TypeVarTuple, Unpack, deprecated +from typing import Any, ClassVar, Final, Generic, Literal, NamedTuple, Protocol, TypedDict, TypeVar, overload, type_check_only +from typing_extensions import TypeAlias, TypeVarTuple, Unpack, deprecated, disjoint_base if sys.version_info >= (3, 11): from enum import StrEnum @@ -153,11 +153,11 @@ __all__ = [ TclError = _tkinter.TclError wantobjects: int -TkVersion: float -TclVersion: float -READABLE = _tkinter.READABLE -WRITABLE = _tkinter.WRITABLE -EXCEPTION = _tkinter.EXCEPTION +TkVersion: Final[float] +TclVersion: Final[float] +READABLE: Final = _tkinter.READABLE +WRITABLE: Final = _tkinter.WRITABLE +EXCEPTION: Final = _tkinter.EXCEPTION # Quick guide for figuring out which widget class to choose: # - Misc: any widget (don't use BaseWidget because Tk doesn't inherit from BaseWidget) @@ -198,7 +198,11 @@ if sys.version_info >= (3, 11): releaselevel: str serial: int - class _VersionInfoType(_VersionInfoTypeBase): ... + if sys.version_info >= (3, 12): + class _VersionInfoType(_VersionInfoTypeBase): ... + else: + @disjoint_base + class _VersionInfoType(_VersionInfoTypeBase): ... if sys.version_info >= (3, 11): class EventType(StrEnum): @@ -321,14 +325,21 @@ class Variable: def trace_add(self, mode: Literal["array", "read", "write", "unset"], callback: Callable[[str, str, str], object]) -> str: ... def trace_remove(self, mode: Literal["array", "read", "write", "unset"], cbname: str) -> None: ... def trace_info(self) -> list[tuple[tuple[Literal["array", "read", "write", "unset"], ...], str]]: ... - @deprecated("use trace_add() instead of trace()") - def trace(self, mode, callback): ... - @deprecated("use trace_add() instead of trace_variable()") - def trace_variable(self, mode, callback): ... - @deprecated("use trace_remove() instead of trace_vdelete()") - def trace_vdelete(self, mode, cbname) -> None: ... - @deprecated("use trace_info() instead of trace_vinfo()") - def trace_vinfo(self): ... + if sys.version_info >= (3, 14): + @deprecated("Deprecated since Python 3.14. Use `trace_add()` instead.") + def trace(self, mode, callback) -> str: ... + @deprecated("Deprecated since Python 3.14. Use `trace_add()` instead.") + def trace_variable(self, mode, callback) -> str: ... + @deprecated("Deprecated since Python 3.14. Use `trace_remove()` instead.") + def trace_vdelete(self, mode, cbname) -> None: ... + @deprecated("Deprecated since Python 3.14. Use `trace_info()` instead.") + def trace_vinfo(self): ... + else: + def trace(self, mode, callback) -> str: ... + def trace_variable(self, mode, callback) -> str: ... + def trace_vdelete(self, mode, cbname) -> None: ... + def trace_vinfo(self): ... + def __eq__(self, other: object) -> bool: ... def __del__(self) -> None: ... __hash__: ClassVar[None] # type: ignore[assignment] @@ -359,8 +370,8 @@ class BooleanVar(Variable): def mainloop(n: int = 0) -> None: ... -getint: Incomplete -getdouble: Incomplete +getint = int +getdouble = float def getboolean(s): ... @@ -3468,7 +3479,7 @@ class Text(Widget, XView, YView): def image_configure( self, index: _TextIndex, - cnf: dict[str, Any] | None = {}, + cnf: dict[str, Any] | None = None, *, align: Literal["baseline", "bottom", "center", "top"] = ..., image: _ImageSpec = ..., diff --git a/mypy/typeshed/stdlib/tkinter/messagebox.pyi b/mypy/typeshed/stdlib/tkinter/messagebox.pyi index 8e5a88f92ea15..cd95f0de5f803 100644 --- a/mypy/typeshed/stdlib/tkinter/messagebox.pyi +++ b/mypy/typeshed/stdlib/tkinter/messagebox.pyi @@ -30,7 +30,7 @@ def showinfo( *, detail: str = ..., icon: Literal["error", "info", "question", "warning"] = ..., - default: Literal["ok"] = ..., + default: Literal["ok"] = "ok", parent: Misc = ..., ) -> str: ... def showwarning( @@ -39,7 +39,7 @@ def showwarning( *, detail: str = ..., icon: Literal["error", "info", "question", "warning"] = ..., - default: Literal["ok"] = ..., + default: Literal["ok"] = "ok", parent: Misc = ..., ) -> str: ... def showerror( @@ -48,7 +48,7 @@ def showerror( *, detail: str = ..., icon: Literal["error", "info", "question", "warning"] = ..., - default: Literal["ok"] = ..., + default: Literal["ok"] = "ok", parent: Misc = ..., ) -> str: ... def askquestion( diff --git a/mypy/typeshed/stdlib/tkinter/ttk.pyi b/mypy/typeshed/stdlib/tkinter/ttk.pyi index c46239df81eb4..86c55eba7006d 100644 --- a/mypy/typeshed/stdlib/tkinter/ttk.pyi +++ b/mypy/typeshed/stdlib/tkinter/ttk.pyi @@ -1,10 +1,11 @@ import _tkinter +import sys import tkinter -from _typeshed import Incomplete, MaybeNone -from collections.abc import Callable +from _typeshed import MaybeNone +from collections.abc import Callable, Iterable from tkinter.font import _FontDescription from typing import Any, Literal, TypedDict, overload, type_check_only -from typing_extensions import TypeAlias +from typing_extensions import Never, TypeAlias, Unpack __all__ = [ "Button", @@ -35,7 +36,7 @@ __all__ = [ ] def tclobjs_to_py(adict: dict[Any, Any]) -> dict[Any, Any]: ... -def setup_master(master=None): ... +def setup_master(master: tkinter.Misc | None = None): ... _Padding: TypeAlias = ( tkinter._ScreenUnits @@ -48,19 +49,153 @@ _Padding: TypeAlias = ( # from ttk_widget (aka ttk::widget) manual page, differs from tkinter._Compound _TtkCompound: TypeAlias = Literal["", "text", "image", tkinter._Compound] +# Last item (option value to apply) varies between different options so use Any. +# It could also be any iterable with items matching the tuple, but that case +# hasn't been added here for consistency with _Padding above. +_Statespec: TypeAlias = tuple[Unpack[tuple[str, ...]], Any] +_ImageStatespec: TypeAlias = tuple[Unpack[tuple[str, ...]], tkinter._ImageSpec] +_VsapiStatespec: TypeAlias = tuple[Unpack[tuple[str, ...]], int] + +class _Layout(TypedDict, total=False): + side: Literal["left", "right", "top", "bottom"] + sticky: str # consists of letters 'n', 's', 'w', 'e', may contain repeats, may be empty + unit: Literal[0, 1] | bool + children: _LayoutSpec + # Note: there seem to be some other undocumented keys sometimes + +# This could be any sequence when passed as a parameter but will always be a list when returned. +_LayoutSpec: TypeAlias = list[tuple[str, _Layout | None]] + +# Keep these in sync with the appropriate methods in Style +class _ElementCreateImageKwargs(TypedDict, total=False): + border: _Padding + height: tkinter._ScreenUnits + padding: _Padding + sticky: str + width: tkinter._ScreenUnits + +_ElementCreateArgsCrossPlatform: TypeAlias = ( + # Could be any sequence here but types are not homogenous so just type it as tuple + tuple[Literal["image"], tkinter._ImageSpec, Unpack[tuple[_ImageStatespec, ...]], _ElementCreateImageKwargs] + | tuple[Literal["from"], str, str] + | tuple[Literal["from"], str] # (fromelement is optional) +) +if sys.platform == "win32" and sys.version_info >= (3, 13): + class _ElementCreateVsapiKwargsPadding(TypedDict, total=False): + padding: _Padding + + class _ElementCreateVsapiKwargsMargin(TypedDict, total=False): + padding: _Padding + + class _ElementCreateVsapiKwargsSize(TypedDict): + width: tkinter._ScreenUnits + height: tkinter._ScreenUnits + + _ElementCreateVsapiKwargsDict: TypeAlias = ( + _ElementCreateVsapiKwargsPadding | _ElementCreateVsapiKwargsMargin | _ElementCreateVsapiKwargsSize + ) + _ElementCreateArgs: TypeAlias = ( # noqa: Y047 # It doesn't recognise the usage below for whatever reason + _ElementCreateArgsCrossPlatform + | tuple[Literal["vsapi"], str, int, _ElementCreateVsapiKwargsDict] + | tuple[Literal["vsapi"], str, int, _VsapiStatespec, _ElementCreateVsapiKwargsDict] + ) +else: + _ElementCreateArgs: TypeAlias = _ElementCreateArgsCrossPlatform +_ThemeSettingsValue = TypedDict( + "_ThemeSettingsValue", + { + "configure": dict[str, Any], + "map": dict[str, Iterable[_Statespec]], + "layout": _LayoutSpec, + "element create": _ElementCreateArgs, + }, + total=False, +) +_ThemeSettings: TypeAlias = dict[str, _ThemeSettingsValue] + class Style: - master: Incomplete + master: tkinter.Misc tk: _tkinter.TkappType def __init__(self, master: tkinter.Misc | None = None) -> None: ... - def configure(self, style, query_opt=None, **kw): ... - def map(self, style, query_opt=None, **kw): ... - def lookup(self, style, option, state=None, default=None): ... - def layout(self, style, layoutspec=None): ... - def element_create(self, elementname, etype, *args, **kw) -> None: ... - def element_names(self): ... - def element_options(self, elementname): ... - def theme_create(self, themename, parent=None, settings=None) -> None: ... - def theme_settings(self, themename, settings) -> None: ... + # For these methods, values given vary between options. Returned values + # seem to be str, but this might not always be the case. + @overload + def configure(self, style: str) -> dict[str, Any] | None: ... # Returns None if no configuration. + @overload + def configure(self, style: str, query_opt: str, **kw: Any) -> Any: ... + @overload + def configure(self, style: str, query_opt: None = None, **kw: Any) -> None: ... + @overload + def map(self, style: str, query_opt: str) -> _Statespec: ... + @overload + def map(self, style: str, query_opt: None = None, **kw: Iterable[_Statespec]) -> dict[str, _Statespec]: ... + def lookup(self, style: str, option: str, state: Iterable[str] | None = None, default: Any | None = None) -> Any: ... + @overload + def layout(self, style: str, layoutspec: _LayoutSpec) -> list[Never]: ... # Always seems to return an empty list + @overload + def layout(self, style: str, layoutspec: None = None) -> _LayoutSpec: ... + @overload + def element_create( + self, + elementname: str, + etype: Literal["image"], + default_image: tkinter._ImageSpec, + /, + *imagespec: _ImageStatespec, + border: _Padding = ..., + height: tkinter._ScreenUnits = ..., + padding: _Padding = ..., + sticky: str = ..., + width: tkinter._ScreenUnits = ..., + ) -> None: ... + @overload + def element_create(self, elementname: str, etype: Literal["from"], themename: str, fromelement: str = ..., /) -> None: ... + if sys.platform == "win32" and sys.version_info >= (3, 13): # and tk version >= 8.6 + # margin, padding, and (width + height) are mutually exclusive. width + # and height must either both be present or not present at all. Note: + # There are other undocumented options if you look at ttk's source code. + @overload + def element_create( + self, + elementname: str, + etype: Literal["vsapi"], + class_: str, + part: int, + vs_statespec: _VsapiStatespec = ..., + /, + *, + padding: _Padding = ..., + ) -> None: ... + @overload + def element_create( + self, + elementname: str, + etype: Literal["vsapi"], + class_: str, + part: int, + vs_statespec: _VsapiStatespec = ..., + /, + *, + margin: _Padding = ..., + ) -> None: ... + @overload + def element_create( + self, + elementname: str, + etype: Literal["vsapi"], + class_: str, + part: int, + vs_statespec: _VsapiStatespec = ..., + /, + *, + width: tkinter._ScreenUnits, + height: tkinter._ScreenUnits, + ) -> None: ... + + def element_names(self) -> tuple[str, ...]: ... + def element_options(self, elementname: str) -> tuple[str, ...]: ... + def theme_create(self, themename: str, parent: str | None = None, settings: _ThemeSettings | None = None) -> None: ... + def theme_settings(self, themename: str, settings: _ThemeSettings) -> None: ... def theme_names(self) -> tuple[str, ...]: ... @overload def theme_use(self, themename: str) -> None: ... @@ -615,7 +750,7 @@ class Panedwindow(Widget, tkinter.PanedWindow): ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def config(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... - forget: Incomplete + forget = tkinter.PanedWindow.forget def insert(self, pos, child, **kw) -> None: ... def pane(self, pane, option=None, **kw): ... def sashpos(self, index, newpos=None): ... diff --git a/mypy/typeshed/stdlib/tokenize.pyi b/mypy/typeshed/stdlib/tokenize.pyi index 1a3a80937f22e..00a24b4eea07d 100644 --- a/mypy/typeshed/stdlib/tokenize.pyi +++ b/mypy/typeshed/stdlib/tokenize.pyi @@ -3,8 +3,8 @@ from _typeshed import FileDescriptorOrPath from collections.abc import Callable, Generator, Iterable, Sequence from re import Pattern from token import * -from typing import Any, NamedTuple, TextIO, type_check_only -from typing_extensions import TypeAlias +from typing import Any, Final, NamedTuple, TextIO, type_check_only +from typing_extensions import TypeAlias, disjoint_base if sys.version_info < (3, 12): # Avoid double assignment to Final name by imports, which pyright objects to. @@ -101,8 +101,8 @@ if sys.version_info >= (3, 13): if sys.version_info >= (3, 14): __all__ += ["TSTRING_START", "TSTRING_MIDDLE", "TSTRING_END"] -cookie_re: Pattern[str] -blank_re: Pattern[bytes] +cookie_re: Final[Pattern[str]] +blank_re: Final[Pattern[bytes]] _Position: TypeAlias = tuple[int, int] @@ -115,9 +115,16 @@ class _TokenInfo(NamedTuple): end: _Position line: str -class TokenInfo(_TokenInfo): - @property - def exact_type(self) -> int: ... +if sys.version_info >= (3, 12): + class TokenInfo(_TokenInfo): + @property + def exact_type(self) -> int: ... + +else: + @disjoint_base + class TokenInfo(_TokenInfo): + @property + def exact_type(self) -> int: ... # Backwards compatible tokens can be sequences of a shorter length too _Token: TypeAlias = TokenInfo | Sequence[int | str | _Position] @@ -151,46 +158,46 @@ def group(*choices: str) -> str: ... # undocumented def any(*choices: str) -> str: ... # undocumented def maybe(*choices: str) -> str: ... # undocumented -Whitespace: str # undocumented -Comment: str # undocumented -Ignore: str # undocumented -Name: str # undocumented - -Hexnumber: str # undocumented -Binnumber: str # undocumented -Octnumber: str # undocumented -Decnumber: str # undocumented -Intnumber: str # undocumented -Exponent: str # undocumented -Pointfloat: str # undocumented -Expfloat: str # undocumented -Floatnumber: str # undocumented -Imagnumber: str # undocumented -Number: str # undocumented +Whitespace: Final[str] # undocumented +Comment: Final[str] # undocumented +Ignore: Final[str] # undocumented +Name: Final[str] # undocumented + +Hexnumber: Final[str] # undocumented +Binnumber: Final[str] # undocumented +Octnumber: Final[str] # undocumented +Decnumber: Final[str] # undocumented +Intnumber: Final[str] # undocumented +Exponent: Final[str] # undocumented +Pointfloat: Final[str] # undocumented +Expfloat: Final[str] # undocumented +Floatnumber: Final[str] # undocumented +Imagnumber: Final[str] # undocumented +Number: Final[str] # undocumented def _all_string_prefixes() -> set[str]: ... # undocumented -StringPrefix: str # undocumented +StringPrefix: Final[str] # undocumented -Single: str # undocumented -Double: str # undocumented -Single3: str # undocumented -Double3: str # undocumented -Triple: str # undocumented -String: str # undocumented +Single: Final[str] # undocumented +Double: Final[str] # undocumented +Single3: Final[str] # undocumented +Double3: Final[str] # undocumented +Triple: Final[str] # undocumented +String: Final[str] # undocumented -Special: str # undocumented -Funny: str # undocumented +Special: Final[str] # undocumented +Funny: Final[str] # undocumented -PlainToken: str # undocumented -Token: str # undocumented +PlainToken: Final[str] # undocumented +Token: Final[str] # undocumented -ContStr: str # undocumented -PseudoExtras: str # undocumented -PseudoToken: str # undocumented +ContStr: Final[str] # undocumented +PseudoExtras: Final[str] # undocumented +PseudoToken: Final[str] # undocumented -endpats: dict[str, str] # undocumented -single_quoted: set[str] # undocumented -triple_quoted: set[str] # undocumented +endpats: Final[dict[str, str]] # undocumented +single_quoted: Final[set[str]] # undocumented +triple_quoted: Final[set[str]] # undocumented -tabsize: int # undocumented +tabsize: Final = 8 # undocumented diff --git a/mypy/typeshed/stdlib/tomllib.pyi b/mypy/typeshed/stdlib/tomllib.pyi index c160ffc38bfdd..4ff4097f8313a 100644 --- a/mypy/typeshed/stdlib/tomllib.pyi +++ b/mypy/typeshed/stdlib/tomllib.pyi @@ -16,7 +16,7 @@ if sys.version_info >= (3, 14): @overload def __init__(self, msg: str, doc: str, pos: int) -> None: ... @overload - @deprecated("Deprecated in Python 3.14; Please set 'msg', 'doc' and 'pos' arguments only.") + @deprecated("Deprecated since Python 3.14. Set the 'msg', 'doc' and 'pos' arguments only.") def __init__(self, msg: str | type = ..., doc: str | type = ..., pos: int | type = ..., *args: Any) -> None: ... else: diff --git a/mypy/typeshed/stdlib/traceback.pyi b/mypy/typeshed/stdlib/traceback.pyi index 4553dbd08384d..d587295cd1cf7 100644 --- a/mypy/typeshed/stdlib/traceback.pyi +++ b/mypy/typeshed/stdlib/traceback.pyi @@ -138,7 +138,7 @@ class TracebackException: @property def exc_type_str(self) -> str: ... @property - @deprecated("Deprecated in 3.13. Use exc_type_str instead.") + @deprecated("Deprecated since Python 3.13. Use `exc_type_str` instead.") def exc_type(self) -> type[BaseException] | None: ... else: exc_type: type[BaseException] @@ -245,6 +245,23 @@ class TracebackException: def print(self, *, file: SupportsWrite[str] | None = None, chain: bool = True) -> None: ... class FrameSummary: + if sys.version_info >= (3, 13): + __slots__ = ( + "filename", + "lineno", + "end_lineno", + "colno", + "end_colno", + "name", + "_lines", + "_lines_dedented", + "locals", + "_code", + ) + elif sys.version_info >= (3, 11): + __slots__ = ("filename", "lineno", "end_lineno", "colno", "end_colno", "name", "_line", "locals") + else: + __slots__ = ("filename", "lineno", "name", "_line", "locals") if sys.version_info >= (3, 11): def __init__( self, diff --git a/mypy/typeshed/stdlib/tracemalloc.pyi b/mypy/typeshed/stdlib/tracemalloc.pyi index 05d98ae127d84..31d8f74456395 100644 --- a/mypy/typeshed/stdlib/tracemalloc.pyi +++ b/mypy/typeshed/stdlib/tracemalloc.pyi @@ -32,6 +32,7 @@ class Filter(BaseFilter): ) -> None: ... class Statistic: + __slots__ = ("traceback", "size", "count") count: int size: int traceback: Traceback @@ -40,6 +41,7 @@ class Statistic: def __hash__(self) -> int: ... class StatisticDiff: + __slots__ = ("traceback", "size", "size_diff", "count", "count_diff") count: int count_diff: int size: int @@ -52,6 +54,7 @@ class StatisticDiff: _FrameTuple: TypeAlias = tuple[str, int] class Frame: + __slots__ = ("_frame",) @property def filename(self) -> str: ... @property @@ -72,6 +75,7 @@ class Frame: _TraceTuple: TypeAlias = tuple[int, int, Sequence[_FrameTuple], int | None] | tuple[int, int, Sequence[_FrameTuple]] class Trace: + __slots__ = ("_trace",) @property def domain(self) -> int: ... @property @@ -83,6 +87,7 @@ class Trace: def __hash__(self) -> int: ... class Traceback(Sequence[Frame]): + __slots__ = ("_frames", "_total_nframe") @property def total_nframe(self) -> int | None: ... def __init__(self, frames: Sequence[_FrameTuple], total_nframe: int | None = None) -> None: ... diff --git a/mypy/typeshed/stdlib/turtle.pyi b/mypy/typeshed/stdlib/turtle.pyi index e41476b73b8c9..0b93429904c53 100644 --- a/mypy/typeshed/stdlib/turtle.pyi +++ b/mypy/typeshed/stdlib/turtle.pyi @@ -4,7 +4,7 @@ from collections.abc import Callable, Generator, Sequence from contextlib import contextmanager from tkinter import Canvas, Frame, Misc, PhotoImage, Scrollbar from typing import Any, ClassVar, Literal, TypedDict, overload, type_check_only -from typing_extensions import Self, TypeAlias, deprecated +from typing_extensions import Self, TypeAlias, deprecated, disjoint_base __all__ = [ "ScrolledCanvas", @@ -163,18 +163,34 @@ class _PenState(TypedDict): _Speed: TypeAlias = str | float _PolygonCoords: TypeAlias = Sequence[tuple[float, float]] -class Vec2D(tuple[float, float]): - def __new__(cls, x: float, y: float) -> Self: ... - def __add__(self, other: tuple[float, float]) -> Vec2D: ... # type: ignore[override] - @overload # type: ignore[override] - def __mul__(self, other: Vec2D) -> float: ... - @overload - def __mul__(self, other: float) -> Vec2D: ... - def __rmul__(self, other: float) -> Vec2D: ... # type: ignore[override] - def __sub__(self, other: tuple[float, float]) -> Vec2D: ... - def __neg__(self) -> Vec2D: ... - def __abs__(self) -> float: ... - def rotate(self, angle: float) -> Vec2D: ... +if sys.version_info >= (3, 12): + class Vec2D(tuple[float, float]): + def __new__(cls, x: float, y: float) -> Self: ... + def __add__(self, other: tuple[float, float]) -> Vec2D: ... # type: ignore[override] + @overload # type: ignore[override] + def __mul__(self, other: Vec2D) -> float: ... + @overload + def __mul__(self, other: float) -> Vec2D: ... + def __rmul__(self, other: float) -> Vec2D: ... # type: ignore[override] + def __sub__(self, other: tuple[float, float]) -> Vec2D: ... + def __neg__(self) -> Vec2D: ... + def __abs__(self) -> float: ... + def rotate(self, angle: float) -> Vec2D: ... + +else: + @disjoint_base + class Vec2D(tuple[float, float]): + def __new__(cls, x: float, y: float) -> Self: ... + def __add__(self, other: tuple[float, float]) -> Vec2D: ... # type: ignore[override] + @overload # type: ignore[override] + def __mul__(self, other: Vec2D) -> float: ... + @overload + def __mul__(self, other: float) -> Vec2D: ... + def __rmul__(self, other: float) -> Vec2D: ... # type: ignore[override] + def __sub__(self, other: tuple[float, float]) -> Vec2D: ... + def __neg__(self) -> Vec2D: ... + def __abs__(self) -> float: ... + def rotate(self, angle: float) -> Vec2D: ... # Does not actually inherit from Canvas, but dynamically gets all methods of Canvas class ScrolledCanvas(Canvas, Frame): # type: ignore[misc] diff --git a/mypy/typeshed/stdlib/types.pyi b/mypy/typeshed/stdlib/types.pyi index 44bd3eeb3f533..591d5da2360dc 100644 --- a/mypy/typeshed/stdlib/types.pyi +++ b/mypy/typeshed/stdlib/types.pyi @@ -17,7 +17,7 @@ from collections.abc import ( ) from importlib.machinery import ModuleSpec from typing import Any, ClassVar, Literal, TypeVar, final, overload -from typing_extensions import ParamSpec, Self, TypeAliasType, TypeVarTuple, deprecated +from typing_extensions import ParamSpec, Self, TypeAliasType, TypeVarTuple, deprecated, disjoint_base if sys.version_info >= (3, 14): from _typeshed import AnnotateFunc @@ -151,7 +151,7 @@ class CodeType: def co_firstlineno(self) -> int: ... if sys.version_info >= (3, 10): @property - @deprecated("Will be removed in Python 3.15. Use the co_lines() method instead.") + @deprecated("Deprecated since Python 3.10; will be removed in Python 3.15. Use `CodeType.co_lines()` instead.") def co_lnotab(self) -> bytes: ... else: @property @@ -331,20 +331,34 @@ class MappingProxyType(Mapping[_KT, _VT_co]): def __or__(self, value: Mapping[_T1, _T2], /) -> dict[_KT | _T1, _VT_co | _T2]: ... def __ror__(self, value: Mapping[_T1, _T2], /) -> dict[_KT | _T1, _VT_co | _T2]: ... -class SimpleNamespace: - __hash__: ClassVar[None] # type: ignore[assignment] - if sys.version_info >= (3, 13): - def __init__(self, mapping_or_iterable: Mapping[str, Any] | Iterable[tuple[str, Any]] = (), /, **kwargs: Any) -> None: ... - else: - def __init__(self, **kwargs: Any) -> None: ... +if sys.version_info >= (3, 12): + @disjoint_base + class SimpleNamespace: + __hash__: ClassVar[None] # type: ignore[assignment] + if sys.version_info >= (3, 13): + def __init__( + self, mapping_or_iterable: Mapping[str, Any] | Iterable[tuple[str, Any]] = (), /, **kwargs: Any + ) -> None: ... + else: + def __init__(self, **kwargs: Any) -> None: ... - def __eq__(self, value: object, /) -> bool: ... - def __getattribute__(self, name: str, /) -> Any: ... - def __setattr__(self, name: str, value: Any, /) -> None: ... - def __delattr__(self, name: str, /) -> None: ... - if sys.version_info >= (3, 13): - def __replace__(self, **kwargs: Any) -> Self: ... + def __eq__(self, value: object, /) -> bool: ... + def __getattribute__(self, name: str, /) -> Any: ... + def __setattr__(self, name: str, value: Any, /) -> None: ... + def __delattr__(self, name: str, /) -> None: ... + if sys.version_info >= (3, 13): + def __replace__(self, **kwargs: Any) -> Self: ... + +else: + class SimpleNamespace: + __hash__: ClassVar[None] # type: ignore[assignment] + def __init__(self, **kwargs: Any) -> None: ... + def __eq__(self, value: object, /) -> bool: ... + def __getattribute__(self, name: str, /) -> Any: ... + def __setattr__(self, name: str, value: Any, /) -> None: ... + def __delattr__(self, name: str, /) -> None: ... +@disjoint_base class ModuleType: __name__: str __file__: str | None @@ -661,7 +675,7 @@ _P = ParamSpec("_P") def coroutine(func: Callable[_P, Generator[Any, Any, _R]]) -> Callable[_P, Awaitable[_R]]: ... @overload def coroutine(func: _Fn) -> _Fn: ... - +@disjoint_base class GenericAlias: @property def __origin__(self) -> type | TypeAliasType: ... diff --git a/mypy/typeshed/stdlib/typing.pyi b/mypy/typeshed/stdlib/typing.pyi index fd9da29addbf4..15a5864613d1f 100644 --- a/mypy/typeshed/stdlib/typing.pyi +++ b/mypy/typeshed/stdlib/typing.pyi @@ -5,7 +5,7 @@ import collections # noqa: F401 # pyright: ignore[reportUnusedImport] import sys import typing_extensions from _collections_abc import dict_items, dict_keys, dict_values -from _typeshed import IdentityFunction, ReadableBuffer, SupportsKeysAndGetItem +from _typeshed import IdentityFunction, ReadableBuffer, SupportsGetItem, SupportsGetItemViewable, SupportsKeysAndGetItem, Viewable from abc import ABCMeta, abstractmethod from re import Match as Match, Pattern as Pattern from types import ( @@ -146,7 +146,9 @@ if sys.version_info >= (3, 13): # from _typeshed import AnnotationForm class Any: ... -class _Final: ... + +class _Final: + __slots__ = ("__weakref__",) def final(f: _T) -> _T: ... @final @@ -229,13 +231,13 @@ _promote = object() # N.B. Keep this definition in sync with typing_extensions._SpecialForm @final class _SpecialForm(_Final): + __slots__ = ("_name", "__doc__", "_getitem") def __getitem__(self, parameters: Any) -> object: ... if sys.version_info >= (3, 10): def __or__(self, other: Any) -> _SpecialForm: ... def __ror__(self, other: Any) -> _SpecialForm: ... Union: _SpecialForm -Generic: _SpecialForm Protocol: _SpecialForm Callable: _SpecialForm Type: _SpecialForm @@ -440,6 +442,20 @@ Annotated: _SpecialForm # Predefined type variables. AnyStr = TypeVar("AnyStr", str, bytes) # noqa: Y001 +@type_check_only +class _Generic: + if sys.version_info < (3, 12): + __slots__ = () + + if sys.version_info >= (3, 10): + @classmethod + def __class_getitem__(cls, args: TypeVar | ParamSpec | tuple[TypeVar | ParamSpec, ...]) -> _Final: ... + else: + @classmethod + def __class_getitem__(cls, args: TypeVar | tuple[TypeVar, ...]) -> _Final: ... + +Generic: type[_Generic] + class _ProtocolMeta(ABCMeta): if sys.version_info >= (3, 12): def __init__(cls, *args: Any, **kwargs: Any) -> None: ... @@ -449,36 +465,43 @@ class _ProtocolMeta(ABCMeta): def runtime_checkable(cls: _TC) -> _TC: ... @runtime_checkable class SupportsInt(Protocol, metaclass=ABCMeta): + __slots__ = () @abstractmethod def __int__(self) -> int: ... @runtime_checkable class SupportsFloat(Protocol, metaclass=ABCMeta): + __slots__ = () @abstractmethod def __float__(self) -> float: ... @runtime_checkable class SupportsComplex(Protocol, metaclass=ABCMeta): + __slots__ = () @abstractmethod def __complex__(self) -> complex: ... @runtime_checkable class SupportsBytes(Protocol, metaclass=ABCMeta): + __slots__ = () @abstractmethod def __bytes__(self) -> bytes: ... @runtime_checkable class SupportsIndex(Protocol, metaclass=ABCMeta): + __slots__ = () @abstractmethod def __index__(self) -> int: ... @runtime_checkable class SupportsAbs(Protocol[_T_co]): + __slots__ = () @abstractmethod def __abs__(self) -> _T_co: ... @runtime_checkable class SupportsRound(Protocol[_T_co]): + __slots__ = () @overload @abstractmethod def __round__(self) -> int: ... @@ -703,11 +726,12 @@ class MutableSet(AbstractSet[_T]): def __isub__(self, it: AbstractSet[Any]) -> typing_extensions.Self: ... class MappingView(Sized): - def __init__(self, mapping: Mapping[Any, Any]) -> None: ... # undocumented + __slots__ = ("_mapping",) + def __init__(self, mapping: Sized) -> None: ... # undocumented def __len__(self) -> int: ... class ItemsView(MappingView, AbstractSet[tuple[_KT_co, _VT_co]], Generic[_KT_co, _VT_co]): - def __init__(self, mapping: Mapping[_KT_co, _VT_co]) -> None: ... # undocumented + def __init__(self, mapping: SupportsGetItemViewable[_KT_co, _VT_co]) -> None: ... # undocumented def __and__(self, other: Iterable[Any]) -> set[tuple[_KT_co, _VT_co]]: ... def __rand__(self, other: Iterable[_T]) -> set[_T]: ... def __contains__(self, item: tuple[object, object]) -> bool: ... # type: ignore[override] @@ -720,7 +744,7 @@ class ItemsView(MappingView, AbstractSet[tuple[_KT_co, _VT_co]], Generic[_KT_co, def __rxor__(self, other: Iterable[_T]) -> set[tuple[_KT_co, _VT_co] | _T]: ... class KeysView(MappingView, AbstractSet[_KT_co]): - def __init__(self, mapping: Mapping[_KT_co, Any]) -> None: ... # undocumented + def __init__(self, mapping: Viewable[_KT_co]) -> None: ... # undocumented def __and__(self, other: Iterable[Any]) -> set[_KT_co]: ... def __rand__(self, other: Iterable[_T]) -> set[_T]: ... def __contains__(self, key: object) -> bool: ... @@ -733,7 +757,7 @@ class KeysView(MappingView, AbstractSet[_KT_co]): def __rxor__(self, other: Iterable[_T]) -> set[_KT_co | _T]: ... class ValuesView(MappingView, Collection[_VT_co]): - def __init__(self, mapping: Mapping[Any, _VT_co]) -> None: ... # undocumented + def __init__(self, mapping: SupportsGetItemViewable[Any, _VT_co]) -> None: ... # undocumented def __contains__(self, value: object) -> bool: ... def __iter__(self) -> Iterator[_VT_co]: ... @@ -801,13 +825,13 @@ class MutableMapping(Mapping[_KT, _VT]): @overload def update(self, m: SupportsKeysAndGetItem[_KT, _VT], /) -> None: ... @overload - def update(self: Mapping[str, _VT], m: SupportsKeysAndGetItem[str, _VT], /, **kwargs: _VT) -> None: ... + def update(self: SupportsGetItem[str, _VT], m: SupportsKeysAndGetItem[str, _VT], /, **kwargs: _VT) -> None: ... @overload def update(self, m: Iterable[tuple[_KT, _VT]], /) -> None: ... @overload - def update(self: Mapping[str, _VT], m: Iterable[tuple[str, _VT]], /, **kwargs: _VT) -> None: ... + def update(self: SupportsGetItem[str, _VT], m: Iterable[tuple[str, _VT]], /, **kwargs: _VT) -> None: ... @overload - def update(self: Mapping[str, _VT], **kwargs: _VT) -> None: ... + def update(self: SupportsGetItem[str, _VT], **kwargs: _VT) -> None: ... Text = str @@ -820,6 +844,7 @@ class IO(Generic[AnyStr]): # At runtime these are all abstract properties, # but making them abstract in the stub is hugely disruptive, for not much gain. # See #8726 + __slots__ = () @property def mode(self) -> str: ... # Usually str, but may be bytes if a bytes path was passed to open(). See #10737. @@ -878,11 +903,13 @@ class IO(Generic[AnyStr]): ) -> None: ... class BinaryIO(IO[bytes]): + __slots__ = () @abstractmethod def __enter__(self) -> BinaryIO: ... class TextIO(IO[str]): # See comment regarding the @properties in the `IO` class + __slots__ = () @property def buffer(self) -> BinaryIO: ... @property @@ -1045,6 +1072,15 @@ if sys.version_info >= (3, 14): else: @final class ForwardRef(_Final): + __slots__ = ( + "__forward_arg__", + "__forward_code__", + "__forward_evaluated__", + "__forward_value__", + "__forward_is_argument__", + "__forward_is_class__", + "__forward_module__", + ) __forward_arg__: str __forward_code__: CodeType __forward_evaluated__: bool diff --git a/mypy/typeshed/stdlib/typing_extensions.pyi b/mypy/typeshed/stdlib/typing_extensions.pyi index 71bf3d87d4996..f5ea13f67733f 100644 --- a/mypy/typeshed/stdlib/typing_extensions.pyi +++ b/mypy/typeshed/stdlib/typing_extensions.pyi @@ -120,6 +120,7 @@ __all__ = [ "clear_overloads", "dataclass_transform", "deprecated", + "disjoint_base", "Doc", "evaluate_forward_ref", "get_overloads", @@ -150,6 +151,7 @@ __all__ = [ "TypeGuard", "TypeIs", "TYPE_CHECKING", + "type_repr", "Never", "NoReturn", "ReadOnly", @@ -219,6 +221,7 @@ runtime = runtime_checkable Final: _SpecialForm def final(f: _F) -> _F: ... +def disjoint_base(cls: _TC) -> _TC: ... Literal: _SpecialForm @@ -405,36 +408,43 @@ else: @runtime_checkable class SupportsInt(Protocol, metaclass=abc.ABCMeta): + __slots__ = () @abc.abstractmethod def __int__(self) -> int: ... @runtime_checkable class SupportsFloat(Protocol, metaclass=abc.ABCMeta): + __slots__ = () @abc.abstractmethod def __float__(self) -> float: ... @runtime_checkable class SupportsComplex(Protocol, metaclass=abc.ABCMeta): + __slots__ = () @abc.abstractmethod def __complex__(self) -> complex: ... @runtime_checkable class SupportsBytes(Protocol, metaclass=abc.ABCMeta): + __slots__ = () @abc.abstractmethod def __bytes__(self) -> bytes: ... @runtime_checkable class SupportsIndex(Protocol, metaclass=abc.ABCMeta): + __slots__ = () @abc.abstractmethod def __index__(self) -> int: ... @runtime_checkable class SupportsAbs(Protocol[_T_co]): + __slots__ = () @abc.abstractmethod def __abs__(self) -> _T_co: ... @runtime_checkable class SupportsRound(Protocol[_T_co]): + __slots__ = () @overload @abc.abstractmethod def __round__(self) -> int: ... @@ -447,11 +457,13 @@ if sys.version_info >= (3, 14): else: @runtime_checkable class Reader(Protocol[_T_co]): + __slots__ = () @abc.abstractmethod def read(self, size: int = ..., /) -> _T_co: ... @runtime_checkable class Writer(Protocol[_T_contra]): + __slots__ = () @abc.abstractmethod def write(self, data: _T_contra, /) -> int: ... @@ -616,7 +628,7 @@ TypeForm: _SpecialForm if sys.version_info >= (3, 14): from typing import evaluate_forward_ref as evaluate_forward_ref - from annotationlib import Format as Format, get_annotations as get_annotations + from annotationlib import Format as Format, get_annotations as get_annotations, type_repr as type_repr else: class Format(enum.IntEnum): VALUE = 1 @@ -684,6 +696,7 @@ else: format: Format | None = None, _recursive_guard: Container[str] = ..., ) -> AnnotationForm: ... + def type_repr(value: object) -> str: ... # PEP 661 class Sentinel: diff --git a/mypy/typeshed/stdlib/unicodedata.pyi b/mypy/typeshed/stdlib/unicodedata.pyi index 77d69edf06af9..9fff042f0b964 100644 --- a/mypy/typeshed/stdlib/unicodedata.pyi +++ b/mypy/typeshed/stdlib/unicodedata.pyi @@ -1,10 +1,10 @@ import sys from _typeshed import ReadOnlyBuffer -from typing import Any, Literal, TypeVar, final, overload +from typing import Any, Final, Literal, TypeVar, final, overload from typing_extensions import TypeAlias ucd_3_2_0: UCD -unidata_version: str +unidata_version: Final[str] if sys.version_info < (3, 10): ucnhash_CAPI: Any diff --git a/mypy/typeshed/stdlib/unittest/loader.pyi b/mypy/typeshed/stdlib/unittest/loader.pyi index 598e3cd84a5e8..81de40c898496 100644 --- a/mypy/typeshed/stdlib/unittest/loader.pyi +++ b/mypy/typeshed/stdlib/unittest/loader.pyi @@ -35,21 +35,38 @@ class TestLoader: defaultTestLoader: TestLoader if sys.version_info < (3, 13): - @deprecated("Deprecated in Python 3.11; removal scheduled for Python 3.13") - def getTestCaseNames( - testCaseClass: type[unittest.case.TestCase], - prefix: str, - sortUsing: _SortComparisonMethod = ..., - testNamePatterns: list[str] | None = None, - ) -> Sequence[str]: ... - @deprecated("Deprecated in Python 3.11; removal scheduled for Python 3.13") - def makeSuite( - testCaseClass: type[unittest.case.TestCase], - prefix: str = "test", - sortUsing: _SortComparisonMethod = ..., - suiteClass: _SuiteClass = ..., - ) -> unittest.suite.TestSuite: ... - @deprecated("Deprecated in Python 3.11; removal scheduled for Python 3.13") - def findTestCases( - module: ModuleType, prefix: str = "test", sortUsing: _SortComparisonMethod = ..., suiteClass: _SuiteClass = ... - ) -> unittest.suite.TestSuite: ... + if sys.version_info >= (3, 11): + @deprecated("Deprecated since Python 3.11; removed in Python 3.13.") + def getTestCaseNames( + testCaseClass: type[unittest.case.TestCase], + prefix: str, + sortUsing: _SortComparisonMethod = ..., + testNamePatterns: list[str] | None = None, + ) -> Sequence[str]: ... + @deprecated("Deprecated since Python 3.11; removed in Python 3.13.") + def makeSuite( + testCaseClass: type[unittest.case.TestCase], + prefix: str = "test", + sortUsing: _SortComparisonMethod = ..., + suiteClass: _SuiteClass = ..., + ) -> unittest.suite.TestSuite: ... + @deprecated("Deprecated since Python 3.11; removed in Python 3.13.") + def findTestCases( + module: ModuleType, prefix: str = "test", sortUsing: _SortComparisonMethod = ..., suiteClass: _SuiteClass = ... + ) -> unittest.suite.TestSuite: ... + else: + def getTestCaseNames( + testCaseClass: type[unittest.case.TestCase], + prefix: str, + sortUsing: _SortComparisonMethod = ..., + testNamePatterns: list[str] | None = None, + ) -> Sequence[str]: ... + def makeSuite( + testCaseClass: type[unittest.case.TestCase], + prefix: str = "test", + sortUsing: _SortComparisonMethod = ..., + suiteClass: _SuiteClass = ..., + ) -> unittest.suite.TestSuite: ... + def findTestCases( + module: ModuleType, prefix: str = "test", sortUsing: _SortComparisonMethod = ..., suiteClass: _SuiteClass = ... + ) -> unittest.suite.TestSuite: ... diff --git a/mypy/typeshed/stdlib/unittest/main.pyi b/mypy/typeshed/stdlib/unittest/main.pyi index 152e9c33209ca..23ead1638ecc2 100644 --- a/mypy/typeshed/stdlib/unittest/main.pyi +++ b/mypy/typeshed/stdlib/unittest/main.pyi @@ -64,8 +64,11 @@ class TestProgram: ) -> None: ... if sys.version_info < (3, 13): - @deprecated("Deprecated in Python 3.11; removal scheduled for Python 3.13") - def usageExit(self, msg: Any = None) -> None: ... + if sys.version_info >= (3, 11): + @deprecated("Deprecated since Python 3.11; removed in Python 3.13.") + def usageExit(self, msg: Any = None) -> None: ... + else: + def usageExit(self, msg: Any = None) -> None: ... def parseArgs(self, argv: list[str]) -> None: ... def createTests(self, from_discovery: bool = False, Loader: unittest.loader.TestLoader | None = None) -> None: ... diff --git a/mypy/typeshed/stdlib/unittest/mock.pyi b/mypy/typeshed/stdlib/unittest/mock.pyi index 6b0941a917190..f4b59e7cab906 100644 --- a/mypy/typeshed/stdlib/unittest/mock.pyi +++ b/mypy/typeshed/stdlib/unittest/mock.pyi @@ -4,7 +4,7 @@ from collections.abc import Awaitable, Callable, Coroutine, Iterable, Mapping, S from contextlib import _GeneratorContextManager from types import TracebackType from typing import Any, ClassVar, Final, Generic, Literal, TypeVar, overload, type_check_only -from typing_extensions import ParamSpec, Self, TypeAlias +from typing_extensions import ParamSpec, Self, TypeAlias, disjoint_base _T = TypeVar("_T") _TT = TypeVar("_TT", bound=type[Any]) @@ -52,7 +52,7 @@ else: "seal", ) -FILTER_DIR: Any +FILTER_DIR: bool # controls the way mock objects respond to `dir` function class _SentinelObject: name: Any @@ -61,36 +61,73 @@ class _SentinelObject: class _Sentinel: def __getattr__(self, name: str) -> Any: ... -sentinel: Any +sentinel: _Sentinel DEFAULT: Any _ArgsKwargs: TypeAlias = tuple[tuple[Any, ...], Mapping[str, Any]] _NameArgsKwargs: TypeAlias = tuple[str, tuple[Any, ...], Mapping[str, Any]] _CallValue: TypeAlias = str | tuple[Any, ...] | Mapping[str, Any] | _ArgsKwargs | _NameArgsKwargs -class _Call(tuple[Any, ...]): - def __new__( - cls, value: _CallValue = (), name: str | None = "", parent: _Call | None = None, two: bool = False, from_kall: bool = True - ) -> Self: ... - def __init__( - self, - value: _CallValue = (), - name: str | None = None, - parent: _Call | None = None, - two: bool = False, - from_kall: bool = True, - ) -> None: ... - __hash__: ClassVar[None] # type: ignore[assignment] - def __eq__(self, other: object) -> bool: ... - def __ne__(self, value: object, /) -> bool: ... - def __call__(self, *args: Any, **kwargs: Any) -> _Call: ... - def __getattr__(self, attr: str) -> Any: ... - def __getattribute__(self, attr: str) -> Any: ... - @property - def args(self) -> tuple[Any, ...]: ... - @property - def kwargs(self) -> Mapping[str, Any]: ... - def call_list(self) -> Any: ... +if sys.version_info >= (3, 12): + class _Call(tuple[Any, ...]): + def __new__( + cls, + value: _CallValue = (), + name: str | None = "", + parent: _Call | None = None, + two: bool = False, + from_kall: bool = True, + ) -> Self: ... + def __init__( + self, + value: _CallValue = (), + name: str | None = None, + parent: _Call | None = None, + two: bool = False, + from_kall: bool = True, + ) -> None: ... + __hash__: ClassVar[None] # type: ignore[assignment] + def __eq__(self, other: object) -> bool: ... + def __ne__(self, value: object, /) -> bool: ... + def __call__(self, *args: Any, **kwargs: Any) -> _Call: ... + def __getattr__(self, attr: str) -> Any: ... + def __getattribute__(self, attr: str) -> Any: ... + @property + def args(self) -> tuple[Any, ...]: ... + @property + def kwargs(self) -> Mapping[str, Any]: ... + def call_list(self) -> Any: ... + +else: + @disjoint_base + class _Call(tuple[Any, ...]): + def __new__( + cls, + value: _CallValue = (), + name: str | None = "", + parent: _Call | None = None, + two: bool = False, + from_kall: bool = True, + ) -> Self: ... + def __init__( + self, + value: _CallValue = (), + name: str | None = None, + parent: _Call | None = None, + two: bool = False, + from_kall: bool = True, + ) -> None: ... + __hash__: ClassVar[None] # type: ignore[assignment] + def __eq__(self, other: object) -> bool: ... + def __ne__(self, value: object, /) -> bool: ... + def __call__(self, *args: Any, **kwargs: Any) -> _Call: ... + def __getattr__(self, attr: str) -> Any: ... + def __getattribute__(self, attr: str) -> Any: ... + @property + def args(self) -> tuple[Any, ...]: ... + @property + def kwargs(self) -> Mapping[str, Any]: ... + def call_list(self) -> Any: ... call: _Call @@ -297,27 +334,32 @@ class _patcher: # Ideally we'd be able to add an overload for it so that the return type is _patch[MagicMock], # but that's impossible with the current type system. @overload - def __call__( + def __call__( # type: ignore[overload-overlap] self, target: str, new: _T, - spec: Any | None = ..., - create: bool = ..., - spec_set: Any | None = ..., - autospec: Any | None = ..., - new_callable: Callable[..., Any] | None = ..., - **kwargs: Any, + spec: Literal[False] | None = None, + create: bool = False, + spec_set: Literal[False] | None = None, + autospec: Literal[False] | None = None, + new_callable: None = None, + *, + unsafe: bool = False, ) -> _patch[_T]: ... @overload def __call__( self, target: str, *, - spec: Any | None = ..., - create: bool = ..., - spec_set: Any | None = ..., - autospec: Any | None = ..., + # If not False or None, this is passed to new_callable + spec: Any | Literal[False] | None = None, + create: bool = False, + # If not False or None, this is passed to new_callable + spec_set: Any | Literal[False] | None = None, + autospec: Literal[False] | None = None, new_callable: Callable[..., _T], + unsafe: bool = False, + # kwargs are passed to new_callable **kwargs: Any, ) -> _patch_pass_arg[_T]: ... @overload @@ -325,25 +367,31 @@ class _patcher: self, target: str, *, - spec: Any | None = ..., - create: bool = ..., - spec_set: Any | None = ..., - autospec: Any | None = ..., - new_callable: None = ..., + spec: Any | bool | None = None, + create: bool = False, + spec_set: Any | bool | None = None, + autospec: Any | bool | None = None, + new_callable: None = None, + unsafe: bool = False, + # kwargs are passed to the MagicMock/AsyncMock constructor **kwargs: Any, ) -> _patch_pass_arg[MagicMock | AsyncMock]: ... + # This overload also covers the case, where new==DEFAULT. In this case, the return type is _patch[Any]. + # Ideally we'd be able to add an overload for it so that the return type is _patch[MagicMock], + # but that's impossible with the current type system. @overload @staticmethod def object( target: Any, attribute: str, new: _T, - spec: Any | None = ..., - create: bool = ..., - spec_set: Any | None = ..., - autospec: Any | None = ..., - new_callable: Callable[..., Any] | None = ..., - **kwargs: Any, + spec: Literal[False] | None = None, + create: bool = False, + spec_set: Literal[False] | None = None, + autospec: Literal[False] | None = None, + new_callable: None = None, + *, + unsafe: bool = False, ) -> _patch[_T]: ... @overload @staticmethod @@ -351,11 +399,15 @@ class _patcher: target: Any, attribute: str, *, - spec: Any | None = ..., - create: bool = ..., - spec_set: Any | None = ..., - autospec: Any | None = ..., + # If not False or None, this is passed to new_callable + spec: Any | Literal[False] | None = None, + create: bool = False, + # If not False or None, this is passed to new_callable + spec_set: Any | Literal[False] | None = None, + autospec: Literal[False] | None = None, new_callable: Callable[..., _T], + unsafe: bool = False, + # kwargs are passed to new_callable **kwargs: Any, ) -> _patch_pass_arg[_T]: ... @overload @@ -364,21 +416,54 @@ class _patcher: target: Any, attribute: str, *, - spec: Any | None = ..., - create: bool = ..., - spec_set: Any | None = ..., - autospec: Any | None = ..., - new_callable: None = ..., + spec: Any | bool | None = None, + create: bool = False, + spec_set: Any | bool | None = None, + autospec: Any | bool | None = None, + new_callable: None = None, + unsafe: bool = False, + # kwargs are passed to the MagicMock/AsyncMock constructor **kwargs: Any, ) -> _patch_pass_arg[MagicMock | AsyncMock]: ... + @overload @staticmethod def multiple( - target: Any, - spec: Any | None = ..., - create: bool = ..., - spec_set: Any | None = ..., - autospec: Any | None = ..., - new_callable: Any | None = ..., + target: Any | str, + # If not False or None, this is passed to new_callable + spec: Any | Literal[False] | None = None, + create: bool = False, + # If not False or None, this is passed to new_callable + spec_set: Any | Literal[False] | None = None, + autospec: Literal[False] | None = None, + *, + new_callable: Callable[..., _T], + # The kwargs must be DEFAULT + **kwargs: Any, + ) -> _patch_pass_arg[_T]: ... + @overload + @staticmethod + def multiple( + target: Any | str, + # If not False or None, this is passed to new_callable + spec: Any | Literal[False] | None, + create: bool, + # If not False or None, this is passed to new_callable + spec_set: Any | Literal[False] | None, + autospec: Literal[False] | None, + new_callable: Callable[..., _T], + # The kwargs must be DEFAULT + **kwargs: Any, + ) -> _patch_pass_arg[_T]: ... + @overload + @staticmethod + def multiple( + target: Any | str, + spec: Any | bool | None = None, + create: bool = False, + spec_set: Any | bool | None = None, + autospec: Any | bool | None = None, + new_callable: None = None, + # The kwargs are the mock objects or DEFAULT **kwargs: Any, ) -> _patch[Any]: ... @staticmethod @@ -428,7 +513,7 @@ class _ANY: def __ne__(self, other: object) -> Literal[False]: ... __hash__: ClassVar[None] # type: ignore[assignment] -ANY: Any +ANY: _ANY if sys.version_info >= (3, 10): def create_autospec( diff --git a/mypy/typeshed/stdlib/unittest/util.pyi b/mypy/typeshed/stdlib/unittest/util.pyi index 945b0cecfed09..31c830e8268a7 100644 --- a/mypy/typeshed/stdlib/unittest/util.pyi +++ b/mypy/typeshed/stdlib/unittest/util.pyi @@ -5,12 +5,12 @@ from typing_extensions import TypeAlias _T = TypeVar("_T") _Mismatch: TypeAlias = tuple[_T, _T, int] -_MAX_LENGTH: Final[int] -_PLACEHOLDER_LEN: Final[int] -_MIN_BEGIN_LEN: Final[int] -_MIN_END_LEN: Final[int] -_MIN_COMMON_LEN: Final[int] -_MIN_DIFF_LEN: Final[int] +_MAX_LENGTH: Final = 80 +_PLACEHOLDER_LEN: Final = 12 +_MIN_BEGIN_LEN: Final = 5 +_MIN_END_LEN: Final = 5 +_MIN_COMMON_LEN: Final = 5 +_MIN_DIFF_LEN: Final = 41 def _shorten(s: str, prefixlen: int, suffixlen: int) -> str: ... def _common_shorten_repr(*args: str) -> tuple[str, ...]: ... diff --git a/mypy/typeshed/stdlib/urllib/parse.pyi b/mypy/typeshed/stdlib/urllib/parse.pyi index a5ed616d25af8..364892ecdf698 100644 --- a/mypy/typeshed/stdlib/urllib/parse.pyi +++ b/mypy/typeshed/stdlib/urllib/parse.pyi @@ -1,7 +1,7 @@ import sys from collections.abc import Iterable, Mapping, Sequence from types import GenericAlias -from typing import Any, AnyStr, Generic, Literal, NamedTuple, Protocol, overload, type_check_only +from typing import Any, AnyStr, Final, Generic, Literal, NamedTuple, Protocol, overload, type_check_only from typing_extensions import TypeAlias __all__ = [ @@ -28,23 +28,26 @@ __all__ = [ "SplitResultBytes", ] -uses_relative: list[str] -uses_netloc: list[str] -uses_params: list[str] -non_hierarchical: list[str] -uses_query: list[str] -uses_fragment: list[str] -scheme_chars: str +uses_relative: Final[list[str]] +uses_netloc: Final[list[str]] +uses_params: Final[list[str]] +non_hierarchical: Final[list[str]] +uses_query: Final[list[str]] +uses_fragment: Final[list[str]] +scheme_chars: Final[str] if sys.version_info < (3, 11): - MAX_CACHE_SIZE: int + MAX_CACHE_SIZE: Final[int] class _ResultMixinStr: + __slots__ = () def encode(self, encoding: str = "ascii", errors: str = "strict") -> _ResultMixinBytes: ... class _ResultMixinBytes: + __slots__ = () def decode(self, encoding: str = "ascii", errors: str = "strict") -> _ResultMixinStr: ... class _NetlocResultMixinBase(Generic[AnyStr]): + __slots__ = () @property def username(self) -> AnyStr | None: ... @property @@ -55,8 +58,11 @@ class _NetlocResultMixinBase(Generic[AnyStr]): def port(self) -> int | None: ... def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... -class _NetlocResultMixinStr(_NetlocResultMixinBase[str], _ResultMixinStr): ... -class _NetlocResultMixinBytes(_NetlocResultMixinBase[bytes], _ResultMixinBytes): ... +class _NetlocResultMixinStr(_NetlocResultMixinBase[str], _ResultMixinStr): + __slots__ = () + +class _NetlocResultMixinBytes(_NetlocResultMixinBase[bytes], _ResultMixinBytes): + __slots__ = () class _DefragResultBase(NamedTuple, Generic[AnyStr]): url: AnyStr diff --git a/mypy/typeshed/stdlib/uuid.pyi b/mypy/typeshed/stdlib/uuid.pyi index 0aa2f76d40cc5..303fb10eaf537 100644 --- a/mypy/typeshed/stdlib/uuid.pyi +++ b/mypy/typeshed/stdlib/uuid.pyi @@ -12,6 +12,7 @@ class SafeUUID(Enum): unknown = None class UUID: + __slots__ = ("int", "is_safe", "__weakref__") def __init__( self, hex: str | None = None, diff --git a/mypy/typeshed/stdlib/venv/__init__.pyi b/mypy/typeshed/stdlib/venv/__init__.pyi index 0f71f0e073f5c..14db88523dba4 100644 --- a/mypy/typeshed/stdlib/venv/__init__.pyi +++ b/mypy/typeshed/stdlib/venv/__init__.pyi @@ -3,10 +3,11 @@ import sys from _typeshed import StrOrBytesPath from collections.abc import Iterable, Sequence from types import SimpleNamespace +from typing import Final logger: logging.Logger -CORE_VENV_DEPS: tuple[str, ...] +CORE_VENV_DEPS: Final[tuple[str, ...]] class EnvBuilder: system_site_packages: bool diff --git a/mypy/typeshed/stdlib/wave.pyi b/mypy/typeshed/stdlib/wave.pyi index ddc6f6bd02a50..fd7dbfade884b 100644 --- a/mypy/typeshed/stdlib/wave.pyi +++ b/mypy/typeshed/stdlib/wave.pyi @@ -1,3 +1,4 @@ +import sys from _typeshed import ReadableBuffer, Unused from typing import IO, Any, BinaryIO, Final, Literal, NamedTuple, NoReturn, overload from typing_extensions import Self, TypeAlias, deprecated @@ -8,7 +9,7 @@ _File: TypeAlias = str | IO[bytes] class Error(Exception): ... -WAVE_FORMAT_PCM: Final = 1 +WAVE_FORMAT_PCM: Final = 0x0001 class _wave_params(NamedTuple): nchannels: int @@ -34,10 +35,15 @@ class Wave_read: def getcomptype(self) -> str: ... def getcompname(self) -> str: ... def getparams(self) -> _wave_params: ... - @deprecated("Deprecated in Python 3.13; removal scheduled for Python 3.15") - def getmarkers(self) -> None: ... - @deprecated("Deprecated in Python 3.13; removal scheduled for Python 3.15") - def getmark(self, id: Any) -> NoReturn: ... + if sys.version_info >= (3, 13): + @deprecated("Deprecated since Python 3.13; will be removed in Python 3.15.") + def getmarkers(self) -> None: ... + @deprecated("Deprecated since Python 3.13; will be removed in Python 3.15.") + def getmark(self, id: Any) -> NoReturn: ... + else: + def getmarkers(self) -> None: ... + def getmark(self, id: Any) -> NoReturn: ... + def setpos(self, pos: int) -> None: ... def readframes(self, nframes: int) -> bytes: ... @@ -59,12 +65,18 @@ class Wave_write: def getcompname(self) -> str: ... def setparams(self, params: _wave_params | tuple[int, int, int, int, str, str]) -> None: ... def getparams(self) -> _wave_params: ... - @deprecated("Deprecated in Python 3.13; removal scheduled for Python 3.15") - def setmark(self, id: Any, pos: Any, name: Any) -> NoReturn: ... - @deprecated("Deprecated in Python 3.13; removal scheduled for Python 3.15") - def getmark(self, id: Any) -> NoReturn: ... - @deprecated("Deprecated in Python 3.13; removal scheduled for Python 3.15") - def getmarkers(self) -> None: ... + if sys.version_info >= (3, 13): + @deprecated("Deprecated since Python 3.13; will be removed in Python 3.15.") + def setmark(self, id: Any, pos: Any, name: Any) -> NoReturn: ... + @deprecated("Deprecated since Python 3.13; will be removed in Python 3.15.") + def getmark(self, id: Any) -> NoReturn: ... + @deprecated("Deprecated since Python 3.13; will be removed in Python 3.15.") + def getmarkers(self) -> None: ... + else: + def setmark(self, id: Any, pos: Any, name: Any) -> NoReturn: ... + def getmark(self, id: Any) -> NoReturn: ... + def getmarkers(self) -> None: ... + def tell(self) -> int: ... def writeframesraw(self, data: ReadableBuffer) -> None: ... def writeframes(self, data: ReadableBuffer) -> None: ... diff --git a/mypy/typeshed/stdlib/weakref.pyi b/mypy/typeshed/stdlib/weakref.pyi index 334fab7e7468c..76ab86b957a13 100644 --- a/mypy/typeshed/stdlib/weakref.pyi +++ b/mypy/typeshed/stdlib/weakref.pyi @@ -4,7 +4,7 @@ from _weakrefset import WeakSet as WeakSet from collections.abc import Callable, Iterable, Iterator, Mapping, MutableMapping from types import GenericAlias from typing import Any, ClassVar, Generic, TypeVar, final, overload -from typing_extensions import ParamSpec, Self +from typing_extensions import ParamSpec, Self, disjoint_base __all__ = [ "ref", @@ -52,6 +52,7 @@ class ProxyType(Generic[_T]): # "weakproxy" def __getattr__(self, attr: str) -> Any: ... __hash__: ClassVar[None] # type: ignore[assignment] +@disjoint_base class ReferenceType(Generic[_T]): # "weakref" __callback__: Callable[[Self], Any] def __new__(cls, o: _T, callback: Callable[[Self], Any] | None = ..., /) -> Self: ... @@ -65,6 +66,7 @@ ref = ReferenceType # everything below here is implemented in weakref.py class WeakMethod(ref[_CallableT]): + __slots__ = ("_func_ref", "_meth_type", "_alive", "__weakref__") def __new__(cls, meth: _CallableT, callback: Callable[[Self], Any] | None = None) -> Self: ... def __call__(self) -> _CallableT | None: ... def __eq__(self, other: object) -> bool: ... @@ -130,6 +132,7 @@ class WeakValueDictionary(MutableMapping[_KT, _VT]): def __ior__(self, other: Iterable[tuple[_KT, _VT]]) -> Self: ... class KeyedRef(ref[_T], Generic[_KT, _T]): + __slots__ = ("key",) key: _KT def __new__(type, ob: _T, callback: Callable[[Self], Any], key: _KT) -> Self: ... def __init__(self, ob: _T, callback: Callable[[Self], Any], key: _KT) -> None: ... @@ -185,6 +188,7 @@ class WeakKeyDictionary(MutableMapping[_KT, _VT]): def __ior__(self, other: Iterable[tuple[_KT, _VT]]) -> Self: ... class finalize(Generic[_P, _T]): + __slots__ = () def __init__(self, obj: _T, func: Callable[_P, Any], /, *args: _P.args, **kwargs: _P.kwargs) -> None: ... def __call__(self, _: Any = None) -> Any | None: ... def detach(self) -> tuple[_T, Callable[_P, Any], tuple[Any, ...], dict[str, Any]] | None: ... diff --git a/mypy/typeshed/stdlib/webbrowser.pyi b/mypy/typeshed/stdlib/webbrowser.pyi index 773786c248219..56c30f8727277 100644 --- a/mypy/typeshed/stdlib/webbrowser.pyi +++ b/mypy/typeshed/stdlib/webbrowser.pyi @@ -64,10 +64,16 @@ if sys.platform == "win32": if sys.platform == "darwin": if sys.version_info < (3, 13): - @deprecated("Deprecated in 3.11, to be removed in 3.13.") - class MacOSX(BaseBrowser): - def __init__(self, name: str) -> None: ... - def open(self, url: str, new: int = 0, autoraise: bool = True) -> bool: ... + if sys.version_info >= (3, 11): + @deprecated("Deprecated since Python 3.11; removed in Python 3.13.") + class MacOSX(BaseBrowser): + def __init__(self, name: str) -> None: ... + def open(self, url: str, new: int = 0, autoraise: bool = True) -> bool: ... + + else: + class MacOSX(BaseBrowser): + def __init__(self, name: str) -> None: ... + def open(self, url: str, new: int = 0, autoraise: bool = True) -> bool: ... class MacOSXOSAScript(BaseBrowser): # In runtime this class does not have `name` and `basename` if sys.version_info >= (3, 11): diff --git a/mypy/typeshed/stdlib/winreg.pyi b/mypy/typeshed/stdlib/winreg.pyi index 0a22bb23d8f66..53457112ee968 100644 --- a/mypy/typeshed/stdlib/winreg.pyi +++ b/mypy/typeshed/stdlib/winreg.pyi @@ -59,13 +59,13 @@ if sys.platform == "win32": def EnableReflectionKey(key: _KeyType, /) -> None: ... def QueryReflectionKey(key: _KeyType, /) -> bool: ... - HKEY_CLASSES_ROOT: int - HKEY_CURRENT_USER: int - HKEY_LOCAL_MACHINE: int - HKEY_USERS: int - HKEY_PERFORMANCE_DATA: int - HKEY_CURRENT_CONFIG: int - HKEY_DYN_DATA: int + HKEY_CLASSES_ROOT: Final[int] + HKEY_CURRENT_USER: Final[int] + HKEY_LOCAL_MACHINE: Final[int] + HKEY_USERS: Final[int] + HKEY_PERFORMANCE_DATA: Final[int] + HKEY_CURRENT_CONFIG: Final[int] + HKEY_DYN_DATA: Final[int] KEY_ALL_ACCESS: Final = 983103 KEY_WRITE: Final = 131078 diff --git a/mypy/typeshed/stdlib/wsgiref/headers.pyi b/mypy/typeshed/stdlib/wsgiref/headers.pyi index 2654d79bf4e53..9febad4b32775 100644 --- a/mypy/typeshed/stdlib/wsgiref/headers.pyi +++ b/mypy/typeshed/stdlib/wsgiref/headers.pyi @@ -1,10 +1,10 @@ from re import Pattern -from typing import overload +from typing import Final, overload from typing_extensions import TypeAlias _HeaderList: TypeAlias = list[tuple[str, str]] -tspecials: Pattern[str] # undocumented +tspecials: Final[Pattern[str]] # undocumented class Headers: def __init__(self, headers: _HeaderList | None = None) -> None: ... diff --git a/mypy/typeshed/stdlib/wsgiref/simple_server.pyi b/mypy/typeshed/stdlib/wsgiref/simple_server.pyi index 547f562cc1d47..bdf58719c8289 100644 --- a/mypy/typeshed/stdlib/wsgiref/simple_server.pyi +++ b/mypy/typeshed/stdlib/wsgiref/simple_server.pyi @@ -1,14 +1,14 @@ from _typeshed.wsgi import ErrorStream, StartResponse, WSGIApplication, WSGIEnvironment from http.server import BaseHTTPRequestHandler, HTTPServer -from typing import TypeVar, overload +from typing import Final, TypeVar, overload from .handlers import SimpleHandler __all__ = ["WSGIServer", "WSGIRequestHandler", "demo_app", "make_server"] -server_version: str # undocumented -sys_version: str # undocumented -software_version: str # undocumented +server_version: Final[str] # undocumented +sys_version: Final[str] # undocumented +software_version: Final[str] # undocumented class ServerHandler(SimpleHandler): # undocumented server_software: str diff --git a/mypy/typeshed/stdlib/xml/dom/NodeFilter.pyi b/mypy/typeshed/stdlib/xml/dom/NodeFilter.pyi index 007df982e06a8..7b301373f5288 100644 --- a/mypy/typeshed/stdlib/xml/dom/NodeFilter.pyi +++ b/mypy/typeshed/stdlib/xml/dom/NodeFilter.pyi @@ -1,22 +1,22 @@ -from typing import Literal +from typing import Final from xml.dom.minidom import Node class NodeFilter: - FILTER_ACCEPT: Literal[1] - FILTER_REJECT: Literal[2] - FILTER_SKIP: Literal[3] + FILTER_ACCEPT: Final = 1 + FILTER_REJECT: Final = 2 + FILTER_SKIP: Final = 3 - SHOW_ALL: int - SHOW_ELEMENT: int - SHOW_ATTRIBUTE: int - SHOW_TEXT: int - SHOW_CDATA_SECTION: int - SHOW_ENTITY_REFERENCE: int - SHOW_ENTITY: int - SHOW_PROCESSING_INSTRUCTION: int - SHOW_COMMENT: int - SHOW_DOCUMENT: int - SHOW_DOCUMENT_TYPE: int - SHOW_DOCUMENT_FRAGMENT: int - SHOW_NOTATION: int + SHOW_ALL: Final = 0xFFFFFFFF + SHOW_ELEMENT: Final = 0x00000001 + SHOW_ATTRIBUTE: Final = 0x00000002 + SHOW_TEXT: Final = 0x00000004 + SHOW_CDATA_SECTION: Final = 0x00000008 + SHOW_ENTITY_REFERENCE: Final = 0x00000010 + SHOW_ENTITY: Final = 0x00000020 + SHOW_PROCESSING_INSTRUCTION: Final = 0x00000040 + SHOW_COMMENT: Final = 0x00000080 + SHOW_DOCUMENT: Final = 0x00000100 + SHOW_DOCUMENT_TYPE: Final = 0x00000200 + SHOW_DOCUMENT_FRAGMENT: Final = 0x00000400 + SHOW_NOTATION: Final = 0x00000800 def acceptNode(self, node: Node) -> int: ... diff --git a/mypy/typeshed/stdlib/xml/dom/__init__.pyi b/mypy/typeshed/stdlib/xml/dom/__init__.pyi index d9615f9aacfea..5dbb6c536f617 100644 --- a/mypy/typeshed/stdlib/xml/dom/__init__.pyi +++ b/mypy/typeshed/stdlib/xml/dom/__init__.pyi @@ -3,18 +3,19 @@ from typing import Any, Final, Literal from .domreg import getDOMImplementation as getDOMImplementation, registerDOMImplementation as registerDOMImplementation class Node: - ELEMENT_NODE: Literal[1] - ATTRIBUTE_NODE: Literal[2] - TEXT_NODE: Literal[3] - CDATA_SECTION_NODE: Literal[4] - ENTITY_REFERENCE_NODE: Literal[5] - ENTITY_NODE: Literal[6] - PROCESSING_INSTRUCTION_NODE: Literal[7] - COMMENT_NODE: Literal[8] - DOCUMENT_NODE: Literal[9] - DOCUMENT_TYPE_NODE: Literal[10] - DOCUMENT_FRAGMENT_NODE: Literal[11] - NOTATION_NODE: Literal[12] + __slots__ = () + ELEMENT_NODE: Final = 1 + ATTRIBUTE_NODE: Final = 2 + TEXT_NODE: Final = 3 + CDATA_SECTION_NODE: Final = 4 + ENTITY_REFERENCE_NODE: Final = 5 + ENTITY_NODE: Final = 6 + PROCESSING_INSTRUCTION_NODE: Final = 7 + COMMENT_NODE: Final = 8 + DOCUMENT_NODE: Final = 9 + DOCUMENT_TYPE_NODE: Final = 10 + DOCUMENT_FRAGMENT_NODE: Final = 11 + NOTATION_NODE: Final = 12 # ExceptionCode INDEX_SIZE_ERR: Final = 1 @@ -88,10 +89,10 @@ class ValidationErr(DOMException): code: Literal[16] class UserDataHandler: - NODE_CLONED: Literal[1] - NODE_IMPORTED: Literal[2] - NODE_DELETED: Literal[3] - NODE_RENAMED: Literal[4] + NODE_CLONED: Final = 1 + NODE_IMPORTED: Final = 2 + NODE_DELETED: Final = 3 + NODE_RENAMED: Final = 4 XML_NAMESPACE: Final = "http://www.w3.org/XML/1998/namespace" XMLNS_NAMESPACE: Final = "http://www.w3.org/2000/xmlns/" diff --git a/mypy/typeshed/stdlib/xml/dom/expatbuilder.pyi b/mypy/typeshed/stdlib/xml/dom/expatbuilder.pyi index 228ad07e15ad4..2b9ac88769700 100644 --- a/mypy/typeshed/stdlib/xml/dom/expatbuilder.pyi +++ b/mypy/typeshed/stdlib/xml/dom/expatbuilder.pyi @@ -1,5 +1,5 @@ from _typeshed import ReadableBuffer, SupportsRead -from typing import Any, NoReturn +from typing import Any, Final, NoReturn from typing_extensions import TypeAlias from xml.dom.minidom import Document, DocumentFragment, DOMImplementation, Element, Node, TypeInfo from xml.dom.xmlbuilder import DOMBuilderFilter, Options @@ -7,16 +7,17 @@ from xml.parsers.expat import XMLParserType _Model: TypeAlias = tuple[int, int, str | None, tuple[Any, ...]] # same as in pyexpat -TEXT_NODE = Node.TEXT_NODE -CDATA_SECTION_NODE = Node.CDATA_SECTION_NODE -DOCUMENT_NODE = Node.DOCUMENT_NODE -FILTER_ACCEPT = DOMBuilderFilter.FILTER_ACCEPT -FILTER_REJECT = DOMBuilderFilter.FILTER_REJECT -FILTER_SKIP = DOMBuilderFilter.FILTER_SKIP -FILTER_INTERRUPT = DOMBuilderFilter.FILTER_INTERRUPT +TEXT_NODE: Final = Node.TEXT_NODE +CDATA_SECTION_NODE: Final = Node.CDATA_SECTION_NODE +DOCUMENT_NODE: Final = Node.DOCUMENT_NODE +FILTER_ACCEPT: Final = DOMBuilderFilter.FILTER_ACCEPT +FILTER_REJECT: Final = DOMBuilderFilter.FILTER_REJECT +FILTER_SKIP: Final = DOMBuilderFilter.FILTER_SKIP +FILTER_INTERRUPT: Final = DOMBuilderFilter.FILTER_INTERRUPT theDOMImplementation: DOMImplementation class ElementInfo: + __slots__ = ("_attr_info", "_model", "tagName") tagName: str def __init__(self, tagName: str, model: _Model | None = None) -> None: ... def getAttributeType(self, aname: str) -> TypeInfo: ... @@ -66,19 +67,23 @@ class ExpatBuilder: def xml_decl_handler(self, version: str, encoding: str | None, standalone: int) -> None: ... class FilterVisibilityController: + __slots__ = ("filter",) filter: DOMBuilderFilter def __init__(self, filter: DOMBuilderFilter) -> None: ... def startContainer(self, node: Node) -> int: ... def acceptNode(self, node: Node) -> int: ... class FilterCrutch: + __slots__ = ("_builder", "_level", "_old_start", "_old_end") def __init__(self, builder: ExpatBuilder) -> None: ... class Rejecter(FilterCrutch): + __slots__ = () def start_element_handler(self, *args: Any) -> None: ... def end_element_handler(self, *args: Any) -> None: ... class Skipper(FilterCrutch): + __slots__ = () def start_element_handler(self, *args: Any) -> None: ... def end_element_handler(self, *args: Any) -> None: ... diff --git a/mypy/typeshed/stdlib/xml/dom/minicompat.pyi b/mypy/typeshed/stdlib/xml/dom/minicompat.pyi index 162f60254a585..6fcaee019dc20 100644 --- a/mypy/typeshed/stdlib/xml/dom/minicompat.pyi +++ b/mypy/typeshed/stdlib/xml/dom/minicompat.pyi @@ -8,11 +8,13 @@ _T = TypeVar("_T") StringTypes: tuple[type[str]] class NodeList(list[_T]): + __slots__ = () @property def length(self) -> int: ... def item(self, index: int) -> _T | None: ... class EmptyNodeList(tuple[()]): + __slots__ = () @property def length(self) -> Literal[0]: ... def item(self, index: int) -> None: ... diff --git a/mypy/typeshed/stdlib/xml/dom/minidom.pyi b/mypy/typeshed/stdlib/xml/dom/minidom.pyi index b9da9f3558ff3..e0431417aa3c0 100644 --- a/mypy/typeshed/stdlib/xml/dom/minidom.pyi +++ b/mypy/typeshed/stdlib/xml/dom/minidom.pyi @@ -188,6 +188,7 @@ _AttrChildrenVar = TypeVar("_AttrChildrenVar", bound=_AttrChildren) _AttrChildrenPlusFragment = TypeVar("_AttrChildrenPlusFragment", bound=_AttrChildren | DocumentFragment) class Attr(Node): + __slots__ = ("_name", "_value", "namespaceURI", "_prefix", "childNodes", "_localName", "ownerDocument", "ownerElement") nodeType: ClassVar[Literal[2]] nodeName: str # same as Attr.name nodeValue: str # same as Attr.value @@ -231,6 +232,7 @@ class Attr(Node): # In the DOM, this interface isn't specific to Attr, but our implementation is # because that's the only place we use it. class NamedNodeMap: + __slots__ = ("_attrs", "_attrsNS", "_ownerElement") def __init__(self, attrs: dict[str, Attr], attrsNS: dict[_NSName, Attr], ownerElement: Element) -> None: ... @property def length(self) -> int: ... @@ -262,6 +264,7 @@ class NamedNodeMap: AttributeList = NamedNodeMap class TypeInfo: + __slots__ = ("namespace", "name") namespace: str | None name: str | None def __init__(self, namespace: Incomplete | None, name: str | None) -> None: ... @@ -270,6 +273,20 @@ _ElementChildrenVar = TypeVar("_ElementChildrenVar", bound=_ElementChildren) _ElementChildrenPlusFragment = TypeVar("_ElementChildrenPlusFragment", bound=_ElementChildren | DocumentFragment) class Element(Node): + __slots__ = ( + "ownerDocument", + "parentNode", + "tagName", + "nodeName", + "prefix", + "namespaceURI", + "_localName", + "childNodes", + "_attrs", + "_attrsNS", + "nextSibling", + "previousSibling", + ) nodeType: ClassVar[Literal[1]] nodeName: str # same as Element.tagName nodeValue: None @@ -331,6 +348,7 @@ class Element(Node): def removeChild(self, oldChild: _ElementChildrenVar) -> _ElementChildrenVar: ... # type: ignore[override] class Childless: + __slots__ = () attributes: None childNodes: EmptyNodeList @property @@ -347,6 +365,7 @@ class Childless: def replaceChild(self, newChild: _NodesThatAreChildren | DocumentFragment, oldChild: _NodesThatAreChildren) -> NoReturn: ... class ProcessingInstruction(Childless, Node): + __slots__ = ("target", "data") nodeType: ClassVar[Literal[7]] nodeName: str # same as ProcessingInstruction.target nodeValue: str # same as ProcessingInstruction.data @@ -373,6 +392,7 @@ class ProcessingInstruction(Childless, Node): def writexml(self, writer: SupportsWrite[str], indent: str = "", addindent: str = "", newl: str = "") -> None: ... class CharacterData(Childless, Node): + __slots__ = ("_data", "ownerDocument", "parentNode", "previousSibling", "nextSibling") nodeValue: str attributes: None @@ -397,6 +417,7 @@ class CharacterData(Childless, Node): def replaceData(self, offset: int, count: int, arg: str) -> None: ... class Text(CharacterData): + __slots__ = () nodeType: ClassVar[Literal[3]] nodeName: Literal["#text"] nodeValue: str # same as CharacterData.data, the content of the text node @@ -448,6 +469,7 @@ class Comment(CharacterData): def writexml(self, writer: SupportsWrite[str], indent: str = "", addindent: str = "", newl: str = "") -> None: ... class CDATASection(Text): + __slots__ = () nodeType: ClassVar[Literal[4]] # type: ignore[assignment] nodeName: Literal["#cdata-section"] # type: ignore[assignment] nodeValue: str # same as CharacterData.data, the content of the CDATA Section @@ -460,6 +482,7 @@ class CDATASection(Text): def writexml(self, writer: SupportsWrite[str], indent: str = "", addindent: str = "", newl: str = "") -> None: ... class ReadOnlySequentialNamedNodeMap(Generic[_N]): + __slots__ = ("_seq",) def __init__(self, seq: Sequence[_N] = ()) -> None: ... def __len__(self) -> int: ... def getNamedItem(self, name: str) -> _N | None: ... @@ -474,6 +497,7 @@ class ReadOnlySequentialNamedNodeMap(Generic[_N]): def length(self) -> int: ... class Identified: + __slots__ = ("publicId", "systemId") publicId: str | None systemId: str | None @@ -565,6 +589,7 @@ class DOMImplementation(DOMImplementationLS): def getInterface(self, feature: str) -> Self | None: ... class ElementInfo: + __slots__ = ("tagName",) tagName: str def __init__(self, name: str) -> None: ... def getAttributeType(self, aname: str) -> TypeInfo: ... @@ -577,6 +602,7 @@ class ElementInfo: _DocumentChildrenPlusFragment = TypeVar("_DocumentChildrenPlusFragment", bound=_DocumentChildren | DocumentFragment) class Document(Node, DocumentLS): + __slots__ = ("_elem_info", "doctype", "_id_search_stack", "childNodes", "_id_cache") nodeType: ClassVar[Literal[9]] nodeName: Literal["#document"] nodeValue: None diff --git a/mypy/typeshed/stdlib/xml/dom/pulldom.pyi b/mypy/typeshed/stdlib/xml/dom/pulldom.pyi index d9458654c1853..df7a3ad0eddb0 100644 --- a/mypy/typeshed/stdlib/xml/dom/pulldom.pyi +++ b/mypy/typeshed/stdlib/xml/dom/pulldom.pyi @@ -99,7 +99,7 @@ class SAX2DOM(PullDOM): def ignorableWhitespace(self, chars: str) -> None: ... def characters(self, chars: str) -> None: ... -default_bufsize: int +default_bufsize: Final[int] def parse( stream_or_string: str | _SupportsReadClose[bytes] | _SupportsReadClose[str], diff --git a/mypy/typeshed/stdlib/xml/dom/xmlbuilder.pyi b/mypy/typeshed/stdlib/xml/dom/xmlbuilder.pyi index 6fb18bbc4eda3..f19f7050b08df 100644 --- a/mypy/typeshed/stdlib/xml/dom/xmlbuilder.pyi +++ b/mypy/typeshed/stdlib/xml/dom/xmlbuilder.pyi @@ -1,5 +1,5 @@ from _typeshed import SupportsRead -from typing import Any, Literal, NoReturn +from typing import Any, Final, Literal, NoReturn from xml.dom.minidom import Document, Node, _DOMErrorHandler __all__ = ["DOMBuilder", "DOMEntityResolver", "DOMInputSource"] @@ -29,10 +29,10 @@ class DOMBuilder: entityResolver: DOMEntityResolver | None errorHandler: _DOMErrorHandler | None filter: DOMBuilderFilter | None - ACTION_REPLACE: Literal[1] - ACTION_APPEND_AS_CHILDREN: Literal[2] - ACTION_INSERT_AFTER: Literal[3] - ACTION_INSERT_BEFORE: Literal[4] + ACTION_REPLACE: Final = 1 + ACTION_APPEND_AS_CHILDREN: Final = 2 + ACTION_INSERT_AFTER: Final = 3 + ACTION_INSERT_BEFORE: Final = 4 def __init__(self) -> None: ... def setFeature(self, name: str, state: int) -> None: ... def supportsFeature(self, name: str) -> bool: ... @@ -44,9 +44,11 @@ class DOMBuilder: def parseWithContext(self, input: DOMInputSource, cnode: Node, action: Literal[1, 2, 3, 4]) -> NoReturn: ... class DOMEntityResolver: + __slots__ = ("_opener",) def resolveEntity(self, publicId: str | None, systemId: str) -> DOMInputSource: ... class DOMInputSource: + __slots__ = ("byteStream", "characterStream", "stringData", "encoding", "publicId", "systemId", "baseURI") byteStream: SupportsRead[bytes] | None characterStream: SupportsRead[str] | None stringData: str | None @@ -56,10 +58,10 @@ class DOMInputSource: baseURI: str | None class DOMBuilderFilter: - FILTER_ACCEPT: Literal[1] - FILTER_REJECT: Literal[2] - FILTER_SKIP: Literal[3] - FILTER_INTERRUPT: Literal[4] + FILTER_ACCEPT: Final = 1 + FILTER_REJECT: Final = 2 + FILTER_SKIP: Final = 3 + FILTER_INTERRUPT: Final = 4 whatToShow: int def acceptNode(self, element: Node) -> Literal[1, 2, 3, 4]: ... def startContainer(self, element: Node) -> Literal[1, 2, 3, 4]: ... @@ -72,8 +74,8 @@ class DocumentLS: def saveXML(self, snode: Node | None) -> str: ... class DOMImplementationLS: - MODE_SYNCHRONOUS: Literal[1] - MODE_ASYNCHRONOUS: Literal[2] + MODE_SYNCHRONOUS: Final = 1 + MODE_ASYNCHRONOUS: Final = 2 def createDOMBuilder(self, mode: Literal[1], schemaType: None) -> DOMBuilder: ... def createDOMWriter(self) -> NoReturn: ... def createDOMInputSource(self) -> DOMInputSource: ... diff --git a/mypy/typeshed/stdlib/xml/etree/ElementInclude.pyi b/mypy/typeshed/stdlib/xml/etree/ElementInclude.pyi index fd829fdaa5ffc..10784e7d40214 100644 --- a/mypy/typeshed/stdlib/xml/etree/ElementInclude.pyi +++ b/mypy/typeshed/stdlib/xml/etree/ElementInclude.pyi @@ -9,9 +9,10 @@ class _Loader(Protocol): @overload def __call__(self, href: FileDescriptorOrPath, parse: Literal["text"], encoding: str | None = None) -> str: ... -XINCLUDE: Final[str] -XINCLUDE_INCLUDE: Final[str] -XINCLUDE_FALLBACK: Final[str] +XINCLUDE: Final = "{http://www.w3.org/2001/XInclude}" + +XINCLUDE_INCLUDE: Final = "{http://www.w3.org/2001/XInclude}include" +XINCLUDE_FALLBACK: Final = "{http://www.w3.org/2001/XInclude}fallback" DEFAULT_MAX_INCLUSION_DEPTH: Final = 6 diff --git a/mypy/typeshed/stdlib/xml/etree/ElementPath.pyi b/mypy/typeshed/stdlib/xml/etree/ElementPath.pyi index ebfb4f1ffbb9c..80f3c55c14899 100644 --- a/mypy/typeshed/stdlib/xml/etree/ElementPath.pyi +++ b/mypy/typeshed/stdlib/xml/etree/ElementPath.pyi @@ -1,10 +1,10 @@ from collections.abc import Callable, Generator, Iterable from re import Pattern -from typing import Any, Literal, TypeVar, overload +from typing import Any, Final, Literal, TypeVar, overload from typing_extensions import TypeAlias from xml.etree.ElementTree import Element -xpath_tokenizer_re: Pattern[str] +xpath_tokenizer_re: Final[Pattern[str]] _Token: TypeAlias = tuple[str, str] _Next: TypeAlias = Callable[[], _Token] @@ -20,7 +20,7 @@ def prepare_descendant(next: _Next, token: _Token) -> _Callback | None: ... def prepare_parent(next: _Next, token: _Token) -> _Callback: ... def prepare_predicate(next: _Next, token: _Token) -> _Callback | None: ... -ops: dict[str, Callable[[_Next, _Token], _Callback | None]] +ops: Final[dict[str, Callable[[_Next, _Token], _Callback | None]]] class _SelectorContext: parent_map: dict[Element, Element] | None diff --git a/mypy/typeshed/stdlib/xml/etree/ElementTree.pyi b/mypy/typeshed/stdlib/xml/etree/ElementTree.pyi index 1d7e1725dd8ee..e8f737778040c 100644 --- a/mypy/typeshed/stdlib/xml/etree/ElementTree.pyi +++ b/mypy/typeshed/stdlib/xml/etree/ElementTree.pyi @@ -3,7 +3,7 @@ from _collections_abc import dict_keys from _typeshed import FileDescriptorOrPath, ReadableBuffer, SupportsRead, SupportsWrite from collections.abc import Callable, Generator, ItemsView, Iterable, Iterator, Mapping, Sequence from typing import Any, Final, Generic, Literal, Protocol, SupportsIndex, TypeVar, overload, type_check_only -from typing_extensions import TypeAlias, TypeGuard, deprecated +from typing_extensions import TypeAlias, TypeGuard, deprecated, disjoint_base from xml.parsers.expat import XMLParserType __all__ = [ @@ -78,14 +78,13 @@ def canonicalize( ) -> None: ... # The tag for Element can be set to the Comment or ProcessingInstruction -# functions defined in this module. _ElementCallable could be a recursive -# type, but defining it that way uncovered a bug in pytype. -_ElementCallable: TypeAlias = Callable[..., Element[Any]] -_CallableElement: TypeAlias = Element[_ElementCallable] +# functions defined in this module. +_ElementCallable: TypeAlias = Callable[..., Element[_ElementCallable]] _Tag = TypeVar("_Tag", default=str, bound=str | _ElementCallable) _OtherTag = TypeVar("_OtherTag", default=str, bound=str | _ElementCallable) +@disjoint_base class Element(Generic[_Tag]): tag: _Tag attrib: dict[str, str] @@ -138,8 +137,8 @@ class Element(Generic[_Tag]): def __bool__(self) -> bool: ... def SubElement(parent: Element, tag: str, attrib: dict[str, str] = ..., **extra: str) -> Element: ... -def Comment(text: str | None = None) -> _CallableElement: ... -def ProcessingInstruction(target: str, text: str | None = None) -> _CallableElement: ... +def Comment(text: str | None = None) -> Element[_ElementCallable]: ... +def ProcessingInstruction(target: str, text: str | None = None) -> Element[_ElementCallable]: ... PI = ProcessingInstruction @@ -182,7 +181,7 @@ class ElementTree(Generic[_Root]): ) -> None: ... def write_c14n(self, file: _FileWriteC14N) -> None: ... -HTML_EMPTY: set[str] +HTML_EMPTY: Final[set[str]] def register_namespace(prefix: str, uri: str) -> None: ... @overload @@ -288,6 +287,7 @@ def fromstringlist(sequence: Sequence[str | ReadableBuffer], parser: XMLParser | # elementfactories. _ElementFactory: TypeAlias = Callable[[Any, dict[Any, Any]], Element] +@disjoint_base class TreeBuilder: # comment_factory can take None because passing None to Comment is not an error def __init__( @@ -353,6 +353,7 @@ _E = TypeVar("_E", default=Element) # The default target is TreeBuilder, which returns Element. # C14NWriterTarget does not implement a close method, so using it results # in a type of XMLParser[None]. +@disjoint_base class XMLParser(Generic[_E]): parser: XMLParserType target: _Target diff --git a/mypy/typeshed/stdlib/xml/sax/__init__.pyi b/mypy/typeshed/stdlib/xml/sax/__init__.pyi index 5a82b48c1e19d..679466fa34d2c 100644 --- a/mypy/typeshed/stdlib/xml/sax/__init__.pyi +++ b/mypy/typeshed/stdlib/xml/sax/__init__.pyi @@ -1,7 +1,7 @@ import sys from _typeshed import ReadableBuffer, StrPath, SupportsRead, _T_co from collections.abc import Iterable -from typing import Protocol, type_check_only +from typing import Final, Protocol, type_check_only from typing_extensions import TypeAlias from xml.sax._exceptions import ( SAXException as SAXException, @@ -19,7 +19,7 @@ class _SupportsReadClose(SupportsRead[_T_co], Protocol[_T_co]): _Source: TypeAlias = StrPath | _SupportsReadClose[bytes] | _SupportsReadClose[str] -default_parser_list: list[str] +default_parser_list: Final[list[str]] def make_parser(parser_list: Iterable[str] = ()) -> XMLReader: ... def parse(source: _Source, handler: ContentHandler, errorHandler: ErrorHandler = ...) -> None: ... diff --git a/mypy/typeshed/stdlib/xml/sax/expatreader.pyi b/mypy/typeshed/stdlib/xml/sax/expatreader.pyi index 012d6c03e1219..3f9573a25f9aa 100644 --- a/mypy/typeshed/stdlib/xml/sax/expatreader.pyi +++ b/mypy/typeshed/stdlib/xml/sax/expatreader.pyi @@ -1,7 +1,7 @@ import sys from _typeshed import ReadableBuffer from collections.abc import Mapping -from typing import Any, Literal, overload +from typing import Any, Final, Literal, overload from typing_extensions import TypeAlias from xml.sax import _Source, xmlreader from xml.sax.handler import _ContentHandlerProtocol @@ -11,7 +11,7 @@ if sys.version_info >= (3, 10): _BoolType: TypeAlias = Literal[0, 1] | bool -version: str +version: Final[str] AttributesImpl = xmlreader.AttributesImpl AttributesNSImpl = xmlreader.AttributesNSImpl diff --git a/mypy/typeshed/stdlib/xml/sax/handler.pyi b/mypy/typeshed/stdlib/xml/sax/handler.pyi index 5509117345968..5ecbfa6f1272c 100644 --- a/mypy/typeshed/stdlib/xml/sax/handler.pyi +++ b/mypy/typeshed/stdlib/xml/sax/handler.pyi @@ -1,8 +1,8 @@ import sys -from typing import Literal, NoReturn, Protocol, type_check_only +from typing import Final, NoReturn, Protocol, type_check_only from xml.sax import xmlreader -version: str +version: Final[str] @type_check_only class _ErrorHandlerProtocol(Protocol): # noqa: Y046 # Protocol is not used @@ -62,20 +62,20 @@ class _EntityResolverProtocol(Protocol): # noqa: Y046 # Protocol is not used class EntityResolver: def resolveEntity(self, publicId: str | None, systemId: str) -> str: ... -feature_namespaces: str -feature_namespace_prefixes: str -feature_string_interning: str -feature_validation: str -feature_external_ges: str -feature_external_pes: str -all_features: list[str] -property_lexical_handler: Literal["http://xml.org/sax/properties/lexical-handler"] -property_declaration_handler: Literal["http://xml.org/sax/properties/declaration-handler"] -property_dom_node: Literal["http://xml.org/sax/properties/dom-node"] -property_xml_string: Literal["http://xml.org/sax/properties/xml-string"] -property_encoding: Literal["http://www.python.org/sax/properties/encoding"] -property_interning_dict: Literal["http://www.python.org/sax/properties/interning-dict"] -all_properties: list[str] +feature_namespaces: Final = "http://xml.org/sax/features/namespaces" +feature_namespace_prefixes: Final = "http://xml.org/sax/features/namespace-prefixes" +feature_string_interning: Final = "http://xml.org/sax/features/string-interning" +feature_validation: Final = "http://xml.org/sax/features/validation" +feature_external_ges: Final[str] # too long string +feature_external_pes: Final[str] # too long string +all_features: Final[list[str]] +property_lexical_handler: Final = "http://xml.org/sax/properties/lexical-handler" +property_declaration_handler: Final = "http://xml.org/sax/properties/declaration-handler" +property_dom_node: Final = "http://xml.org/sax/properties/dom-node" +property_xml_string: Final = "http://xml.org/sax/properties/xml-string" +property_encoding: Final = "http://www.python.org/sax/properties/encoding" +property_interning_dict: Final[str] # too long string +all_properties: Final[list[str]] if sys.version_info >= (3, 10): class LexicalHandler: diff --git a/mypy/typeshed/stdlib/zipfile/__init__.pyi b/mypy/typeshed/stdlib/zipfile/__init__.pyi index 73e3a92fd0e29..e573d04dba051 100644 --- a/mypy/typeshed/stdlib/zipfile/__init__.pyi +++ b/mypy/typeshed/stdlib/zipfile/__init__.pyi @@ -161,7 +161,7 @@ class ZipFile: def __init__( self, file: StrPath | _ZipWritable, - mode: Literal["w", "x"] = ..., + mode: Literal["w", "x"], compression: int = 0, allowZip64: bool = True, compresslevel: int | None = None, @@ -173,7 +173,7 @@ class ZipFile: def __init__( self, file: StrPath | _ZipReadableTellable, - mode: Literal["a"] = ..., + mode: Literal["a"], compression: int = 0, allowZip64: bool = True, compresslevel: int | None = None, @@ -208,7 +208,7 @@ class ZipFile: def __init__( self, file: StrPath | _ZipWritable, - mode: Literal["w", "x"] = ..., + mode: Literal["w", "x"], compression: int = 0, allowZip64: bool = True, compresslevel: int | None = None, @@ -219,7 +219,7 @@ class ZipFile: def __init__( self, file: StrPath | _ZipReadableTellable, - mode: Literal["a"] = ..., + mode: Literal["a"], compression: int = 0, allowZip64: bool = True, compresslevel: int | None = None, @@ -272,6 +272,29 @@ class PyZipFile(ZipFile): def writepy(self, pathname: str, basename: str = "", filterfunc: Callable[[str], bool] | None = None) -> None: ... class ZipInfo: + __slots__ = ( + "orig_filename", + "filename", + "date_time", + "compress_type", + "compress_level", + "comment", + "extra", + "create_system", + "create_version", + "extract_version", + "reserved", + "flag_bits", + "volume", + "internal_attr", + "external_attr", + "header_offset", + "CRC", + "compress_size", + "file_size", + "_raw_time", + "_end_offset", + ) filename: str date_time: _DateTuple compress_type: int diff --git a/mypy/typeshed/stdlib/zipfile/_path/glob.pyi b/mypy/typeshed/stdlib/zipfile/_path/glob.pyi index f25ae71725c02..f6a661be8cdf4 100644 --- a/mypy/typeshed/stdlib/zipfile/_path/glob.pyi +++ b/mypy/typeshed/stdlib/zipfile/_path/glob.pyi @@ -4,7 +4,11 @@ from re import Match if sys.version_info >= (3, 13): class Translator: - def __init__(self, seps: str = ...) -> None: ... + if sys.platform == "win32": + def __init__(self, seps: str = "\\/") -> None: ... + else: + def __init__(self, seps: str = "/") -> None: ... + def translate(self, pattern: str) -> str: ... def extend(self, pattern: str) -> str: ... def match_dirs(self, pattern: str) -> str: ... diff --git a/mypy/typeshed/stdlib/zoneinfo/__init__.pyi b/mypy/typeshed/stdlib/zoneinfo/__init__.pyi index e9f54fbf2a26c..b7433f835f83d 100644 --- a/mypy/typeshed/stdlib/zoneinfo/__init__.pyi +++ b/mypy/typeshed/stdlib/zoneinfo/__init__.pyi @@ -1,7 +1,7 @@ import sys from collections.abc import Iterable from datetime import datetime, timedelta, tzinfo -from typing_extensions import Self +from typing_extensions import Self, disjoint_base from zoneinfo._common import ZoneInfoNotFoundError as ZoneInfoNotFoundError, _IOBytes from zoneinfo._tzpath import ( TZPATH as TZPATH, @@ -12,6 +12,7 @@ from zoneinfo._tzpath import ( __all__ = ["ZoneInfo", "reset_tzpath", "available_timezones", "TZPATH", "ZoneInfoNotFoundError", "InvalidTZPathWarning"] +@disjoint_base class ZoneInfo(tzinfo): @property def key(self) -> str: ... diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index 2910e59b91739..93b67bfa813a8 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -1479,9 +1479,9 @@ y: str if isinstance(x, int): reveal_type(x) [out] -_testIsInstanceAdHocIntersectionWithStrAndBytes.py:3: error: Subclass of "str" and "bytes" cannot exist: would have incompatible method signatures +_testIsInstanceAdHocIntersectionWithStrAndBytes.py:3: error: Subclass of "str" and "bytes" cannot exist: have distinct disjoint bases _testIsInstanceAdHocIntersectionWithStrAndBytes.py:4: error: Statement is unreachable -_testIsInstanceAdHocIntersectionWithStrAndBytes.py:6: error: Subclass of "str" and "int" cannot exist: would have incompatible method signatures +_testIsInstanceAdHocIntersectionWithStrAndBytes.py:6: error: Subclass of "str" and "int" cannot exist: have distinct disjoint bases _testIsInstanceAdHocIntersectionWithStrAndBytes.py:7: error: Statement is unreachable [case testAsyncioFutureWait] From 0afa33dcdffa770de5fadb73e8210f0d5724611c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 1 Sep 2025 14:25:20 +0100 Subject: [PATCH 228/424] Mypy micro-optimizations (batch 3/3) (#19770) Several mypy micro-optimizations. Together with batches 1 and 2 these improve self check performance by 1.8%. This mostly avoids some allocations of nested function objects. --- mypy/meet.py | 98 +++++++++++++++++++++++++++++----------------------- 1 file changed, 55 insertions(+), 43 deletions(-) diff --git a/mypy/meet.py b/mypy/meet.py index 349c15e668c3c..353af59367ad1 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -306,6 +306,19 @@ def is_none_object_overlap(t1: ProperType, t2: ProperType) -> bool: ) +def are_related_types( + left: Type, right: Type, *, proper_subtype: bool, ignore_promotions: bool +) -> bool: + if proper_subtype: + return is_proper_subtype( + left, right, ignore_promotions=ignore_promotions + ) or is_proper_subtype(right, left, ignore_promotions=ignore_promotions) + else: + return is_subtype(left, right, ignore_promotions=ignore_promotions) or is_subtype( + right, left, ignore_promotions=ignore_promotions + ) + + def is_overlapping_types( left: Type, right: Type, @@ -329,27 +342,13 @@ def is_overlapping_types( if seen_types is None: seen_types = set() - if (left, right) in seen_types: + elif (left, right) in seen_types: return True if isinstance(left, TypeAliasType) and isinstance(right, TypeAliasType): seen_types.add((left, right)) left, right = get_proper_types((left, right)) - def _is_overlapping_types(left: Type, right: Type) -> bool: - """Encode the kind of overlapping check to perform. - - This function mostly exists, so we don't have to repeat keyword arguments everywhere. - """ - return is_overlapping_types( - left, - right, - ignore_promotions=ignore_promotions, - prohibit_none_typevar_overlap=prohibit_none_typevar_overlap, - overlap_for_overloads=overlap_for_overloads, - seen_types=seen_types.copy(), - ) - # We should never encounter this type. if isinstance(left, PartialType) or isinstance(right, PartialType): assert False, "Unexpectedly encountered partial type" @@ -399,13 +398,9 @@ def _is_overlapping_types(left: Type, right: Type) -> bool: if is_none_object_overlap(left, right) or is_none_object_overlap(right, left): return False - def _is_subtype(left: Type, right: Type) -> bool: - if overlap_for_overloads: - return is_proper_subtype(left, right, ignore_promotions=ignore_promotions) - else: - return is_subtype(left, right, ignore_promotions=ignore_promotions) - - if _is_subtype(left, right) or _is_subtype(right, left): + if are_related_types( + left, right, proper_subtype=overlap_for_overloads, ignore_promotions=ignore_promotions + ): return True # See the docstring for 'get_possible_variants' for more info on what the @@ -428,6 +423,20 @@ def _is_subtype(left: Type, right: Type) -> bool: if is_none_typevarlike_overlap(left, right) or is_none_typevarlike_overlap(right, left): return False + def _is_overlapping_types(left: Type, right: Type) -> bool: + """Encode the kind of overlapping check to perform. + + This function mostly exists, so we don't have to repeat keyword arguments everywhere. + """ + return is_overlapping_types( + left, + right, + ignore_promotions=ignore_promotions, + prohibit_none_typevar_overlap=prohibit_none_typevar_overlap, + overlap_for_overloads=overlap_for_overloads, + seen_types=seen_types.copy(), + ) + if ( len(left_possible) > 1 or len(right_possible) > 1 @@ -483,27 +492,28 @@ def _is_subtype(left: Type, right: Type) -> bool: if isinstance(left, TypeType) and isinstance(right, TypeType): return _is_overlapping_types(left.item, right.item) - def _type_object_overlap(left: Type, right: Type) -> bool: - """Special cases for type object types overlaps.""" - # TODO: these checks are a bit in gray area, adjust if they cause problems. - left, right = get_proper_types((left, right)) - # 1. Type[C] vs Callable[..., C] overlap even if the latter is not class object. - if isinstance(left, TypeType) and isinstance(right, CallableType): - return _is_overlapping_types(left.item, right.ret_type) - # 2. Type[C] vs Meta, where Meta is a metaclass for C. - if isinstance(left, TypeType) and isinstance(right, Instance): - if isinstance(left.item, Instance): - left_meta = left.item.type.metaclass_type - if left_meta is not None: - return _is_overlapping_types(left_meta, right) - # builtins.type (default metaclass) overlaps with all metaclasses - return right.type.has_base("builtins.type") - elif isinstance(left.item, AnyType): - return right.type.has_base("builtins.type") - # 3. Callable[..., C] vs Meta is considered below, when we switch to fallbacks. - return False - if isinstance(left, TypeType) or isinstance(right, TypeType): + + def _type_object_overlap(left: Type, right: Type) -> bool: + """Special cases for type object types overlaps.""" + # TODO: these checks are a bit in gray area, adjust if they cause problems. + left, right = get_proper_types((left, right)) + # 1. Type[C] vs Callable[..., C] overlap even if the latter is not class object. + if isinstance(left, TypeType) and isinstance(right, CallableType): + return _is_overlapping_types(left.item, right.ret_type) + # 2. Type[C] vs Meta, where Meta is a metaclass for C. + if isinstance(left, TypeType) and isinstance(right, Instance): + if isinstance(left.item, Instance): + left_meta = left.item.type.metaclass_type + if left_meta is not None: + return _is_overlapping_types(left_meta, right) + # builtins.type (default metaclass) overlaps with all metaclasses + return right.type.has_base("builtins.type") + elif isinstance(left.item, AnyType): + return right.type.has_base("builtins.type") + # 3. Callable[..., C] vs Meta is considered below, when we switch to fallbacks. + return False + return _type_object_overlap(left, right) or _type_object_overlap(right, left) if isinstance(left, Parameters) and isinstance(right, Parameters): @@ -564,7 +574,9 @@ def _type_object_overlap(left: Type, right: Type) -> bool: if isinstance(left, Instance) and isinstance(right, Instance): # First we need to handle promotions and structural compatibility for instances # that came as fallbacks, so simply call is_subtype() to avoid code duplication. - if _is_subtype(left, right) or _is_subtype(right, left): + if are_related_types( + left, right, proper_subtype=overlap_for_overloads, ignore_promotions=ignore_promotions + ): return True if right.type.fullname == "builtins.int" and left.type.fullname in MYPYC_NATIVE_INT_NAMES: From acacc9eba333701efbc1beedfe87245d4743bf94 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 1 Sep 2025 14:57:59 +0100 Subject: [PATCH 229/424] [mypyc] Fix C function signature (#19773) --- mypyc/lib-rt/CPy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index b4d3a0013ae79..5dec7509ac7b9 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -67,7 +67,7 @@ typedef struct tuple_T4CIOO { // System-wide empty tuple constant extern PyObject * __mypyc_empty_tuple__; -static inline PyObject *CPyTuple_LoadEmptyTupleConstant() { +static inline PyObject *CPyTuple_LoadEmptyTupleConstant(void) { #if !CPY_3_12_FEATURES Py_INCREF(__mypyc_empty_tuple__); #endif From 23965ab27f78058c1d4e464cdccd070bf3250bfa Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 1 Sep 2025 15:46:08 +0100 Subject: [PATCH 230/424] [mypyc] Speed up unary "not" (#19774) Specialize "not" for common primitive types, optional types and native instance types. Also specialize variable-length tuple in a boolean context (while working on "not" I noticed that this wasn't specialized). This appears to speed up self check by 0.7%, but this is only barely above the noise floor. --- mypyc/irbuild/ll_builder.py | 49 +++++++- mypyc/test-data/irbuild-bool.test | 178 ++++++++++++++++++++++++++++++ mypyc/test-data/irbuild-i64.test | 8 ++ mypyc/test-data/run-bools.test | 112 +++++++++++++++++++ 4 files changed, 342 insertions(+), 5 deletions(-) diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index 475d490a48f2e..4b85c13892c1d 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -94,7 +94,6 @@ c_pyssize_t_rprimitive, c_size_t_rprimitive, check_native_int_range, - dict_rprimitive, float_rprimitive, int_rprimitive, is_bool_or_bit_rprimitive, @@ -110,13 +109,13 @@ is_list_rprimitive, is_none_rprimitive, is_object_rprimitive, + is_optional_type, is_set_rprimitive, is_short_int_rprimitive, is_str_rprimitive, is_tagged, is_tuple_rprimitive, is_uint8_rprimitive, - list_rprimitive, none_rprimitive, object_pointer_rprimitive, object_rprimitive, @@ -1684,6 +1683,44 @@ def unary_not(self, value: Value, line: int, *, likely_bool: bool = False) -> Va if is_bool_or_bit_rprimitive(typ): mask = Integer(1, typ, line) return self.int_op(typ, value, mask, IntOp.XOR, line) + if is_tagged(typ) or is_fixed_width_rtype(typ): + return self.binary_op(value, Integer(0), "==", line) + if ( + is_str_rprimitive(typ) + or is_list_rprimitive(typ) + or is_tuple_rprimitive(typ) + or is_dict_rprimitive(typ) + or isinstance(typ, RInstance) + ): + bool_val = self.bool_value(value) + return self.unary_not(bool_val, line) + if is_optional_type(typ): + value_typ = optional_value_type(typ) + assert value_typ + if ( + is_str_rprimitive(value_typ) + or is_list_rprimitive(value_typ) + or is_tuple_rprimitive(value_typ) + or is_dict_rprimitive(value_typ) + or isinstance(value_typ, RInstance) + ): + # 'X | None' type: Check for None first and then specialize for X. + res = Register(bit_rprimitive) + cmp = self.add(ComparisonOp(value, self.none_object(), ComparisonOp.EQ, line)) + none, not_none, out = BasicBlock(), BasicBlock(), BasicBlock() + self.add(Branch(cmp, none, not_none, Branch.BOOL)) + self.activate_block(none) + self.add(Assign(res, self.true())) + self.goto(out) + self.activate_block(not_none) + val = self.unary_not( + self.unbox_or_cast(value, value_typ, line, can_borrow=True, unchecked=True), + line, + ) + self.add(Assign(res, val)) + self.goto(out) + self.activate_block(out) + return res if likely_bool and is_object_rprimitive(typ): # First quickly check if it's a bool, and otherwise fall back to generic op. res = Register(bit_rprimitive) @@ -1882,10 +1919,12 @@ def bool_value(self, value: Value) -> Value: elif is_fixed_width_rtype(value.type): zero = Integer(0, value.type) result = self.add(ComparisonOp(value, zero, ComparisonOp.NEQ)) - elif is_same_type(value.type, str_rprimitive): + elif is_str_rprimitive(value.type): result = self.call_c(str_check_if_true, [value], value.line) - elif is_same_type(value.type, list_rprimitive) or is_same_type( - value.type, dict_rprimitive + elif ( + is_list_rprimitive(value.type) + or is_dict_rprimitive(value.type) + or is_tuple_rprimitive(value.type) ): length = self.builtin_len(value, value.line) zero = Integer(0) diff --git a/mypyc/test-data/irbuild-bool.test b/mypyc/test-data/irbuild-bool.test index 9810daf487fae..5eac6d8db24f7 100644 --- a/mypyc/test-data/irbuild-bool.test +++ b/mypyc/test-data/irbuild-bool.test @@ -473,3 +473,181 @@ L0: r0 = x == y r1 = r0 ^ 1 return r1 + +[case testUnaryNotWithPrimitiveTypes] +def not_obj(x: object) -> bool: + return not x + +def not_int(x: int) -> bool: + return not x + +def not_str(x: str) -> bool: + return not x + +def not_list(x: list[int]) -> bool: + return not x + +def not_tuple(x: tuple[int, ...]) -> bool: + return not x + +def not_dict(x: dict[str, int]) -> bool: + return not x +[out] +def not_obj(x): + x :: object + r0 :: i32 + r1 :: bit + r2 :: bool +L0: + r0 = PyObject_Not(x) + r1 = r0 >= 0 :: signed + r2 = truncate r0: i32 to builtins.bool + return r2 +def not_int(x): + x :: int + r0 :: bit +L0: + r0 = int_eq x, 0 + return r0 +def not_str(x): + x :: str + r0, r1 :: bit +L0: + r0 = CPyStr_IsTrue(x) + r1 = r0 ^ 1 + return r1 +def not_list(x): + x :: list + r0 :: native_int + r1 :: short_int + r2, r3 :: bit +L0: + r0 = var_object_size x + r1 = r0 << 1 + r2 = int_ne r1, 0 + r3 = r2 ^ 1 + return r3 +def not_tuple(x): + x :: tuple + r0 :: native_int + r1 :: short_int + r2, r3 :: bit +L0: + r0 = var_object_size x + r1 = r0 << 1 + r2 = int_ne r1, 0 + r3 = r2 ^ 1 + return r3 +def not_dict(x): + x :: dict + r0 :: native_int + r1 :: short_int + r2, r3 :: bit +L0: + r0 = PyDict_Size(x) + r1 = r0 << 1 + r2 = int_ne r1, 0 + r3 = r2 ^ 1 + return r3 + +[case testUnaryNotWithNativeClass] +from __future__ import annotations + +class C: + def __bool__(self) -> bool: + return True + +def not_c(x: C) -> bool: + return not x + +def not_c_opt(x: C | None) -> bool: + return not x +[out] +def C.__bool__(self): + self :: __main__.C +L0: + return 1 +def not_c(x): + x :: __main__.C + r0, r1 :: bool +L0: + r0 = x.__bool__() + r1 = r0 ^ 1 + return r1 +def not_c_opt(x): + x :: union[__main__.C, None] + r0 :: object + r1, r2 :: bit + r3 :: __main__.C + r4, r5 :: bool +L0: + r0 = load_address _Py_NoneStruct + r1 = x == r0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = 1 + goto L3 +L2: + r3 = unchecked borrow cast(__main__.C, x) + r4 = r3.__bool__() + r5 = r4 ^ 1 + r2 = r5 +L3: + keep_alive x + return r2 + +[case testUnaryNotWithOptionalPrimitiveTypes] +from __future__ import annotations + +def not_str(x: str | None) -> bool: + return not x + +def not_list(x: list[int] | None) -> bool: + return not x +[out] +def not_str(x): + x :: union[str, None] + r0 :: object + r1, r2 :: bit + r3 :: str + r4, r5 :: bit +L0: + r0 = load_address _Py_NoneStruct + r1 = x == r0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = 1 + goto L3 +L2: + r3 = unchecked borrow cast(str, x) + r4 = CPyStr_IsTrue(r3) + r5 = r4 ^ 1 + r2 = r5 +L3: + keep_alive x + return r2 +def not_list(x): + x :: union[list, None] + r0 :: object + r1, r2 :: bit + r3 :: list + r4 :: native_int + r5 :: short_int + r6, r7 :: bit +L0: + r0 = load_address _Py_NoneStruct + r1 = x == r0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = 1 + goto L3 +L2: + r3 = unchecked borrow cast(list, x) + r4 = var_object_size r3 + r5 = r4 << 1 + r6 = int_ne r5, 0 + r7 = r6 ^ 1 + r2 = r7 +L3: + keep_alive x + return r2 diff --git a/mypyc/test-data/irbuild-i64.test b/mypyc/test-data/irbuild-i64.test index e55c3bfe2acc2..955f8b658f0e6 100644 --- a/mypyc/test-data/irbuild-i64.test +++ b/mypyc/test-data/irbuild-i64.test @@ -947,6 +947,8 @@ def f(x: i64) -> i64: elif not x: return 6 return 3 +def unary_not(x: i64) -> bool: + return not x [out] def f(x): x :: i64 @@ -964,6 +966,12 @@ L3: L4: L5: return 3 +def unary_not(x): + x :: i64 + r0 :: bit +L0: + r0 = x == 0 + return r0 [case testI64AssignMixed_64bit] from mypy_extensions import i64 diff --git a/mypyc/test-data/run-bools.test b/mypyc/test-data/run-bools.test index b34fedebaa9fc..45bf861e71e32 100644 --- a/mypyc/test-data/run-bools.test +++ b/mypyc/test-data/run-bools.test @@ -15,6 +15,8 @@ True False [case testBoolOps] +from __future__ import annotations + from typing import Optional, Any MYPY = False if MYPY: @@ -117,6 +119,29 @@ def test_optional_to_bool() -> None: assert not optional_to_bool3(F(False)) assert optional_to_bool3(F(True)) +def not_c(c: C) -> bool: + return not c + +def not_c_opt(c: C | None) -> bool: + return not c + +def not_d(d: D) -> bool: + return not d + +def not_d_opt(d: D | None) -> bool: + return not d + +def test_not_instance() -> None: + assert not not_c(C()) + assert not_c_opt(None) + assert not not_c_opt(C()) + + assert not_d(D(False)) + assert not not_d(D(True)) + assert not_d_opt(D(False)) + assert not_d_opt(None) + assert not not_d_opt(D(True)) + def test_any_to_bool() -> None: a: Any = int() b: Any = a + 1 @@ -222,6 +247,93 @@ def test_mixed_comparisons_i64() -> None: assert lt_mixed_i64(x, n) == (int(x) < n) assert gt_mixed_i64(n, x) == (n > int(x)) +def not_object(x: object) -> bool: + return not x + +def not_str(x: str) -> bool: + return not x + +def not_int(x: int) -> bool: + return not x + +def not_list(x: list[int]) -> bool: + return not x + +def not_tuple(x: tuple[int, ...]) -> bool: + return not x + +def not_dict(x: dict[str, int]) -> bool: + return not x + +def test_not_object() -> None: + assert not_object(None) + assert not_object([]) + assert not_object(0) + assert not not_object(1) + assert not not_object([1]) + +def test_not_str() -> None: + assert not_str(str()) + assert not not_str('x' + str()) + +def test_not_int() -> None: + assert not_int(int('0')) + assert not not_int(int('1')) + assert not not_int(int('-1')) + +def test_not_list() -> None: + assert not_list([]) + assert not not_list([1]) + +def test_not_tuple() -> None: + assert not_tuple(()) + assert not not_tuple((1,)) + +def test_not_dict() -> None: + assert not_dict({}) + assert not not_dict({'x': 1}) + +def not_str_opt(x: str | None) -> bool: + return not x + +def not_int_opt(x: int | None) -> bool: + return not x + +def not_list_opt(x: list[int] | None) -> bool: + return not x + +def not_tuple_opt(x: tuple[int, ...] | None) -> bool: + return not x + +def not_dict_opt(x: dict[str, int] | None) -> bool: + return not x + +def test_not_str_opt() -> None: + assert not_str_opt(str()) + assert not_str_opt(None) + assert not not_str_opt('x' + str()) + +def test_not_int_opt() -> None: + assert not_int_opt(int('0')) + assert not_int_opt(None) + assert not not_int_opt(int('1')) + assert not not_int_opt(int('-1')) + +def test_not_list_opt() -> None: + assert not_list_opt([]) + assert not_list_opt(None) + assert not not_list_opt([1]) + +def test_not_tuple_opt() -> None: + assert not_tuple_opt(()) + assert not_tuple_opt(None) + assert not not_tuple_opt((1,)) + +def test_not_dict_opt() -> None: + assert not_dict_opt({}) + assert not_dict_opt(None) + assert not not_dict_opt({'x': 1}) + [case testBoolMixInt] def test_mix() -> None: y = False From b8ee1f5d6c9c73b58cb6ea108f088eb001b4b4ec Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 1 Sep 2025 18:11:40 +0100 Subject: [PATCH 231/424] Use dedicated tags for most common instances (#19762) This uses dedicated secondary type tags to 5 most common instances. I also move the instance cache from `checker.py` to `types.py` so it is easier to share. The latter however requires couple tweaks to not break builtins fixtures in tests (see changes in `build.py` and `checkexpr.py`). This makes cache another ~20% smaller (so that with this PR FF is 4.5x smaller than JSON), and also this makes `mypy -c 'import torch'` almost 10% faster with warm cache (when one uses `--fixed-format-cache` obviously). I don't see any visible effect on cold cache runs. --- mypy/build.py | 5 ++- mypy/checker.py | 37 ++++++++++------------- mypy/checkexpr.py | 3 +- mypy/types.py | 77 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 99 insertions(+), 23 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 39199b39b6adc..4ccc3dec408eb 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -91,7 +91,7 @@ from mypy.renaming import LimitedVariableRenameVisitor, VariableRenameVisitor from mypy.stats import dump_type_stats from mypy.stubinfo import is_module_from_legacy_bundled_package, stub_distribution_name -from mypy.types import Type +from mypy.types import Type, instance_cache from mypy.typestate import reset_global_state, type_state from mypy.util import json_dumps, json_loads from mypy.version import __version__ @@ -180,6 +180,9 @@ def build( # fields for callers that want the traditional API. messages = [] + # This is mostly for the benefit of tests that use builtins fixtures. + instance_cache.reset() + def default_flush_errors( filename: str | None, new_messages: list[str], is_serious: bool ) -> None: diff --git a/mypy/checker.py b/mypy/checker.py index 12a86fe6fba12..77822b7068ae9 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -229,6 +229,7 @@ flatten_nested_unions, get_proper_type, get_proper_types, + instance_cache, is_literal_type, is_named_instance, ) @@ -467,12 +468,6 @@ def __init__( self, self.msg, self.plugin, per_line_checking_time_ns ) - self._str_type: Instance | None = None - self._function_type: Instance | None = None - self._int_type: Instance | None = None - self._bool_type: Instance | None = None - self._object_type: Instance | None = None - self.pattern_checker = PatternChecker(self, self.msg, self.plugin, options) self._unique_id = 0 @@ -7460,25 +7455,25 @@ def named_type(self, name: str) -> Instance: For example, named_type('builtins.object') produces the 'object' type. """ if name == "builtins.str": - if self._str_type is None: - self._str_type = self._named_type(name) - return self._str_type + if instance_cache.str_type is None: + instance_cache.str_type = self._named_type(name) + return instance_cache.str_type if name == "builtins.function": - if self._function_type is None: - self._function_type = self._named_type(name) - return self._function_type + if instance_cache.function_type is None: + instance_cache.function_type = self._named_type(name) + return instance_cache.function_type if name == "builtins.int": - if self._int_type is None: - self._int_type = self._named_type(name) - return self._int_type + if instance_cache.int_type is None: + instance_cache.int_type = self._named_type(name) + return instance_cache.int_type if name == "builtins.bool": - if self._bool_type is None: - self._bool_type = self._named_type(name) - return self._bool_type + if instance_cache.bool_type is None: + instance_cache.bool_type = self._named_type(name) + return instance_cache.bool_type if name == "builtins.object": - if self._object_type is None: - self._object_type = self._named_type(name) - return self._object_type + if instance_cache.object_type is None: + instance_cache.object_type = self._named_type(name) + return instance_cache.object_type return self._named_type(name) def _named_type(self, name: str) -> Instance: diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 250decd7567e5..2e5cf6e544d5b 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -494,7 +494,8 @@ def module_type(self, node: MypyFile) -> Instance: # In test cases might 'types' may not be available. # Fall back to a dummy 'object' type instead to # avoid a crash. - result = self.named_type("builtins.object") + # Make a copy so that we don't set extra_attrs (below) on a shared instance. + result = self.named_type("builtins.object").copy_modified() module_attrs: dict[str, Type] = {} immutable = set() for name, n in node.names.items(): diff --git a/mypy/types.py b/mypy/types.py index c23997d069d4b..3f4bd94b5b24b 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1708,6 +1708,23 @@ def deserialize(cls, data: JsonDict | str) -> Instance: def write(self, data: Buffer) -> None: write_tag(data, INSTANCE) + if not self.args and not self.last_known_value and not self.extra_attrs: + type_ref = self.type.fullname + if type_ref == "builtins.str": + write_tag(data, INSTANCE_STR) + elif type_ref == "builtins.function": + write_tag(data, INSTANCE_FUNCTION) + elif type_ref == "builtins.int": + write_tag(data, INSTANCE_INT) + elif type_ref == "builtins.bool": + write_tag(data, INSTANCE_BOOL) + elif type_ref == "builtins.object": + write_tag(data, INSTANCE_OBJECT) + else: + write_tag(data, INSTANCE_SIMPLE) + write_str(data, type_ref) + return + write_tag(data, INSTANCE_GENERIC) write_str(data, self.type.fullname) write_type_list(data, self.args) write_type_opt(data, self.last_known_value) @@ -1719,6 +1736,39 @@ def write(self, data: Buffer) -> None: @classmethod def read(cls, data: Buffer) -> Instance: + tag = read_tag(data) + # This is quite verbose, but this is very hot code, so we are not + # using dictionary lookups here. + if tag == INSTANCE_STR: + if instance_cache.str_type is None: + instance_cache.str_type = Instance(NOT_READY, []) + instance_cache.str_type.type_ref = "builtins.str" + return instance_cache.str_type + if tag == INSTANCE_FUNCTION: + if instance_cache.function_type is None: + instance_cache.function_type = Instance(NOT_READY, []) + instance_cache.function_type.type_ref = "builtins.function" + return instance_cache.function_type + if tag == INSTANCE_INT: + if instance_cache.int_type is None: + instance_cache.int_type = Instance(NOT_READY, []) + instance_cache.int_type.type_ref = "builtins.int" + return instance_cache.int_type + if tag == INSTANCE_BOOL: + if instance_cache.bool_type is None: + instance_cache.bool_type = Instance(NOT_READY, []) + instance_cache.bool_type.type_ref = "builtins.bool" + return instance_cache.bool_type + if tag == INSTANCE_OBJECT: + if instance_cache.object_type is None: + instance_cache.object_type = Instance(NOT_READY, []) + instance_cache.object_type.type_ref = "builtins.object" + return instance_cache.object_type + if tag == INSTANCE_SIMPLE: + inst = Instance(NOT_READY, []) + inst.type_ref = read_str(data) + return inst + assert tag == INSTANCE_GENERIC type_ref = read_str(data) inst = Instance(NOT_READY, read_type_list(data)) inst.type_ref = type_ref @@ -1769,6 +1819,25 @@ def is_singleton_type(self) -> bool: ) +class InstanceCache: + def __init__(self) -> None: + self.str_type: Instance | None = None + self.function_type: Instance | None = None + self.int_type: Instance | None = None + self.bool_type: Instance | None = None + self.object_type: Instance | None = None + + def reset(self) -> None: + self.str_type = None + self.function_type = None + self.int_type = None + self.bool_type = None + self.object_type = None + + +instance_cache: Final = InstanceCache() + + class FunctionLike(ProperType): """Abstract base class for function types.""" @@ -4142,6 +4211,14 @@ def type_vars_as_args(type_vars: Sequence[TypeVarLikeType]) -> tuple[Type, ...]: TYPE_TYPE: Final[Tag] = 18 PARAMETERS: Final[Tag] = 19 +INSTANCE_STR: Final[Tag] = 101 +INSTANCE_FUNCTION: Final[Tag] = 102 +INSTANCE_INT: Final[Tag] = 103 +INSTANCE_BOOL: Final[Tag] = 104 +INSTANCE_OBJECT: Final[Tag] = 105 +INSTANCE_SIMPLE: Final[Tag] = 106 +INSTANCE_GENERIC: Final[Tag] = 107 + def read_type(data: Buffer) -> Type: tag = read_tag(data) From e35e05f12fd6e682689abe0b39ec1ee07eefdf88 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 2 Sep 2025 13:53:24 +0100 Subject: [PATCH 232/424] [mypyc] Allow defining a single-item free "list" for a native class (#19785) It's quite common to have a class where we almost always have at most a single allocated instance (per thread). Now these instances can be allocated more quickly, by reusing the memory that was used for the most recently freed instance (a separate memory block is reused for each thread on free-threaded builds). It's used like this (only the value 1 is supported for now): ``` from mypy_extensions import mypyc_attr @mypyc_attr(free_list=1) class Foo: ... ``` This makes a microbenchmark that only allocates and immediately frees simple objects repeatedly around 3.8x faster. It's probably worth extending this to support larger free lists in the future. We can later look into enabling this automatically for certain native classes based on profile information. --- mypyc/irbuild/prepare.py | 12 +++++++++- mypyc/irbuild/util.py | 9 +++++-- mypyc/test-data/irbuild-classes.test | 20 ++++++++++++++++ mypyc/test-data/run-classes.test | 36 ++++++++++++++++++++++++++++ 4 files changed, 74 insertions(+), 3 deletions(-) diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index 95c8c448d642c..f47dcb52ceb70 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -351,13 +351,23 @@ def prepare_class_def( ir = mapper.type_to_ir[cdef.info] info = cdef.info - attrs = get_mypyc_attrs(cdef) + attrs, attrs_lines = get_mypyc_attrs(cdef) if attrs.get("allow_interpreted_subclasses") is True: ir.allow_interpreted_subclasses = True if attrs.get("serializable") is True: # Supports copy.copy and pickle (including subclasses) ir._serializable = True + free_list_len = attrs.get("free_list_len") + if free_list_len is not None: + line = attrs_lines["free_list_len"] + if ir.is_trait: + errors.error('"free_list_len" can\'t be used with traits', path, line) + if free_list_len == 1: + ir.reuse_freed_instance = True + else: + errors.error(f'Unsupported value for "free_list_len": {free_list_len}', path, line) + # Check for subclassing from builtin types for cls in info.mro: # Special case exceptions and dicts diff --git a/mypyc/irbuild/util.py b/mypyc/irbuild/util.py index 757b49c68c83b..eca2cac7e9dba 100644 --- a/mypyc/irbuild/util.py +++ b/mypyc/irbuild/util.py @@ -96,6 +96,8 @@ def get_mypyc_attr_literal(e: Expression) -> Any: return False elif isinstance(e, RefExpr) and e.fullname == "builtins.None": return None + elif isinstance(e, IntExpr): + return e.value return NotImplemented @@ -110,9 +112,10 @@ def get_mypyc_attr_call(d: Expression) -> CallExpr | None: return None -def get_mypyc_attrs(stmt: ClassDef | Decorator) -> dict[str, Any]: +def get_mypyc_attrs(stmt: ClassDef | Decorator) -> tuple[dict[str, Any], dict[str, int]]: """Collect all the mypyc_attr attributes on a class definition or a function.""" attrs: dict[str, Any] = {} + lines: dict[str, int] = {} for dec in stmt.decorators: d = get_mypyc_attr_call(dec) if d: @@ -120,10 +123,12 @@ def get_mypyc_attrs(stmt: ClassDef | Decorator) -> dict[str, Any]: if name is None: if isinstance(arg, StrExpr): attrs[arg.value] = True + lines[arg.value] = d.line else: attrs[name] = get_mypyc_attr_literal(arg) + lines[name] = d.line - return attrs + return attrs, lines def is_extension_class(path: str, cdef: ClassDef, errors: Errors) -> bool: diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index bb55958dc6dcc..2f59bb000220e 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -1734,3 +1734,23 @@ class NonNative: class InheritsPython(dict): def __new__(cls) -> InheritsPython: return super().__new__(cls) # E: super().__new__() not supported for classes inheriting from non-native classes + +[case testClassWithFreeList] +from mypy_extensions import mypyc_attr, trait + +@mypyc_attr(free_list_len=1) +class UsesFreeList: + pass + +@mypyc_attr(free_list_len=None) +class NoFreeList: + pass + +@mypyc_attr(free_list_len=2) # E: Unsupported value for "free_list_len": 2 +class FreeListError: + pass + +@trait +@mypyc_attr(free_list_len=1) # E: "free_list_len" can't be used with traits +class NonNative: + pass diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index b25dc9458fd16..79ad2fa0a03b7 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -3794,3 +3794,39 @@ assert t.native == 43 assert t.generic == "{}" assert t.bitfield == 0x0C assert t.default == 10 + +[case testPerTypeFreeList] +from __future__ import annotations + +from mypy_extensions import mypyc_attr + +a = [] + +@mypyc_attr(free_list=1) +class Foo: + def __init__(self, x: int) -> None: + self.x = x + a.append(x) + +def test_alloc() -> None: + x: Foo | None + y: Foo | None + + x = Foo(1) + assert x.x == 1 + x = None + + x = Foo(2) + assert x.x == 2 + y = Foo(3) + assert x.x == 2 + assert y.x == 3 + x = None + y = None + assert a == [1, 2, 3] + + x = Foo(4) + assert x.x == 4 + y = Foo(5) + assert x.x == 4 + assert y.x == 5 From 341d3ccd07e591220e1c806bb775c22d9d0dbcef Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Tue, 2 Sep 2025 17:11:56 +0200 Subject: [PATCH 233/424] Consider non-empty enums assignable to Self (#19779) Fixes #18345. Fixes #16558. See the linked ticket for reasoning - enums with members are implicitly final and should be treated exactly as if they were decorated with `@final`. --- mypy/typeanal.py | 2 +- test-data/unit/check-selftype.test | 32 ++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index af70c52180aa1..7429030573a3c 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -779,7 +779,7 @@ def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Typ if self.api.type.has_base("builtins.type"): self.fail("Self type cannot be used in a metaclass", t) if self.api.type.self_type is not None: - if self.api.type.is_final: + if self.api.type.is_final or self.api.type.is_enum and self.api.type.enum_members: return fill_typevars(self.api.type) return self.api.type.self_type.copy_modified(line=t.line, column=t.column) # TODO: verify this is unreachable and replace with an assert? diff --git a/test-data/unit/check-selftype.test b/test-data/unit/check-selftype.test index 6481a17669445..89603efafddd6 100644 --- a/test-data/unit/check-selftype.test +++ b/test-data/unit/check-selftype.test @@ -2346,3 +2346,35 @@ gc: G[D2] reveal_type(gb.test()) # N: Revealed type is "typing.Sequence[__main__.D1]" reveal_type(gc.test()) # N: Revealed type is "builtins.list[__main__.D2]" [builtins fixtures/list.pyi] + +[case testEnumImplicitlyFinalForSelfType] +from enum import Enum +from typing import Self + +# This enum has members and so is implicitly final. +# Foo and Self are interchangeable within the class. +class Foo(Enum): + A = 1 + + @classmethod + def foo(cls) -> Self: + return Foo.A + + @classmethod + def foo2(cls) -> Self: + return cls.bar() + + @classmethod + def bar(cls) -> Foo: + ... + +# This enum is empty and should not be assignable to Self +class Bar(Enum): + @classmethod + def foo(cls) -> Self: + return cls.bar() # E: Incompatible return value type (got "Bar", expected "Self") + + @classmethod + def bar(cls) -> Bar: + ... +[builtins fixtures/classmethod.pyi] From 7e7d7a7a3e09350f7e2beff0cde10d997539ef6e Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Tue, 2 Sep 2025 17:41:15 +0200 Subject: [PATCH 234/424] [mypyc] Fix object finalization (#19749) Fixes https://github.com/mypyc/mypyc/issues/1127 --- mypy/test/helpers.py | 3 +++ mypyc/codegen/emitclass.py | 35 +++++++++++++++----------------- mypyc/test-data/run-classes.test | 2 +- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/mypy/test/helpers.py b/mypy/test/helpers.py index ae432ff6981bc..36ad5ad4ec1a2 100644 --- a/mypy/test/helpers.py +++ b/mypy/test/helpers.py @@ -233,6 +233,9 @@ def clean_up(a: list[str]) -> list[str]: for p in prefix, prefix.replace(os.sep, "/"): if p != "/" and p != "//" and p != "\\" and p != "\\\\": ss = ss.replace(p, "") + # Replace memory address with zeros + if "at 0x" in ss: + ss = re.sub(r"(at 0x)\w+>", r"\g<1>000000000000>", ss) # Ignore spaces at end of line. ss = re.sub(" +$", "", ss) # Remove pwd from driver.py's path diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 0931c849131dc..3103a0dcb5e42 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -889,8 +889,21 @@ def generate_dealloc_for_class( emitter.emit_line(f"{dealloc_func_name}({cl.struct_name(emitter.names)} *self)") emitter.emit_line("{") if has_tp_finalize: - emitter.emit_line("if (!PyObject_GC_IsFinalized((PyObject *)self)) {") - emitter.emit_line("Py_TYPE(self)->tp_finalize((PyObject *)self);") + emitter.emit_line("PyObject *type, *value, *traceback;") + emitter.emit_line("PyErr_Fetch(&type, &value, &traceback);") + emitter.emit_line("int res = PyObject_CallFinalizerFromDealloc((PyObject *)self);") + # CPython interpreter uses PyErr_WriteUnraisable: https://docs.python.org/3/c-api/exceptions.html#c.PyErr_WriteUnraisable + # However, the message is slightly different due to the way mypyc compiles classes. + # CPython interpreter prints: Exception ignored in: + # mypyc prints: Exception ignored in: + emitter.emit_line("if (PyErr_Occurred() != NULL) {") + # Don't untrack instance if error occurred + emitter.emit_line("PyErr_WriteUnraisable((PyObject *)self);") + emitter.emit_line("res = -1;") + emitter.emit_line("}") + emitter.emit_line("PyErr_Restore(type, value, traceback);") + emitter.emit_line("if (res < 0) {") + emitter.emit_line("goto done;") emitter.emit_line("}") emitter.emit_line("PyObject_GC_UnTrack(self);") if cl.reuse_freed_instance: @@ -900,6 +913,7 @@ def generate_dealloc_for_class( emitter.emit_line(f"{clear_func_name}(self);") emitter.emit_line("Py_TYPE(self)->tp_free((PyObject *)self);") emitter.emit_line("CPy_TRASHCAN_END(self)") + emitter.emit_line("done: ;") emitter.emit_line("}") @@ -930,8 +944,6 @@ def generate_finalize_for_class( emitter.emit_line("static void") emitter.emit_line(f"{finalize_func_name}(PyObject *self)") emitter.emit_line("{") - emitter.emit_line("PyObject *type, *value, *traceback;") - emitter.emit_line("PyErr_Fetch(&type, &value, &traceback);") emitter.emit_line( "{}{}{}(self);".format( emitter.get_group_prefix(del_method.decl), @@ -939,21 +951,6 @@ def generate_finalize_for_class( del_method.cname(emitter.names), ) ) - emitter.emit_line("if (PyErr_Occurred() != NULL) {") - emitter.emit_line('PyObject *del_str = PyUnicode_FromString("__del__");') - emitter.emit_line( - "PyObject *del_method = (del_str == NULL) ? NULL : _PyType_Lookup(Py_TYPE(self), del_str);" - ) - # CPython interpreter uses PyErr_WriteUnraisable: https://docs.python.org/3/c-api/exceptions.html#c.PyErr_WriteUnraisable - # However, the message is slightly different due to the way mypyc compiles classes. - # CPython interpreter prints: Exception ignored in: - # mypyc prints: Exception ignored in: - emitter.emit_line("PyErr_WriteUnraisable(del_method);") - emitter.emit_line("Py_XDECREF(del_method);") - emitter.emit_line("Py_XDECREF(del_str);") - emitter.emit_line("}") - # PyErr_Restore also clears exception raised in __del__. - emitter.emit_line("PyErr_Restore(type, value, traceback);") emitter.emit_line("}") diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 79ad2fa0a03b7..46d5aaa0cbcbd 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -3110,7 +3110,7 @@ f = native.F() del f [out] -Exception ignored in: +Exception ignored in: Traceback (most recent call last): File "native.py", line 5, in __del__ raise Exception("e2") From 2ae06676f36d36a6ea573fd1e7679421aca3b1f5 Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Tue, 2 Sep 2025 20:51:48 +0200 Subject: [PATCH 235/424] Add await to empty context hack (#19777) Fixes #19716. It is a follow-up to #19767 and was missed there due to malformed test stubs (the same testcase fails when run by full mypy against typeshed, I did not notice missing AwaitExpr because of the passing test...). I synced typevar variance with typeshed definitions and added AwaitExpr to the list of context-dependent exprs. Cc @ilevkivskyi --- mypy/checker.py | 7 +++- test-data/unit/check-inference-context.test | 2 + test-data/unit/fixtures/typing-async.pyi | 45 ++++++++++++--------- 3 files changed, 34 insertions(+), 20 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 77822b7068ae9..ba821df621e56 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -80,6 +80,7 @@ AssertStmt, AssignmentExpr, AssignmentStmt, + AwaitExpr, Block, BreakStmt, BytesExpr, @@ -4924,7 +4925,11 @@ def check_return_stmt(self, s: ReturnStmt) -> None: allow_none_func_call = is_lambda or declared_none_return or declared_any_return # Return with a value. - if isinstance(s.expr, (CallExpr, ListExpr, TupleExpr, DictExpr, SetExpr, OpExpr)): + if ( + isinstance(s.expr, (CallExpr, ListExpr, TupleExpr, DictExpr, SetExpr, OpExpr)) + or isinstance(s.expr, AwaitExpr) + and isinstance(s.expr.expr, CallExpr) + ): # For expressions that (strongly) depend on type context (i.e. those that # are handled like a function call), we allow fallback to empty type context # in case of errors, this improves user experience in some cases, diff --git a/test-data/unit/check-inference-context.test b/test-data/unit/check-inference-context.test index 7dbbd68c4215d..a41ee5f59670e 100644 --- a/test-data/unit/check-inference-context.test +++ b/test-data/unit/check-inference-context.test @@ -1582,3 +1582,5 @@ async def inner(c: Cls[T]) -> Optional[T]: async def outer(c: Cls[T]) -> Optional[T]: return await inner(c) +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] diff --git a/test-data/unit/fixtures/typing-async.pyi b/test-data/unit/fixtures/typing-async.pyi index 03728f8223162..7ce2821d29168 100644 --- a/test-data/unit/fixtures/typing-async.pyi +++ b/test-data/unit/fixtures/typing-async.pyi @@ -28,7 +28,9 @@ Self = 0 T = TypeVar('T') T_co = TypeVar('T_co', covariant=True) +R_co = TypeVar('R_co', covariant=True) T_contra = TypeVar('T_contra', contravariant=True) +S_contra = TypeVar('S_contra', contravariant=True) U = TypeVar('U') V = TypeVar('V') S = TypeVar('S') @@ -49,9 +51,9 @@ class Iterator(Iterable[T_co], Protocol): @abstractmethod def __next__(self) -> T_co: pass -class Generator(Iterator[T], Generic[T, U, V]): +class Generator(Iterator[T_co], Generic[T_co, S_contra, R_co]): @abstractmethod - def send(self, value: U) -> T: pass + def send(self, value: S_contra) -> T_co: pass @abstractmethod def throw(self, typ: Any, val: Any=None, tb: Any=None) -> None: pass @@ -60,34 +62,39 @@ class Generator(Iterator[T], Generic[T, U, V]): def close(self) -> None: pass @abstractmethod - def __iter__(self) -> 'Generator[T, U, V]': pass + def __iter__(self) -> 'Generator[T_co, S_contra, R_co]': pass -class AsyncGenerator(AsyncIterator[T], Generic[T, U]): +class AsyncGenerator(AsyncIterator[T_co], Generic[T_co, S_contra]): @abstractmethod - def __anext__(self) -> Awaitable[T]: pass + def __anext__(self) -> Awaitable[T_co]: pass @abstractmethod - def asend(self, value: U) -> Awaitable[T]: pass + def asend(self, value: S_contra) -> Awaitable[T_co]: pass @abstractmethod - def athrow(self, typ: Any, val: Any=None, tb: Any=None) -> Awaitable[T]: pass + def athrow(self, typ: Any, val: Any=None, tb: Any=None) -> Awaitable[T_co]: pass @abstractmethod - def aclose(self) -> Awaitable[T]: pass + def aclose(self) -> Awaitable[T_co]: pass @abstractmethod - def __aiter__(self) -> 'AsyncGenerator[T, U]': pass + def __aiter__(self) -> 'AsyncGenerator[T_co, S_contra]': pass -class Awaitable(Protocol[T]): +class Awaitable(Protocol[T_co]): @abstractmethod - def __await__(self) -> Generator[Any, Any, T]: pass + def __await__(self) -> Generator[Any, Any, T_co]: pass -class AwaitableGenerator(Generator[T, U, V], Awaitable[V], Generic[T, U, V, S], metaclass=ABCMeta): +class AwaitableGenerator( + Awaitable[R_co], + Generator[T_co, S_contra, R_co], + Generic[T_co, S_contra, R_co, S], + metaclass=ABCMeta +): pass -class Coroutine(Awaitable[V], Generic[T, U, V]): +class Coroutine(Awaitable[R_co], Generic[T_co, S_contra, R_co]): @abstractmethod - def send(self, value: U) -> T: pass + def send(self, value: S_contra) -> T_co: pass @abstractmethod def throw(self, typ: Any, val: Any=None, tb: Any=None) -> None: pass @@ -95,14 +102,14 @@ class Coroutine(Awaitable[V], Generic[T, U, V]): @abstractmethod def close(self) -> None: pass -class AsyncIterable(Protocol[T]): +class AsyncIterable(Protocol[T_co]): @abstractmethod - def __aiter__(self) -> 'AsyncIterator[T]': pass + def __aiter__(self) -> 'AsyncIterator[T_co]': pass -class AsyncIterator(AsyncIterable[T], Protocol): - def __aiter__(self) -> 'AsyncIterator[T]': return self +class AsyncIterator(AsyncIterable[T_co], Protocol): + def __aiter__(self) -> 'AsyncIterator[T_co]': return self @abstractmethod - def __anext__(self) -> Awaitable[T]: pass + def __anext__(self) -> Awaitable[T_co]: pass class Sequence(Iterable[T_co], Container[T_co]): @abstractmethod From d40f63cde42adeb0ac7e5fde70a12c26c6679053 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 3 Sep 2025 13:02:24 +0100 Subject: [PATCH 236/424] [mypyc] Allow per-class free list to be used with inheritance (#19790) It's still unsupported if interpreted subclasses are allowed. --- mypyc/codegen/emitclass.py | 4 -- mypyc/irbuild/prepare.py | 6 +++ mypyc/test-data/irbuild-classes.test | 4 ++ mypyc/test-data/run-classes.test | 63 +++++++++++++++++++++++++++- 4 files changed, 72 insertions(+), 5 deletions(-) diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 3103a0dcb5e42..94f32b3224a98 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -205,10 +205,6 @@ def generate_class_reuse( TODO: Generalize to support a free list with up to N objects. """ assert cl.reuse_freed_instance - - # The free list implementation doesn't support class hierarchies - assert cl.is_final_class or cl.children == [] - context = c_emitter.context name = cl.name_prefix(c_emitter.names) + "_free_instance" struct_name = cl.struct_name(c_emitter.names) diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index f47dcb52ceb70..61e3e5b95cf43 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -363,6 +363,12 @@ def prepare_class_def( line = attrs_lines["free_list_len"] if ir.is_trait: errors.error('"free_list_len" can\'t be used with traits', path, line) + if ir.allow_interpreted_subclasses: + errors.error( + '"free_list_len" can\'t be used in a class that allows interpreted subclasses', + path, + line, + ) if free_list_len == 1: ir.reuse_freed_instance = True else: diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index 2f59bb000220e..78ca7b68cefbc 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -1754,3 +1754,7 @@ class FreeListError: @mypyc_attr(free_list_len=1) # E: "free_list_len" can't be used with traits class NonNative: pass + +@mypyc_attr(free_list_len=1, allow_interpreted_subclasses=True) # E: "free_list_len" can't be used in a class that allows interpreted subclasses +class InterpSub: + pass diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 46d5aaa0cbcbd..6c4ddc03887ab 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -3802,7 +3802,7 @@ from mypy_extensions import mypyc_attr a = [] -@mypyc_attr(free_list=1) +@mypyc_attr(free_list_len=1) class Foo: def __init__(self, x: int) -> None: self.x = x @@ -3830,3 +3830,64 @@ def test_alloc() -> None: y = Foo(5) assert x.x == 4 assert y.x == 5 + +@mypyc_attr(free_list_len=1) +class Base: + def __init__(self, x: str) -> None: + self.x = x + +class Deriv(Base): + def __init__(self, x: str, y: str) -> None: + super().__init__(x) + self.y = y + +@mypyc_attr(free_list_len=1) +class Deriv2(Base): + def __init__(self, x: str, y: str) -> None: + super().__init__(x) + self.y = y + +def test_inheritance() -> None: + x: Base | None + y: Base | None + x = Base('x' + str()) + y = Base('y' + str()) + y = None + d = Deriv('a' + str(), 'b' + str()) + assert type(d) is Deriv + assert d.x == 'a' + assert d.y == 'b' + assert x.x == 'x' + y = Base('z' + str()) + assert d.x == 'a' + assert d.y == 'b' + assert y.x == 'z' + x = None + y = None + +def test_inheritance_2() -> None: + x: Base | None + y: Base | None + d: Deriv2 | None + x = Base('x' + str()) + y = Base('y' + str()) + y = None + d = Deriv2('a' + str(), 'b' + str()) + assert type(d) is Deriv2 + assert d.x == 'a' + assert d.y == 'b' + assert x.x == 'x' + d = None + d = Deriv2('c' + str(), 'd' + str()) + assert type(d) is Deriv2 + assert d.x == 'c' + assert d.y == 'd' + assert x.x == 'x' + y = Base('z' + str()) + assert type(y) is Base + assert d.x == 'c' + assert d.y == 'd' + assert y.x == 'z' + x = None + y = None + d = None From 39427835ea4ab9e5735e7e5956063026d1bcc4bb Mon Sep 17 00:00:00 2001 From: Chainfire Date: Wed, 3 Sep 2025 17:58:12 +0200 Subject: [PATCH 237/424] [mypyc] Fix subclass processing in detect_undefined_bitmap (#19787) Incorrect processing in detect_undefined_bitmap could cause a ValueError exception in emit_undefined_attr_check. --- mypyc/analysis/attrdefined.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/analysis/attrdefined.py b/mypyc/analysis/attrdefined.py index 5be57d767e355..1dfd33630f1c0 100644 --- a/mypyc/analysis/attrdefined.py +++ b/mypyc/analysis/attrdefined.py @@ -422,7 +422,7 @@ def detect_undefined_bitmap(cl: ClassIR, seen: set[ClassIR]) -> None: return seen.add(cl) for base in cl.base_mro[1:]: - detect_undefined_bitmap(cl, seen) + detect_undefined_bitmap(base, seen) if len(cl.base_mro) > 1: cl.bitmap_attrs.extend(cl.base_mro[1].bitmap_attrs) From 4007c7d5760cd87d6b309d358ffea414fbac975e Mon Sep 17 00:00:00 2001 From: Kevin Kannammalil Date: Thu, 4 Sep 2025 11:34:13 -0400 Subject: [PATCH 238/424] Bump version to 1.19.0+dev (#19793) The release branch has been cut: https://github.com/python/mypy/tree/release-1.18 So this PR increases the dev version --- mypy/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/version.py b/mypy/version.py index bb6a9582e74e5..af216bddded1a 100644 --- a/mypy/version.py +++ b/mypy/version.py @@ -8,7 +8,7 @@ # - Release versions have the form "1.2.3". # - Dev versions have the form "1.2.3+dev" (PLUS sign to conform to PEP 440). # - Before 1.0 we had the form "0.NNN". -__version__ = "1.18.0+dev" +__version__ = "1.19.0+dev" base_version = __version__ mypy_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) From 645b421fa23a3f0d63f651059fa1e8318a6f214a Mon Sep 17 00:00:00 2001 From: Joren Hammudoglu Date: Thu, 4 Sep 2025 19:18:10 +0200 Subject: [PATCH 239/424] [stubtest] temporary `--ignore-disjoint-bases` flag (#19740) closes #19737 ref: https://github.com/python/mypy/issues/19737#issuecomment-3224801978 --- It's not the prettiest code, but since it will be removed once PEP 800 gets accepted (or rejected), I figured it would be best to keep it simple, so that we can easily revert it. --- mypy/stubtest.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index 31b3fd20b0025..d4f96a3d9389f 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -140,6 +140,11 @@ def is_positional_only_related(self) -> bool: # TODO: This is hacky, use error codes or something more resilient return "should be positional" in self.message + def is_disjoint_base_related(self) -> bool: + """Whether or not the error is related to @disjoint_base.""" + # TODO: This is hacky, use error codes or something more resilient + return "@disjoint_base" in self.message + def get_description(self, concise: bool = False) -> str: """Returns a description of the error. @@ -2181,6 +2186,7 @@ class _Arguments: concise: bool ignore_missing_stub: bool ignore_positional_only: bool + ignore_disjoint_bases: bool allowlist: list[str] generate_allowlist: bool ignore_unused_allowlist: bool @@ -2274,6 +2280,8 @@ def warning_callback(msg: str) -> None: continue if args.ignore_positional_only and error.is_positional_only_related(): continue + if args.ignore_disjoint_bases and error.is_disjoint_base_related(): + continue if error.object_desc in allowlist: allowlist[error.object_desc] = True continue @@ -2364,6 +2372,12 @@ def parse_options(args: list[str]) -> _Arguments: action="store_true", help="Ignore errors for whether an argument should or shouldn't be positional-only", ) + # TODO: Remove once PEP 800 is accepted + parser.add_argument( + "--ignore-disjoint-bases", + action="store_true", + help="Disable checks for PEP 800 @disjoint_base classes", + ) parser.add_argument( "--allowlist", "--whitelist", From d33c147138f56a78d3fca8e2f7b61be46677b13c Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 5 Sep 2025 00:50:31 +0100 Subject: [PATCH 240/424] Make --allow-redefinition-new argument public (#19796) It is time to announce this (as still experimental obviously). --- mypy/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/main.py b/mypy/main.py index 706d1daef6802..4ca1bde73d400 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -886,7 +886,7 @@ def add_invertible_flag( "--allow-redefinition-new", default=False, strict_flag=False, - help=argparse.SUPPRESS, # This is still very experimental + help="Allow more flexible variable redefinition semantics (experimental)", group=strictness_group, ) From 7e446b414917cbc32c0832236822611a4ff72993 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 5 Sep 2025 12:49:37 +0100 Subject: [PATCH 241/424] [mypyc] Add type annotations to tests (#19794) Missing type annotations can compromise test coverage. My eventual goal is to require annotations by default in all run tests. --- mypyc/test-data/fixtures/ir.py | 4 +- mypyc/test-data/fixtures/typing-full.pyi | 6 +- mypyc/test-data/run-dunders.test | 48 +++++++--- mypyc/test-data/run-singledispatch.test | 108 ++++++++++++++--------- 4 files changed, 106 insertions(+), 60 deletions(-) diff --git a/mypyc/test-data/fixtures/ir.py b/mypyc/test-data/fixtures/ir.py index fb5512b772793..075a0eec28d20 100644 --- a/mypyc/test-data/fixtures/ir.py +++ b/mypyc/test-data/fixtures/ir.py @@ -243,7 +243,7 @@ def __add__(self, value: List[_S], /) -> List[_S | _T]: ... def __iadd__(self, value: Iterable[_T], /) -> List[_T]: ... # type: ignore[misc] def append(self, x: _T) -> None: pass def pop(self, i: int = -1) -> _T: pass - def count(self, _T) -> int: pass + def count(self, x: _T) -> int: pass def extend(self, l: Iterable[_T]) -> None: pass def insert(self, i: int, x: _T) -> None: pass def sort(self) -> None: pass @@ -365,7 +365,7 @@ def reversed(object: Sequence[_T]) -> Iterator[_T]: ... def id(o: object) -> int: pass # This type is obviously wrong but the test stubs don't have Sized anymore def len(o: object) -> int: pass -def print(*object) -> None: pass +def print(*args: object) -> None: pass def isinstance(x: object, t: object) -> bool: pass def iter(i: Iterable[_T]) -> Iterator[_T]: pass @overload diff --git a/mypyc/test-data/fixtures/typing-full.pyi b/mypyc/test-data/fixtures/typing-full.pyi index 8d89e4f93bc9b..25aaaf700d051 100644 --- a/mypyc/test-data/fixtures/typing-full.pyi +++ b/mypyc/test-data/fixtures/typing-full.pyi @@ -11,10 +11,10 @@ from abc import abstractmethod, ABCMeta class GenericMeta(type): pass class _SpecialForm: - def __getitem__(self, index): ... + def __getitem__(self, index: Any) -> Any: ... class TypeVar: - def __init__(self, name, *args, bound=None): ... - def __or__(self, other): ... + def __init__(self, name: str, *args: Any, bound: Any = None): ... + def __or__(self, other: Any) -> Any: ... cast = 0 overload = 0 diff --git a/mypyc/test-data/run-dunders.test b/mypyc/test-data/run-dunders.test index ec992afbfbd1e..a3ec06763d75b 100644 --- a/mypyc/test-data/run-dunders.test +++ b/mypyc/test-data/run-dunders.test @@ -185,7 +185,7 @@ class SeqError: def __contains__(self, x: int) -> bool: raise RuntimeError() - def __len__(self): + def __len__(self) -> int: return -5 def any_seq_error() -> Any: @@ -545,6 +545,7 @@ def test_type_mismatch_fall_back_to_reverse() -> None: assert F()**G() == -6 [case testDundersBinaryNotImplemented] +# mypy: allow-untyped-defs from typing import Any, Union from testutil import assertRaises @@ -617,15 +618,28 @@ def test_unannotated_add() -> None: with assertRaises(TypeError, "unsupported operand type(s) for +: 'F' and 'str'"): o + 'x' + o2: Any = F(4) + assert o2 + 5 == 9 + with assertRaises(TypeError, "unsupported operand type(s) for +: 'F' and 'str'"): + o2 + 'x' + def test_unannotated_add_and_radd_1() -> None: o = F(4) assert o + G() == 5 + o2: Any = F(4) + assert o2 + G() == 5 + def test_unannotated_radd() -> None: assert 'x' + G() == 'a' with assertRaises(TypeError, "unsupported operand type(s) for +: 'int' and 'G'"): 1 + G() + o: Any = G() + assert 'x' + o == 'a' + with assertRaises(TypeError, "unsupported operand type(s) for +: 'int' and 'G'"): + 1 + o + class H: def __add__(self, x): if isinstance(x, int): @@ -644,40 +658,48 @@ def test_unannotated_add_and_radd_2() -> None: with assertRaises(TypeError, "unsupported operand type(s) for +: 'int' and 'H'"): 1 + h + h2: Any = H() + assert h + 5 == 6 + assert 'x' + h == 22 + with assertRaises(TypeError, "unsupported operand type(s) for +: 'int' and 'H'"): + 1 + h + # TODO: Inheritance [case testDifferentReverseDunders] +from typing import Any + class C: # __radd__ and __rsub__ are tested elsewhere - def __rmul__(self, x): + def __rmul__(self, x: Any) -> int: return 1 - def __rtruediv__(self, x): + def __rtruediv__(self, x: Any) -> int: return 2 - def __rmod__(self, x): + def __rmod__(self, x: Any) -> int: return 3 - def __rfloordiv__(self, x): + def __rfloordiv__(self, x: Any) -> int: return 4 - def __rlshift__(self, x): + def __rlshift__(self, x: Any) -> int: return 5 - def __rrshift__(self, x): + def __rrshift__(self, x: Any) -> int: return 6 - def __rand__(self, x): + def __rand__(self, x: Any) -> int: return 7 - def __ror__(self, x): + def __ror__(self, x: Any) -> int: return 8 - def __rxor__(self, x): + def __rxor__(self, x: Any) -> int: return 9 - def __rmatmul__(self, x): + def __rmatmul__(self, x: Any) -> int: return 10 def test_reverse_dunders() -> None: @@ -803,10 +825,10 @@ def test_error() -> None: c += 'x' class BadInplaceAdd: - def __init__(self): + def __init__(self) -> None: self.x = 0 - def __iadd__(self, x): + def __iadd__(self, x: int) -> Any: self.x += x def test_in_place_operator_returns_none() -> None: diff --git a/mypyc/test-data/run-singledispatch.test b/mypyc/test-data/run-singledispatch.test index a119c325984ae..03b937261e22a 100644 --- a/mypyc/test-data/run-singledispatch.test +++ b/mypyc/test-data/run-singledispatch.test @@ -4,9 +4,10 @@ [case testSpecializedImplementationUsed] from functools import singledispatch +from typing import Any @singledispatch -def fun(arg) -> bool: +def fun(arg: Any) -> bool: return False @fun.register @@ -19,11 +20,13 @@ def test_specialize() -> None: [case testSubclassesOfExpectedTypeUseSpecialized] from functools import singledispatch +from typing import Any + class A: pass class B(A): pass @singledispatch -def fun(arg) -> bool: +def fun(arg: Any) -> bool: return False @fun.register @@ -36,11 +39,13 @@ def test_specialize() -> None: [case testSuperclassImplementationNotUsedWhenSubclassHasImplementation] from functools import singledispatch +from typing import Any + class A: pass class B(A): pass @singledispatch -def fun(arg) -> bool: +def fun(arg: Any) -> bool: # shouldn't be using this assert False @@ -58,9 +63,10 @@ def test_specialize() -> None: [case testMultipleUnderscoreFunctionsIsntError] from functools import singledispatch +from typing import Any @singledispatch -def fun(arg) -> str: +def fun(arg: Any) -> str: return 'default' @fun.register @@ -72,7 +78,7 @@ def _(arg: int) -> str: return 'int' # extra function to make sure all 3 underscore functions aren't treated as one OverloadedFuncDef -def a(b): pass +def a(b: Any) -> Any: pass @fun.register def _(arg: list) -> str: @@ -86,10 +92,12 @@ def test_singledispatch() -> None: [case testCanRegisterCompiledClasses] from functools import singledispatch +from typing import Any + class A: pass @singledispatch -def fun(arg) -> bool: +def fun(arg: Any) -> bool: return False @fun.register def fun_specialized(arg: A) -> bool: @@ -101,13 +109,14 @@ def test_singledispatch() -> None: [case testTypeUsedAsArgumentToRegister] from functools import singledispatch +from typing import Any @singledispatch -def fun(arg) -> bool: +def fun(arg: Any) -> bool: return False @fun.register(int) -def fun_specialized(arg) -> bool: +def fun_specialized(arg: Any) -> bool: return True def test_singledispatch() -> None: @@ -116,12 +125,13 @@ def test_singledispatch() -> None: [case testUseRegisterAsAFunction] from functools import singledispatch +from typing import Any @singledispatch -def fun(arg) -> bool: +def fun(arg: Any) -> bool: return False -def fun_specialized_impl(arg) -> bool: +def fun_specialized_impl(arg: Any) -> bool: return True fun.register(int, fun_specialized_impl) @@ -132,13 +142,14 @@ def test_singledispatch() -> None: [case testRegisterDoesntChangeFunction] from functools import singledispatch +from typing import Any @singledispatch -def fun(arg) -> bool: +def fun(arg: Any) -> bool: return False @fun.register(int) -def fun_specialized(arg) -> bool: +def fun_specialized(arg: Any) -> bool: return True def test_singledispatch() -> None: @@ -147,9 +158,10 @@ def test_singledispatch() -> None: # TODO: turn this into a mypy error [case testNoneIsntATypeWhenUsedAsArgumentToRegister] from functools import singledispatch +from typing import Any @singledispatch -def fun(arg) -> bool: +def fun(arg: Any) -> bool: return False def test_argument() -> None: @@ -163,14 +175,15 @@ def test_argument() -> None: [case testRegisteringTheSameFunctionSeveralTimes] from functools import singledispatch +from typing import Any @singledispatch -def fun(arg) -> bool: +def fun(arg: Any) -> bool: return False @fun.register(int) @fun.register(str) -def fun_specialized(arg) -> bool: +def fun_specialized(arg: Any) -> bool: return True def test_singledispatch() -> None: @@ -179,11 +192,13 @@ def test_singledispatch() -> None: assert not fun([1, 2]) [case testTypeIsAnABC] +# mypy: allow-untyped-defs from functools import singledispatch from collections.abc import Mapping @singledispatch def fun(arg) -> bool: + # TODO: Adding an Any parameter annotation breaks the test case return False @fun.register @@ -241,6 +256,7 @@ def test_singledispatchmethod() -> None: [case testSingledispatchTreeSumAndEqual] from functools import singledispatch +from typing import cast class Tree: pass @@ -286,10 +302,10 @@ def build(n: int) -> Tree: return Leaf() return Node(n, build(n - 1), build(n - 1)) -def test_sum_and_equal(): +def test_sum_and_equal() -> None: tree = build(5) tree2 = build(5) - tree2.right.right.right.value = 10 + cast(Node, cast(Node, cast(Node, cast(Node, tree2).right).right).right).value = 10 assert calc_sum(tree) == 57 assert calc_sum(tree2) == 65 assert equal(tree, tree) @@ -354,7 +370,7 @@ def verify_typevarexpr(stub: TypeVarExpr, a: MaybeMissing[Any], b: List[str]) -> if False: yield None -def verify_list(stub, a, b) -> List[str]: +def verify_list(stub: Any, a: Any, b: Any) -> List[str]: """Helper function that converts iterator of errors to list of messages""" return list(err.msg for err in verify(stub, a, b)) @@ -368,31 +384,33 @@ def test_verify() -> None: [case testArgsInRegisteredImplNamedDifferentlyFromMainFunction] from functools import singledispatch +from typing import Any @singledispatch -def f(a) -> bool: +def f(a: Any) -> bool: return False @f.register def g(b: int) -> bool: return True -def test_singledispatch(): +def test_singledispatch() -> None: assert f(5) assert not f('a') [case testKeywordArguments] from functools import singledispatch +from typing import Any @singledispatch -def f(arg, *, kwarg: int = 0) -> int: +def f(arg: Any, *, kwarg: int = 0) -> int: return kwarg + 10 @f.register def g(arg: int, *, kwarg: int = 5) -> int: return kwarg - 10 -def test_keywords(): +def test_keywords() -> None: assert f('a') == 10 assert f('a', kwarg=3) == 13 assert f('a', kwarg=7) == 17 @@ -413,14 +431,14 @@ def f(arg: Any) -> Iterable[int]: def g(arg: str) -> Iterable[int]: return [0] -def test_iterables(): +def test_iterables() -> None: assert f(1) != [1] assert list(f(1)) == [1] assert f('a') == [0] [case testRegisterUsedAtSameTimeAsOtherDecorators] from functools import singledispatch -from typing import TypeVar +from typing import TypeVar, Any class A: pass class B: pass @@ -431,7 +449,7 @@ def decorator(f: T) -> T: return f @singledispatch -def f(arg) -> int: +def f(arg: Any) -> int: return 0 @f.register @@ -439,7 +457,7 @@ def f(arg) -> int: def h(arg: str) -> int: return 2 -def test_singledispatch(): +def test_singledispatch() -> None: assert f(1) == 0 assert f('a') == 2 @@ -450,12 +468,12 @@ from typing import Callable, Any class A: pass def decorator(f: Callable[[Any], int]) -> Callable[[Any], int]: - def wrapper(x) -> int: + def wrapper(x: Any) -> int: return f(x) * 7 return wrapper @singledispatch -def f(arg) -> int: +def f(arg: Any) -> int: return 10 @f.register @@ -464,17 +482,19 @@ def h(arg: str) -> int: return 5 -def test_singledispatch(): +def test_singledispatch() -> None: assert f('a') == 35 assert f(A()) == 10 [case testMoreSpecificTypeBeforeLessSpecificType] from functools import singledispatch +from typing import Any + class A: pass class B(A): pass @singledispatch -def f(arg) -> str: +def f(arg: Any) -> str: return 'default' @f.register @@ -485,20 +505,21 @@ def g(arg: B) -> str: def h(arg: A) -> str: return 'a' -def test_singledispatch(): +def test_singledispatch() -> None: assert f(B()) == 'b' assert f(A()) == 'a' assert f(5) == 'default' [case testMultipleRelatedClassesBeingRegistered] from functools import singledispatch +from typing import Any class A: pass class B(A): pass class C(B): pass @singledispatch -def f(arg) -> str: return 'default' +def f(arg: Any) -> str: return 'default' @f.register def _(arg: A) -> str: return 'a' @@ -509,7 +530,7 @@ def _(arg: C) -> str: return 'c' @f.register def _(arg: B) -> str: return 'b' -def test_singledispatch(): +def test_singledispatch() -> None: assert f(A()) == 'a' assert f(B()) == 'b' assert f(C()) == 'c' @@ -525,7 +546,7 @@ def a(arg: A) -> int: def _(arg: C) -> int: return 3 -def test_singledispatch(): +def test_singledispatch() -> None: assert f(B()) == 1 assert f(A()) == 2 assert f(C()) == 3 @@ -539,7 +560,7 @@ class B(A): pass class C(B): pass @singledispatch -def f(arg) -> int: +def f(arg: object) -> int: return 0 @f.register @@ -549,6 +570,7 @@ def g(arg: B) -> int: [case testOrderCanOnlyBeDeterminedFromMRONotIsinstanceChecks] from mypy_extensions import trait from functools import singledispatch +from typing import Any @trait class A: pass @@ -558,9 +580,8 @@ class AB(A, B): pass class BA(B, A): pass @singledispatch -def f(arg) -> str: +def f(arg: Any) -> str: return "default" - pass @f.register def fa(arg: A) -> str: @@ -570,18 +591,19 @@ def fa(arg: A) -> str: def fb(arg: B) -> str: return "b" -def test_singledispatch(): +def test_singledispatch() -> None: assert f(AB()) == "a" assert f(BA()) == "b" [case testCallingFunctionBeforeAllImplementationsRegistered] from functools import singledispatch +from typing import Any class A: pass class B(A): pass @singledispatch -def f(arg) -> str: +def f(arg: Any) -> str: return 'default' assert f(A()) == 'default' @@ -609,6 +631,7 @@ def test_final() -> None: [case testDynamicallyRegisteringFunctionFromInterpretedCode] from functools import singledispatch +from typing import Any class A: pass class B(A): pass @@ -616,7 +639,7 @@ class C(B): pass class D(C): pass @singledispatch -def f(arg) -> str: +def f(arg: Any) -> str: return "default" @f.register @@ -647,9 +670,10 @@ assert c(A()) == 'c' [case testMalformedDynamicRegisterCall] from functools import singledispatch +from typing import Any @singledispatch -def f(arg) -> None: +def f(arg: Any) -> None: pass [file register.py] from native import f @@ -667,7 +691,7 @@ import register from functools import singledispatch @singledispatch -def f(arg) -> str: +def f(arg: object) -> str: return 'default' [file register.py] From 8f2371a565eb9c29f03922b26da8eab054fbbcf8 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Fri, 5 Sep 2025 11:15:36 -0400 Subject: [PATCH 242/424] feat: new mypyc primitives for weakref.proxy (#19217) This PR adds 2 new weakref primitives for weakref.proxy (1 and 2 arg) --- mypyc/primitives/weakref_ops.py | 18 ++++++++++ mypyc/test-data/fixtures/ir.py | 1 + mypyc/test-data/irbuild-weakref.test | 52 ++++++++++++++++++++++++++++ mypyc/test-data/run-weakref.test | 44 +++++++++++++++++------ test-data/unit/lib-stub/_weakref.pyi | 11 ++++++ test-data/unit/lib-stub/weakref.pyi | 14 +++++++- 6 files changed, 128 insertions(+), 12 deletions(-) create mode 100644 test-data/unit/lib-stub/_weakref.pyi diff --git a/mypyc/primitives/weakref_ops.py b/mypyc/primitives/weakref_ops.py index a7ac035b22a4e..21379d3b2c820 100644 --- a/mypyc/primitives/weakref_ops.py +++ b/mypyc/primitives/weakref_ops.py @@ -20,3 +20,21 @@ c_function_name="PyWeakref_NewRef", error_kind=ERR_MAGIC, ) + +new_proxy_op = function_op( + name="_weakref.proxy", + arg_types=[object_rprimitive], + return_type=object_rprimitive, + c_function_name="PyWeakref_NewProxy", + extra_int_constants=[(0, pointer_rprimitive)], + error_kind=ERR_MAGIC, +) + +new_proxy_with_callback_op = function_op( + name="_weakref.proxy", + arg_types=[object_rprimitive, object_rprimitive], + # steals=[True, False], + return_type=object_rprimitive, + c_function_name="PyWeakref_NewProxy", + error_kind=ERR_MAGIC, +) diff --git a/mypyc/test-data/fixtures/ir.py b/mypyc/test-data/fixtures/ir.py index 075a0eec28d20..a4b4f3ce2b1fc 100644 --- a/mypyc/test-data/fixtures/ir.py +++ b/mypyc/test-data/fixtures/ir.py @@ -343,6 +343,7 @@ class RuntimeError(Exception): pass class UnicodeEncodeError(RuntimeError): pass class UnicodeDecodeError(RuntimeError): pass class NotImplementedError(RuntimeError): pass +class ReferenceError(Exception): pass class StopIteration(Exception): value: Any diff --git a/mypyc/test-data/irbuild-weakref.test b/mypyc/test-data/irbuild-weakref.test index 58ac6417d2970..2180b1e747aab 100644 --- a/mypyc/test-data/irbuild-weakref.test +++ b/mypyc/test-data/irbuild-weakref.test @@ -49,3 +49,55 @@ def f(x, cb): L0: r0 = PyWeakref_NewRef(x, cb) return r0 + +[case testWeakrefProxy] +import weakref +from typing import Any, Callable +def f(x: object) -> object: + return weakref.proxy(x) + +[out] +def f(x): + x, r0 :: object +L0: + r0 = PyWeakref_NewProxy(x, 0) + return r0 + +[case testWeakrefProxyCallback] +import weakref +from typing import Any, Callable +def f(x: object, cb: Callable[[object], Any]) -> object: + return weakref.proxy(x, cb) + +[out] +def f(x, cb): + x, cb, r0 :: object +L0: + r0 = PyWeakref_NewProxy(x, cb) + return r0 + +[case testFromWeakrefProxy] +from typing import Any, Callable +from weakref import proxy +def f(x: object) -> object: + return proxy(x) + +[out] +def f(x): + x, r0 :: object +L0: + r0 = PyWeakref_NewProxy(x, 0) + return r0 + +[case testFromWeakrefProxyCallback] +from typing import Any, Callable +from weakref import proxy +def f(x: object, cb: Callable[[object], Any]) -> object: + return proxy(x, cb) + +[out] +def f(x, cb): + x, cb, r0 :: object +L0: + r0 = PyWeakref_NewProxy(x, cb) + return r0 diff --git a/mypyc/test-data/run-weakref.test b/mypyc/test-data/run-weakref.test index 902c9e407ff45..0a0e180d635d7 100644 --- a/mypyc/test-data/run-weakref.test +++ b/mypyc/test-data/run-weakref.test @@ -1,30 +1,52 @@ # Test cases for weakrefs (compile and run) [case testWeakrefRef] -from weakref import ref +# mypy: disable-error-code="union-attr" +from weakref import proxy, ref from mypy_extensions import mypyc_attr +from testutil import assertRaises +from typing import Optional @mypyc_attr(native_class=False) class Object: """some random weakreffable object""" - pass + def some_meth(self) -> int: + return 1 -def test_weakref_ref(): - obj = Object() +_callback_called_cache = {"ref": False, "proxy": False} + +def test_weakref_ref() -> None: + obj: Optional[Object] = Object() r = ref(obj) assert r() is obj obj = None assert r() is None, r() -def test_weakref_ref_with_callback(): - obj = Object() - r = ref(obj, lambda x: x) +def test_weakref_ref_with_callback() -> None: + obj: Optional[Object] = Object() + r = ref(obj, lambda x: _callback_called_cache.__setitem__("ref", True)) assert r() is obj obj = None assert r() is None, r() + assert _callback_called_cache["ref"] is True -[file driver.py] -from native import test_weakref_ref, test_weakref_ref_with_callback +def test_weakref_proxy() -> None: + obj: Optional[Object] = Object() + p = proxy(obj) + assert obj.some_meth() == 1 + assert p.some_meth() == 1 + obj.some_meth() + obj = None + with assertRaises(ReferenceError): + p.some_meth() -test_weakref_ref() -test_weakref_ref_with_callback() +def test_weakref_proxy_with_callback() -> None: + obj: Optional[Object] = Object() + p = proxy(obj, lambda x: _callback_called_cache.__setitem__("proxy", True)) + assert obj.some_meth() == 1 + assert p.some_meth() == 1 + obj.some_meth() + obj = None + with assertRaises(ReferenceError): + p.some_meth() + assert _callback_called_cache["proxy"] is True diff --git a/test-data/unit/lib-stub/_weakref.pyi b/test-data/unit/lib-stub/_weakref.pyi new file mode 100644 index 0000000000000..50c59b65e2677 --- /dev/null +++ b/test-data/unit/lib-stub/_weakref.pyi @@ -0,0 +1,11 @@ +from typing import Any, Callable, TypeVar, overload +from weakref import CallableProxyType, ProxyType + +_C = TypeVar("_C", bound=Callable[..., Any]) +_T = TypeVar("_T") + +# Return CallableProxyType if object is callable, ProxyType otherwise +@overload +def proxy(object: _C, callback: Callable[[CallableProxyType[_C]], Any] | None = None, /) -> CallableProxyType[_C]: ... +@overload +def proxy(object: _T, callback: Callable[[ProxyType[_T]], Any] | None = None, /) -> ProxyType[_T]: ... diff --git a/test-data/unit/lib-stub/weakref.pyi b/test-data/unit/lib-stub/weakref.pyi index 34e01f4d48f1f..7d11b65d45481 100644 --- a/test-data/unit/lib-stub/weakref.pyi +++ b/test-data/unit/lib-stub/weakref.pyi @@ -1,11 +1,23 @@ +from _weakref import proxy from collections.abc import Callable -from typing import Any, Generic, TypeVar +from typing import Any, ClassVar, Generic, TypeVar, final from typing_extensions import Self +_C = TypeVar("_C", bound=Callable[..., Any]) _T = TypeVar("_T") class ReferenceType(Generic[_T]): # "weakref" __callback__: Callable[[Self], Any] def __new__(cls, o: _T, callback: Callable[[Self], Any] | None = ..., /) -> Self: ... + def __call__(self) -> _T | None: ... ref = ReferenceType + +@final +class CallableProxyType(Generic[_C]): # "weakcallableproxy" + def __eq__(self, value: object, /) -> bool: ... + def __getattr__(self, attr: str) -> Any: ... + __call__: _C + __hash__: ClassVar[None] # type: ignore[assignment] + +__all__ = ["proxy"] From 309b01e287e3afabeb888572455f9bf55d86acad Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 6 Sep 2025 01:16:36 +0100 Subject: [PATCH 243/424] Make untyped_calls_exclude invalidate cache (#19801) Fixes https://github.com/python/mypy/issues/16652 This is minor and straightforward, so I am going to just merge it (assuming everything passes). --- mypy/options.py | 1 + test-data/unit/check-incremental.test | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/mypy/options.py b/mypy/options.py index b3dc9639a41d1..5aced56c940f8 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -74,6 +74,7 @@ class BuildType: "disable_memoryview_promotion", "strict_bytes", "fixed_format_cache", + "untyped_calls_exclude", } ) - {"debug_cache"} diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index defe7402730f0..76532e6eba4ad 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -6865,6 +6865,24 @@ if int(): [out2] main:6: error: Incompatible types in assignment (expression has type "str", variable has type "int") +[case testUntypedCallsExcludeAffectsCache] +# flags: --disallow-untyped-calls --untyped-calls-exclude=mod.Super +# flags2: --disallow-untyped-calls --untyped-calls-exclude=mod +# flags3: --disallow-untyped-calls --untyped-calls-exclude=mod.Super +import mod +[file mod.py] +class Super: + def draw(self): + ... +class Class(Super): + ... +Class().draw() +[out] +tmp/mod.py:6: error: Call to untyped function "draw" in typed context +[out2] +[out3] +tmp/mod.py:6: error: Call to untyped function "draw" in typed context + [case testMethodMakeBoundIncremental] from a import A a = A() From bf77aab8033ae1d8adcc3039c2923832a65da1df Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 6 Sep 2025 12:15:24 +0100 Subject: [PATCH 244/424] Traverse ParamSpec prefix where we should (#19800) Fixes https://github.com/python/mypy/issues/18087 I started from fixing an incremental crash from `fixup.py`, but then noticed we don't traverse `ParamSpec` prefix in multiple places where we should, so I fixed most of those as well. --- mypy/checkexpr.py | 2 +- mypy/erasetype.py | 1 + mypy/fixup.py | 1 + mypy/indirection.py | 1 + mypy/server/astdiff.py | 1 + mypy/server/astmerge.py | 1 + mypy/server/deps.py | 17 ++----- mypy/type_visitor.py | 2 +- mypy/typeanal.py | 2 +- mypy/typetraverser.py | 1 + test-data/unit/check-incremental.test | 67 +++++++++++++++++++++++++++ test-data/unit/fine-grained.test | 33 +++++++++++++ 12 files changed, 114 insertions(+), 15 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 2e5cf6e544d5b..835eeb7253945 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -6435,7 +6435,7 @@ def visit_type_var(self, t: TypeVarType) -> bool: def visit_param_spec(self, t: ParamSpecType) -> bool: default = [t.default] if t.has_default() else [] - return self.query_types([t.upper_bound, *default]) + return self.query_types([t.upper_bound, *default, t.prefix]) def visit_type_var_tuple(self, t: TypeVarTupleType) -> bool: default = [t.default] if t.has_default() else [] diff --git a/mypy/erasetype.py b/mypy/erasetype.py index 3f33ea1648f08..6645bcf916d90 100644 --- a/mypy/erasetype.py +++ b/mypy/erasetype.py @@ -222,6 +222,7 @@ def visit_type_var_tuple(self, t: TypeVarTupleType) -> Type: return t def visit_param_spec(self, t: ParamSpecType) -> Type: + # TODO: we should probably preserve prefix here. if self.erase_id is None or self.erase_id(t.id): return self.replacement return t diff --git a/mypy/fixup.py b/mypy/fixup.py index bec5929ad4b14..260c0f84cf1b9 100644 --- a/mypy/fixup.py +++ b/mypy/fixup.py @@ -347,6 +347,7 @@ def visit_type_var(self, tvt: TypeVarType) -> None: def visit_param_spec(self, p: ParamSpecType) -> None: p.upper_bound.accept(self) p.default.accept(self) + p.prefix.accept(self) def visit_type_var_tuple(self, t: TypeVarTupleType) -> None: t.tuple_fallback.accept(self) diff --git a/mypy/indirection.py b/mypy/indirection.py index 06a158818fbed..88258b94d94a1 100644 --- a/mypy/indirection.py +++ b/mypy/indirection.py @@ -93,6 +93,7 @@ def visit_type_var(self, t: types.TypeVarType) -> None: def visit_param_spec(self, t: types.ParamSpecType) -> None: self._visit(t.upper_bound) self._visit(t.default) + self._visit(t.prefix) def visit_type_var_tuple(self, t: types.TypeVarTupleType) -> None: self._visit(t.upper_bound) diff --git a/mypy/server/astdiff.py b/mypy/server/astdiff.py index 1df85a163e0fd..25542ce37588f 100644 --- a/mypy/server/astdiff.py +++ b/mypy/server/astdiff.py @@ -435,6 +435,7 @@ def visit_param_spec(self, typ: ParamSpecType) -> SnapshotItem: typ.flavor, snapshot_type(typ.upper_bound), snapshot_type(typ.default), + snapshot_type(typ.prefix), ) def visit_type_var_tuple(self, typ: TypeVarTupleType) -> SnapshotItem: diff --git a/mypy/server/astmerge.py b/mypy/server/astmerge.py index 33e2d2b799cbe..cda1d20fb8e4f 100644 --- a/mypy/server/astmerge.py +++ b/mypy/server/astmerge.py @@ -489,6 +489,7 @@ def visit_type_var(self, typ: TypeVarType) -> None: def visit_param_spec(self, typ: ParamSpecType) -> None: typ.upper_bound.accept(self) typ.default.accept(self) + typ.prefix.accept(self) def visit_type_var_tuple(self, typ: TypeVarTupleType) -> None: typ.upper_bound.accept(self) diff --git a/mypy/server/deps.py b/mypy/server/deps.py index b994a214f67ab..9d4445a1f7e74 100644 --- a/mypy/server/deps.py +++ b/mypy/server/deps.py @@ -1037,10 +1037,8 @@ def visit_type_var(self, typ: TypeVarType) -> list[str]: triggers = [] if typ.fullname: triggers.append(make_trigger(typ.fullname)) - if typ.upper_bound: - triggers.extend(self.get_type_triggers(typ.upper_bound)) - if typ.default: - triggers.extend(self.get_type_triggers(typ.default)) + triggers.extend(self.get_type_triggers(typ.upper_bound)) + triggers.extend(self.get_type_triggers(typ.default)) for val in typ.values: triggers.extend(self.get_type_triggers(val)) return triggers @@ -1049,22 +1047,17 @@ def visit_param_spec(self, typ: ParamSpecType) -> list[str]: triggers = [] if typ.fullname: triggers.append(make_trigger(typ.fullname)) - if typ.upper_bound: - triggers.extend(self.get_type_triggers(typ.upper_bound)) - if typ.default: - triggers.extend(self.get_type_triggers(typ.default)) triggers.extend(self.get_type_triggers(typ.upper_bound)) + triggers.extend(self.get_type_triggers(typ.default)) + triggers.extend(self.get_type_triggers(typ.prefix)) return triggers def visit_type_var_tuple(self, typ: TypeVarTupleType) -> list[str]: triggers = [] if typ.fullname: triggers.append(make_trigger(typ.fullname)) - if typ.upper_bound: - triggers.extend(self.get_type_triggers(typ.upper_bound)) - if typ.default: - triggers.extend(self.get_type_triggers(typ.default)) triggers.extend(self.get_type_triggers(typ.upper_bound)) + triggers.extend(self.get_type_triggers(typ.default)) return triggers def visit_unpack_type(self, typ: UnpackType) -> list[str]: diff --git a/mypy/type_visitor.py b/mypy/type_visitor.py index 65051ddbab674..15494393cae66 100644 --- a/mypy/type_visitor.py +++ b/mypy/type_visitor.py @@ -525,7 +525,7 @@ def visit_type_var(self, t: TypeVarType, /) -> bool: return self.query_types([t.upper_bound, t.default] + t.values) def visit_param_spec(self, t: ParamSpecType, /) -> bool: - return self.query_types([t.upper_bound, t.default]) + return self.query_types([t.upper_bound, t.default, t.prefix]) def visit_type_var_tuple(self, t: TypeVarTupleType, /) -> bool: return self.query_types([t.upper_bound, t.default]) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 7429030573a3c..6587304147635 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -2615,7 +2615,7 @@ def visit_type_var(self, t: TypeVarType) -> None: self.process_types([t.upper_bound, t.default] + t.values) def visit_param_spec(self, t: ParamSpecType) -> None: - self.process_types([t.upper_bound, t.default]) + self.process_types([t.upper_bound, t.default, t.prefix]) def visit_type_var_tuple(self, t: TypeVarTupleType) -> None: self.process_types([t.upper_bound, t.default]) diff --git a/mypy/typetraverser.py b/mypy/typetraverser.py index 047c5caf6daea..abd0f6bf3bdfe 100644 --- a/mypy/typetraverser.py +++ b/mypy/typetraverser.py @@ -64,6 +64,7 @@ def visit_type_var(self, t: TypeVarType, /) -> None: t.default.accept(self) def visit_param_spec(self, t: ParamSpecType, /) -> None: + # TODO: do we need to traverse prefix here? t.default.accept(self) def visit_parameters(self, t: Parameters, /) -> None: diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 76532e6eba4ad..9f5c811dc0a12 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -6915,3 +6915,70 @@ import does_not_exist [builtins fixtures/ops.pyi] [out] [out2] + +[case testIncrementalNoCrashOnParamSpecPrefixUpdateMethod] +import impl +[file impl.py] +from typing_extensions import ParamSpec +from lib import Sub + +P = ParamSpec("P") +class Impl(Sub[P]): + def test(self, *args: P.args, **kwargs: P.kwargs) -> None: + self.meth(1, *args, **kwargs) + +[file impl.py.2] +from typing_extensions import ParamSpec +from lib import Sub + +P = ParamSpec("P") +class Impl(Sub[P]): + def test(self, *args: P.args, **kwargs: P.kwargs) -> None: + self.meth("no", *args, **kwargs) + +[file lib.py] +from typing import Generic +from typing_extensions import ParamSpec, Concatenate + +P = ParamSpec("P") +class Base(Generic[P]): + def meth(self, *args: P.args, **kwargs: P.kwargs) -> None: ... +class Sub(Base[Concatenate[int, P]]): ... +[builtins fixtures/paramspec.pyi] +[out] +[out2] +tmp/impl.py:7: error: Argument 1 to "meth" of "Base" has incompatible type "str"; expected "int" + +[case testIncrementalNoCrashOnParamSpecPrefixUpdateMethodAlias] +import impl +[file impl.py] +from typing_extensions import ParamSpec +from lib import Sub + +P = ParamSpec("P") +class Impl(Sub[P]): + def test(self, *args: P.args, **kwargs: P.kwargs) -> None: + self.alias(1, *args, **kwargs) + +[file impl.py.2] +from typing_extensions import ParamSpec +from lib import Sub + +P = ParamSpec("P") +class Impl(Sub[P]): + def test(self, *args: P.args, **kwargs: P.kwargs) -> None: + self.alias("no", *args, **kwargs) + +[file lib.py] +from typing import Generic +from typing_extensions import ParamSpec, Concatenate + +P = ParamSpec("P") +class Base(Generic[P]): + def meth(self, *args: P.args, **kwargs: P.kwargs) -> None: ... + alias = meth +class Sub(Base[Concatenate[int, P]]): ... +[builtins fixtures/paramspec.pyi] +[out] +[out2] +tmp/impl.py:7: error: Argument 1 has incompatible type "str"; expected "int" diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index 888b7bc7e97f1..1bddee0e5ed27 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -11485,3 +11485,36 @@ class A: [out] == main:3: error: Too few arguments + +[case testFineGrainedParamSpecPrefixUpdateMethod] +import impl +[file impl.py] +from typing_extensions import ParamSpec +from lib import Sub + +P = ParamSpec("P") +class Impl(Sub[P]): + def test(self, *args: P.args, **kwargs: P.kwargs) -> None: + self.meth(1, *args, **kwargs) + +[file lib.py] +from typing import Generic +from typing_extensions import ParamSpec, Concatenate + +P = ParamSpec("P") +class Base(Generic[P]): + def meth(self, *args: P.args, **kwargs: P.kwargs) -> None: ... +class Sub(Base[Concatenate[int, P]]): ... + +[file lib.py.2] +from typing import Generic +from typing_extensions import ParamSpec, Concatenate + +P = ParamSpec("P") +class Base(Generic[P]): + def meth(self, *args: P.args, **kwargs: P.kwargs) -> None: ... +class Sub(Base[Concatenate[str, P]]): ... +[builtins fixtures/paramspec.pyi] +[out] +== +impl.py:7: error: Argument 1 to "meth" of "Base" has incompatible type "int"; expected "str" From 8437cf5d43cd099d10838f0bc9cadd9eed94425d Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Sun, 7 Sep 2025 02:40:07 +0200 Subject: [PATCH 245/424] Do not report exhaustive-match after deferral (#19804) Fixes #19791 --- mypy/checker.py | 4 ++-- test-data/unit/check-python310.test | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index ba821df621e56..5843148d4b4e7 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -5682,8 +5682,8 @@ def visit_match_stmt(self, s: MatchStmt) -> None: self.push_type_map(else_map, from_assignment=False) unmatched_types = else_map - if unmatched_types is not None: - for typ in list(unmatched_types.values()): + if unmatched_types is not None and not self.current_node_deferred: + for typ in unmatched_types.values(): self.msg.match_statement_inexhaustive_match(typ, s) # This is needed due to a quirk in frame_context. Without it types will stay narrowed diff --git a/test-data/unit/check-python310.test b/test-data/unit/check-python310.test index 5c495d2ed863b..7d76c09b61514 100644 --- a/test-data/unit/check-python310.test +++ b/test-data/unit/check-python310.test @@ -2975,3 +2975,18 @@ val: int = 8 match val: case FOO: # E: Cannot assign to final name "FOO" pass + +[case testMatchExhaustivenessWithDeferral] +# flags: --enable-error-code exhaustive-match +from typing import Literal +import unknown_module # E: Cannot find implementation or library stub for module named "unknown_module" \ + # N: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +def foo(e: Literal[0, 1]) -> None: + match e: + case 0: + defer + case 1: + ... + +defer = unknown_module.foo From 60639c5c07230c498c71d8d22bf16a4ee769777f Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Mon, 8 Sep 2025 03:41:35 +0700 Subject: [PATCH 246/424] Update options.py: typo `scrict_equality` and phrasing (#19806) Fixes a typo `scrict_equality` which meant `strict_equality`. While in there, I also changed a preposition to be more specific. --- mypy/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/options.py b/mypy/options.py index 5aced56c940f8..b1456934c6c9e 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -232,7 +232,7 @@ def __init__(self) -> None: # This makes 1 == '1', 1 in ['1'], and 1 is '1' errors. self.strict_equality = False - # Extend the logic of `scrict_equality` for comparisons with `None`. + # Extend the logic of `strict_equality` to comparisons with `None`. self.strict_equality_for_none = False # Disable treating bytearray and memoryview as subtypes of bytes From 9edd29ae2b8fe8411964a0dd91ac2d067f17c006 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 8 Sep 2025 10:46:07 +0100 Subject: [PATCH 247/424] Inverse interface freshness logic (#19809) Fixes https://github.com/python/mypy/issues/9554 This is another case where I am surprised id didn't work like this in the first place. Right now the freshness info originates from the dependency itself (like trust me, I am fresh, whatever it means). IMO this doesn't make much sense, instead a dependent should verify whether all dependencies are the same it last seen them. On the surface the idea is simple, but there are couple tricky parts: * This requires splitting `write_cache()` in two phases: first write all data files in an SCC (or at least serialize them), the write all meta files. I didn't find any elegant way to do the split, but it is probably fine, as we already have this untyped meta JSON in few places. * I am adding plugin data (used by mypyc separate compilation currently) as part of interface hash. It is not documented whether it should be this way or not, but I would say it should, and this is essentially how mypyc expects it (group name will appear in `#include <__native_group_name.h>`, so it _is_ a part of the interface). It used to work ~accidentally because we check plugin data in `find_cache_meta()` that is called before setting `interface_hash`, not in `validate_meta()`. --- mypy/build.py | 88 +++++++++++++++++---------- mypyc/test-data/run-multimodule.test | 2 +- test-data/unit/check-incremental.test | 18 +++++- 3 files changed, 73 insertions(+), 35 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 4ccc3dec408eb..2271365f3affc 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -335,6 +335,7 @@ class CacheMeta(NamedTuple): # dep_prios and dep_lines are in parallel with dependencies + suppressed dep_prios: list[int] dep_lines: list[int] + dep_hashes: dict[str, str] interface_hash: str # hash representing the public interface version_id: str # mypy version for cache invalidation ignore_all: bool # if errors were ignored @@ -373,6 +374,7 @@ def cache_meta_from_dict(meta: dict[str, Any], data_json: str) -> CacheMeta: meta.get("options"), meta.get("dep_prios", []), meta.get("dep_lines", []), + meta.get("dep_hashes", {}), meta.get("interface_hash", ""), meta.get("version_id", sentinel), meta.get("ignore_all", True), @@ -890,8 +892,6 @@ def log(self, *message: str) -> None: self.stderr.flush() def log_fine_grained(self, *message: str) -> None: - import mypy.build - if self.verbosity() >= 1: self.log("fine-grained:", *message) elif mypy.build.DEBUG_FINE_GRAINED: @@ -1500,6 +1500,7 @@ def validate_meta( "options": (manager.options.clone_for_module(id).select_options_affecting_cache()), "dep_prios": meta.dep_prios, "dep_lines": meta.dep_lines, + "dep_hashes": meta.dep_hashes, "interface_hash": meta.interface_hash, "version_id": manager.version_id, "ignore_all": meta.ignore_all, @@ -1543,7 +1544,7 @@ def write_cache( source_hash: str, ignore_all: bool, manager: BuildManager, -) -> tuple[str, CacheMeta | None]: +) -> tuple[str, tuple[dict[str, Any], str, str] | None]: """Write cache files for a module. Note that this mypy's behavior is still correct when any given @@ -1564,9 +1565,9 @@ def write_cache( manager: the build manager (for pyversion, log/trace) Returns: - A tuple containing the interface hash and CacheMeta - corresponding to the metadata that was written (the latter may - be None if the cache could not be written). + A tuple containing the interface hash and inner tuple with cache meta JSON + that should be written and paths to cache files (inner tuple may be None, + if the cache data could not be written). """ metastore = manager.metastore # For Bazel we use relative paths and zero mtimes. @@ -1581,6 +1582,8 @@ def write_cache( if bazel: tree.path = path + plugin_data = manager.plugin.report_config_data(ReportConfigContext(id, path, is_check=False)) + # Serialize data and analyze interface if manager.options.fixed_format_cache: data_io = Buffer() @@ -1589,9 +1592,7 @@ def write_cache( else: data = tree.serialize() data_bytes = json_dumps(data, manager.options.debug_cache) - interface_hash = hash_digest(data_bytes) - - plugin_data = manager.plugin.report_config_data(ReportConfigContext(id, path, is_check=False)) + interface_hash = hash_digest(data_bytes + json_dumps(plugin_data)) # Obtain and set up metadata st = manager.get_stat(path) @@ -1659,8 +1660,14 @@ def write_cache( "ignore_all": ignore_all, "plugin_data": plugin_data, } + return interface_hash, (meta, meta_json, data_json) + +def write_cache_meta( + meta: dict[str, Any], manager: BuildManager, meta_json: str, data_json: str +) -> CacheMeta: # Write meta cache file + metastore = manager.metastore meta_str = json_dumps(meta, manager.options.debug_cache) if not metastore.write(meta_json, meta_str): # Most likely the error is the replace() call @@ -1668,7 +1675,7 @@ def write_cache( # The next run will simply find the cache entry out of date. manager.log(f"Error writing meta JSON file {meta_json}") - return interface_hash, cache_meta_from_dict(meta, data_json) + return cache_meta_from_dict(meta, data_json) def delete_cache(id: str, path: str, manager: BuildManager) -> None: @@ -1867,6 +1874,9 @@ class State: # Map each dependency to the line number where it is first imported dep_line_map: dict[str, int] + # Map from dependency id to its last observed interface hash + dep_hashes: dict[str, str] = {} + # Parent package, its parent, etc. ancestors: list[str] | None = None @@ -1879,9 +1889,6 @@ class State: # If caller_state is set, the line number in the caller where the import occurred caller_line = 0 - # If True, indicate that the public interface of this module is unchanged - externally_same = True - # Contains a hash of the public interface in incremental mode interface_hash: str = "" @@ -1994,6 +2001,7 @@ def __init__( self.priorities = {id: pri for id, pri in zip(all_deps, self.meta.dep_prios)} assert len(all_deps) == len(self.meta.dep_lines) self.dep_line_map = {id: line for id, line in zip(all_deps, self.meta.dep_lines)} + self.dep_hashes = self.meta.dep_hashes if temporary: self.load_tree(temporary=True) if not manager.use_fine_grained_cache(): @@ -2046,26 +2054,17 @@ def is_fresh(self) -> bool: """Return whether the cache data for this file is fresh.""" # NOTE: self.dependencies may differ from # self.meta.dependencies when a dependency is dropped due to - # suppression by silent mode. However when a suppressed + # suppression by silent mode. However, when a suppressed # dependency is added back we find out later in the process. - return ( - self.meta is not None - and self.is_interface_fresh() - and self.dependencies == self.meta.dependencies - ) - - def is_interface_fresh(self) -> bool: - return self.externally_same + return self.meta is not None and self.dependencies == self.meta.dependencies def mark_as_rechecked(self) -> None: """Marks this module as having been fully re-analyzed by the type-checker.""" self.manager.rechecked_modules.add(self.id) - def mark_interface_stale(self, *, on_errors: bool = False) -> None: + def mark_interface_stale(self) -> None: """Marks this module as having a stale public interface, and discards the cache data.""" - self.externally_same = False - if not on_errors: - self.manager.stale_modules.add(self.id) + self.manager.stale_modules.add(self.id) def check_blockers(self) -> None: """Raise CompileError if a blocking error is detected.""" @@ -2507,7 +2506,7 @@ def valid_references(self) -> set[str]: return valid_refs - def write_cache(self) -> None: + def write_cache(self) -> tuple[dict[str, Any], str, str] | None: assert self.tree is not None, "Internal error: method must be called on parsed file only" # We don't support writing cache files in fine-grained incremental mode. if ( @@ -2525,20 +2524,19 @@ def write_cache(self) -> None: except Exception: print(f"Error serializing {self.id}", file=self.manager.stdout) raise # Propagate to display traceback - return + return None is_errors = self.transitive_error if is_errors: delete_cache(self.id, self.path, self.manager) self.meta = None - self.mark_interface_stale(on_errors=True) - return + return None dep_prios = self.dependency_priorities() dep_lines = self.dependency_lines() assert self.source_hash is not None assert len(set(self.dependencies)) == len( self.dependencies ), f"Duplicates in dependencies list for {self.id} ({self.dependencies})" - new_interface_hash, self.meta = write_cache( + new_interface_hash, meta_tuple = write_cache( self.id, self.path, self.tree, @@ -2557,6 +2555,7 @@ def write_cache(self) -> None: self.manager.log(f"Cached module {self.id} has changed interface") self.mark_interface_stale() self.interface_hash = new_interface_hash + return meta_tuple def verify_dependencies(self, suppressed_only: bool = False) -> None: """Report errors for import targets in modules that don't exist. @@ -3287,7 +3286,19 @@ def process_graph(graph: Graph, manager: BuildManager) -> None: for id in scc: deps.update(graph[id].dependencies) deps -= ascc - stale_deps = {id for id in deps if id in graph and not graph[id].is_interface_fresh()} + + # Verify that interfaces of dependencies still present in graph are up-to-date (fresh). + # Note: if a dependency is not in graph anymore, it should be considered interface-stale. + # This is important to trigger any relevant updates from indirect dependencies that were + # removed in load_graph(). + stale_deps = set() + for id in ascc: + for dep in graph[id].dep_hashes: + if dep not in graph: + stale_deps.add(dep) + continue + if graph[dep].interface_hash != graph[id].dep_hashes[dep]: + stale_deps.add(dep) fresh = fresh and not stale_deps undeps = set() if fresh: @@ -3518,14 +3529,25 @@ def process_stale_scc(graph: Graph, scc: list[str], manager: BuildManager) -> No if any(manager.errors.is_errors_for_file(graph[id].xpath) for id in stale): for id in stale: graph[id].transitive_error = True + meta_tuples = {} for id in stale: if graph[id].xpath not in manager.errors.ignored_files: errors = manager.errors.file_messages( graph[id].xpath, formatter=manager.error_formatter ) manager.flush_errors(manager.errors.simplify_path(graph[id].xpath), errors, False) - graph[id].write_cache() + meta_tuples[id] = graph[id].write_cache() graph[id].mark_as_rechecked() + for id in stale: + meta_tuple = meta_tuples[id] + if meta_tuple is None: + graph[id].meta = None + continue + meta, meta_json, data_json = meta_tuple + meta["dep_hashes"] = { + dep: graph[dep].interface_hash for dep in graph[id].dependencies if dep in graph + } + graph[id].meta = write_cache_meta(meta, manager, meta_json, data_json) def sorted_components( diff --git a/mypyc/test-data/run-multimodule.test b/mypyc/test-data/run-multimodule.test index 5112e126169f5..4208af0f04c81 100644 --- a/mypyc/test-data/run-multimodule.test +++ b/mypyc/test-data/run-multimodule.test @@ -816,7 +816,7 @@ def foo() -> int: return 10 [file driver.py] import native -[rechecked native, other_a] +[rechecked other_a] [case testSeparateCompilationWithUndefinedAttribute] from other_a import A diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 9f5c811dc0a12..d1155f54a75d1 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -205,7 +205,7 @@ def foo() -> int: return "foo" return inner2() -[rechecked mod1, mod2] +[rechecked mod2] [stale] [out2] tmp/mod2.py:4: error: Incompatible return value type (got "str", expected "int") @@ -6982,3 +6982,19 @@ class Sub(Base[Concatenate[int, P]]): ... [out] [out2] tmp/impl.py:7: error: Argument 1 has incompatible type "str"; expected "int" + +[case testIncrementalDifferentSourcesFreshnessCorrect] +# cmd: mypy -m foo bar +# cmd2: mypy -m foo +# cmd3: mypy -m foo bar +[file foo.py] +foo = 5 +[file foo.py.2] +foo = None +[file bar.py] +from foo import foo +bar: int = foo +[out] +[out2] +[out3] +tmp/bar.py:2: error: Incompatible types in assignment (expression has type "None", variable has type "int") From f09aa57e345eb3cf252e2922f0f832b86360faf0 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 8 Sep 2025 23:46:51 +0100 Subject: [PATCH 248/424] Try some aliases speed-up (#19810) This makes self-check almost 2% faster on my desktop (Python 3.12, compiled -O2). Inspired by a slight regression in https://github.com/python/mypy/pull/19798 I decided to re-think how we detect/label the recursive types. The new algorithm is not 100% equivalent to old one, but should be much faster. The main semantic difference is this: ```python A = list[B1] B1 = list[B2] B2 = list[B1] ``` previously all three aliases where labeled as recursive, now only last two are. Which is kind of correct if you think about it for some time, there is nothing genuinely recursive in `A` by itself. As a result: * We have somewhat more verbose `reveal_type()` for recursive types after fine-grained increments. Excessive use of `get_proper_type()` in the daemon code is a known issue. I will take a look at it when I will have a chance. * I cleaned up some of relevant visitors to be more consistent with recursive aliases. * I also do couple cleanups/speedups in the type queries while I am at it. If there are no comments/objections, I will merge it later today. Then I will merge https://github.com/python/mypy/pull/19798, and then _maybe_ an equivalent optimization for recursive instances like `class str(Sequence[str]): ...` --- mypy/constraints.py | 6 +-- mypy/indirection.py | 3 +- mypy/mixedtraverser.py | 2 + mypy/semanal_typeargs.py | 20 ++++---- mypy/stats.py | 6 +-- mypy/test/testtypes.py | 12 ----- mypy/type_visitor.py | 33 ++++++------ mypy/typeanal.py | 22 +++----- mypy/typeops.py | 6 +-- mypy/types.py | 87 ++++++++++++-------------------- test-data/unit/fine-grained.test | 4 +- 11 files changed, 80 insertions(+), 121 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index 6416791fa74a8..96c0c7ccaf35e 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -21,6 +21,7 @@ ArgKind, TypeInfo, ) +from mypy.type_visitor import ALL_STRATEGY, BoolTypeQuery from mypy.types import ( TUPLE_LIKE_INSTANCE_NAMES, AnyType, @@ -41,7 +42,6 @@ TypeAliasType, TypedDictType, TypeOfAny, - TypeQuery, TypeType, TypeVarId, TypeVarLikeType, @@ -670,9 +670,9 @@ def is_complete_type(typ: Type) -> bool: return typ.accept(CompleteTypeVisitor()) -class CompleteTypeVisitor(TypeQuery[bool]): +class CompleteTypeVisitor(BoolTypeQuery): def __init__(self) -> None: - super().__init__(all) + super().__init__(ALL_STRATEGY) def visit_uninhabited_type(self, t: UninhabitedType) -> bool: return False diff --git a/mypy/indirection.py b/mypy/indirection.py index 88258b94d94a1..4e566194632b1 100644 --- a/mypy/indirection.py +++ b/mypy/indirection.py @@ -39,8 +39,7 @@ def find_modules(self, typs: Iterable[types.Type]) -> set[str]: def _visit(self, typ: types.Type) -> None: if isinstance(typ, types.TypeAliasType): # Avoid infinite recursion for recursive type aliases. - if typ not in self.seen_aliases: - self.seen_aliases.add(typ) + self.seen_aliases.add(typ) typ.accept(self) def _visit_type_tuple(self, typs: tuple[types.Type, ...]) -> None: diff --git a/mypy/mixedtraverser.py b/mypy/mixedtraverser.py index 324e8a87c1bd6..f47d762934bc7 100644 --- a/mypy/mixedtraverser.py +++ b/mypy/mixedtraverser.py @@ -47,6 +47,8 @@ def visit_class_def(self, o: ClassDef, /) -> None: if info: for base in info.bases: base.accept(self) + if info.special_alias: + info.special_alias.accept(self) def visit_type_alias_expr(self, o: TypeAliasExpr, /) -> None: super().visit_type_alias_expr(o) diff --git a/mypy/semanal_typeargs.py b/mypy/semanal_typeargs.py index be39a8259c2e4..686e7a57042d9 100644 --- a/mypy/semanal_typeargs.py +++ b/mypy/semanal_typeargs.py @@ -83,12 +83,11 @@ def visit_block(self, o: Block) -> None: def visit_type_alias_type(self, t: TypeAliasType) -> None: super().visit_type_alias_type(t) - if t in self.seen_aliases: - # Avoid infinite recursion on recursive type aliases. - # Note: it is fine to skip the aliases we have already seen in non-recursive - # types, since errors there have already been reported. - return - self.seen_aliases.add(t) + if t.is_recursive: + if t in self.seen_aliases: + # Avoid infinite recursion on recursive type aliases. + return + self.seen_aliases.add(t) assert t.alias is not None, f"Unfixed type alias {t.type_ref}" is_error, is_invalid = self.validate_args( t.alias.name, tuple(t.args), t.alias.alias_tvars, t @@ -101,9 +100,12 @@ def visit_type_alias_type(self, t: TypeAliasType) -> None: if not is_error: # If there was already an error for the alias itself, there is no point in checking # the expansion, most likely it will result in the same kind of error. - get_proper_type(t).accept(self) - if t.alias is not None: - t.alias.accept(self) + if t.args: + # Since we always allow unbounded type variables in alias definitions, we need + # to verify the arguments satisfy the upper bounds of the expansion as well. + get_proper_type(t).accept(self) + if t.is_recursive: + self.seen_aliases.discard(t) def visit_tuple_type(self, t: TupleType) -> None: t.items = flatten_nested_tuples(t.items) diff --git a/mypy/stats.py b/mypy/stats.py index 6bad400ce5d57..e3499d2345635 100644 --- a/mypy/stats.py +++ b/mypy/stats.py @@ -43,6 +43,7 @@ YieldFromExpr, ) from mypy.traverser import TraverserVisitor +from mypy.type_visitor import ANY_STRATEGY, BoolTypeQuery from mypy.typeanal import collect_all_inner_types from mypy.types import ( AnyType, @@ -52,7 +53,6 @@ TupleType, Type, TypeOfAny, - TypeQuery, TypeVarType, get_proper_type, get_proper_types, @@ -453,9 +453,9 @@ def is_imprecise(t: Type) -> bool: return t.accept(HasAnyQuery()) -class HasAnyQuery(TypeQuery[bool]): +class HasAnyQuery(BoolTypeQuery): def __init__(self) -> None: - super().__init__(any) + super().__init__(ANY_STRATEGY) def visit_any(self, t: AnyType) -> bool: return not is_special_form_any(t) diff --git a/mypy/test/testtypes.py b/mypy/test/testtypes.py index 0fe41bc28ecd1..fc68d9aa6eac2 100644 --- a/mypy/test/testtypes.py +++ b/mypy/test/testtypes.py @@ -201,18 +201,6 @@ def test_type_alias_expand_once(self) -> None: assert get_proper_type(A) == target assert get_proper_type(target) == target - def test_type_alias_expand_all(self) -> None: - A, _ = self.fx.def_alias_1(self.fx.a) - assert A.expand_all_if_possible() is None - A, _ = self.fx.def_alias_2(self.fx.a) - assert A.expand_all_if_possible() is None - - B = self.fx.non_rec_alias(self.fx.a) - C = self.fx.non_rec_alias(TupleType([B, B], Instance(self.fx.std_tuplei, [B]))) - assert C.expand_all_if_possible() == TupleType( - [self.fx.a, self.fx.a], Instance(self.fx.std_tuplei, [self.fx.a]) - ) - def test_recursive_nested_in_non_recursive(self) -> None: A, _ = self.fx.def_alias_1(self.fx.a) T = TypeVarType( diff --git a/mypy/type_visitor.py b/mypy/type_visitor.py index 15494393cae66..86ef6ade84718 100644 --- a/mypy/type_visitor.py +++ b/mypy/type_visitor.py @@ -15,7 +15,7 @@ from abc import abstractmethod from collections.abc import Iterable, Sequence -from typing import Any, Callable, Final, Generic, TypeVar, cast +from typing import Any, Final, Generic, TypeVar, cast from mypy_extensions import mypyc_attr, trait @@ -353,16 +353,19 @@ class TypeQuery(SyntheticTypeVisitor[T]): # TODO: check that we don't have existing violations of this rule. """ - def __init__(self, strategy: Callable[[list[T]], T]) -> None: - self.strategy = strategy + def __init__(self) -> None: # Keep track of the type aliases already visited. This is needed to avoid # infinite recursion on types like A = Union[int, List[A]]. - self.seen_aliases: set[TypeAliasType] = set() + self.seen_aliases: set[TypeAliasType] | None = None # By default, we eagerly expand type aliases, and query also types in the # alias target. In most cases this is a desired behavior, but we may want # to skip targets in some cases (e.g. when collecting type variables). self.skip_alias_target = False + @abstractmethod + def strategy(self, items: list[T]) -> T: + raise NotImplementedError + def visit_unbound_type(self, t: UnboundType, /) -> T: return self.query_types(t.args) @@ -440,14 +443,15 @@ def visit_placeholder_type(self, t: PlaceholderType, /) -> T: return self.query_types(t.args) def visit_type_alias_type(self, t: TypeAliasType, /) -> T: - # Skip type aliases already visited types to avoid infinite recursion. - # TODO: Ideally we should fire subvisitors here (or use caching) if we care - # about duplicates. - if t in self.seen_aliases: - return self.strategy([]) - self.seen_aliases.add(t) if self.skip_alias_target: return self.query_types(t.args) + # Skip type aliases already visited types to avoid infinite recursion + # (also use this as a simple-minded cache). + if self.seen_aliases is None: + self.seen_aliases = set() + elif t in self.seen_aliases: + return self.strategy([]) + self.seen_aliases.add(t) return get_proper_type(t).accept(self) def query_types(self, types: Iterable[Type]) -> T: @@ -580,16 +584,15 @@ def visit_placeholder_type(self, t: PlaceholderType, /) -> bool: return self.query_types(t.args) def visit_type_alias_type(self, t: TypeAliasType, /) -> bool: - # Skip type aliases already visited types to avoid infinite recursion. - # TODO: Ideally we should fire subvisitors here (or use caching) if we care - # about duplicates. + if self.skip_alias_target: + return self.query_types(t.args) + # Skip type aliases already visited types to avoid infinite recursion + # (also use this as a simple-minded cache). if self.seen_aliases is None: self.seen_aliases = set() elif t in self.seen_aliases: return self.default self.seen_aliases.add(t) - if self.skip_alias_target: - return self.query_types(t.args) return get_proper_type(t).accept(self) def query_types(self, types: list[Type] | tuple[Type, ...]) -> bool: diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 6587304147635..81fb87fbf9ee1 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -2377,9 +2377,9 @@ def has_explicit_any(t: Type) -> bool: return t.accept(HasExplicitAny()) -class HasExplicitAny(TypeQuery[bool]): +class HasExplicitAny(BoolTypeQuery): def __init__(self) -> None: - super().__init__(any) + super().__init__(ANY_STRATEGY) def visit_any(self, t: AnyType) -> bool: return t.type_of_any == TypeOfAny.explicit @@ -2418,15 +2418,11 @@ def collect_all_inner_types(t: Type) -> list[Type]: class CollectAllInnerTypesQuery(TypeQuery[list[Type]]): - def __init__(self) -> None: - super().__init__(self.combine_lists_strategy) - def query_types(self, types: Iterable[Type]) -> list[Type]: return self.strategy([t.accept(self) for t in types]) + list(types) - @classmethod - def combine_lists_strategy(cls, it: Iterable[list[Type]]) -> list[Type]: - return list(itertools.chain.from_iterable(it)) + def strategy(self, items: Iterable[list[Type]]) -> list[Type]: + return list(itertools.chain.from_iterable(items)) def make_optional_type(t: Type) -> Type: @@ -2556,7 +2552,6 @@ def __init__(self, api: SemanticAnalyzerCoreInterface, scope: TypeVarLikeScope) self.scope = scope self.type_var_likes: list[tuple[str, TypeVarLikeExpr]] = [] self.has_self_type = False - self.seen_aliases: set[TypeAliasType] | None = None self.include_callables = True def _seems_like_callable(self, type: UnboundType) -> bool: @@ -2653,7 +2648,8 @@ def visit_union_type(self, t: UnionType) -> None: self.process_types(t.items) def visit_overloaded(self, t: Overloaded) -> None: - self.process_types(t.items) # type: ignore[arg-type] + for it in t.items: + it.accept(self) def visit_type_type(self, t: TypeType) -> None: t.item.accept(self) @@ -2665,12 +2661,6 @@ def visit_placeholder_type(self, t: PlaceholderType) -> None: return self.process_types(t.args) def visit_type_alias_type(self, t: TypeAliasType) -> None: - # Skip type aliases in already visited types to avoid infinite recursion. - if self.seen_aliases is None: - self.seen_aliases = set() - elif t in self.seen_aliases: - return - self.seen_aliases.add(t) self.process_types(t.args) def process_types(self, types: list[Type] | tuple[Type, ...]) -> None: diff --git a/mypy/typeops.py b/mypy/typeops.py index 87a4d8cefd133..298ad4d16f8c6 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -1114,12 +1114,12 @@ def get_all_type_vars(tp: Type) -> list[TypeVarLikeType]: class TypeVarExtractor(TypeQuery[list[TypeVarLikeType]]): def __init__(self, include_all: bool = False) -> None: - super().__init__(self._merge) + super().__init__() self.include_all = include_all - def _merge(self, iter: Iterable[list[TypeVarLikeType]]) -> list[TypeVarLikeType]: + def strategy(self, items: Iterable[list[TypeVarLikeType]]) -> list[TypeVarLikeType]: out = [] - for item in iter: + for item in items: out.extend(item) return out diff --git a/mypy/types.py b/mypy/types.py index 3f4bd94b5b24b..e0e897e04cadf 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -369,29 +369,6 @@ def _expand_once(self) -> Type: return self.alias.target.accept(InstantiateAliasVisitor(mapping)) - def _partial_expansion(self, nothing_args: bool = False) -> tuple[ProperType, bool]: - # Private method mostly for debugging and testing. - unroller = UnrollAliasVisitor(set(), {}) - if nothing_args: - alias = self.copy_modified(args=[UninhabitedType()] * len(self.args)) - else: - alias = self - unrolled = alias.accept(unroller) - assert isinstance(unrolled, ProperType) - return unrolled, unroller.recursed - - def expand_all_if_possible(self, nothing_args: bool = False) -> ProperType | None: - """Attempt a full expansion of the type alias (including nested aliases). - - If the expansion is not possible, i.e. the alias is (mutually-)recursive, - return None. If nothing_args is True, replace all type arguments with an - UninhabitedType() (used to detect recursively defined aliases). - """ - unrolled, recursed = self._partial_expansion(nothing_args=nothing_args) - if recursed: - return None - return unrolled - @property def is_recursive(self) -> bool: """Whether this type alias is recursive. @@ -404,7 +381,7 @@ def is_recursive(self) -> bool: assert self.alias is not None, "Unfixed type alias" is_recursive = self.alias._is_recursive if is_recursive is None: - is_recursive = self.expand_all_if_possible(nothing_args=True) is None + is_recursive = self.alias in self.alias.target.accept(CollectAliasesVisitor()) # We cache the value on the underlying TypeAlias node as an optimization, # since the value is the same for all instances of the same alias. self.alias._is_recursive = is_recursive @@ -3654,8 +3631,8 @@ class TypeStrVisitor(SyntheticTypeVisitor[str]): def __init__(self, id_mapper: IdMapper | None = None, *, options: Options) -> None: self.id_mapper = id_mapper - self.any_as_dots = False self.options = options + self.dotted_aliases: set[TypeAliasType] | None = None def visit_unbound_type(self, t: UnboundType, /) -> str: s = t.name + "?" @@ -3674,8 +3651,6 @@ def visit_callable_argument(self, t: CallableArgument, /) -> str: return f"{t.constructor}({typ}, {t.name})" def visit_any(self, t: AnyType, /) -> str: - if self.any_as_dots and t.type_of_any == TypeOfAny.special_form: - return "..." return "Any" def visit_none_type(self, t: NoneType, /) -> str: @@ -3902,13 +3877,18 @@ def visit_placeholder_type(self, t: PlaceholderType, /) -> str: return f"" def visit_type_alias_type(self, t: TypeAliasType, /) -> str: - if t.alias is not None: - unrolled, recursed = t._partial_expansion() - self.any_as_dots = recursed - type_str = unrolled.accept(self) - self.any_as_dots = False - return type_str - return "" + if t.alias is None: + return "" + if not t.is_recursive: + return get_proper_type(t).accept(self) + if self.dotted_aliases is None: + self.dotted_aliases = set() + elif t in self.dotted_aliases: + return "..." + self.dotted_aliases.add(t) + type_str = get_proper_type(t).accept(self) + self.dotted_aliases.discard(t) + return type_str def visit_unpack_type(self, t: UnpackType, /) -> str: return f"Unpack[{t.type.accept(self)}]" @@ -3943,28 +3923,23 @@ def visit_type_list(self, t: TypeList, /) -> Type: return t -class UnrollAliasVisitor(TrivialSyntheticTypeTranslator): - def __init__( - self, initial_aliases: set[TypeAliasType], cache: dict[Type, Type] | None - ) -> None: - assert cache is not None - super().__init__(cache) - self.recursed = False - self.initial_aliases = initial_aliases - - def visit_type_alias_type(self, t: TypeAliasType) -> Type: - if t in self.initial_aliases: - self.recursed = True - return AnyType(TypeOfAny.special_form) - # Create a new visitor on encountering a new type alias, so that an alias like - # A = Tuple[B, B] - # B = int - # will not be detected as recursive on the second encounter of B. - subvisitor = UnrollAliasVisitor(self.initial_aliases | {t}, self.cache) - result = get_proper_type(t).accept(subvisitor) - if subvisitor.recursed: - self.recursed = True - return result +class CollectAliasesVisitor(TypeQuery[list[mypy.nodes.TypeAlias]]): + def __init__(self) -> None: + super().__init__() + self.seen_alias_nodes: set[mypy.nodes.TypeAlias] = set() + + def strategy(self, items: list[list[mypy.nodes.TypeAlias]]) -> list[mypy.nodes.TypeAlias]: + out = [] + for item in items: + out.extend(item) + return out + + def visit_type_alias_type(self, t: TypeAliasType, /) -> list[mypy.nodes.TypeAlias]: + assert t.alias is not None + if t.alias not in self.seen_alias_nodes: + self.seen_alias_nodes.add(t.alias) + return [t.alias] + t.alias.target.accept(self) + return [] def is_named_instance(t: Type, fullnames: str | tuple[str, ...]) -> TypeGuard[Instance]: diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index 1bddee0e5ed27..4a30c8a3828f0 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -3528,9 +3528,9 @@ reveal_type(a.n) [out] == == -c.py:4: note: Revealed type is "tuple[Union[tuple[Union[..., None], builtins.int, fallback=b.M], None], builtins.int, fallback=a.N]" +c.py:4: note: Revealed type is "tuple[Union[tuple[Union[tuple[Union[..., None], builtins.int, fallback=a.N], None], builtins.int, fallback=b.M], None], builtins.int, fallback=a.N]" c.py:5: error: Incompatible types in assignment (expression has type "Optional[N]", variable has type "int") -c.py:7: note: Revealed type is "tuple[Union[tuple[Union[..., None], builtins.int, fallback=b.M], None], builtins.int, fallback=a.N]" +c.py:7: note: Revealed type is "tuple[Union[tuple[Union[tuple[Union[..., None], builtins.int, fallback=a.N], None], builtins.int, fallback=b.M], None], builtins.int, fallback=a.N]" [case testTupleTypeUpdateNonRecursiveToRecursiveFine] import c From ccd290c5b3a752668623576a5e1aa29e78f53307 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 9 Sep 2025 00:29:47 +0100 Subject: [PATCH 249/424] Re-work indirect dependencies (#19798) Wow, this was quite a ride. Indirect dependencies were always supported kind of on best effort. This PR puts them on some principled foundation. It fixes three crashes and three stale types reported. All tests are quite weird/obscure, they are designed to expose the flaws in current logic (plus one test that passes on master, but it covers important corner case, so I add it just in case ). A short summary of various fixes (in arbitrary order): * Update many outdated comments and docstrings * Missing transitive dependency is now considered stale * Handle transitive generic bases in indirection visitor * Handle chained alias targets in indirection visitor * Always record original aliases during semantic analysis * Replace ad-hoc `module_refs` logic in type checker with more principled one during semantic analysis * Delete `qualified_tvars` as a concept, they are not needed since long ago * Remove ad-hoc handling for `TypeInfo`s from `build.py` * Support symbols with setter type different from getter type In general the logic should be more simple/straightforward now: * Get all symbols we try to access in a module and record the modules they were defined in (note this automatically handles problem with possible excessive `get_proper_type()` calls). * Get all types in a type map, for each type _transitively_ find all named types in them (thus aggregating all interfaces the type depends on) Note since this makes the algorithm correct, it may also make it slower (most notably because we must visit generic bases). I tried to offset this by couple optimizations, hopefully performance impact will be minimal. On my machine slow down is ~0.6% --- mypy/build.py | 58 +++---- mypy/checker.py | 6 +- mypy/checkexpr.py | 33 ---- mypy/checkmember.py | 2 + mypy/fixup.py | 2 +- mypy/indirection.py | 101 ++++++----- mypy/nodes.py | 19 +- mypy/semanal.py | 77 +++++---- mypy/server/deps.py | 15 +- mypy/test/typefixture.py | 6 +- test-data/unit/check-incremental.test | 239 ++++++++++++++++++++++++++ 11 files changed, 402 insertions(+), 156 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 2271365f3affc..84dbf2b2df880 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -47,7 +47,7 @@ from mypy.graph_utils import prepare_sccs, strongly_connected_components, topsort from mypy.indirection import TypeIndirectionVisitor from mypy.messages import MessageBuilder -from mypy.nodes import Import, ImportAll, ImportBase, ImportFrom, MypyFile, SymbolTable, TypeInfo +from mypy.nodes import Import, ImportAll, ImportBase, ImportFrom, MypyFile, SymbolTable from mypy.partially_defined import PossiblyUndefinedVariableVisitor from mypy.semanal import SemanticAnalyzer from mypy.semanal_pass1 import SemanticAnalyzerPreAnalysis @@ -1765,26 +1765,24 @@ def delete_cache(id: str, path: str, manager: BuildManager) -> None: For single nodes, processing is simple. If the node was cached, we deserialize the cache data and fix up cross-references. Otherwise, we -do semantic analysis followed by type checking. We also handle (c) -above; if a module has valid cache data *but* any of its -dependencies was processed from source, then the module should be -processed from source. - -A relatively simple optimization (outside SCCs) we might do in the -future is as follows: if a node's cache data is valid, but one or more -of its dependencies are out of date so we have to re-parse the node -from source, once we have fully type-checked the node, we can decide -whether its symbol table actually changed compared to the cache data -(by reading the cache data and comparing it to the data we would be -writing). If there is no change we can declare the node up to date, -and any node that depends (and for which we have cached data, and -whose other dependencies are up to date) on it won't need to be -re-parsed from source. +do semantic analysis followed by type checking. Once we (re-)processed +an SCC we check whether its interface (symbol table) is still fresh +(matches previous cached value). If it is not, we consider dependent SCCs +stale so that they need to be re-parsed as well. + +Note on indirect dependencies: normally dependencies are determined from +imports, but since our interfaces are "opaque" (i.e. symbol tables can +contain cross-references as well as types identified by name), these are not +enough. We *must* also add "indirect" dependencies from symbols and types to +their definitions. For this purpose, we record all accessed symbols during +semantic analysis, and after we finished processing a module, we traverse its +type map, and for each type we find (transitively) on which named types it +depends. Import cycles ------------- -Finally we have to decide how to handle (c), import cycles. Here +Finally we have to decide how to handle (b), import cycles. Here we'll need a modified version of the original state machine (build.py), but we only need to do this per SCC, and we won't have to deal with changes to the list of nodes while we're processing it. @@ -2409,21 +2407,15 @@ def finish_passes(self) -> None: # We should always patch indirect dependencies, even in full (non-incremental) builds, # because the cache still may be written, and it must be correct. - # TODO: find a more robust way to traverse *all* relevant types? - all_types = list(self.type_map().values()) - for _, sym, _ in self.tree.local_definitions(): - if sym.type is not None: - all_types.append(sym.type) - if isinstance(sym.node, TypeInfo): - # TypeInfo symbols have some extra relevant types. - all_types.extend(sym.node.bases) - if sym.node.metaclass_type: - all_types.append(sym.node.metaclass_type) - if sym.node.typeddict_type: - all_types.append(sym.node.typeddict_type) - if sym.node.tuple_type: - all_types.append(sym.node.tuple_type) - self._patch_indirect_dependencies(self.type_checker().module_refs, all_types) + self._patch_indirect_dependencies( + # Two possible sources of indirect dependencies: + # * Symbols not directly imported in this module but accessed via an attribute + # or via a re-export (vast majority of these recorded in semantic analysis). + # * For each expression type we need to record definitions of type components + # since "meaning" of the type may be updated when definitions are updated. + self.tree.module_refs | self.type_checker().module_refs, + set(self.type_map().values()), + ) if self.options.dump_inference_stats: dump_type_stats( @@ -2452,7 +2444,7 @@ def free_state(self) -> None: self._type_checker.reset() self._type_checker = None - def _patch_indirect_dependencies(self, module_refs: set[str], types: list[Type]) -> None: + def _patch_indirect_dependencies(self, module_refs: set[str], types: set[Type]) -> None: assert None not in types valid = self.valid_references() diff --git a/mypy/checker.py b/mypy/checker.py index 5843148d4b4e7..96b55f321a73c 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -378,11 +378,9 @@ class TypeChecker(NodeVisitor[None], TypeCheckerSharedApi): inferred_attribute_types: dict[Var, Type] | None = None # Don't infer partial None types if we are processing assignment from Union no_partial_types: bool = False - - # The set of all dependencies (suppressed or not) that this module accesses, either - # directly or indirectly. + # Extra module references not detected during semantic analysis (these are rare cases + # e.g. access to class-level import via instance). module_refs: set[str] - # A map from variable nodes to a snapshot of the frame ids of the # frames that were active when the variable was declared. This can # be used to determine nearest common ancestor frame of a variable's diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 835eeb7253945..73282c94be4eb 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -198,7 +198,6 @@ ) from mypy.typestate import type_state from mypy.typevars import fill_typevars -from mypy.util import split_module_names from mypy.visitor import ExpressionVisitor # Type of callback user for checking individual function arguments. See @@ -248,36 +247,6 @@ def allow_fast_container_literal(t: Type) -> bool: ) -def extract_refexpr_names(expr: RefExpr, output: set[str]) -> None: - """Recursively extracts all module references from a reference expression. - - Note that currently, the only two subclasses of RefExpr are NameExpr and - MemberExpr.""" - while isinstance(expr.node, MypyFile) or expr.fullname: - if isinstance(expr.node, MypyFile) and expr.fullname: - # If it's None, something's wrong (perhaps due to an - # import cycle or a suppressed error). For now we just - # skip it. - output.add(expr.fullname) - - if isinstance(expr, NameExpr): - is_suppressed_import = isinstance(expr.node, Var) and expr.node.is_suppressed_import - if isinstance(expr.node, TypeInfo): - # Reference to a class or a nested class - output.update(split_module_names(expr.node.module_name)) - elif "." in expr.fullname and not is_suppressed_import: - # Everything else (that is not a silenced import within a class) - output.add(expr.fullname.rsplit(".", 1)[0]) - break - elif isinstance(expr, MemberExpr): - if isinstance(expr.expr, RefExpr): - expr = expr.expr - else: - break - else: - raise AssertionError(f"Unknown RefExpr subclass: {type(expr)}") - - class Finished(Exception): """Raised if we can terminate overload argument check early (no match).""" @@ -370,7 +339,6 @@ def visit_name_expr(self, e: NameExpr) -> Type: It can be of any kind: local, member or global. """ - extract_refexpr_names(e, self.chk.module_refs) result = self.analyze_ref_expr(e) narrowed = self.narrow_type_from_binder(e, result) self.chk.check_deprecated(e.node, e) @@ -3344,7 +3312,6 @@ def check_union_call( def visit_member_expr(self, e: MemberExpr, is_lvalue: bool = False) -> Type: """Visit member expression (of form e.id).""" - extract_refexpr_names(e, self.chk.module_refs) result = self.analyze_ordinary_member_access(e, is_lvalue) narrowed = self.narrow_type_from_binder(e, result) self.chk.warn_deprecated(e.node, e) diff --git a/mypy/checkmember.py b/mypy/checkmember.py index e7de1b7a304fe..f19a76ec6a342 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -543,6 +543,8 @@ def analyze_member_var_access( if isinstance(v, FuncDef): assert False, "Did not expect a function" if isinstance(v, MypyFile): + # Special case: accessing module on instances is allowed, but will not + # be recorded by semantic analyzer. mx.chk.module_refs.add(v.fullname) if isinstance(vv, (TypeInfo, TypeAlias, MypyFile, TypeVarLikeExpr)): diff --git a/mypy/fixup.py b/mypy/fixup.py index 260c0f84cf1b9..d0205f64b7207 100644 --- a/mypy/fixup.py +++ b/mypy/fixup.py @@ -441,4 +441,4 @@ def missing_info(modules: dict[str, MypyFile]) -> TypeInfo: def missing_alias() -> TypeAlias: suggestion = _SUGGESTION.format("alias") - return TypeAlias(AnyType(TypeOfAny.special_form), suggestion, line=-1, column=-1) + return TypeAlias(AnyType(TypeOfAny.special_form), suggestion, "", line=-1, column=-1) diff --git a/mypy/indirection.py b/mypy/indirection.py index 4e566194632b1..95023e303cbdc 100644 --- a/mypy/indirection.py +++ b/mypy/indirection.py @@ -4,17 +4,6 @@ import mypy.types as types from mypy.types import TypeVisitor -from mypy.util import split_module_names - - -def extract_module_names(type_name: str | None) -> list[str]: - """Returns the module names of a fully qualified type name.""" - if type_name is not None: - # Discard the first one, which is just the qualified name of the type - possible_module_names = split_module_names(type_name) - return possible_module_names[1:] - else: - return [] class TypeIndirectionVisitor(TypeVisitor[None]): @@ -23,49 +12,57 @@ class TypeIndirectionVisitor(TypeVisitor[None]): def __init__(self) -> None: # Module references are collected here self.modules: set[str] = set() - # User to avoid infinite recursion with recursive type aliases - self.seen_aliases: set[types.TypeAliasType] = set() - # Used to avoid redundant work - self.seen_fullnames: set[str] = set() + # User to avoid infinite recursion with recursive types + self.seen_types: set[types.TypeAliasType | types.Instance] = set() def find_modules(self, typs: Iterable[types.Type]) -> set[str]: self.modules = set() - self.seen_fullnames = set() - self.seen_aliases = set() + self.seen_types = set() for typ in typs: self._visit(typ) return self.modules def _visit(self, typ: types.Type) -> None: - if isinstance(typ, types.TypeAliasType): - # Avoid infinite recursion for recursive type aliases. - self.seen_aliases.add(typ) + # Note: instances are needed for `class str(Sequence[str]): ...` + if ( + isinstance(typ, types.TypeAliasType) + or isinstance(typ, types.ProperType) + and isinstance(typ, types.Instance) + ): + # Avoid infinite recursion for recursive types. + if typ in self.seen_types: + return + self.seen_types.add(typ) typ.accept(self) def _visit_type_tuple(self, typs: tuple[types.Type, ...]) -> None: # Micro-optimization: Specialized version of _visit for lists for typ in typs: - if isinstance(typ, types.TypeAliasType): - # Avoid infinite recursion for recursive type aliases. - if typ in self.seen_aliases: + if ( + isinstance(typ, types.TypeAliasType) + or isinstance(typ, types.ProperType) + and isinstance(typ, types.Instance) + ): + # Avoid infinite recursion for recursive types. + if typ in self.seen_types: continue - self.seen_aliases.add(typ) + self.seen_types.add(typ) typ.accept(self) def _visit_type_list(self, typs: list[types.Type]) -> None: # Micro-optimization: Specialized version of _visit for tuples for typ in typs: - if isinstance(typ, types.TypeAliasType): - # Avoid infinite recursion for recursive type aliases. - if typ in self.seen_aliases: + if ( + isinstance(typ, types.TypeAliasType) + or isinstance(typ, types.ProperType) + and isinstance(typ, types.Instance) + ): + # Avoid infinite recursion for recursive types. + if typ in self.seen_types: continue - self.seen_aliases.add(typ) + self.seen_types.add(typ) typ.accept(self) - def _visit_module_name(self, module_name: str) -> None: - if module_name not in self.modules: - self.modules.update(split_module_names(module_name)) - def visit_unbound_type(self, t: types.UnboundType) -> None: self._visit_type_tuple(t.args) @@ -105,27 +102,36 @@ def visit_parameters(self, t: types.Parameters) -> None: self._visit_type_list(t.arg_types) def visit_instance(self, t: types.Instance) -> None: + # Instance is named, record its definition and continue digging into + # components that constitute semantic meaning of this type: bases, metaclass, + # tuple type, and typeddict type. + # Note: we cannot simply record the MRO, in case an intermediate base contains + # a reference to type alias, this affects meaning of map_instance_to_supertype(), + # see e.g. testDoubleReexportGenericUpdated. self._visit_type_tuple(t.args) if t.type: - # Uses of a class depend on everything in the MRO, - # as changes to classes in the MRO can add types to methods, - # change property types, change the MRO itself, etc. + # Important optimization: instead of simply recording the definition and + # recursing into bases, record the MRO and only traverse generic bases. for s in t.type.mro: - self._visit_module_name(s.module_name) - if t.type.metaclass_type is not None: - self._visit_module_name(t.type.metaclass_type.type.module_name) + self.modules.add(s.module_name) + for base in s.bases: + if base.args: + self._visit_type_tuple(base.args) + if t.type.metaclass_type: + self._visit(t.type.metaclass_type) + if t.type.typeddict_type: + self._visit(t.type.typeddict_type) + if t.type.tuple_type: + self._visit(t.type.tuple_type) def visit_callable_type(self, t: types.CallableType) -> None: self._visit_type_list(t.arg_types) self._visit(t.ret_type) - if t.definition is not None: - fullname = t.definition.fullname - if fullname not in self.seen_fullnames: - self.modules.update(extract_module_names(t.definition.fullname)) - self.seen_fullnames.add(fullname) + self._visit_type_tuple(t.variables) def visit_overloaded(self, t: types.Overloaded) -> None: - self._visit_type_list(list(t.items)) + for item in t.items: + self._visit(item) self._visit(t.fallback) def visit_tuple_type(self, t: types.TupleType) -> None: @@ -149,4 +155,9 @@ def visit_type_type(self, t: types.TypeType) -> None: self._visit(t.item) def visit_type_alias_type(self, t: types.TypeAliasType) -> None: - self._visit(types.get_proper_type(t)) + # Type alias is named, record its definition and continue digging into + # components that constitute semantic meaning of this type: target and args. + if t.alias: + self.modules.add(t.alias.module) + self._visit(t.alias.target) + self._visit_type_list(t.args) diff --git a/mypy/nodes.py b/mypy/nodes.py index 9cfc61c80b3e7..7480745c6aa1c 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -288,6 +288,7 @@ class MypyFile(SymbolNode): "path", "defs", "alias_deps", + "module_refs", "is_bom", "names", "imports", @@ -311,6 +312,9 @@ class MypyFile(SymbolNode): defs: list[Statement] # Type alias dependencies as mapping from target to set of alias full names alias_deps: defaultdict[str, set[str]] + # The set of all dependencies (suppressed or not) that this module accesses, either + # directly or indirectly. + module_refs: set[str] # Is there a UTF-8 BOM at the start? is_bom: bool names: SymbolTable @@ -351,6 +355,7 @@ def __init__( self.imports = imports self.is_bom = is_bom self.alias_deps = defaultdict(set) + self.module_refs = set() self.plugin_deps = {} if ignored_lines: self.ignored_lines = ignored_lines @@ -4121,7 +4126,8 @@ def f(x: B[T]) -> T: ... # without T, Any would be used here target: The target type. For generic aliases contains bound type variables as nested types (currently TypeVar and ParamSpec are supported). _fullname: Qualified name of this type alias. This is used in particular - to track fine grained dependencies from aliases. + to track fine-grained dependencies from aliases. + module: Module where the alias was defined. alias_tvars: Type variables used to define this alias. normalized: Used to distinguish between `A = List`, and `A = list`. Both are internally stored using `builtins.list` (because `typing.List` is @@ -4135,6 +4141,7 @@ def f(x: B[T]) -> T: ... # without T, Any would be used here __slots__ = ( "target", "_fullname", + "module", "alias_tvars", "no_args", "normalized", @@ -4150,6 +4157,7 @@ def __init__( self, target: mypy.types.Type, fullname: str, + module: str, line: int, column: int, *, @@ -4160,6 +4168,7 @@ def __init__( python_3_12_type_alias: bool = False, ) -> None: self._fullname = fullname + self.module = module self.target = target if alias_tvars is None: alias_tvars = [] @@ -4194,6 +4203,7 @@ def from_tuple_type(cls, info: TypeInfo) -> TypeAlias: ) ), info.fullname, + info.module_name, info.line, info.column, ) @@ -4215,6 +4225,7 @@ def from_typeddict_type(cls, info: TypeInfo) -> TypeAlias: ) ), info.fullname, + info.module_name, info.line, info.column, ) @@ -4238,6 +4249,7 @@ def serialize(self) -> JsonDict: data: JsonDict = { ".class": "TypeAlias", "fullname": self._fullname, + "module": self.module, "target": self.target.serialize(), "alias_tvars": [v.serialize() for v in self.alias_tvars], "no_args": self.no_args, @@ -4252,6 +4264,7 @@ def serialize(self) -> JsonDict: def deserialize(cls, data: JsonDict) -> TypeAlias: assert data[".class"] == "TypeAlias" fullname = data["fullname"] + module = data["module"] alias_tvars = [mypy.types.deserialize_type(v) for v in data["alias_tvars"]] assert all(isinstance(t, mypy.types.TypeVarLikeType) for t in alias_tvars) target = mypy.types.deserialize_type(data["target"]) @@ -4263,6 +4276,7 @@ def deserialize(cls, data: JsonDict) -> TypeAlias: return cls( target, fullname, + module, line, column, alias_tvars=cast(list[mypy.types.TypeVarLikeType], alias_tvars), @@ -4274,6 +4288,7 @@ def deserialize(cls, data: JsonDict) -> TypeAlias: def write(self, data: Buffer) -> None: write_tag(data, TYPE_ALIAS) write_str(data, self._fullname) + write_str(data, self.module) self.target.write(data) mypy.types.write_type_list(data, self.alias_tvars) write_int(data, self.line) @@ -4285,11 +4300,13 @@ def write(self, data: Buffer) -> None: @classmethod def read(cls, data: Buffer) -> TypeAlias: fullname = read_str(data) + module = read_str(data) target = mypy.types.read_type(data) alias_tvars = [mypy.types.read_type_var_like(data) for _ in range(read_int(data))] return TypeAlias( target, fullname, + module, read_int(data), read_int(data), alias_tvars=alias_tvars, diff --git a/mypy/semanal.py b/mypy/semanal.py index 50ee3b5324633..b3fd1b98bfd20 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -809,6 +809,7 @@ def create_alias(self, tree: MypyFile, target_name: str, alias: str, name: str) alias_node = TypeAlias( target, alias, + tree.fullname, line=-1, column=-1, # there is no context no_args=True, @@ -3885,16 +3886,15 @@ def analyze_alias( declared_type_vars: TypeVarLikeList | None = None, all_declared_type_params_names: list[str] | None = None, python_3_12_type_alias: bool = False, - ) -> tuple[Type | None, list[TypeVarLikeType], set[str], list[str], bool]: + ) -> tuple[Type | None, list[TypeVarLikeType], set[str], bool]: """Check if 'rvalue' is a valid type allowed for aliasing (e.g. not a type variable). - If yes, return the corresponding type, a list of - qualified type variable names for generic aliases, a set of names the alias depends on, - and a list of type variables if the alias is generic. - A schematic example for the dependencies: + If yes, return the corresponding type, a list of type variables for generic aliases, + a set of names the alias depends on, and True if the original type has empty tuple index. + An example for the dependencies: A = int B = str - analyze_alias(Dict[A, B])[2] == {'__main__.A', '__main__.B'} + analyze_alias(dict[A, B])[2] == {'__main__.A', '__main__.B'} """ dynamic = bool(self.function_stack and self.function_stack[-1].is_dynamic()) global_scope = not self.type and not self.function_stack @@ -3906,10 +3906,9 @@ def analyze_alias( self.fail( "Invalid type alias: expression is not a valid type", rvalue, code=codes.VALID_TYPE ) - return None, [], set(), [], False + return None, [], set(), False found_type_vars = self.find_type_var_likes(typ) - tvar_defs: list[TypeVarLikeType] = [] namespace = self.qualified_name(name) alias_type_vars = found_type_vars if declared_type_vars is None else declared_type_vars with self.tvar_scope_frame(self.tvar_scope.class_frame(namespace)): @@ -3945,9 +3944,8 @@ def analyze_alias( variadic = True new_tvar_defs.append(td) - qualified_tvars = [node.fullname for _name, node in alias_type_vars] empty_tuple_index = typ.empty_tuple_index if isinstance(typ, UnboundType) else False - return analyzed, new_tvar_defs, depends_on, qualified_tvars, empty_tuple_index + return analyzed, new_tvar_defs, depends_on, empty_tuple_index def is_pep_613(self, s: AssignmentStmt) -> bool: if s.unanalyzed_type is not None and isinstance(s.unanalyzed_type, UnboundType): @@ -4042,11 +4040,10 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> bool: res = NoneType() alias_tvars: list[TypeVarLikeType] = [] depends_on: set[str] = set() - qualified_tvars: list[str] = [] empty_tuple_index = False else: tag = self.track_incomplete_refs() - res, alias_tvars, depends_on, qualified_tvars, empty_tuple_index = self.analyze_alias( + res, alias_tvars, depends_on, empty_tuple_index = self.analyze_alias( lvalue.name, rvalue, allow_placeholder=True, @@ -4070,12 +4067,6 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> bool: self.mark_incomplete(lvalue.name, rvalue, becomes_typeinfo=True) return True self.add_type_alias_deps(depends_on) - # In addition to the aliases used, we add deps on unbound - # type variables, since they are erased from target type. - self.add_type_alias_deps(qualified_tvars) - # The above are only direct deps on other aliases. - # For subscripted aliases, type deps from expansion are added in deps.py - # (because the type is stored). check_for_explicit_any(res, self.options, self.is_typeshed_stub_file, self.msg, context=s) # When this type alias gets "inlined", the Any is not explicit anymore, # so we need to replace it with non-explicit Anys. @@ -4106,6 +4097,7 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> bool: alias_node = TypeAlias( res, self.qualified_name(lvalue.name), + self.cur_mod_id, s.line, s.column, alias_tvars=alias_tvars, @@ -5577,7 +5569,7 @@ def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None: return tag = self.track_incomplete_refs() - res, alias_tvars, depends_on, qualified_tvars, empty_tuple_index = self.analyze_alias( + res, alias_tvars, depends_on, empty_tuple_index = self.analyze_alias( s.name.name, s.value.expr(), allow_placeholder=True, @@ -5606,12 +5598,6 @@ def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None: return self.add_type_alias_deps(depends_on) - # In addition to the aliases used, we add deps on unbound - # type variables, since they are erased from target type. - self.add_type_alias_deps(qualified_tvars) - # The above are only direct deps on other aliases. - # For subscripted aliases, type deps from expansion are added in deps.py - # (because the type is stored). check_for_explicit_any( res, self.options, self.is_typeshed_stub_file, self.msg, context=s ) @@ -5627,6 +5613,7 @@ def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None: alias_node = TypeAlias( res, self.qualified_name(s.name.name), + self.cur_mod_id, s.line, s.column, alias_tvars=alias_tvars, @@ -5941,6 +5928,8 @@ def visit_member_expr(self, expr: MemberExpr) -> None: if isinstance(sym.node, PlaceholderNode): self.process_placeholder(expr.name, "attribute", expr) return + if sym.node is not None: + self.record_imported_symbol(sym.node) expr.kind = sym.kind expr.fullname = sym.fullname or "" expr.node = sym.node @@ -5971,8 +5960,7 @@ def visit_member_expr(self, expr: MemberExpr) -> None: if type_info: n = type_info.names.get(expr.name) if n is not None and isinstance(n.node, (MypyFile, TypeInfo, TypeAlias)): - if not n: - return + self.record_imported_symbol(n.node) expr.kind = n.kind expr.fullname = n.fullname or "" expr.node = n.node @@ -6292,6 +6280,37 @@ def visit_class_pattern(self, p: ClassPattern) -> None: def lookup( self, name: str, ctx: Context, suppress_errors: bool = False + ) -> SymbolTableNode | None: + node = self._lookup(name, ctx, suppress_errors) + if node is not None and node.node is not None: + # This call is unfortunate from performance point of view, but + # needed for rare cases like e.g. testIncrementalChangingAlias. + self.record_imported_symbol(node.node) + return node + + def record_imported_symbol(self, node: SymbolNode) -> None: + """If the symbol was not defined in current module, add its module to module_refs.""" + if not node.fullname: + return + if isinstance(node, MypyFile): + fullname = node.fullname + elif isinstance(node, TypeInfo): + fullname = node.module_name + elif isinstance(node, TypeAlias): + fullname = node.module + elif isinstance(node, (Var, FuncDef, OverloadedFuncDef)) and node.info: + fullname = node.info.module_name + else: + fullname = node.fullname.rsplit(".")[0] + if fullname == self.cur_mod_id: + return + while "." in fullname and fullname not in self.modules: + fullname = fullname.rsplit(".")[0] + if fullname != self.cur_mod_id: + self.cur_mod_node.module_refs.add(fullname) + + def _lookup( + self, name: str, ctx: Context, suppress_errors: bool = False ) -> SymbolTableNode | None: """Look up an unqualified (no dots) name in all active namespaces. @@ -6500,6 +6519,8 @@ def lookup_qualified( self.name_not_defined(name, ctx, namespace=namespace) return None sym = nextsym + if sym is not None and sym.node is not None: + self.record_imported_symbol(sym.node) return sym def lookup_type_node(self, expr: Expression) -> SymbolTableNode | None: @@ -7546,8 +7567,6 @@ def add_type_alias_deps( If `target` is None, then the target node used will be the current scope. """ if not aliases_used: - # A basic optimization to avoid adding targets with no dependencies to - # the `alias_deps` dict. return if target is None: target = self.scope.current_target() diff --git a/mypy/server/deps.py b/mypy/server/deps.py index 9d4445a1f7e74..076d95e2baf98 100644 --- a/mypy/server/deps.py +++ b/mypy/server/deps.py @@ -227,14 +227,17 @@ def __init__( self.scope = Scope() self.type_map = type_map # This attribute holds a mapping from target to names of type aliases - # it depends on. These need to be processed specially, since they are - # only present in expanded form in symbol tables. For example, after: - # A = List[int] + # it depends on. These need to be processed specially, since they may + # appear in expanded form in symbol tables, because of a get_proper_type() + # somewhere. For example, after: + # A = int # x: A - # The module symbol table will just have a Var `x` with type `List[int]`, - # and the dependency of `x` on `A` is lost. Therefore the alias dependencies + # the module symbol table will just have a Var `x` with type `int`, + # and the dependency of `x` on `A` is lost. Therefore, the alias dependencies # are preserved at alias expansion points in `semanal.py`, stored as an attribute # on MypyFile, and then passed here. + # TODO: fine-grained is more susceptible to this partially because we are reckless + # about get_proper_type() in *this specific file*. self.alias_deps = alias_deps self.map: dict[str, set[str]] = {} self.is_class = False @@ -979,8 +982,6 @@ def visit_type_alias_type(self, typ: TypeAliasType) -> list[str]: triggers = [trigger] for arg in typ.args: triggers.extend(self.get_type_triggers(arg)) - # TODO: Now that type aliases are its own kind of types we can simplify - # the logic to rely on intermediate dependencies (like for instance types). triggers.extend(self.get_type_triggers(typ.alias.target)) return triggers diff --git a/mypy/test/typefixture.py b/mypy/test/typefixture.py index 0defcdaebc990..f70c8b94f09c5 100644 --- a/mypy/test/typefixture.py +++ b/mypy/test/typefixture.py @@ -374,7 +374,7 @@ def def_alias_1(self, base: Instance) -> tuple[TypeAliasType, Type]: target = Instance( self.std_tuplei, [UnionType([base, A])] ) # A = Tuple[Union[base, A], ...] - AN = TypeAlias(target, "__main__.A", -1, -1) + AN = TypeAlias(target, "__main__.A", "__main__", -1, -1) A.alias = AN return A, target @@ -383,7 +383,7 @@ def def_alias_2(self, base: Instance) -> tuple[TypeAliasType, Type]: target = UnionType( [base, Instance(self.std_tuplei, [A])] ) # A = Union[base, Tuple[A, ...]] - AN = TypeAlias(target, "__main__.A", -1, -1) + AN = TypeAlias(target, "__main__.A", "__main__", -1, -1) A.alias = AN return A, target @@ -393,7 +393,7 @@ def non_rec_alias( alias_tvars: list[TypeVarLikeType] | None = None, args: list[Type] | None = None, ) -> TypeAliasType: - AN = TypeAlias(target, "__main__.A", -1, -1, alias_tvars=alias_tvars) + AN = TypeAlias(target, "__main__.A", "__main__", -1, -1, alias_tvars=alias_tvars) if args is None: args = [] return TypeAliasType(AN, args) diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index d1155f54a75d1..06f228721a868 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -6171,6 +6171,118 @@ class Base: [out2] main:5: error: Call to abstract method "meth" of "Base" with trivial body via super() is unsafe +[case testLiteralCoarseGrainedChainedAliases] +from mod1 import Alias1 +from typing import Literal +x: Alias1 +def expect_int(x: int) -> None: pass +expect_int(x) +[file mod1.py] +from mod2 import Alias2 +Alias1 = Alias2 +[file mod2.py] +from mod3 import Alias3 +Alias2 = Alias3 +[file mod3.py] +from typing import Literal +Alias3 = int +[file mod3.py.2] +from typing import Literal +Alias3 = str +[builtins fixtures/tuple.pyi] +[out] +[out2] +main:5: error: Argument 1 to "expect_int" has incompatible type "str"; expected "int" + +[case testLiteralCoarseGrainedChainedAliases2] +from mod1 import Alias1 +from typing import Literal +x: Alias1 +def expect_3(x: Literal[3]) -> None: pass +expect_3(x) +[file mod1.py] +from mod2 import Alias2 +Alias1 = Alias2 +[file mod2.py] +from mod3 import Alias3 +Alias2 = Alias3 +[file mod3.py] +from typing import Literal +Alias3 = Literal[3] +[file mod3.py.2] +from typing import Literal +Alias3 = Literal[4] +[builtins fixtures/tuple.pyi] +[out] +[out2] +main:5: error: Argument 1 to "expect_3" has incompatible type "Literal[4]"; expected "Literal[3]" + +[case testDoubleReexportFunctionUpdated] +import m + +[file m.py] +import f +[file m.py.3] +import f +reveal_type(f.foo) + +[file f.py] +import c +def foo(arg: c.C) -> None: pass + +[file c.py] +from types import C + +[file types.py] +import pb1 +C = pb1.C +[file types.py.2] +import pb1, pb2 +C = pb2.C + +[file pb1.py] +class C: ... +[file pb2.py] +class C: ... +[out] +[out2] +[out3] +tmp/m.py:2: note: Revealed type is "def (arg: pb2.C)" + +[case testDoubleReexportGenericUpdated] +import m + +[file m.py] +import f +[file m.py.3] +import f +x: f.F +reveal_type(x[0]) + +[file f.py] +import c +class FB(list[c.C]): ... +class F(FB): ... + +[file c.py] +from types import C + +[file types.py] +import pb1 +C = pb1.C +[file types.py.2] +import pb1, pb2 +C = pb2.C + +[file pb1.py] +class C: ... +[file pb2.py] +class C: ... +[out] +[out2] +[out3] +tmp/m.py:3: note: Revealed type is "pb2.C" + [case testNoCrashDoubleReexportFunctionEmpty] import m @@ -6203,6 +6315,38 @@ class C: ... [out2] [out3] +[case testNoCrashDoubleReexportAliasEmpty] +import m + +[file m.py] +import f +[file m.py.3] +import f +# modify + +[file f.py] +import c +D = list[c.C] + +[file c.py] +from types import C + +[file types.py] +import pb1 +C = pb1.C +[file types.py.2] +import pb2 +C = pb2.C + +[file pb1.py] +class C: ... +[file pb2.py.2] +class C: ... +[file pb1.py.2] +[out] +[out2] +[out3] + [case testNoCrashDoubleReexportBaseEmpty] import m @@ -6235,6 +6379,69 @@ class C: ... [out2] [out3] +[case testNoCrashDoubleReexportBaseEmpty2] +import m + +[file m.py] +import f +[file m.py.3] +import f +# modify + +[file f.py] +import c +class D(c.C): pass + +[file c.py] +from types import C + +[file types.py] +import pb1 +C = pb1.C +[file types.py.2] +import pb2 +C = pb2.C + +[file pb1.py] +class C: ... +[file pb2.py.2] +class C: ... +[file pb1.py.2] +[out] +[out2] +[out3] + +[case testDoubleReexportMetaUpdated] +import m +class C(metaclass=m.M): ... + +[file m.py] +from types import M + +[file types.py] +class M(type): ... +[file types.py.2] +class M: ... +[out] +[out2] +main:2: error: Metaclasses not inheriting from "type" are not supported + +[case testIncrementalOkChangeWithSave2] +import mod1 +x: int = mod1.x + +[file mod1.py] +from mod2 import x + +[file mod2.py] +x = 1 + +[file mod2.py.2] +x = "no way" +[out] +[out2] +main:2: error: Incompatible types in assignment (expression has type "str", variable has type "int") + [case testNoCrashDoubleReexportMetaEmpty] import m @@ -6267,6 +6474,38 @@ class C(type): ... [out2] [out3] +[case testNoCrashDoubleReexportMetaEmpty2] +import m + +[file m.py] +import f +[file m.py.3] +import f +# modify + +[file f.py] +import c +class D(metaclass=c.C): pass + +[file c.py] +from types import C + +[file types.py] +import pb1 +C = pb1.C +[file types.py.2] +import pb2 +C = pb2.C + +[file pb1.py] +class C(type): ... +[file pb2.py.2] +class C(type): ... +[file pb1.py.2] +[out] +[out2] +[out3] + [case testNoCrashDoubleReexportTypedDictEmpty] import m From 4f872d7076c6d0803b6d12d824d33a477ce36449 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Tue, 9 Sep 2025 12:04:36 -0400 Subject: [PATCH 250/424] [mypyc] feat: cache len for container creation from expressions with length known at compile time (#19503) Currently, if a user uses an immutable type as the sequence input for a for loop, the length is checked once at each iteration which, while necessary for some container types such as list and dictionaries, is not necessary for iterating over immutable types tuple, str, and bytes. This PR modifies the codebase such that the length is only checked at the first iteration, and reused from there. Also, in cases where a simple genexp is the input argument for a tuple, the length is currently checked one additional time before entering the iteration (this is done to determine how to size the new tuple). In those cases, we don't even need a length check at the first iteration step, and can reuse the result of that first `len` call (or compile-time determined constant) instead. Lastly, in cases where a tuple is created from a genexp and the length of the genexp is knowable at compile time, this PR replaces PyList_AsTuple with the tuple constructor fast-path. --- mypyc/irbuild/for_helpers.py | 62 ++++- mypyc/test-data/irbuild-generics.test | 117 ++++---- mypyc/test-data/irbuild-lists.test | 342 +++++++++++++---------- mypyc/test-data/irbuild-tuple.test | 377 ++++++++++++++------------ mypyc/test-data/run-lists.test | 6 + mypyc/test-data/run-tuples.test | 7 + 6 files changed, 529 insertions(+), 382 deletions(-) diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index 762b41866a057..5edee6cb4df40 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -11,17 +11,22 @@ from mypy.nodes import ( ARG_POS, + BytesExpr, CallExpr, DictionaryComprehension, Expression, GeneratorExpr, + ListExpr, Lvalue, MemberExpr, NameExpr, RefExpr, SetExpr, + StarExpr, + StrExpr, TupleExpr, TypeAlias, + Var, ) from mypyc.ir.ops import ( ERR_NEVER, @@ -152,6 +157,7 @@ def for_loop_helper_with_index( expr_reg: Value, body_insts: Callable[[Value], None], line: int, + length: Value, ) -> None: """Generate IR for a sequence iteration. @@ -173,7 +179,7 @@ def for_loop_helper_with_index( condition_block = BasicBlock() for_gen = ForSequence(builder, index, body_block, exit_block, line, False) - for_gen.init(expr_reg, target_type, reverse=False) + for_gen.init(expr_reg, target_type, reverse=False, length=length) builder.push_loop_stack(step_block, exit_block) @@ -227,7 +233,9 @@ def sequence_from_generator_preallocate_helper( rtype = builder.node_type(gen.sequences[0]) if is_sequence_rprimitive(rtype): sequence = builder.accept(gen.sequences[0]) - length = builder.builder.builtin_len(sequence, gen.line, use_pyssize_t=True) + length = get_expr_length_value( + builder, gen.sequences[0], sequence, gen.line, use_pyssize_t=True + ) target_op = empty_op_llbuilder(length, gen.line) def set_item(item_index: Value) -> None: @@ -235,7 +243,7 @@ def set_item(item_index: Value) -> None: builder.call_c(set_item_op, [target_op, item_index, e], gen.line) for_loop_helper_with_index( - builder, gen.indices[0], gen.sequences[0], sequence, set_item, gen.line + builder, gen.indices[0], gen.sequences[0], sequence, set_item, gen.line, length ) return target_op @@ -788,9 +796,13 @@ class ForSequence(ForGenerator): length_reg: Value | AssignmentTarget | None - def init(self, expr_reg: Value, target_type: RType, reverse: bool) -> None: + def init( + self, expr_reg: Value, target_type: RType, reverse: bool, length: Value | None = None + ) -> None: assert is_sequence_rprimitive(expr_reg.type), expr_reg builder = self.builder + # Record a Value indicating the length of the sequence, if known at compile time. + self.length = length self.reverse = reverse # Define target to contain the expression, along with the index that will be used # for the for-loop. If we are inside of a generator function, spill these into the @@ -798,7 +810,7 @@ def init(self, expr_reg: Value, target_type: RType, reverse: bool) -> None: self.expr_target = builder.maybe_spill(expr_reg) if is_immutable_rprimitive(expr_reg.type): # If the expression is an immutable type, we can load the length just once. - self.length_reg = builder.maybe_spill(self.load_len(self.expr_target)) + self.length_reg = builder.maybe_spill(self.length or self.load_len(self.expr_target)) else: # Otherwise, even if the length is known, we must recalculate the length # at every iteration for compatibility with python semantics. @@ -1166,3 +1178,43 @@ def gen_step(self) -> None: def gen_cleanup(self) -> None: for gen in self.gens: gen.gen_cleanup() + + +def get_expr_length(expr: Expression) -> int | None: + if isinstance(expr, (StrExpr, BytesExpr)): + return len(expr.value) + elif isinstance(expr, (ListExpr, TupleExpr)): + # if there are no star expressions, or we know the length of them, + # we know the length of the expression + stars = [get_expr_length(i) for i in expr.items if isinstance(i, StarExpr)] + if None not in stars: + other = sum(not isinstance(i, StarExpr) for i in expr.items) + return other + sum(stars) # type: ignore [arg-type] + elif isinstance(expr, StarExpr): + return get_expr_length(expr.expr) + elif ( + isinstance(expr, RefExpr) + and isinstance(expr.node, Var) + and expr.node.is_final + and isinstance(expr.node.final_value, str) + and expr.node.has_explicit_value + ): + return len(expr.node.final_value) + # TODO: extend this, passing length of listcomp and genexp should have worthwhile + # performance boost and can be (sometimes) figured out pretty easily. set and dict + # comps *can* be done as well but will need special logic to consider the possibility + # of key conflicts. Range, enumerate, zip are all simple logic. + return None + + +def get_expr_length_value( + builder: IRBuilder, expr: Expression, expr_reg: Value, line: int, use_pyssize_t: bool +) -> Value: + rtype = builder.node_type(expr) + assert is_sequence_rprimitive(rtype), rtype + length = get_expr_length(expr) + if length is None: + # We cannot compute the length at compile time, so we will fetch it. + return builder.builder.builtin_len(expr_reg, line, use_pyssize_t=use_pyssize_t) + # The expression result is known at compile time, so we can use a constant. + return Integer(length, c_pyssize_t_rprimitive if use_pyssize_t else short_int_rprimitive) diff --git a/mypyc/test-data/irbuild-generics.test b/mypyc/test-data/irbuild-generics.test index 96437a0079c96..9ec29182e89b6 100644 --- a/mypyc/test-data/irbuild-generics.test +++ b/mypyc/test-data/irbuild-generics.test @@ -678,84 +678,83 @@ def inner_deco_obj.__call__(__mypyc_self__, args, kwargs): r0 :: __main__.deco_env r1 :: native_int r2 :: list - r3, r4 :: native_int - r5 :: bit - r6, x :: object - r7 :: native_int + r3 :: native_int + r4 :: bit + r5, x :: object + r6 :: native_int can_listcomp :: list - r8 :: dict - r9 :: short_int - r10 :: native_int - r11 :: object - r12 :: tuple[bool, short_int, object, object] - r13 :: short_int - r14 :: bool - r15, r16 :: object - r17, k :: str + r7 :: dict + r8 :: short_int + r9 :: native_int + r10 :: object + r11 :: tuple[bool, short_int, object, object] + r12 :: short_int + r13 :: bool + r14, r15 :: object + r16, k :: str v :: object - r18 :: i32 - r19, r20, r21 :: bit + r17 :: i32 + r18, r19, r20 :: bit can_dictcomp :: dict - r22, can_iter, r23, can_use_keys, r24, can_use_values :: list - r25 :: object - r26 :: dict - r27 :: object - r28 :: int + r21, can_iter, r22, can_use_keys, r23, can_use_values :: list + r24 :: object + r25 :: dict + r26 :: object + r27 :: int L0: r0 = __mypyc_self__.__mypyc_env__ r1 = var_object_size args r2 = PyList_New(r1) - r3 = var_object_size args - r4 = 0 + r3 = 0 L1: - r5 = r4 < r3 :: signed - if r5 goto L2 else goto L4 :: bool + r4 = r3 < r1 :: signed + if r4 goto L2 else goto L4 :: bool L2: - r6 = CPySequenceTuple_GetItemUnsafe(args, r4) - x = r6 - CPyList_SetItemUnsafe(r2, r4, x) + r5 = CPySequenceTuple_GetItemUnsafe(args, r3) + x = r5 + CPyList_SetItemUnsafe(r2, r3, x) L3: - r7 = r4 + 1 - r4 = r7 + r6 = r3 + 1 + r3 = r6 goto L1 L4: can_listcomp = r2 - r8 = PyDict_New() - r9 = 0 - r10 = PyDict_Size(kwargs) - r11 = CPyDict_GetItemsIter(kwargs) + r7 = PyDict_New() + r8 = 0 + r9 = PyDict_Size(kwargs) + r10 = CPyDict_GetItemsIter(kwargs) L5: - r12 = CPyDict_NextItem(r11, r9) - r13 = r12[1] - r9 = r13 - r14 = r12[0] - if r14 goto L6 else goto L8 :: bool + r11 = CPyDict_NextItem(r10, r8) + r12 = r11[1] + r8 = r12 + r13 = r11[0] + if r13 goto L6 else goto L8 :: bool L6: - r15 = r12[2] - r16 = r12[3] - r17 = cast(str, r15) - k = r17 - v = r16 - r18 = PyDict_SetItem(r8, k, v) - r19 = r18 >= 0 :: signed + r14 = r11[2] + r15 = r11[3] + r16 = cast(str, r14) + k = r16 + v = r15 + r17 = PyDict_SetItem(r7, k, v) + r18 = r17 >= 0 :: signed L7: - r20 = CPyDict_CheckSize(kwargs, r10) + r19 = CPyDict_CheckSize(kwargs, r9) goto L5 L8: - r21 = CPy_NoErrOccurred() + r20 = CPy_NoErrOccurred() L9: - can_dictcomp = r8 - r22 = PySequence_List(kwargs) - can_iter = r22 - r23 = CPyDict_Keys(kwargs) - can_use_keys = r23 - r24 = CPyDict_Values(kwargs) - can_use_values = r24 - r25 = r0.func - r26 = PyDict_Copy(kwargs) - r27 = PyObject_Call(r25, args, r26) - r28 = unbox(int, r27) - return r28 + can_dictcomp = r7 + r21 = PySequence_List(kwargs) + can_iter = r21 + r22 = CPyDict_Keys(kwargs) + can_use_keys = r22 + r23 = CPyDict_Values(kwargs) + can_use_values = r23 + r24 = r0.func + r25 = PyDict_Copy(kwargs) + r26 = PyObject_Call(r24, args, r25) + r27 = unbox(int, r26) + return r27 def deco(func): func :: object r0 :: __main__.deco_env diff --git a/mypyc/test-data/irbuild-lists.test b/mypyc/test-data/irbuild-lists.test index d83fb88390db4..2f5b3b39319e7 100644 --- a/mypyc/test-data/irbuild-lists.test +++ b/mypyc/test-data/irbuild-lists.test @@ -594,10 +594,8 @@ def test(): r3 :: list r4 :: native_int r5 :: bit - r6 :: native_int - r7 :: bit - r8, x, r9 :: str - r10 :: native_int + r6, x, r7 :: str + r8 :: native_int a :: list L0: r0 = 'abc' @@ -605,20 +603,18 @@ L0: r1 = CPyStr_Size_size_t(source) r2 = r1 >= 0 :: signed r3 = PyList_New(r1) - r4 = CPyStr_Size_size_t(source) - r5 = r4 >= 0 :: signed - r6 = 0 + r4 = 0 L1: - r7 = r6 < r4 :: signed - if r7 goto L2 else goto L4 :: bool + r5 = r4 < r1 :: signed + if r5 goto L2 else goto L4 :: bool L2: - r8 = CPyStr_GetItemUnsafe(source, r6) - x = r8 - r9 = f2(x) - CPyList_SetItemUnsafe(r3, r6, r9) + r6 = CPyStr_GetItemUnsafe(source, r4) + x = r6 + r7 = f2(x) + CPyList_SetItemUnsafe(r3, r4, r7) L3: - r10 = r6 + 1 - r6 = r10 + r8 = r4 + 1 + r4 = r8 goto L1 L4: a = r3 @@ -639,38 +635,30 @@ L0: return r1 def test(): r0 :: str - r1 :: native_int - r2 :: bit - r3 :: list - r4 :: native_int - r5 :: bit + r1 :: list + r2 :: native_int + r3 :: bit + r4, x, r5 :: str r6 :: native_int - r7 :: bit - r8, x, r9 :: str - r10 :: native_int a :: list L0: r0 = 'abc' - r1 = CPyStr_Size_size_t(r0) - r2 = r1 >= 0 :: signed - r3 = PyList_New(r1) - r4 = CPyStr_Size_size_t(r0) - r5 = r4 >= 0 :: signed - r6 = 0 + r1 = PyList_New(3) + r2 = 0 L1: - r7 = r6 < r4 :: signed - if r7 goto L2 else goto L4 :: bool + r3 = r2 < 3 :: signed + if r3 goto L2 else goto L4 :: bool L2: - r8 = CPyStr_GetItemUnsafe(r0, r6) - x = r8 - r9 = f2(x) - CPyList_SetItemUnsafe(r3, r6, r9) + r4 = CPyStr_GetItemUnsafe(r0, r2) + x = r4 + r5 = f2(x) + CPyList_SetItemUnsafe(r1, r2, r5) L3: - r10 = r6 + 1 - r6 = r10 + r6 = r2 + 1 + r2 = r6 goto L1 L4: - a = r3 + a = r1 return 1 [case testListBuiltFromFinalStr] @@ -692,38 +680,30 @@ L0: return r1 def test(): r0 :: str - r1 :: native_int - r2 :: bit - r3 :: list - r4 :: native_int - r5 :: bit + r1 :: list + r2 :: native_int + r3 :: bit + r4, x, r5 :: str r6 :: native_int - r7 :: bit - r8, x, r9 :: str - r10 :: native_int a :: list L0: r0 = 'abc' - r1 = CPyStr_Size_size_t(r0) - r2 = r1 >= 0 :: signed - r3 = PyList_New(r1) - r4 = CPyStr_Size_size_t(r0) - r5 = r4 >= 0 :: signed - r6 = 0 + r1 = PyList_New(3) + r2 = 0 L1: - r7 = r6 < r4 :: signed - if r7 goto L2 else goto L4 :: bool + r3 = r2 < 3 :: signed + if r3 goto L2 else goto L4 :: bool L2: - r8 = CPyStr_GetItemUnsafe(r0, r6) - x = r8 - r9 = f2(x) - CPyList_SetItemUnsafe(r3, r6, r9) + r4 = CPyStr_GetItemUnsafe(r0, r2) + x = r4 + r5 = f2(x) + CPyList_SetItemUnsafe(r1, r2, r5) L3: - r10 = r6 + 1 - r6 = r10 + r6 = r2 + 1 + r2 = r6 goto L1 L4: - a = r3 + a = r1 return 1 [case testListBuiltFromBytes_64bit] @@ -744,48 +724,47 @@ def test(): r0, source :: bytes r1 :: native_int r2 :: list - r3, r4 :: native_int - r5, r6, r7 :: bit - r8, r9, r10, r11 :: int - r12 :: object - r13, x, r14 :: int - r15 :: object - r16 :: native_int + r3 :: native_int + r4, r5, r6 :: bit + r7, r8, r9, r10 :: int + r11 :: object + r12, x, r13 :: int + r14 :: object + r15 :: native_int a :: list L0: r0 = b'abc' source = r0 r1 = var_object_size source r2 = PyList_New(r1) - r3 = var_object_size source - r4 = 0 + r3 = 0 L1: - r5 = r4 < r3 :: signed - if r5 goto L2 else goto L8 :: bool + r4 = r3 < r1 :: signed + if r4 goto L2 else goto L8 :: bool L2: - r6 = r4 <= 4611686018427387903 :: signed - if r6 goto L3 else goto L4 :: bool + r5 = r3 <= 4611686018427387903 :: signed + if r5 goto L3 else goto L4 :: bool L3: - r7 = r4 >= -4611686018427387904 :: signed - if r7 goto L5 else goto L4 :: bool + r6 = r3 >= -4611686018427387904 :: signed + if r6 goto L5 else goto L4 :: bool L4: - r8 = CPyTagged_FromInt64(r4) - r9 = r8 + r7 = CPyTagged_FromInt64(r3) + r8 = r7 goto L6 L5: - r10 = r4 << 1 - r9 = r10 + r9 = r3 << 1 + r8 = r9 L6: - r11 = CPyBytes_GetItem(source, r9) - r12 = box(int, r11) - r13 = unbox(int, r12) - x = r13 - r14 = f2(x) - r15 = box(int, r14) - CPyList_SetItemUnsafe(r2, r4, r15) + r10 = CPyBytes_GetItem(source, r8) + r11 = box(int, r10) + r12 = unbox(int, r11) + x = r12 + r13 = f2(x) + r14 = box(int, r13) + CPyList_SetItemUnsafe(r2, r3, r14) L7: - r16 = r4 + 1 - r4 = r16 + r15 = r3 + 1 + r3 = r15 goto L1 L8: a = r2 @@ -806,52 +785,49 @@ L0: return r0 def test(): r0 :: bytes - r1 :: native_int - r2 :: list - r3, r4 :: native_int - r5, r6, r7 :: bit - r8, r9, r10, r11 :: int - r12 :: object - r13, x, r14 :: int - r15 :: object - r16 :: native_int + r1 :: list + r2 :: native_int + r3, r4, r5 :: bit + r6, r7, r8, r9 :: int + r10 :: object + r11, x, r12 :: int + r13 :: object + r14 :: native_int a :: list L0: r0 = b'abc' - r1 = var_object_size r0 - r2 = PyList_New(r1) - r3 = var_object_size r0 - r4 = 0 + r1 = PyList_New(3) + r2 = 0 L1: - r5 = r4 < r3 :: signed - if r5 goto L2 else goto L8 :: bool + r3 = r2 < 3 :: signed + if r3 goto L2 else goto L8 :: bool L2: - r6 = r4 <= 4611686018427387903 :: signed - if r6 goto L3 else goto L4 :: bool + r4 = r2 <= 4611686018427387903 :: signed + if r4 goto L3 else goto L4 :: bool L3: - r7 = r4 >= -4611686018427387904 :: signed - if r7 goto L5 else goto L4 :: bool + r5 = r2 >= -4611686018427387904 :: signed + if r5 goto L5 else goto L4 :: bool L4: - r8 = CPyTagged_FromInt64(r4) - r9 = r8 + r6 = CPyTagged_FromInt64(r2) + r7 = r6 goto L6 L5: - r10 = r4 << 1 - r9 = r10 + r8 = r2 << 1 + r7 = r8 L6: - r11 = CPyBytes_GetItem(r0, r9) - r12 = box(int, r11) - r13 = unbox(int, r12) - x = r13 - r14 = f2(x) - r15 = box(int, r14) - CPyList_SetItemUnsafe(r2, r4, r15) + r9 = CPyBytes_GetItem(r0, r7) + r10 = box(int, r9) + r11 = unbox(int, r10) + x = r11 + r12 = f2(x) + r13 = box(int, r12) + CPyList_SetItemUnsafe(r1, r2, r13) L7: - r16 = r4 + 1 - r4 = r16 + r14 = r2 + 1 + r2 = r14 goto L1 L8: - a = r2 + a = r1 return 1 [case testListBuiltFromFinalBytes_64bit] @@ -876,13 +852,13 @@ def test(): r1 :: bool r2 :: native_int r3 :: list - r4, r5 :: native_int - r6, r7, r8 :: bit - r9, r10, r11, r12 :: int - r13 :: object - r14, x, r15 :: int - r16 :: object - r17 :: native_int + r4 :: native_int + r5, r6, r7 :: bit + r8, r9, r10, r11 :: int + r12 :: object + r13, x, r14 :: int + r15 :: object + r16 :: native_int a :: list L0: r0 = __main__.source :: static @@ -893,36 +869,102 @@ L1: L2: r2 = var_object_size r0 r3 = PyList_New(r2) - r4 = var_object_size r0 - r5 = 0 + r4 = 0 L3: - r6 = r5 < r4 :: signed - if r6 goto L4 else goto L10 :: bool + r5 = r4 < r2 :: signed + if r5 goto L4 else goto L10 :: bool L4: - r7 = r5 <= 4611686018427387903 :: signed - if r7 goto L5 else goto L6 :: bool + r6 = r4 <= 4611686018427387903 :: signed + if r6 goto L5 else goto L6 :: bool L5: - r8 = r5 >= -4611686018427387904 :: signed - if r8 goto L7 else goto L6 :: bool + r7 = r4 >= -4611686018427387904 :: signed + if r7 goto L7 else goto L6 :: bool L6: - r9 = CPyTagged_FromInt64(r5) - r10 = r9 + r8 = CPyTagged_FromInt64(r4) + r9 = r8 goto L8 L7: - r11 = r5 << 1 - r10 = r11 + r10 = r4 << 1 + r9 = r10 L8: - r12 = CPyBytes_GetItem(r0, r10) - r13 = box(int, r12) - r14 = unbox(int, r13) - x = r14 - r15 = f2(x) - r16 = box(int, r15) - CPyList_SetItemUnsafe(r3, r5, r16) + r11 = CPyBytes_GetItem(r0, r9) + r12 = box(int, r11) + r13 = unbox(int, r12) + x = r13 + r14 = f2(x) + r15 = box(int, r14) + CPyList_SetItemUnsafe(r3, r4, r15) L9: - r17 = r5 + 1 - r5 = r17 + r16 = r4 + 1 + r4 = r16 goto L3 L10: a = r3 return 1 + +[case testListBuiltFromStars] +from typing import Final + +abc: Final = "abc" + +def test() -> None: + a = [str(x) for x in [*abc, *"def", *b"ghi", ("j", "k"), *("l", "m", "n")]] + +[out] +def test(): + r0, r1 :: str + r2 :: bytes + r3, r4 :: str + r5 :: tuple[str, str] + r6, r7, r8 :: str + r9 :: tuple[str, str, str] + r10 :: list + r11, r12, r13, r14 :: object + r15 :: i32 + r16 :: bit + r17, r18 :: object + r19 :: list + r20, r21 :: native_int + r22 :: bit + r23, x :: object + r24 :: str + r25 :: native_int + a :: list +L0: + r0 = 'abc' + r1 = 'def' + r2 = b'ghi' + r3 = 'j' + r4 = 'k' + r5 = (r3, r4) + r6 = 'l' + r7 = 'm' + r8 = 'n' + r9 = (r6, r7, r8) + r10 = PyList_New(0) + r11 = CPyList_Extend(r10, r0) + r12 = CPyList_Extend(r10, r1) + r13 = CPyList_Extend(r10, r2) + r14 = box(tuple[str, str], r5) + r15 = PyList_Append(r10, r14) + r16 = r15 >= 0 :: signed + r17 = box(tuple[str, str, str], r9) + r18 = CPyList_Extend(r10, r17) + r19 = PyList_New(13) + r20 = 0 +L1: + r21 = var_object_size r10 + r22 = r20 < r21 :: signed + if r22 goto L2 else goto L4 :: bool +L2: + r23 = list_get_item_unsafe r10, r20 + x = r23 + r24 = PyObject_Str(x) + CPyList_SetItemUnsafe(r19, r20, r24) +L3: + r25 = r20 + 1 + r20 = r25 + goto L1 +L4: + a = r19 + return 1 diff --git a/mypyc/test-data/irbuild-tuple.test b/mypyc/test-data/irbuild-tuple.test index 00ea7f074a5d8..081cc1b174c9b 100644 --- a/mypyc/test-data/irbuild-tuple.test +++ b/mypyc/test-data/irbuild-tuple.test @@ -365,10 +365,8 @@ def test(): r3 :: tuple r4 :: native_int r5 :: bit - r6 :: native_int - r7 :: bit - r8, x, r9 :: str - r10 :: native_int + r6, x, r7 :: str + r8 :: native_int a :: tuple L0: r0 = 'abc' @@ -376,20 +374,18 @@ L0: r1 = CPyStr_Size_size_t(source) r2 = r1 >= 0 :: signed r3 = PyTuple_New(r1) - r4 = CPyStr_Size_size_t(source) - r5 = r4 >= 0 :: signed - r6 = 0 + r4 = 0 L1: - r7 = r6 < r4 :: signed - if r7 goto L2 else goto L4 :: bool + r5 = r4 < r1 :: signed + if r5 goto L2 else goto L4 :: bool L2: - r8 = CPyStr_GetItemUnsafe(source, r6) - x = r8 - r9 = f2(x) - CPySequenceTuple_SetItemUnsafe(r3, r6, r9) + r6 = CPyStr_GetItemUnsafe(source, r4) + x = r6 + r7 = f2(x) + CPySequenceTuple_SetItemUnsafe(r3, r4, r7) L3: - r10 = r6 + 1 - r6 = r10 + r8 = r4 + 1 + r4 = r8 goto L1 L4: a = r3 @@ -411,38 +407,30 @@ L0: return r1 def test(): r0 :: str - r1 :: native_int - r2 :: bit - r3 :: tuple - r4 :: native_int - r5 :: bit + r1 :: tuple + r2 :: native_int + r3 :: bit + r4, x, r5 :: str r6 :: native_int - r7 :: bit - r8, x, r9 :: str - r10 :: native_int a :: tuple L0: r0 = 'abc' - r1 = CPyStr_Size_size_t(r0) - r2 = r1 >= 0 :: signed - r3 = PyTuple_New(r1) - r4 = CPyStr_Size_size_t(r0) - r5 = r4 >= 0 :: signed - r6 = 0 + r1 = PyTuple_New(3) + r2 = 0 L1: - r7 = r6 < r4 :: signed - if r7 goto L2 else goto L4 :: bool + r3 = r2 < 3 :: signed + if r3 goto L2 else goto L4 :: bool L2: - r8 = CPyStr_GetItemUnsafe(r0, r6) - x = r8 - r9 = f2(x) - CPySequenceTuple_SetItemUnsafe(r3, r6, r9) + r4 = CPyStr_GetItemUnsafe(r0, r2) + x = r4 + r5 = f2(x) + CPySequenceTuple_SetItemUnsafe(r1, r2, r5) L3: - r10 = r6 + 1 - r6 = r10 + r6 = r2 + 1 + r2 = r6 goto L1 L4: - a = r3 + a = r1 return 1 [case testTupleBuiltFromFinalStr] @@ -464,38 +452,30 @@ L0: return r1 def test(): r0 :: str - r1 :: native_int - r2 :: bit - r3 :: tuple - r4 :: native_int - r5 :: bit + r1 :: tuple + r2 :: native_int + r3 :: bit + r4, x, r5 :: str r6 :: native_int - r7 :: bit - r8, x, r9 :: str - r10 :: native_int a :: tuple L0: r0 = 'abc' - r1 = CPyStr_Size_size_t(r0) - r2 = r1 >= 0 :: signed - r3 = PyTuple_New(r1) - r4 = CPyStr_Size_size_t(r0) - r5 = r4 >= 0 :: signed - r6 = 0 + r1 = PyTuple_New(3) + r2 = 0 L1: - r7 = r6 < r4 :: signed - if r7 goto L2 else goto L4 :: bool + r3 = r2 < 3 :: signed + if r3 goto L2 else goto L4 :: bool L2: - r8 = CPyStr_GetItemUnsafe(r0, r6) - x = r8 - r9 = f2(x) - CPySequenceTuple_SetItemUnsafe(r3, r6, r9) + r4 = CPyStr_GetItemUnsafe(r0, r2) + x = r4 + r5 = f2(x) + CPySequenceTuple_SetItemUnsafe(r1, r2, r5) L3: - r10 = r6 + 1 - r6 = r10 + r6 = r2 + 1 + r2 = r6 goto L1 L4: - a = r3 + a = r1 return 1 [case testTupleBuiltFromBytes_64bit] @@ -516,48 +496,47 @@ def test(): r0, source :: bytes r1 :: native_int r2 :: tuple - r3, r4 :: native_int - r5, r6, r7 :: bit - r8, r9, r10, r11 :: int - r12 :: object - r13, x, r14 :: int - r15 :: object - r16 :: native_int + r3 :: native_int + r4, r5, r6 :: bit + r7, r8, r9, r10 :: int + r11 :: object + r12, x, r13 :: int + r14 :: object + r15 :: native_int a :: tuple L0: r0 = b'abc' source = r0 r1 = var_object_size source r2 = PyTuple_New(r1) - r3 = var_object_size source - r4 = 0 + r3 = 0 L1: - r5 = r4 < r3 :: signed - if r5 goto L2 else goto L8 :: bool + r4 = r3 < r1 :: signed + if r4 goto L2 else goto L8 :: bool L2: - r6 = r4 <= 4611686018427387903 :: signed - if r6 goto L3 else goto L4 :: bool + r5 = r3 <= 4611686018427387903 :: signed + if r5 goto L3 else goto L4 :: bool L3: - r7 = r4 >= -4611686018427387904 :: signed - if r7 goto L5 else goto L4 :: bool + r6 = r3 >= -4611686018427387904 :: signed + if r6 goto L5 else goto L4 :: bool L4: - r8 = CPyTagged_FromInt64(r4) - r9 = r8 + r7 = CPyTagged_FromInt64(r3) + r8 = r7 goto L6 L5: - r10 = r4 << 1 - r9 = r10 + r9 = r3 << 1 + r8 = r9 L6: - r11 = CPyBytes_GetItem(source, r9) - r12 = box(int, r11) - r13 = unbox(int, r12) - x = r13 - r14 = f2(x) - r15 = box(int, r14) - CPySequenceTuple_SetItemUnsafe(r2, r4, r15) + r10 = CPyBytes_GetItem(source, r8) + r11 = box(int, r10) + r12 = unbox(int, r11) + x = r12 + r13 = f2(x) + r14 = box(int, r13) + CPySequenceTuple_SetItemUnsafe(r2, r3, r14) L7: - r16 = r4 + 1 - r4 = r16 + r15 = r3 + 1 + r3 = r15 goto L1 L8: a = r2 @@ -578,52 +557,49 @@ L0: return r0 def test(): r0 :: bytes - r1 :: native_int - r2 :: tuple - r3, r4 :: native_int - r5, r6, r7 :: bit - r8, r9, r10, r11 :: int - r12 :: object - r13, x, r14 :: int - r15 :: object - r16 :: native_int + r1 :: tuple + r2 :: native_int + r3, r4, r5 :: bit + r6, r7, r8, r9 :: int + r10 :: object + r11, x, r12 :: int + r13 :: object + r14 :: native_int a :: tuple L0: r0 = b'abc' - r1 = var_object_size r0 - r2 = PyTuple_New(r1) - r3 = var_object_size r0 - r4 = 0 + r1 = PyTuple_New(3) + r2 = 0 L1: - r5 = r4 < r3 :: signed - if r5 goto L2 else goto L8 :: bool + r3 = r2 < 3 :: signed + if r3 goto L2 else goto L8 :: bool L2: - r6 = r4 <= 4611686018427387903 :: signed - if r6 goto L3 else goto L4 :: bool + r4 = r2 <= 4611686018427387903 :: signed + if r4 goto L3 else goto L4 :: bool L3: - r7 = r4 >= -4611686018427387904 :: signed - if r7 goto L5 else goto L4 :: bool + r5 = r2 >= -4611686018427387904 :: signed + if r5 goto L5 else goto L4 :: bool L4: - r8 = CPyTagged_FromInt64(r4) - r9 = r8 + r6 = CPyTagged_FromInt64(r2) + r7 = r6 goto L6 L5: - r10 = r4 << 1 - r9 = r10 + r8 = r2 << 1 + r7 = r8 L6: - r11 = CPyBytes_GetItem(r0, r9) - r12 = box(int, r11) - r13 = unbox(int, r12) - x = r13 - r14 = f2(x) - r15 = box(int, r14) - CPySequenceTuple_SetItemUnsafe(r2, r4, r15) + r9 = CPyBytes_GetItem(r0, r7) + r10 = box(int, r9) + r11 = unbox(int, r10) + x = r11 + r12 = f2(x) + r13 = box(int, r12) + CPySequenceTuple_SetItemUnsafe(r1, r2, r13) L7: - r16 = r4 + 1 - r4 = r16 + r14 = r2 + 1 + r2 = r14 goto L1 L8: - a = r2 + a = r1 return 1 [case testTupleBuiltFromFinalBytes_64bit] @@ -648,13 +624,13 @@ def test(): r1 :: bool r2 :: native_int r3 :: tuple - r4, r5 :: native_int - r6, r7, r8 :: bit - r9, r10, r11, r12 :: int - r13 :: object - r14, x, r15 :: int - r16 :: object - r17 :: native_int + r4 :: native_int + r5, r6, r7 :: bit + r8, r9, r10, r11 :: int + r12 :: object + r13, x, r14 :: int + r15 :: object + r16 :: native_int a :: tuple L0: r0 = __main__.source :: static @@ -665,35 +641,34 @@ L1: L2: r2 = var_object_size r0 r3 = PyTuple_New(r2) - r4 = var_object_size r0 - r5 = 0 + r4 = 0 L3: - r6 = r5 < r4 :: signed - if r6 goto L4 else goto L10 :: bool + r5 = r4 < r2 :: signed + if r5 goto L4 else goto L10 :: bool L4: - r7 = r5 <= 4611686018427387903 :: signed - if r7 goto L5 else goto L6 :: bool + r6 = r4 <= 4611686018427387903 :: signed + if r6 goto L5 else goto L6 :: bool L5: - r8 = r5 >= -4611686018427387904 :: signed - if r8 goto L7 else goto L6 :: bool + r7 = r4 >= -4611686018427387904 :: signed + if r7 goto L7 else goto L6 :: bool L6: - r9 = CPyTagged_FromInt64(r5) - r10 = r9 + r8 = CPyTagged_FromInt64(r4) + r9 = r8 goto L8 L7: - r11 = r5 << 1 - r10 = r11 + r10 = r4 << 1 + r9 = r10 L8: - r12 = CPyBytes_GetItem(r0, r10) - r13 = box(int, r12) - r14 = unbox(int, r13) - x = r14 - r15 = f2(x) - r16 = box(int, r15) - CPySequenceTuple_SetItemUnsafe(r3, r5, r16) + r11 = CPyBytes_GetItem(r0, r9) + r12 = box(int, r11) + r13 = unbox(int, r12) + x = r13 + r14 = f2(x) + r15 = box(int, r14) + CPySequenceTuple_SetItemUnsafe(r3, r4, r15) L9: - r17 = r5 + 1 - r5 = r17 + r16 = r4 + 1 + r4 = r16 goto L3 L10: a = r3 @@ -825,36 +800,102 @@ def test(source): source :: tuple r0 :: native_int r1 :: tuple - r2, r3 :: native_int - r4 :: bit - r5 :: object - r6, x, r7 :: bool - r8 :: object - r9 :: native_int + r2 :: native_int + r3 :: bit + r4 :: object + r5, x, r6 :: bool + r7 :: object + r8 :: native_int a :: tuple L0: r0 = var_object_size source r1 = PyTuple_New(r0) - r2 = var_object_size source - r3 = 0 + r2 = 0 L1: - r4 = r3 < r2 :: signed - if r4 goto L2 else goto L4 :: bool + r3 = r2 < r0 :: signed + if r3 goto L2 else goto L4 :: bool L2: - r5 = CPySequenceTuple_GetItemUnsafe(source, r3) - r6 = unbox(bool, r5) - x = r6 - r7 = f(x) - r8 = box(bool, r7) - CPySequenceTuple_SetItemUnsafe(r1, r3, r8) + r4 = CPySequenceTuple_GetItemUnsafe(source, r2) + r5 = unbox(bool, r4) + x = r5 + r6 = f(x) + r7 = box(bool, r6) + CPySequenceTuple_SetItemUnsafe(r1, r2, r7) L3: - r9 = r3 + 1 - r3 = r9 + r8 = r2 + 1 + r2 = r8 goto L1 L4: a = r1 return 1 +[case testTupleBuiltFromStars] +from typing import Final + +abc: Final = "abc" + +def test() -> None: + a = tuple(str(x) for x in [*abc, *"def", *b"ghi", ("j", "k"), *("l", "m", "n")]) + +[out] +def test(): + r0, r1 :: str + r2 :: bytes + r3, r4 :: str + r5 :: tuple[str, str] + r6, r7, r8 :: str + r9 :: tuple[str, str, str] + r10 :: list + r11, r12, r13, r14 :: object + r15 :: i32 + r16 :: bit + r17, r18 :: object + r19 :: tuple + r20, r21 :: native_int + r22 :: bit + r23, x :: object + r24 :: str + r25 :: native_int + a :: tuple +L0: + r0 = 'abc' + r1 = 'def' + r2 = b'ghi' + r3 = 'j' + r4 = 'k' + r5 = (r3, r4) + r6 = 'l' + r7 = 'm' + r8 = 'n' + r9 = (r6, r7, r8) + r10 = PyList_New(0) + r11 = CPyList_Extend(r10, r0) + r12 = CPyList_Extend(r10, r1) + r13 = CPyList_Extend(r10, r2) + r14 = box(tuple[str, str], r5) + r15 = PyList_Append(r10, r14) + r16 = r15 >= 0 :: signed + r17 = box(tuple[str, str, str], r9) + r18 = CPyList_Extend(r10, r17) + r19 = PyTuple_New(13) + r20 = 0 +L1: + r21 = var_object_size r10 + r22 = r20 < r21 :: signed + if r22 goto L2 else goto L4 :: bool +L2: + r23 = list_get_item_unsafe r10, r20 + x = r23 + r24 = PyObject_Str(x) + CPySequenceTuple_SetItemUnsafe(r19, r20, r24) +L3: + r25 = r20 + 1 + r20 = r25 + goto L1 +L4: + a = r19 + return 1 + [case testTupleAdd] from typing import Tuple def f(a: Tuple[int, ...], b: Tuple[int, ...]) -> None: diff --git a/mypyc/test-data/run-lists.test b/mypyc/test-data/run-lists.test index 1569579c1156d..40ca1b6e005f9 100644 --- a/mypyc/test-data/run-lists.test +++ b/mypyc/test-data/run-lists.test @@ -479,6 +479,8 @@ def test_in_operator_various_cases() -> None: assert list_in_mixed(type) [case testListBuiltFromGenerator] +from typing import Final +abc: Final = "abc" def test_from_gen() -> None: source_a = ["a", "b", "c"] a = list(x + "f2" for x in source_a) @@ -498,6 +500,10 @@ def test_from_gen() -> None: source_str = "abcd" f = list("str:" + x for x in source_str) assert f == ["str:a", "str:b", "str:c", "str:d"] +def test_known_length() -> None: + # not built from generator but doesnt need its own test either + built = [str(x) for x in [*abc, *"def", *b"ghi", ("j", "k"), *("l", "m", "n")]] + assert built == ['a', 'b', 'c', 'd', 'e', 'f', '103', '104', '105', "('j', 'k')", 'l', 'm', 'n'] [case testNext] from typing import List diff --git a/mypyc/test-data/run-tuples.test b/mypyc/test-data/run-tuples.test index f5e1733d429b2..e2e8358bb43e2 100644 --- a/mypyc/test-data/run-tuples.test +++ b/mypyc/test-data/run-tuples.test @@ -270,6 +270,11 @@ def test_slicing() -> None: def f8(val: int) -> bool: return val % 2 == 0 +abc: Final = "abc" + +def known_length() -> tuple[str, ...]: + return tuple(str(x) for x in [*abc, *"def", *b"ghi", ("j", "k"), *("l", "m", "n")]) + def test_sequence_generator() -> None: source_list = [1, 2, 3] a = tuple(f8(x) for x in source_list) @@ -287,6 +292,8 @@ def test_sequence_generator() -> None: b = tuple('s:' + x for x in source_str) assert b == ('s:a', 's:b', 's:b', 's:c') + assert known_length() == ('a', 'b', 'c', 'd', 'e', 'f', '103', '104', '105', "('j', 'k')", 'l', 'm', 'n') + TUPLE: Final[Tuple[str, ...]] = ('x', 'y') def test_final_boxed_tuple() -> None: From fd0526545419028090f064ba4c1fa6e576ccdd6b Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 9 Sep 2025 21:10:52 +0100 Subject: [PATCH 251/424] Expose --fixed-format-cache if compiled (#19815) --- mypy/main.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mypy/main.py b/mypy/main.py index 4ca1bde73d400..d5bbca7043058 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -1064,7 +1064,13 @@ def add_invertible_flag( help="Include fine-grained dependency information in the cache for the mypy daemon", ) incremental_group.add_argument( - "--fixed-format-cache", action="store_true", help=argparse.SUPPRESS + "--fixed-format-cache", + action="store_true", + help=( + "Use experimental fast and compact fixed format cache" + if compilation_status == "yes" + else argparse.SUPPRESS + ), ) incremental_group.add_argument( "--skip-version-check", From 33660dbb669ab5ce75d1ff7030c62b036e134588 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 10 Sep 2025 14:34:00 +0100 Subject: [PATCH 252/424] Do not serialize line/column for type aliases (#19821) It looks like it is actually not used for anything now. Also it may unnecessarily invalidate cache and mask possible bugs. --- mypy/nodes.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index 7480745c6aa1c..040f3fc28dce0 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -4254,8 +4254,6 @@ def serialize(self) -> JsonDict: "alias_tvars": [v.serialize() for v in self.alias_tvars], "no_args": self.no_args, "normalized": self.normalized, - "line": self.line, - "column": self.column, "python_3_12_type_alias": self.python_3_12_type_alias, } return data @@ -4270,15 +4268,13 @@ def deserialize(cls, data: JsonDict) -> TypeAlias: target = mypy.types.deserialize_type(data["target"]) no_args = data["no_args"] normalized = data["normalized"] - line = data["line"] - column = data["column"] python_3_12_type_alias = data["python_3_12_type_alias"] return cls( target, fullname, module, - line, - column, + -1, + -1, alias_tvars=cast(list[mypy.types.TypeVarLikeType], alias_tvars), no_args=no_args, normalized=normalized, @@ -4291,8 +4287,6 @@ def write(self, data: Buffer) -> None: write_str(data, self.module) self.target.write(data) mypy.types.write_type_list(data, self.alias_tvars) - write_int(data, self.line) - write_int(data, self.column) write_bool(data, self.no_args) write_bool(data, self.normalized) write_bool(data, self.python_3_12_type_alias) @@ -4307,8 +4301,8 @@ def read(cls, data: Buffer) -> TypeAlias: target, fullname, module, - read_int(data), - read_int(data), + -1, + -1, alias_tvars=alias_tvars, no_args=read_bool(data), normalized=read_bool(data), From 9ae3e9aa160c11b99960f12eef111e4a3197b7d3 Mon Sep 17 00:00:00 2001 From: Kevin Kannammalil Date: Wed, 10 Sep 2025 13:04:02 -0400 Subject: [PATCH 253/424] Initial changelog for release 1.18 (#19818) --- CHANGELOG.md | 239 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 239 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bdb888ff9d6e..76643a0b805ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,245 @@ ## Next Release +## Mypy 1.18 (Unreleased) + +We’ve just uploaded mypy 1.18 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). +Mypy is a static type checker for Python. This release includes new features and bug fixes. +You can install it as follows: + + python3 -m pip install -U mypy + +You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). + +### `--allow-redefinition-new` + +TODO by Jukka + +This feature was contributed by Jukka Lehtosalo. + +### Fixed‑Format Cache (experimental) + +TODO by Jukka + +This feature was contributed by Ivan Levkivskyi (PR [19668](https://github.com/python/mypy/pull/19668), [19735](https://github.com/python/mypy/pull/19735), [19750](https://github.com/python/mypy/pull/19750), [19681](https://github.com/python/mypy/pull/19681), [19752](https://github.com/python/mypy/pull/19752), [19815](https://github.com/python/mypy/pull/19815)) + +### Disjoint Base Classes (@disjoint_base, PEP 800) + +Mypy now implements PEP 800 Disjoint bases: it understands the @disjoint_base marker, rejects class definitions that combine incompatible disjoint bases, and exploits the fact that such classes cannot exist in reachability and narrowing logic + +This feature was contributed by Jelle Zijlstra (PR [19678](https://github.com/python/mypy/pull/19678)). + +### Mypy Performance Improvements + +Mypy 1.18 includes numerous performance improvements, resulting in a 38% overall speedup compared to 1.17 + +- Improve self check performance by 1.8% (Jukka Lehtosalo, PR [19768](https://github.com/python/mypy/pull/19768), [19769](https://github.com/python/mypy/pull/19769), [19770](https://github.com/python/mypy/pull/19770)) +- Use fast Python wrappers in native_internal (Ivan Levkivskyi, PR [19765](https://github.com/python/mypy/pull/19765)) +- Use macros in native_internal hot paths (Ivan Levkivskyi, PR [19757](https://github.com/python/mypy/pull/19757)) +- Special‑case certain Enum method calls for speed (Ivan Levkivskyi, PR [19634](https://github.com/python/mypy/pull/19634)) +- Two additional micro‑optimizations (Ivan Levkivskyi, PR [19627](https://github.com/python/mypy/pull/19627)) +- Another set of micro‑optimizations (Ivan Levkivskyi, PR [19633](https://github.com/python/mypy/pull/19633)) +- Cache common instances (Ivan Levkivskyi, PR [19621](https://github.com/python/mypy/pull/19621)) +- Skip more method bodies in third‑party libraries for speed (Ivan Levkivskyi, PR [19586](https://github.com/python/mypy/pull/19586)) +- Avoid using a dict in CallableType (Ivan Levkivskyi, PR [19580](https://github.com/python/mypy/pull/19580)) +- Use cache for DictExpr (Ivan Levkivskyi, PR [19536](https://github.com/python/mypy/pull/19536)) +- Use cache for OpExpr (Ivan Levkivskyi, PR [19523](https://github.com/python/mypy/pull/19523)) +- Simple call‑expression cache (Ivan Levkivskyi, PR [19505](https://github.com/python/mypy/pull/19505)) +- Cache type_object_type() (Ivan Levkivskyi, PR [19514](https://github.com/python/mypy/pull/19514)) +- Avoid duplicate visits in boolean‑op checking (Ivan Levkivskyi, PR [19515](https://github.com/python/mypy/pull/19515)) +- Optimize generic inference passes (Ivan Levkivskyi, PR [19501](https://github.com/python/mypy/pull/19501)) +- Speed up default plugin (Jukka Lehtosalo, PR [19462](https://github.com/python/mypy/pull/19462)) +- Micro‑optimize ExpandTypeVisitor (Jukka Lehtosalo, PR [19461](https://github.com/python/mypy/pull/19461)) +- Micro‑optimize type indirection visitor (Jukka Lehtosalo, PR [19460](https://github.com/python/mypy/pull/19460)) +- Micro‑optimize chained plugin (Jukka Lehtosalo, PR [19464](https://github.com/python/mypy/pull/19464)) +- Avoid temporary set creation in is_proper_subtype (Jukka Lehtosalo, PR [19463](https://github.com/python/mypy/pull/19463)) +- Subtype checking micro‑optimization (Jukka Lehtosalo, PR [19384](https://github.com/python/mypy/pull/19384)) +- Speed up default plugin (earlier pass) (Jukka Lehtosalo, PR [19385](https://github.com/python/mypy/pull/19385)) +- Remove nested imports from default plugin (Ivan Levkivskyi, PR [19388](https://github.com/python/mypy/pull/19388)) +- is_subtype: return early where possible (Stanislav Terliakov, PR [19400](https://github.com/python/mypy/pull/19400)) +- Deduplicate fast_container_type / fast_dict_type items before join (Stanislav Terliakov, PR [19409](https://github.com/python/mypy/pull/19409)) +- Speed up type checking by caching argument inference context (Jukka Lehtosalo, PR [19323](https://github.com/python/mypy/pull/19323)) +- Optimize bind_self() and deprecation checks (Ivan Levkivskyi, PR [19556](https://github.com/python/mypy/pull/19556)) +- Keep trivial instances/aliases during expansion (Ivan Levkivskyi, PR [19543](https://github.com/python/mypy/pull/19543)) + +### Stubtest Improvements +- Add temporary --ignore-disjoint-bases flag to ease PEP 800 migration (Joren Hammudoglu, PR [19740](https://github.com/python/mypy/pull/19740)) +- Flag redundant uses of @disjoint_base (Jelle Zijlstra, PR [19715](https://github.com/python/mypy/pull/19715)) +- Improve signatures for `__init__` of C classes (Stephen Morton, PR [18259](https://github.com/python/mypy/pull/18259)) +- Handle overloads with mixed pos‑only parameters (Stephen Morton, PR [18287](https://github.com/python/mypy/pull/18287)) +- Use “parameter” (not “argument”) in error messages (PrinceNaroliya, PR [19707](https://github.com/python/mypy/pull/19707)) +- Don’t require @disjoint_base when `__slots__` imply finality (Jelle Zijlstra, PR [19701](https://github.com/python/mypy/pull/19701)) +- Allow runtime‑existing aliases of @type_check_only types (Brian Schubert, PR [19568](https://github.com/python/mypy/pull/19568)) +- More detailed checking of type objects in stubtest (Stephen Morton, PR [18251](https://github.com/python/mypy/pull/18251)) +- Support running stubtest in non-UTF8 terminals (Stanislav Terliakovm, PR [19085](https://github.com/python/mypy/pull/19085)) + +### Mypyc Improvements + +- Fix subclass processing in detect_undefined_bitmap (Chainfire, PR [19787](https://github.com/python/mypy/pull/19787)) +- Fix C function signature emission (Jukka Lehtosalo, PR [19773](https://github.com/python/mypy/pull/19773)) +- Use defined `__new__` in tp_new and constructor (Piotr Sawicki, PR [19739](https://github.com/python/mypy/pull/19739)) +- Speed up implicit `__ne__` (Jukka Lehtosalo, PR [19759](https://github.com/python/mypy/pull/19759)) +- Speed up equality with optional str/bytes (Jukka Lehtosalo, PR [19758](https://github.com/python/mypy/pull/19758)) +- Add `__mypyc_empty_tuple__` constant (BobTheBuidler, PR [19654](https://github.com/python/mypy/pull/19654)) +- Add PyObject_CallObject fast‑path op for fn(*args) (BobTheBuidler, PR [19631](https://github.com/python/mypy/pull/19631)) +- Add **kwargs star2 fast‑path (follow‑up to starargs) (BobTheBuidler, PR [19630](https://github.com/python/mypy/pull/19630)) +- Optimize type(x), x.`__class__`, and `.__name__` (Jukka Lehtosalo, PR [19691](https://github.com/python/mypy/pull/19691), [19683](https://github.com/python/mypy/pull/19683)) +- Specialize bytes.decode for common encodings (Jukka Lehtosalo, PR [19688](https://github.com/python/mypy/pull/19688)) +- Speed up in against final fixed‑length tuples (Jukka Lehtosalo, PR [19682](https://github.com/python/mypy/pull/19682)) +- Optimize f‑string building from Final values (BobTheBuidler, PR [19611](https://github.com/python/mypy/pull/19611)) +- Add exact_dict_set_item_op (BobTheBuidler, PR [19657](https://github.com/python/mypy/pull/19657)) +- Cache len() when iterating over immutable types (BobTheBuidler, PR [19656](https://github.com/python/mypy/pull/19656)) +- Add stararg fast‑path for tuple calls fn(*args) (BobTheBuidler, PR [19623](https://github.com/python/mypy/pull/19623)) +- Include more operations in the mypyc trace log (Jukka Lehtosalo, PR [19647](https://github.com/python/mypy/pull/19647)) +- Add prefix to attributes of generator classes (Piotr Sawicki, PR [19535](https://github.com/python/mypy/pull/19535)) +- Fix segfault from heap type objects with static tp_doc (Brian Schubert, PR [19636](https://github.com/python/mypy/pull/19636)) +- Unwrap NewType to its base type for optimized paths (BobTheBuidler, PR [19497](https://github.com/python/mypy/pull/19497)) +- Enable free‑threading when compiling multiple modules (Jukka Lehtosalo, PR [19541](https://github.com/python/mypy/pull/19541)) +- Make type objects immortal under free‑threading (Jukka Lehtosalo, PR [19538](https://github.com/python/mypy/pull/19538)) +- Fix list.pop primitive on free‑threaded builds (Jukka Lehtosalo, PR [19522](https://github.com/python/mypy/pull/19522)) +- Generate an export table only for separate compilation (Jukka Lehtosalo, PR [19521](https://github.com/python/mypy/pull/19521)) +- Add primitives for isinstance of built‑in types (Piotr Sawicki, PR [19435](https://github.com/python/mypy/pull/19435)) +- Add SetElement op to initialize struct values (Jukka Lehtosalo, PR [19437](https://github.com/python/mypy/pull/19437)) +- Simplify IR for for loops over strings (Jukka Lehtosalo, PR [19434](https://github.com/python/mypy/pull/19434)) +- Use native integers for some sequence indexing (Jukka Lehtosalo, PR [19426](https://github.com/python/mypy/pull/19426)) +- Remove unused CPyList_GetItemUnsafe primitive (Jukka Lehtosalo, PR [19424](https://github.com/python/mypy/pull/19424)) +- Add native‑int helper methods in IR builder (Jukka Lehtosalo, PR [19423](https://github.com/python/mypy/pull/19423)) +- Use PyList_Check for isinstance(obj, list) (Piotr Sawicki, PR [19416](https://github.com/python/mypy/pull/19416)) +- Speed up for loops over native generators (Jukka Lehtosalo, PR [19415](https://github.com/python/mypy/pull/19415)) +- Report error on reserved method names (Piotr Sawicki, PR [19407](https://github.com/python/mypy/pull/19407)) +- Add is_bool_or_bit_rprimitive (Piotr Sawicki, PR [19406](https://github.com/python/mypy/pull/19406)) +- Faster string equality primitive (Jukka Lehtosalo, PR [19402](https://github.com/python/mypy/pull/19402)) +- Speed up native‑to‑native calls using await (Jukka Lehtosalo, PR [19398](https://github.com/python/mypy/pull/19398)) +- Raise NameError on undefined names (Piotr Sawicki, PR [19395](https://github.com/python/mypy/pull/19395)) +- Simplify comparison of tuple elements (Piotr Sawicki, PR [19396](https://github.com/python/mypy/pull/19396)) +- Use per‑type freelists for nested functions (Jukka Lehtosalo, PR [19390](https://github.com/python/mypy/pull/19390)) +- Call generator helper directly in await expressions (Jukka Lehtosalo, PR [19376](https://github.com/python/mypy/pull/19376)) +- Generate introspection signatures for compiled functions (Brian Schubert, PR [19307](https://github.com/python/mypy/pull/19307)) +- Support C string literals in IR (Jukka Lehtosalo, PR [19383](https://github.com/python/mypy/pull/19383)) +- Fix error‑value check for GetAttr that allows nullable values (Jukka Lehtosalo, PR [19378](https://github.com/python/mypy/pull/19378)) +- Fix comparison of tuples with different lengths (Piotr Sawicki, PR [19372](https://github.com/python/mypy/pull/19372)) +- Speed up generator allocation with per‑type freelists (Jukka Lehtosalo, PR [19316](https://github.com/python/mypy/pull/19316)) +- Implement list.clear() primitive (Jahongir Qurbonov, PR [19344](https://github.com/python/mypy/pull/19344)) +- New primitives for weakref.proxy (BobTheBuidler, PR [19217](https://github.com/python/mypy/pull/19217)) +- New primitive for weakref.ref (BobTheBuidler, PR [19099](https://github.com/python/mypy/pull/19099)) +- New primitive for str.count (BobTheBuidler, PR [19264](https://github.com/python/mypy/pull/19264)) +- Tracing/tooling: optionally log sampled operation traces (Jukka Lehtosalo, PR [19457](https://github.com/python/mypy/pull/19457)) +- Tracing/tooling: script to compile with trace logging and run mypy (Jukka Lehtosalo, PR [19475](https://github.com/python/mypy/pull/19475)) + + +### Documentation Updates + +- Add idlemypyextension to IDE integrations (CoolCat467, PR [18615](https://github.com/python/mypy/pull/18615)) +- Document that object is often preferable to Any in APIs (wyattscarpenter, PR [19103](https://github.com/python/mypy/pull/19103)) +- Include a detailed listing of flags enabled by --strict (wyattscarpenter, PR [19062](https://github.com/python/mypy/pull/19062)) +- Update “common issues” (reveal_type/reveal_locals; note on orjson) (wyattscarpenter, PR [19059](https://github.com/python/mypy/pull/19059), [19058](https://github.com/python/mypy/pull/19058)) + +### Other Notable Improvements + +- Remove deprecated --new-type-inference flag (the new algorithm has long been default) (Ivan Levkivskyi, PR [19570](https://github.com/python/mypy/pull/19570)) +- Use empty context as a fallback for return expressions when outer context misleads inference (Ivan Levkivskyi, PR [19767](https://github.com/python/mypy/pull/19767)) +- Support --strict-equality checks involving None (Christoph Tyralla, PR [19718](https://github.com/python/mypy/pull/19718)) +- Don’t show import‑related errors after a module‑level assert False (Stanislav Terliakov, PR [19347](https://github.com/python/mypy/pull/19347)) +- Fix forward refs in type parameters of over‑parameterized PEP 695 aliases (Brian Schubert, PR [19725](https://github.com/python/mypy/pull/19725)) +- Don’t expand PEP 695 aliases when checking node fullnames (Brian Schubert, PR [19699](https://github.com/python/mypy/pull/19699)) +- Don’t use outer context for or expression inference when LHS is Any (Stanislav Terliakov, PR [19748](https://github.com/python/mypy/pull/19748)) +- Interpret bare ClassVar as inferred (not Any) (Ivan Levkivskyi, PR [19573](https://github.com/python/mypy/pull/19573)) +- Recognize buffer protocol special methods (Brian Schubert, PR [19581](https://github.com/python/mypy/pull/19581)) +- Add temporary named expressions for match subjects (Stanislav Terliakov, PR [18446](https://github.com/python/mypy/pull/18446)) +- Support attribute access on enum members correctly (Stanislav Terliakov, PR [19422](https://github.com/python/mypy/pull/19422)) +- Check `__slots__` assignments on self types (Stanislav Terliakov, PR [19332](https://github.com/python/mypy/pull/19332)) +- Move self‑argument checks after decorator application (Stanislav Terliakov, PR [19490](https://github.com/python/mypy/pull/19490)) +- Infer empty list for `__slots__` and module `__all__` (Stanislav Terliakov, PR [19348](https://github.com/python/mypy/pull/19348)) +- Use normalized tuples for fallback calculation (Stanislav Terliakov, PR [19111](https://github.com/python/mypy/pull/19111)) +- Preserve literals when joining Literal with Instance that has matching last_known_value (Stanislav Terliakov, PR [19279](https://github.com/python/mypy/pull/19279)) +- Allow adjacent conditionally‑defined overloads (Stanislav Terliakov, PR [19042](https://github.com/python/mypy/pull/19042)) +- Check property decorators more strictly (Stanislav Terliakov, PR [19313](https://github.com/python/mypy/pull/19313)) +- Support properties with generic setters (Ivan Levkivskyi, PR [19298](https://github.com/python/mypy/pull/19298)) +- Generalize class/static method and property alias support (Ivan Levkivskyi, PR [19297](https://github.com/python/mypy/pull/19297)) +- Re‑widen custom properties after narrowing (Ivan Levkivskyi, PR [19296](https://github.com/python/mypy/pull/19296)) +- Avoid erasing type objects when checking runtime cover (Shantanu, PR [19320](https://github.com/python/mypy/pull/19320)) +- Include tuple fallback in constraints built from tuple types (Stanislav Terliakov, PR [19100](https://github.com/python/mypy/pull/19100)) +- Somewhat better isinstance support on old‑style unions (Shantanu, PR [19714](https://github.com/python/mypy/pull/19714)) +- Improve promotions inside unions (Christoph Tyralla, PR [19245](https://github.com/python/mypy/pull/19245)) +- Uninhabited types should have all attributes (Ivan Levkivskyi, PR [19300](https://github.com/python/mypy/pull/19300)) +- Metaclass conflict checks improved (Robsdedude, PR [17682](https://github.com/python/mypy/pull/17682)) +- Metaclass resolution algorithm fixes (Robsdedude, PR [17713](https://github.com/python/mypy/pull/17713)) +- PEP 702 @deprecated: handle “combined” overloads (Christoph Tyralla, PR [19626](https://github.com/python/mypy/pull/19626)) +- PEP 702 @deprecated: include overloads in snapshot descriptions (Christoph Tyralla, PR [19613](https://github.com/python/mypy/pull/19613)) +- Ignore overload implementation when checking `__OP__` / `__rOP__` compatibility (Stanislav Terliakov, PR [18502](https://github.com/python/mypy/pull/18502)) +- Fix unwrapping of assignment expressions in match subject (Marc Mueller, PR [19742](https://github.com/python/mypy/pull/19742)) +- Omit errors for class patterns against object (Marc Mueller, PR [19709](https://github.com/python/mypy/pull/19709)) +- Remove unnecessary error for certain match class patterns (Marc Mueller, PR [19708](https://github.com/python/mypy/pull/19708)) +- Use union type for captured vars in or pattern (Marc Mueller, PR [19710](https://github.com/python/mypy/pull/19710)) +- Prevent final reassignment inside match case (Omer Hadari, PR [19496](https://github.com/python/mypy/pull/19496)) +- Support _value_ as a fallback for ellipsis Enum members (Stanislav Terliakov, PR [19352](https://github.com/python/mypy/pull/19352)) +- Sort arguments in TypedDict overlap messages (Marc Mueller, PR [19666](https://github.com/python/mypy/pull/19666)) +- Reset to previous statement on leaving return in semanal (Stanislav Terliakov, PR [19642](https://github.com/python/mypy/pull/19642)) +- Add ambiguous to UninhabitedType identity for better messaging (Stanislav Terliakov, PR [19648](https://github.com/python/mypy/pull/19648)) +- Further fix overload diagnostics for varargs/kwargs (Shantanu, PR [19619](https://github.com/python/mypy/pull/19619)) +- Fix overload diagnostics when vararg and varkwarg both match (Shantanu, PR [19614](https://github.com/python/mypy/pull/19614)) +- Show type variable name in “Cannot infer type argument” (Brian Schubert, PR [19290](https://github.com/python/mypy/pull/19290)) +- Fail gracefully on unsupported template strings (PEP 750) (Brian Schubert, PR [19700](https://github.com/python/mypy/pull/19700)) +- Revert colored argparse help for Python 3.14 (Marc Mueller, PR [19721](https://github.com/python/mypy/pull/19721)) +- Support type‑checking a code fragment in the profile script (Jukka Lehtosalo, PR [19379](https://github.com/python/mypy/pull/19379)) +- Fix C compiler flags in the profile self‑check script (Jukka Lehtosalo, PR [19326](https://github.com/python/mypy/pull/19326)) +- Add a script for profiling self‑check (Linux only) (Jukka Lehtosalo, PR [19322](https://github.com/python/mypy/pull/19322)) +- Retry PyPI upload script: skip existing files on retry (Jukka Lehtosalo, PR [19305](https://github.com/python/mypy/pull/19305)) +- Update stubinfo for latest typeshed (Shantanu, PR [19771](https://github.com/python/mypy/pull/19771)) +- Fix crash with variadic tuple arguments to a generic type (Randolf Scholz, PR [19705](https://github.com/python/mypy/pull/19705)) +- Fix crash when enable_error_code in pyproject.toml has wrong type (wyattscarpenter, PR [19494](https://github.com/python/mypy/pull/19494)) +- Fix dict assignment to a wider context when an incompatible same‑shape TypedDict exists (Stanislav Terliakov, PR [19592](https://github.com/python/mypy/pull/19592)) +- Prevent crash for dataclass with PEP 695 TypeVarTuple on Python 3.13+ (Stanislav Terliakov, PR [19565](https://github.com/python/mypy/pull/19565)) +- Fix constructor type for subclasses of Any (Ivan Levkivskyi, PR [19295](https://github.com/python/mypy/pull/19295)) +- Fix TypeGuard/TypeIs being forgotten when semanal defers (Brian Schubert, PR [19325](https://github.com/python/mypy/pull/19325)) +- Fix TypeIs negative narrowing for unions of generics (Brian Schubert, PR [18193](https://github.com/python/mypy/pull/18193)) +- dmypy suggest: fix incorrect signature suggestion when a type matches a module name (Brian Schubert, PR [18937](https://github.com/python/mypy/pull/18937)) +- dmypy suggest: fix interaction with `__new__` (Stanislav Terliakov, PR [18966](https://github.com/python/mypy/pull/18966)) +- dmypy suggest: support Callable / callable Protocols in decorator unwrapping (Anthony Sottile, PR [19072](https://github.com/python/mypy/pull/19072)) +- Fix missing error when redeclaring a type variable in a nested generic class (Brian Schubert, PR [18883](https://github.com/python/mypy/pull/18883)) +- Fix for overloaded type object erasure (Shantanu, PR [19338](https://github.com/python/mypy/pull/19338)) +- Fix TypeGuard with call on temporary object (Saul Shanabrook, PR [19577](https://github.com/python/mypy/pull/19577)) +- Fix crash on settable property alias (Ivan Levkivskyi, PR [19615](https://github.com/python/mypy/pull/19615)) + +### Typeshed Updates + +Please see [git log](https://github.com/python/typeshed/commits/main?after=2480d7e7c74493a024eaf254c5d2c6f452c80ee2+0&branch=main&path=stdlib) for full list of standard library typeshed stub changes. + +### Acknowledgements + +Thanks to all mypy contributors who contributed to this release: + +- Ali Hamdan +- Anthony Sottile +- BobTheBuidler +- Brian Schubert +- Chainfire +- Charlie Denton +- Christoph Tyralla +- CoolCat467 +- Daniel Hnyk +- Emily +- Emma Smith +- Ethan Sarp +- Ivan Levkivskyi +- Jahongir Qurbonov +- Jelle Zijlstra +- Joren Hammudoglu +- Jukka Lehtosalo +- Marc Mueller +- Omer Hadari +- Piotr Sawicki +- PrinceNaroliya +- Randolf Scholz +- Robsdedude +- Saul Shanabrook +- Shantanu +- Stanislav Terliakov +- Stephen Morton +- wyattscarpenter + +I’d also like to thank my employer, Dropbox, for supporting mypy development. + ## Mypy 1.17 We’ve just uploaded mypy 1.17 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). From 73fa69ed3d7fe0f80d74874ec0d9c738e8674bd1 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 11 Sep 2025 14:03:58 +0100 Subject: [PATCH 254/424] Updates to 1.18 changelog (#19826) Did various edits, added a few additional sections, and reordered some sections. --- CHANGELOG.md | 335 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 206 insertions(+), 129 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76643a0b805ca..5266a86c725ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,154 +5,244 @@ ## Mypy 1.18 (Unreleased) We’ve just uploaded mypy 1.18 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). -Mypy is a static type checker for Python. This release includes new features and bug fixes. -You can install it as follows: +Mypy is a static type checker for Python. This release includes new features, performance +improvements and bug fixes. You can install it as follows: python3 -m pip install -U mypy You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). -### `--allow-redefinition-new` +### Mypy Performance Improvements + +Mypy 1.18 includes numerous performance improvements, resulting in about 40% speedup +compared to 1.17 when type checking mypy itself. In extreme cases, the improvement +can be 10x or higher. The list below is an overview of the various mypy optimizations. +Many mypyc improvements (discussed in a separate section below) also improve performance. -TODO by Jukka +Type caching optimizations have a small risk of causing regressions. When +reporting issues with unexpected inferred types, please also check if +`--disable-expression-cache` will work around the issue, as it turns off some of +these optimizations. + +- Improve self check performance by 1.8% (Jukka Lehtosalo, PR [19768](https://github.com/python/mypy/pull/19768), [19769](https://github.com/python/mypy/pull/19769), [19770](https://github.com/python/mypy/pull/19770)) +- Optimize fixed-format deserialization (Ivan Levkivskyi, PR [19765](https://github.com/python/mypy/pull/19765)) +- Use macros to optimize fixed-format deserialization (Ivan Levkivskyi, PR [19757](https://github.com/python/mypy/pull/19757)) +- Two additional micro‑optimizations (Ivan Levkivskyi, PR [19627](https://github.com/python/mypy/pull/19627)) +- Another set of micro‑optimizations (Ivan Levkivskyi, PR [19633](https://github.com/python/mypy/pull/19633)) +- Cache common types (Ivan Levkivskyi, PR [19621](https://github.com/python/mypy/pull/19621)) +- Skip more method bodies in third‑party libraries for speed (Ivan Levkivskyi, PR [19586](https://github.com/python/mypy/pull/19586)) +- Simplify the representation of callable types (Ivan Levkivskyi, PR [19580](https://github.com/python/mypy/pull/19580)) +- Add cache for types of some expressions (Ivan Levkivskyi, PR [19505](https://github.com/python/mypy/pull/19505)) +- Use cache for dictionary expressions (Ivan Levkivskyi, PR [19536](https://github.com/python/mypy/pull/19536)) +- Use cache for binary operations (Ivan Levkivskyi, PR [19523](https://github.com/python/mypy/pull/19523)) +- Cache types of type objects (Ivan Levkivskyi, PR [19514](https://github.com/python/mypy/pull/19514)) +- Avoid duplicate work when checking boolean operations (Ivan Levkivskyi, PR [19515](https://github.com/python/mypy/pull/19515)) +- Optimize generic inference passes (Ivan Levkivskyi, PR [19501](https://github.com/python/mypy/pull/19501)) +- Speed up the default plugin (Jukka Lehtosalo, PRs [19385](https://github.com/python/mypy/pull/19385) and [19462](https://github.com/python/mypy/pull/19462)) +- Remove nested imports from the default plugin (Ivan Levkivskyi, PR [19388](https://github.com/python/mypy/pull/19388)) +- Micro‑optimize type expansion (Jukka Lehtosalo, PR [19461](https://github.com/python/mypy/pull/19461)) +- Micro‑optimize type indirection (Jukka Lehtosalo, PR [19460](https://github.com/python/mypy/pull/19460)) +- Micro‑optimize the plugin framework (Jukka Lehtosalo, PR [19464](https://github.com/python/mypy/pull/19464)) +- Avoid temporary set creation in subtype checking (Jukka Lehtosalo, PR [19463](https://github.com/python/mypy/pull/19463)) +- Subtype checking micro‑optimization (Jukka Lehtosalo, PR [19384](https://github.com/python/mypy/pull/19384)) +- Return early where possible in subtype check (Stanislav Terliakov, PR [19400](https://github.com/python/mypy/pull/19400)) +- Deduplicate some types before joining (Stanislav Terliakov, PR [19409](https://github.com/python/mypy/pull/19409)) +- Speed up type checking by caching argument inference context (Jukka Lehtosalo, PR [19323](https://github.com/python/mypy/pull/19323)) +- Optimize binding method self argument type and deprecation checks (Ivan Levkivskyi, PR [19556](https://github.com/python/mypy/pull/19556)) +- Keep trivial instance types/aliases during expansion (Ivan Levkivskyi, PR [19543](https://github.com/python/mypy/pull/19543)) + +### Fixed‑Format Cache (Experimental) + +Mypy now supports a new cache format used for faster incremental builds. It makes +incremental builds up to twice as fast. The feature is experimental and +currently only supported when using a compiled version of mypy. Use `--fixed-format-cache` +to enable the new format, or `fixed_format_cache = True` in a configuration file. + +We plan to enable this by default in a future mypy release, and we'll eventually +deprecate and remove support for the original JSON-based format. + +Unlike the JSON-based cache format, the new binary format is currently +not easy to parse and inspect by mypy users. We are planning to provide a tool to +convert fixed-format cache files to JSON, but details of the output JSON may be +different from the current JSON format. If you rely on being able to inspect +mypy cache files, we recommend creating a GitHub issue and explaining your use +case, so that we can more likely provide support for it. (Using +`MypyFile.read(binary_data)` to inspect cache data may be sufficient to support +some use cases.) + +This feature was contributed by Ivan Levkivskyi (PR [19668](https://github.com/python/mypy/pull/19668), [19735](https://github.com/python/mypy/pull/19735), [19750](https://github.com/python/mypy/pull/19750), [19681](https://github.com/python/mypy/pull/19681), [19752](https://github.com/python/mypy/pull/19752), [19815](https://github.com/python/mypy/pull/19815)). + +### Flexible Variable Definitions: Update + +Mypy 1.16.0 introduced `--allow-redefinition-new`, which allows redefining variables +with different types, and inferring union types for variables from multiple assignments. +The feature is now documented in the `--help` output, but the feature is still experimental. + +We are planning to enable this by default in mypy 2.0, and we will also deprecate the +older `--allow-redefinition` flag. Since the new behavior differs significantly from +the older flag, we encourage users of `--allow-redefinition` to experiment with +`--allow-redefinition-new` and create a GitHub issue if the new functionality doesn't +support some important use cases. This feature was contributed by Jukka Lehtosalo. -### Fixed‑Format Cache (experimental) +### Inferred Type for Bare ClassVar + +A ClassVar without an explicit type annotation now causes the type of the variable +to be inferred from the initializer: + + +```python +from typing import ClassVar + +class Item: + # Type of 'next_id' is now 'int' (it was 'Any') + next_id: ClassVar = 1 -TODO by Jukka + ... +``` -This feature was contributed by Ivan Levkivskyi (PR [19668](https://github.com/python/mypy/pull/19668), [19735](https://github.com/python/mypy/pull/19735), [19750](https://github.com/python/mypy/pull/19750), [19681](https://github.com/python/mypy/pull/19681), [19752](https://github.com/python/mypy/pull/19752), [19815](https://github.com/python/mypy/pull/19815)) +This feature was contributed by Ivan Levkivskyi (PR [19573](https://github.com/python/mypy/pull/19573)). ### Disjoint Base Classes (@disjoint_base, PEP 800) -Mypy now implements PEP 800 Disjoint bases: it understands the @disjoint_base marker, rejects class definitions that combine incompatible disjoint bases, and exploits the fact that such classes cannot exist in reachability and narrowing logic +Mypy now understands disjoint bases (PEP 800): it recognizes the `@disjoint_base` +decorator, and rejects class definitions that combine mutually incompatible base classes, +and takes advantage of the fact that such classes cannot exist in reachability and +narrowing logic. + +This class definition will now generate an error: + +```python +# Error: Class "Bad" has incompatible disjoint bases +class Bad(str, Exception): + ... +``` This feature was contributed by Jelle Zijlstra (PR [19678](https://github.com/python/mypy/pull/19678)). -### Mypy Performance Improvements +### Miscellaneous New Mypy Features -Mypy 1.18 includes numerous performance improvements, resulting in a 38% overall speedup compared to 1.17 +- Add `--strict-equality-for-none` to flag non-overlapping comparisons involving None (Christoph Tyralla, PR [19718](https://github.com/python/mypy/pull/19718)) +- Don’t show import‑related errors after a module‑level assert such as `assert sys.platform == "linux"` that is always false (Stanislav Terliakov, PR [19347](https://github.com/python/mypy/pull/19347)) -- Improve self check performance by 1.8% (Jukka Lehtosalo, PR [19768](https://github.com/python/mypy/pull/19768), [19769](https://github.com/python/mypy/pull/19769), [19770](https://github.com/python/mypy/pull/19770)) -- Use fast Python wrappers in native_internal (Ivan Levkivskyi, PR [19765](https://github.com/python/mypy/pull/19765)) -- Use macros in native_internal hot paths (Ivan Levkivskyi, PR [19757](https://github.com/python/mypy/pull/19757)) -- Special‑case certain Enum method calls for speed (Ivan Levkivskyi, PR [19634](https://github.com/python/mypy/pull/19634)) -- Two additional micro‑optimizations (Ivan Levkivskyi, PR [19627](https://github.com/python/mypy/pull/19627)) -- Another set of micro‑optimizations (Ivan Levkivskyi, PR [19633](https://github.com/python/mypy/pull/19633)) -- Cache common instances (Ivan Levkivskyi, PR [19621](https://github.com/python/mypy/pull/19621)) -- Skip more method bodies in third‑party libraries for speed (Ivan Levkivskyi, PR [19586](https://github.com/python/mypy/pull/19586)) -- Avoid using a dict in CallableType (Ivan Levkivskyi, PR [19580](https://github.com/python/mypy/pull/19580)) -- Use cache for DictExpr (Ivan Levkivskyi, PR [19536](https://github.com/python/mypy/pull/19536)) -- Use cache for OpExpr (Ivan Levkivskyi, PR [19523](https://github.com/python/mypy/pull/19523)) -- Simple call‑expression cache (Ivan Levkivskyi, PR [19505](https://github.com/python/mypy/pull/19505)) -- Cache type_object_type() (Ivan Levkivskyi, PR [19514](https://github.com/python/mypy/pull/19514)) -- Avoid duplicate visits in boolean‑op checking (Ivan Levkivskyi, PR [19515](https://github.com/python/mypy/pull/19515)) -- Optimize generic inference passes (Ivan Levkivskyi, PR [19501](https://github.com/python/mypy/pull/19501)) -- Speed up default plugin (Jukka Lehtosalo, PR [19462](https://github.com/python/mypy/pull/19462)) -- Micro‑optimize ExpandTypeVisitor (Jukka Lehtosalo, PR [19461](https://github.com/python/mypy/pull/19461)) -- Micro‑optimize type indirection visitor (Jukka Lehtosalo, PR [19460](https://github.com/python/mypy/pull/19460)) -- Micro‑optimize chained plugin (Jukka Lehtosalo, PR [19464](https://github.com/python/mypy/pull/19464)) -- Avoid temporary set creation in is_proper_subtype (Jukka Lehtosalo, PR [19463](https://github.com/python/mypy/pull/19463)) -- Subtype checking micro‑optimization (Jukka Lehtosalo, PR [19384](https://github.com/python/mypy/pull/19384)) -- Speed up default plugin (earlier pass) (Jukka Lehtosalo, PR [19385](https://github.com/python/mypy/pull/19385)) -- Remove nested imports from default plugin (Ivan Levkivskyi, PR [19388](https://github.com/python/mypy/pull/19388)) -- is_subtype: return early where possible (Stanislav Terliakov, PR [19400](https://github.com/python/mypy/pull/19400)) -- Deduplicate fast_container_type / fast_dict_type items before join (Stanislav Terliakov, PR [19409](https://github.com/python/mypy/pull/19409)) -- Speed up type checking by caching argument inference context (Jukka Lehtosalo, PR [19323](https://github.com/python/mypy/pull/19323)) -- Optimize bind_self() and deprecation checks (Ivan Levkivskyi, PR [19556](https://github.com/python/mypy/pull/19556)) -- Keep trivial instances/aliases during expansion (Ivan Levkivskyi, PR [19543](https://github.com/python/mypy/pull/19543)) +### Improvements to Match Statements -### Stubtest Improvements -- Add temporary --ignore-disjoint-bases flag to ease PEP 800 migration (Joren Hammudoglu, PR [19740](https://github.com/python/mypy/pull/19740)) -- Flag redundant uses of @disjoint_base (Jelle Zijlstra, PR [19715](https://github.com/python/mypy/pull/19715)) -- Improve signatures for `__init__` of C classes (Stephen Morton, PR [18259](https://github.com/python/mypy/pull/18259)) -- Handle overloads with mixed pos‑only parameters (Stephen Morton, PR [18287](https://github.com/python/mypy/pull/18287)) -- Use “parameter” (not “argument”) in error messages (PrinceNaroliya, PR [19707](https://github.com/python/mypy/pull/19707)) -- Don’t require @disjoint_base when `__slots__` imply finality (Jelle Zijlstra, PR [19701](https://github.com/python/mypy/pull/19701)) -- Allow runtime‑existing aliases of @type_check_only types (Brian Schubert, PR [19568](https://github.com/python/mypy/pull/19568)) -- More detailed checking of type objects in stubtest (Stephen Morton, PR [18251](https://github.com/python/mypy/pull/18251)) -- Support running stubtest in non-UTF8 terminals (Stanislav Terliakovm, PR [19085](https://github.com/python/mypy/pull/19085)) +- Add temporary named expressions for match subjects (Stanislav Terliakov, PR [18446](https://github.com/python/mypy/pull/18446)) +- Fix unwrapping of assignment expressions in match subject (Marc Mueller, PR [19742](https://github.com/python/mypy/pull/19742)) +- Omit errors for class patterns against object (Marc Mueller, PR [19709](https://github.com/python/mypy/pull/19709)) +- Remove unnecessary error for certain match class patterns (Marc Mueller, PR [19708](https://github.com/python/mypy/pull/19708)) +- Use union type for captured vars in or pattern (Marc Mueller, PR [19710](https://github.com/python/mypy/pull/19710)) +- Prevent final reassignment inside match case (Omer Hadari, PR [19496](https://github.com/python/mypy/pull/19496)) -### Mypyc Improvements +### Fixes to Crashes -- Fix subclass processing in detect_undefined_bitmap (Chainfire, PR [19787](https://github.com/python/mypy/pull/19787)) -- Fix C function signature emission (Jukka Lehtosalo, PR [19773](https://github.com/python/mypy/pull/19773)) -- Use defined `__new__` in tp_new and constructor (Piotr Sawicki, PR [19739](https://github.com/python/mypy/pull/19739)) -- Speed up implicit `__ne__` (Jukka Lehtosalo, PR [19759](https://github.com/python/mypy/pull/19759)) -- Speed up equality with optional str/bytes (Jukka Lehtosalo, PR [19758](https://github.com/python/mypy/pull/19758)) -- Add `__mypyc_empty_tuple__` constant (BobTheBuidler, PR [19654](https://github.com/python/mypy/pull/19654)) -- Add PyObject_CallObject fast‑path op for fn(*args) (BobTheBuidler, PR [19631](https://github.com/python/mypy/pull/19631)) -- Add **kwargs star2 fast‑path (follow‑up to starargs) (BobTheBuidler, PR [19630](https://github.com/python/mypy/pull/19630)) -- Optimize type(x), x.`__class__`, and `.__name__` (Jukka Lehtosalo, PR [19691](https://github.com/python/mypy/pull/19691), [19683](https://github.com/python/mypy/pull/19683)) -- Specialize bytes.decode for common encodings (Jukka Lehtosalo, PR [19688](https://github.com/python/mypy/pull/19688)) -- Speed up in against final fixed‑length tuples (Jukka Lehtosalo, PR [19682](https://github.com/python/mypy/pull/19682)) -- Optimize f‑string building from Final values (BobTheBuidler, PR [19611](https://github.com/python/mypy/pull/19611)) -- Add exact_dict_set_item_op (BobTheBuidler, PR [19657](https://github.com/python/mypy/pull/19657)) -- Cache len() when iterating over immutable types (BobTheBuidler, PR [19656](https://github.com/python/mypy/pull/19656)) -- Add stararg fast‑path for tuple calls fn(*args) (BobTheBuidler, PR [19623](https://github.com/python/mypy/pull/19623)) -- Include more operations in the mypyc trace log (Jukka Lehtosalo, PR [19647](https://github.com/python/mypy/pull/19647)) -- Add prefix to attributes of generator classes (Piotr Sawicki, PR [19535](https://github.com/python/mypy/pull/19535)) -- Fix segfault from heap type objects with static tp_doc (Brian Schubert, PR [19636](https://github.com/python/mypy/pull/19636)) -- Unwrap NewType to its base type for optimized paths (BobTheBuidler, PR [19497](https://github.com/python/mypy/pull/19497)) +- Fix crash with variadic tuple arguments to a generic type (Randolf Scholz, PR [19705](https://github.com/python/mypy/pull/19705)) +- Fix crash when enable_error_code in pyproject.toml has wrong type (wyattscarpenter, PR [19494](https://github.com/python/mypy/pull/19494)) +- Prevent crash for dataclass with PEP 695 TypeVarTuple on Python 3.13+ (Stanislav Terliakov, PR [19565](https://github.com/python/mypy/pull/19565)) +- Fix crash on settable property alias (Ivan Levkivskyi, PR [19615](https://github.com/python/mypy/pull/19615)) + +### Experimental Free-threading Support for Mypyc + +All mypyc tests now pass on free-threading Python 3.14 release candidate builds. The performance +of various micro-benchmarks scale well across multiple threads. + +Free-threading support is still experimental. Note that native attribute access +(get and set), list item access and certain other operations are still +unsafe when there are race conditions. This will likely change in the future. +You can follow the +[area-free-threading label](https://github.com/mypyc/mypyc/issues?q=is%3Aissue%20state%3Aopen%20label%3Aarea-free-threading) +in the mypyc issues tracker to follow progress. + +Related PRs: - Enable free‑threading when compiling multiple modules (Jukka Lehtosalo, PR [19541](https://github.com/python/mypy/pull/19541)) +- Fix `list.pop` on free‑threaded builds (Jukka Lehtosalo, PR [19522](https://github.com/python/mypy/pull/19522)) - Make type objects immortal under free‑threading (Jukka Lehtosalo, PR [19538](https://github.com/python/mypy/pull/19538)) -- Fix list.pop primitive on free‑threaded builds (Jukka Lehtosalo, PR [19522](https://github.com/python/mypy/pull/19522)) + +### Mypyc: Support `__new__` + +Mypyc now has rudimentary support for user-defined `__new__` methods. + +This feature was contributed by Piotr Sawicki (PR [19739](https://github.com/python/mypy/pull/19739)). + +### Mypyc: Faster Generators and Async Functions + +Generators and calls of async functions are now faster, sometimes by 2x or more. + +Related PRs: +- Speed up for loops over native generators (Jukka Lehtosalo, PR [19415](https://github.com/python/mypy/pull/19415)) +- Speed up native‑to‑native calls using await (Jukka Lehtosalo, PR [19398](https://github.com/python/mypy/pull/19398)) +- Call generator helper directly in await expressions (Jukka Lehtosalo, PR [19376](https://github.com/python/mypy/pull/19376)) +- Speed up generator allocation with per‑type freelists (Jukka Lehtosalo, PR [19316](https://github.com/python/mypy/pull/19316)) + +### Miscellaneous Mypyc Improvements + +- Special‑case certain Enum method calls for speed (Ivan Levkivskyi, PR [19634](https://github.com/python/mypy/pull/19634)) +- Fix issues related to subclassing and undefined attribute tracking (Chainfire, PR [19787](https://github.com/python/mypy/pull/19787)) +- Fix invalid C function signature (Jukka Lehtosalo, PR [19773](https://github.com/python/mypy/pull/19773)) +- Speed up implicit `__ne__` (Jukka Lehtosalo, PR [19759](https://github.com/python/mypy/pull/19759)) +- Speed up equality with optional str/bytes types (Jukka Lehtosalo, PR [19758](https://github.com/python/mypy/pull/19758)) +- Speed up access to empty tuples (BobTheBuidler, PR [19654](https://github.com/python/mypy/pull/19654)) +- Speed up calls with `*args` (BobTheBuidler, PRs [19623](https://github.com/python/mypy/pull/19623) and [19631](https://github.com/python/mypy/pull/19631)) +- Speed up calls with `**kwargs` (BobTheBuidler, PR [19630](https://github.com/python/mypy/pull/19630)) +- Optimize `type(x)`, `x.__class__`, and `.__name__` (Jukka Lehtosalo, PR [19691](https://github.com/python/mypy/pull/19691), [19683](https://github.com/python/mypy/pull/19683)) +- Specialize `bytes.decode` for common encodings (Jukka Lehtosalo, PR [19688](https://github.com/python/mypy/pull/19688)) +- Speed up `in` operations using final fixed‑length tuples (Jukka Lehtosalo, PR [19682](https://github.com/python/mypy/pull/19682)) +- Optimize f‑string building from final values (BobTheBuidler, PR [19611](https://github.com/python/mypy/pull/19611)) +- Add dictionary set item for exact dict instances (BobTheBuidler, PR [19657](https://github.com/python/mypy/pull/19657)) +- Cache length when iterating over immutable types (BobTheBuidler, PR [19656](https://github.com/python/mypy/pull/19656)) +- Fix name conflict related to attributes of generator classes (Piotr Sawicki, PR [19535](https://github.com/python/mypy/pull/19535)) +- Fix segfault from heap type objects with a static docstring (Brian Schubert, PR [19636](https://github.com/python/mypy/pull/19636)) +- Unwrap NewType to its base type for additional optimizations (BobTheBuidler, PR [19497](https://github.com/python/mypy/pull/19497)) - Generate an export table only for separate compilation (Jukka Lehtosalo, PR [19521](https://github.com/python/mypy/pull/19521)) -- Add primitives for isinstance of built‑in types (Piotr Sawicki, PR [19435](https://github.com/python/mypy/pull/19435)) -- Add SetElement op to initialize struct values (Jukka Lehtosalo, PR [19437](https://github.com/python/mypy/pull/19437)) -- Simplify IR for for loops over strings (Jukka Lehtosalo, PR [19434](https://github.com/python/mypy/pull/19434)) +- Speed up `isinstance` with built‑in types (Piotr Sawicki, PR [19435](https://github.com/python/mypy/pull/19435)) - Use native integers for some sequence indexing (Jukka Lehtosalo, PR [19426](https://github.com/python/mypy/pull/19426)) -- Remove unused CPyList_GetItemUnsafe primitive (Jukka Lehtosalo, PR [19424](https://github.com/python/mypy/pull/19424)) -- Add native‑int helper methods in IR builder (Jukka Lehtosalo, PR [19423](https://github.com/python/mypy/pull/19423)) -- Use PyList_Check for isinstance(obj, list) (Piotr Sawicki, PR [19416](https://github.com/python/mypy/pull/19416)) -- Speed up for loops over native generators (Jukka Lehtosalo, PR [19415](https://github.com/python/mypy/pull/19415)) +- Speed up `isinstance(obj, list)` (Piotr Sawicki, PR [19416](https://github.com/python/mypy/pull/19416)) - Report error on reserved method names (Piotr Sawicki, PR [19407](https://github.com/python/mypy/pull/19407)) -- Add is_bool_or_bit_rprimitive (Piotr Sawicki, PR [19406](https://github.com/python/mypy/pull/19406)) -- Faster string equality primitive (Jukka Lehtosalo, PR [19402](https://github.com/python/mypy/pull/19402)) -- Speed up native‑to‑native calls using await (Jukka Lehtosalo, PR [19398](https://github.com/python/mypy/pull/19398)) -- Raise NameError on undefined names (Piotr Sawicki, PR [19395](https://github.com/python/mypy/pull/19395)) -- Simplify comparison of tuple elements (Piotr Sawicki, PR [19396](https://github.com/python/mypy/pull/19396)) +- Speed up string equality (Jukka Lehtosalo, PR [19402](https://github.com/python/mypy/pull/19402)) +- Raise `NameError` on undefined names (Piotr Sawicki, PR [19395](https://github.com/python/mypy/pull/19395)) - Use per‑type freelists for nested functions (Jukka Lehtosalo, PR [19390](https://github.com/python/mypy/pull/19390)) -- Call generator helper directly in await expressions (Jukka Lehtosalo, PR [19376](https://github.com/python/mypy/pull/19376)) +- Simplify comparison of tuple elements (Piotr Sawicki, PR [19396](https://github.com/python/mypy/pull/19396)) - Generate introspection signatures for compiled functions (Brian Schubert, PR [19307](https://github.com/python/mypy/pull/19307)) -- Support C string literals in IR (Jukka Lehtosalo, PR [19383](https://github.com/python/mypy/pull/19383)) -- Fix error‑value check for GetAttr that allows nullable values (Jukka Lehtosalo, PR [19378](https://github.com/python/mypy/pull/19378)) +- Fix undefined attribute checking special case (Jukka Lehtosalo, PR [19378](https://github.com/python/mypy/pull/19378)) - Fix comparison of tuples with different lengths (Piotr Sawicki, PR [19372](https://github.com/python/mypy/pull/19372)) -- Speed up generator allocation with per‑type freelists (Jukka Lehtosalo, PR [19316](https://github.com/python/mypy/pull/19316)) -- Implement list.clear() primitive (Jahongir Qurbonov, PR [19344](https://github.com/python/mypy/pull/19344)) -- New primitives for weakref.proxy (BobTheBuidler, PR [19217](https://github.com/python/mypy/pull/19217)) -- New primitive for weakref.ref (BobTheBuidler, PR [19099](https://github.com/python/mypy/pull/19099)) -- New primitive for str.count (BobTheBuidler, PR [19264](https://github.com/python/mypy/pull/19264)) -- Tracing/tooling: optionally log sampled operation traces (Jukka Lehtosalo, PR [19457](https://github.com/python/mypy/pull/19457)) -- Tracing/tooling: script to compile with trace logging and run mypy (Jukka Lehtosalo, PR [19475](https://github.com/python/mypy/pull/19475)) +- Speed up `list.clear` (Jahongir Qurbonov, PR [19344](https://github.com/python/mypy/pull/19344)) +- Speed up `weakref.proxy` (BobTheBuidler, PR [19217](https://github.com/python/mypy/pull/19217)) +- Speed up `weakref.ref` (BobTheBuidler, PR [19099](https://github.com/python/mypy/pull/19099)) +- Speed up `str.count` (BobTheBuidler, PR [19264](https://github.com/python/mypy/pull/19264)) +### Stubtest Improvements +- Add temporary `--ignore-disjoint-bases` flag to ease PEP 800 migration (Joren Hammudoglu, PR [19740](https://github.com/python/mypy/pull/19740)) +- Flag redundant uses of `@disjoint_base` (Jelle Zijlstra, PR [19715](https://github.com/python/mypy/pull/19715)) +- Improve signatures for `__init__` of C extension classes (Stephen Morton, PR [18259](https://github.com/python/mypy/pull/18259)) +- Handle overloads with mixed positional‑only parameters (Stephen Morton, PR [18287](https://github.com/python/mypy/pull/18287)) +- Use “parameter” (not “argument”) in error messages (PrinceNaroliya, PR [19707](https://github.com/python/mypy/pull/19707)) +- Don’t require `@disjoint_base` when `__slots__` imply finality (Jelle Zijlstra, PR [19701](https://github.com/python/mypy/pull/19701)) +- Allow runtime‑existing aliases of `@type_check_only` types (Brian Schubert, PR [19568](https://github.com/python/mypy/pull/19568)) +- More detailed checking of type objects in stubtest (Stephen Morton, PR [18251](https://github.com/python/mypy/pull/18251)) +- Support running stubtest in non-UTF8 terminals (Stanislav Terliakov, PR [19085](https://github.com/python/mypy/pull/19085)) ### Documentation Updates - Add idlemypyextension to IDE integrations (CoolCat467, PR [18615](https://github.com/python/mypy/pull/18615)) -- Document that object is often preferable to Any in APIs (wyattscarpenter, PR [19103](https://github.com/python/mypy/pull/19103)) -- Include a detailed listing of flags enabled by --strict (wyattscarpenter, PR [19062](https://github.com/python/mypy/pull/19062)) +- Document that `object` is often preferable to `Any` in APIs (wyattscarpenter, PR [19103](https://github.com/python/mypy/pull/19103)) +- Include a detailed listing of flags enabled by `--strict` (wyattscarpenter, PR [19062](https://github.com/python/mypy/pull/19062)) - Update “common issues” (reveal_type/reveal_locals; note on orjson) (wyattscarpenter, PR [19059](https://github.com/python/mypy/pull/19059), [19058](https://github.com/python/mypy/pull/19058)) -### Other Notable Improvements +### Other Notable Fixes and Improvements -- Remove deprecated --new-type-inference flag (the new algorithm has long been default) (Ivan Levkivskyi, PR [19570](https://github.com/python/mypy/pull/19570)) +- Remove deprecated `--new-type-inference` flag (the new algorithm has long been default) (Ivan Levkivskyi, PR [19570](https://github.com/python/mypy/pull/19570)) - Use empty context as a fallback for return expressions when outer context misleads inference (Ivan Levkivskyi, PR [19767](https://github.com/python/mypy/pull/19767)) -- Support --strict-equality checks involving None (Christoph Tyralla, PR [19718](https://github.com/python/mypy/pull/19718)) -- Don’t show import‑related errors after a module‑level assert False (Stanislav Terliakov, PR [19347](https://github.com/python/mypy/pull/19347)) -- Fix forward refs in type parameters of over‑parameterized PEP 695 aliases (Brian Schubert, PR [19725](https://github.com/python/mypy/pull/19725)) +- Fix forward references in type parameters of over‑parameterized PEP 695 aliases (Brian Schubert, PR [19725](https://github.com/python/mypy/pull/19725)) - Don’t expand PEP 695 aliases when checking node fullnames (Brian Schubert, PR [19699](https://github.com/python/mypy/pull/19699)) -- Don’t use outer context for or expression inference when LHS is Any (Stanislav Terliakov, PR [19748](https://github.com/python/mypy/pull/19748)) -- Interpret bare ClassVar as inferred (not Any) (Ivan Levkivskyi, PR [19573](https://github.com/python/mypy/pull/19573)) +- Don’t use outer context for 'or' expression inference when LHS is Any (Stanislav Terliakov, PR [19748](https://github.com/python/mypy/pull/19748)) - Recognize buffer protocol special methods (Brian Schubert, PR [19581](https://github.com/python/mypy/pull/19581)) -- Add temporary named expressions for match subjects (Stanislav Terliakov, PR [18446](https://github.com/python/mypy/pull/18446)) - Support attribute access on enum members correctly (Stanislav Terliakov, PR [19422](https://github.com/python/mypy/pull/19422)) - Check `__slots__` assignments on self types (Stanislav Terliakov, PR [19332](https://github.com/python/mypy/pull/19332)) - Move self‑argument checks after decorator application (Stanislav Terliakov, PR [19490](https://github.com/python/mypy/pull/19490)) - Infer empty list for `__slots__` and module `__all__` (Stanislav Terliakov, PR [19348](https://github.com/python/mypy/pull/19348)) - Use normalized tuples for fallback calculation (Stanislav Terliakov, PR [19111](https://github.com/python/mypy/pull/19111)) -- Preserve literals when joining Literal with Instance that has matching last_known_value (Stanislav Terliakov, PR [19279](https://github.com/python/mypy/pull/19279)) +- Preserve literals when joining similar types (Stanislav Terliakov, PR [19279](https://github.com/python/mypy/pull/19279)) - Allow adjacent conditionally‑defined overloads (Stanislav Terliakov, PR [19042](https://github.com/python/mypy/pull/19042)) - Check property decorators more strictly (Stanislav Terliakov, PR [19313](https://github.com/python/mypy/pull/19313)) - Support properties with generic setters (Ivan Levkivskyi, PR [19298](https://github.com/python/mypy/pull/19298)) @@ -162,45 +252,32 @@ Mypy 1.18 includes numerous performance improvements, resulting in a 38% overall - Include tuple fallback in constraints built from tuple types (Stanislav Terliakov, PR [19100](https://github.com/python/mypy/pull/19100)) - Somewhat better isinstance support on old‑style unions (Shantanu, PR [19714](https://github.com/python/mypy/pull/19714)) - Improve promotions inside unions (Christoph Tyralla, PR [19245](https://github.com/python/mypy/pull/19245)) -- Uninhabited types should have all attributes (Ivan Levkivskyi, PR [19300](https://github.com/python/mypy/pull/19300)) -- Metaclass conflict checks improved (Robsdedude, PR [17682](https://github.com/python/mypy/pull/17682)) -- Metaclass resolution algorithm fixes (Robsdedude, PR [17713](https://github.com/python/mypy/pull/17713)) +- Treat uninhabited types as having all attributes (Ivan Levkivskyi, PR [19300](https://github.com/python/mypy/pull/19300)) +- Improve metaclass conflict checks (Robsdedude, PR [17682](https://github.com/python/mypy/pull/17682)) +- Fixes to metaclass resolution algorithm (Robsdedude, PR [17713](https://github.com/python/mypy/pull/17713)) - PEP 702 @deprecated: handle “combined” overloads (Christoph Tyralla, PR [19626](https://github.com/python/mypy/pull/19626)) - PEP 702 @deprecated: include overloads in snapshot descriptions (Christoph Tyralla, PR [19613](https://github.com/python/mypy/pull/19613)) - Ignore overload implementation when checking `__OP__` / `__rOP__` compatibility (Stanislav Terliakov, PR [18502](https://github.com/python/mypy/pull/18502)) -- Fix unwrapping of assignment expressions in match subject (Marc Mueller, PR [19742](https://github.com/python/mypy/pull/19742)) -- Omit errors for class patterns against object (Marc Mueller, PR [19709](https://github.com/python/mypy/pull/19709)) -- Remove unnecessary error for certain match class patterns (Marc Mueller, PR [19708](https://github.com/python/mypy/pull/19708)) -- Use union type for captured vars in or pattern (Marc Mueller, PR [19710](https://github.com/python/mypy/pull/19710)) -- Prevent final reassignment inside match case (Omer Hadari, PR [19496](https://github.com/python/mypy/pull/19496)) -- Support _value_ as a fallback for ellipsis Enum members (Stanislav Terliakov, PR [19352](https://github.com/python/mypy/pull/19352)) +- Support `_value_` as a fallback for ellipsis Enum members (Stanislav Terliakov, PR [19352](https://github.com/python/mypy/pull/19352)) - Sort arguments in TypedDict overlap messages (Marc Mueller, PR [19666](https://github.com/python/mypy/pull/19666)) -- Reset to previous statement on leaving return in semanal (Stanislav Terliakov, PR [19642](https://github.com/python/mypy/pull/19642)) -- Add ambiguous to UninhabitedType identity for better messaging (Stanislav Terliakov, PR [19648](https://github.com/python/mypy/pull/19648)) -- Further fix overload diagnostics for varargs/kwargs (Shantanu, PR [19619](https://github.com/python/mypy/pull/19619)) -- Fix overload diagnostics when vararg and varkwarg both match (Shantanu, PR [19614](https://github.com/python/mypy/pull/19614)) -- Show type variable name in “Cannot infer type argument” (Brian Schubert, PR [19290](https://github.com/python/mypy/pull/19290)) +- Fix handling of implicit return in lambda (Stanislav Terliakov, PR [19642](https://github.com/python/mypy/pull/19642)) +- Improve behavior of uninhabited types (Stanislav Terliakov, PR [19648](https://github.com/python/mypy/pull/19648)) +- Fix overload diagnostics when `*args` and `**kwargs` both match (Shantanu, PR [19614](https://github.com/python/mypy/pull/19614)) +- Further fix overload diagnostics for `*args`/`**kwargs` (Shantanu, PR [19619](https://github.com/python/mypy/pull/19619)) +- Show type variable name in "Cannot infer type argument" (Brian Schubert, PR [19290](https://github.com/python/mypy/pull/19290)) - Fail gracefully on unsupported template strings (PEP 750) (Brian Schubert, PR [19700](https://github.com/python/mypy/pull/19700)) - Revert colored argparse help for Python 3.14 (Marc Mueller, PR [19721](https://github.com/python/mypy/pull/19721)) -- Support type‑checking a code fragment in the profile script (Jukka Lehtosalo, PR [19379](https://github.com/python/mypy/pull/19379)) -- Fix C compiler flags in the profile self‑check script (Jukka Lehtosalo, PR [19326](https://github.com/python/mypy/pull/19326)) -- Add a script for profiling self‑check (Linux only) (Jukka Lehtosalo, PR [19322](https://github.com/python/mypy/pull/19322)) -- Retry PyPI upload script: skip existing files on retry (Jukka Lehtosalo, PR [19305](https://github.com/python/mypy/pull/19305)) - Update stubinfo for latest typeshed (Shantanu, PR [19771](https://github.com/python/mypy/pull/19771)) -- Fix crash with variadic tuple arguments to a generic type (Randolf Scholz, PR [19705](https://github.com/python/mypy/pull/19705)) -- Fix crash when enable_error_code in pyproject.toml has wrong type (wyattscarpenter, PR [19494](https://github.com/python/mypy/pull/19494)) -- Fix dict assignment to a wider context when an incompatible same‑shape TypedDict exists (Stanislav Terliakov, PR [19592](https://github.com/python/mypy/pull/19592)) -- Prevent crash for dataclass with PEP 695 TypeVarTuple on Python 3.13+ (Stanislav Terliakov, PR [19565](https://github.com/python/mypy/pull/19565)) +- Fix dict assignment when an incompatible same‑shape TypedDict exists (Stanislav Terliakov, PR [19592](https://github.com/python/mypy/pull/19592)) - Fix constructor type for subclasses of Any (Ivan Levkivskyi, PR [19295](https://github.com/python/mypy/pull/19295)) -- Fix TypeGuard/TypeIs being forgotten when semanal defers (Brian Schubert, PR [19325](https://github.com/python/mypy/pull/19325)) +- Fix TypeGuard/TypeIs being forgotten in some cases (Brian Schubert, PR [19325](https://github.com/python/mypy/pull/19325)) - Fix TypeIs negative narrowing for unions of generics (Brian Schubert, PR [18193](https://github.com/python/mypy/pull/18193)) -- dmypy suggest: fix incorrect signature suggestion when a type matches a module name (Brian Schubert, PR [18937](https://github.com/python/mypy/pull/18937)) -- dmypy suggest: fix interaction with `__new__` (Stanislav Terliakov, PR [18966](https://github.com/python/mypy/pull/18966)) -- dmypy suggest: support Callable / callable Protocols in decorator unwrapping (Anthony Sottile, PR [19072](https://github.com/python/mypy/pull/19072)) +- dmypy suggest: Fix incorrect signature suggestion when a type matches a module name (Brian Schubert, PR [18937](https://github.com/python/mypy/pull/18937)) +- dmypy suggest: Fix interaction with `__new__` (Stanislav Terliakov, PR [18966](https://github.com/python/mypy/pull/18966)) +- dmypy suggest: Support Callable / callable Protocols in decorator unwrapping (Anthony Sottile, PR [19072](https://github.com/python/mypy/pull/19072)) - Fix missing error when redeclaring a type variable in a nested generic class (Brian Schubert, PR [18883](https://github.com/python/mypy/pull/18883)) - Fix for overloaded type object erasure (Shantanu, PR [19338](https://github.com/python/mypy/pull/19338)) - Fix TypeGuard with call on temporary object (Saul Shanabrook, PR [19577](https://github.com/python/mypy/pull/19577)) -- Fix crash on settable property alias (Ivan Levkivskyi, PR [19615](https://github.com/python/mypy/pull/19615)) ### Typeshed Updates From f0863a551ad1ee7f0116cf2580cdb19ffbbbf9c3 Mon Sep 17 00:00:00 2001 From: Kevin Kannammalil Date: Thu, 11 Sep 2025 11:19:13 -0400 Subject: [PATCH 255/424] Removed Unreleased in the Changelog for Release 1.18 (#19827) Remove Unreleased from section title --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5266a86c725ec..3e6f8c2cac38b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Next Release -## Mypy 1.18 (Unreleased) +## Mypy 1.18 We’ve just uploaded mypy 1.18 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). Mypy is a static type checker for Python. This release includes new features, performance From 83d186a9cc76956534332b3618dea7f662237daf Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 12 Sep 2025 01:12:58 +0100 Subject: [PATCH 256/424] Remove some effectively dead code from build.py (#19833) We can't have `fresh` True when `stale_deps` is non-empty. I guess this part is a leftover from `--quick-and-dirty` times. --- mypy/build.py | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 84dbf2b2df880..5ccf1c86e7e3e 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -3303,34 +3303,7 @@ def process_graph(graph: Graph, manager: BuildManager) -> None: if undeps: fresh = False if fresh: - # All cache files are fresh. Check that no dependency's - # cache file is newer than any scc node's cache file. - oldest_in_scc = min(graph[id].xmeta.data_mtime for id in scc) - viable = {id for id in stale_deps if graph[id].meta is not None} - newest_in_deps = ( - 0 if not viable else max(graph[dep].xmeta.data_mtime for dep in viable) - ) - if manager.options.verbosity >= 3: # Dump all mtimes for extreme debugging. - all_ids = sorted(ascc | viable, key=lambda id: graph[id].xmeta.data_mtime) - for id in all_ids: - if id in scc: - if graph[id].xmeta.data_mtime < newest_in_deps: - key = "*id:" - else: - key = "id:" - else: - if graph[id].xmeta.data_mtime > oldest_in_scc: - key = "+dep:" - else: - key = "dep:" - manager.trace(" %5s %.0f %s" % (key, graph[id].xmeta.data_mtime, id)) - # If equal, give the benefit of the doubt, due to 1-sec time granularity - # (on some platforms). - if oldest_in_scc < newest_in_deps: - fresh = False - fresh_msg = f"out of date by {newest_in_deps - oldest_in_scc:.0f} seconds" - else: - fresh_msg = "fresh" + fresh_msg = "fresh" elif undeps: fresh_msg = f"stale due to changed suppression ({' '.join(sorted(undeps))})" elif stale_scc: From 4939b116adbd8550342ca79c87bb01a3c15c044f Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 12 Sep 2025 12:47:05 +0100 Subject: [PATCH 257/424] [mypyc] Fix crash with NewType and other non-class types in incremental builds (#19837) Fixes https://github.com/mypyc/mypyc/issues/1138. Also fix similar issue with named tuples and TypedDicts. --- mypyc/irbuild/prepare.py | 8 ++++- mypyc/test-data/run-multimodule.test | 48 ++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index 61e3e5b95cf43..20f2aeef8e6ef 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -159,7 +159,13 @@ def load_type_map(mapper: Mapper, modules: list[MypyFile], deser_ctx: DeserMaps) """Populate a Mapper with deserialized IR from a list of modules.""" for module in modules: for node in module.names.values(): - if isinstance(node.node, TypeInfo) and is_from_module(node.node, module): + if ( + isinstance(node.node, TypeInfo) + and is_from_module(node.node, module) + and not node.node.is_newtype + and not node.node.is_named_tuple + and node.node.typeddict_type is None + ): ir = deser_ctx.classes[node.node.fullname] mapper.type_to_ir[node.node] = ir mapper.symbol_fullnames.add(node.node.fullname) diff --git a/mypyc/test-data/run-multimodule.test b/mypyc/test-data/run-multimodule.test index 4208af0f04c81..9323612cb4fbc 100644 --- a/mypyc/test-data/run-multimodule.test +++ b/mypyc/test-data/run-multimodule.test @@ -902,3 +902,51 @@ import native [out2] 0 None + +[case testIncrementalCompilationWithNonClassTypeDef] +import other_a +[file other_a.py] +from other_b import MyInt +[file other_a.py.2] +from other_b import MyInt, NT, TD +i = MyInt(42) + +def f(x: MyInt) -> int: + return x + 1 + +def g(x: int) -> MyInt: + return MyInt(x + 2) + +print(i) +print(f(i)) +print(g(13)) + +def make_nt(x: int) -> NT: + return NT(x=MyInt(x)) + +print(make_nt(4)) + +def make_td(x: int) -> TD: + return {"x": MyInt(x)} + +print(make_td(5)) + +[file other_b.py] +from typing import NewType, NamedTuple, TypedDict +from enum import Enum + +MyInt = NewType("MyInt", int) +NT = NamedTuple("NT", [("x", MyInt)]) +TD = TypedDict("TD", {"x": MyInt}) + +[file driver.py] +import native + +[typing fixtures/typing-full.pyi] +[out] +[out2] +42 +43 +15 +NT(x=4) +{'x': 5} From f8f618a79606d42bb0362361ec5d1d6c300f66e9 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Fri, 12 Sep 2025 19:26:54 +0700 Subject: [PATCH 258/424] Update main.py: remove superfluous `--experimental` flag (#19831) This commit removes the --experimental flag, completing a TODO from 2018-03-16. It seems like this flag is unused and undocumented and the task of removing it "after a short transition" simply slipped under everyone's radar. --- mypy/main.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index d5bbca7043058..150d388af84c1 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -1204,13 +1204,6 @@ def add_invertible_flag( ) if server_options: - # TODO: This flag is superfluous; remove after a short transition (2018-03-16) - other_group.add_argument( - "--experimental", - action="store_true", - dest="fine_grained_incremental", - help="Enable fine-grained incremental mode", - ) other_group.add_argument( "--use-fine-grained-cache", action="store_true", From 8bfecd4e9fbbcb26390b4b851bf2c6f9e9e34e00 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 12 Sep 2025 15:22:15 +0100 Subject: [PATCH 259/424] Write cache for modules with errors (#19820) This is required for https://github.com/python/mypy/issues/933 Here I use a very simple-minded approach, errors are serialized simply as a list of strings. In near future I may switch to serializing `ErrorInfo`s (as this has some other benefits). Note that many tests have `[stale ...]` checks updated because previously modules with errors were not included in the list. I double-checked each test that the new values are correct. Note we still don't write cache if there were blockers in an SCC (like a syntax error). --- mypy/build.py | 62 +++++------------- mypy/test/testcheck.py | 44 ++++--------- test-data/unit/check-incremental.test | 94 ++++++++++++++++++++------- test-data/unit/check-serialize.test | 2 +- 4 files changed, 101 insertions(+), 101 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 5ccf1c86e7e3e..2d3296a4713ee 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -15,7 +15,6 @@ import collections import contextlib -import errno import gc import json import os @@ -337,6 +336,7 @@ class CacheMeta(NamedTuple): dep_lines: list[int] dep_hashes: dict[str, str] interface_hash: str # hash representing the public interface + error_lines: list[str] version_id: str # mypy version for cache invalidation ignore_all: bool # if errors were ignored plugin_data: Any # config data from plugins @@ -376,6 +376,7 @@ def cache_meta_from_dict(meta: dict[str, Any], data_json: str) -> CacheMeta: meta.get("dep_lines", []), meta.get("dep_hashes", {}), meta.get("interface_hash", ""), + meta.get("error_lines", []), meta.get("version_id", sentinel), meta.get("ignore_all", True), meta.get("plugin_data", None), @@ -1502,6 +1503,7 @@ def validate_meta( "dep_lines": meta.dep_lines, "dep_hashes": meta.dep_hashes, "interface_hash": meta.interface_hash, + "error_lines": meta.error_lines, "version_id": manager.version_id, "ignore_all": meta.ignore_all, "plugin_data": meta.plugin_data, @@ -1678,28 +1680,6 @@ def write_cache_meta( return cache_meta_from_dict(meta, data_json) -def delete_cache(id: str, path: str, manager: BuildManager) -> None: - """Delete cache files for a module. - - The cache files for a module are deleted when mypy finds errors there. - This avoids inconsistent states with cache files from different mypy runs, - see #4043 for an example. - """ - # We don't delete .deps files on errors, since the dependencies - # are mostly generated from other files and the metadata is - # tracked separately. - meta_path, data_path, _ = get_cache_names(id, path, manager.options) - cache_paths = [meta_path, data_path] - manager.log(f"Deleting {id} {path} {' '.join(x for x in cache_paths if x)}") - - for filename in cache_paths: - try: - manager.metastore.remove(filename) - except OSError as e: - if e.errno != errno.ENOENT: - manager.log(f"Error deleting cache file {filename}: {e.strerror}") - - """Dependency manager. Design @@ -1875,6 +1855,9 @@ class State: # Map from dependency id to its last observed interface hash dep_hashes: dict[str, str] = {} + # List of errors reported for this file last time. + error_lines: list[str] = [] + # Parent package, its parent, etc. ancestors: list[str] | None = None @@ -1896,9 +1879,6 @@ class State: # Whether to ignore all errors ignore_all = False - # Whether the module has an error or any of its dependencies have one. - transitive_error = False - # Errors reported before semantic analysis, to allow fine-grained # mode to keep reporting them. early_errors: list[ErrorInfo] @@ -2000,6 +1980,7 @@ def __init__( assert len(all_deps) == len(self.meta.dep_lines) self.dep_line_map = {id: line for id, line in zip(all_deps, self.meta.dep_lines)} self.dep_hashes = self.meta.dep_hashes + self.error_lines = self.meta.error_lines if temporary: self.load_tree(temporary=True) if not manager.use_fine_grained_cache(): @@ -2517,11 +2498,6 @@ def write_cache(self) -> tuple[dict[str, Any], str, str] | None: print(f"Error serializing {self.id}", file=self.manager.stdout) raise # Propagate to display traceback return None - is_errors = self.transitive_error - if is_errors: - delete_cache(self.id, self.path, self.manager) - self.meta = None - return None dep_prios = self.dependency_priorities() dep_lines = self.dependency_lines() assert self.source_hash is not None @@ -3315,15 +3291,14 @@ def process_graph(graph: Graph, manager: BuildManager) -> None: else: fresh_msg = f"stale due to deps ({' '.join(sorted(stale_deps))})" - # Initialize transitive_error for all SCC members from union - # of transitive_error of dependencies. - if any(graph[dep].transitive_error for dep in deps if dep in graph): - for id in scc: - graph[id].transitive_error = True - scc_str = " ".join(scc) if fresh: manager.trace(f"Queuing {fresh_msg} SCC ({scc_str})") + for id in scc: + if graph[id].error_lines: + manager.flush_errors( + manager.errors.simplify_path(graph[id].xpath), graph[id].error_lines, False + ) fresh_scc_queue.append(scc) else: if fresh_scc_queue: @@ -3335,11 +3310,6 @@ def process_graph(graph: Graph, manager: BuildManager) -> None: # single fresh SCC. This is intentional -- we don't need those modules # loaded if there are no more stale SCCs to be rechecked. # - # Also note we shouldn't have to worry about transitive_error here, - # since modules with transitive errors aren't written to the cache, - # and if any dependencies were changed, this SCC would be stale. - # (Also, in quick_and_dirty mode we don't care about transitive errors.) - # # TODO: see if it's possible to determine if we need to process only a # _subset_ of the past SCCs instead of having to process them all. if ( @@ -3491,16 +3461,17 @@ def process_stale_scc(graph: Graph, scc: list[str], manager: BuildManager) -> No for id in stale: graph[id].generate_unused_ignore_notes() graph[id].generate_ignore_without_code_notes() - if any(manager.errors.is_errors_for_file(graph[id].xpath) for id in stale): - for id in stale: - graph[id].transitive_error = True + + # Flush errors, and write cache in two phases: first data files, then meta files. meta_tuples = {} + errors_by_id = {} for id in stale: if graph[id].xpath not in manager.errors.ignored_files: errors = manager.errors.file_messages( graph[id].xpath, formatter=manager.error_formatter ) manager.flush_errors(manager.errors.simplify_path(graph[id].xpath), errors, False) + errors_by_id[id] = errors meta_tuples[id] = graph[id].write_cache() graph[id].mark_as_rechecked() for id in stale: @@ -3512,6 +3483,7 @@ def process_stale_scc(graph: Graph, scc: list[str], manager: BuildManager) -> No meta["dep_hashes"] = { dep: graph[dep].interface_hash for dep in graph[id].dependencies if dep in graph } + meta["error_lines"] = errors_by_id.get(id, []) graph[id].meta = write_cache_meta(meta, manager, meta_json, data_json) diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index 04ef5370d381f..73f33c0323af9 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -9,7 +9,6 @@ from pathlib import Path from mypy import build -from mypy.build import Graph from mypy.errors import CompileError from mypy.modulefinder import BuildSource, FindModuleCache, SearchPaths from mypy.test.config import test_data_prefix, test_temp_dir @@ -164,11 +163,13 @@ def run_case_once( sys.path.insert(0, plugin_dir) res = None + blocker = False try: res = build.build(sources=sources, options=options, alt_lib_path=test_temp_dir) a = res.errors except CompileError as e: a = e.messages + blocker = True finally: assert sys.path[0] == plugin_dir del sys.path[0] @@ -199,7 +200,7 @@ def run_case_once( if res: if options.cache_dir != os.devnull: - self.verify_cache(module_data, res.errors, res.manager, res.graph) + self.verify_cache(module_data, res.manager, blocker) name = "targets" if incremental_step: @@ -229,42 +230,23 @@ def run_case_once( check_test_output_files(testcase, incremental_step, strip_prefix="tmp/") def verify_cache( - self, - module_data: list[tuple[str, str, str]], - a: list[str], - manager: build.BuildManager, - graph: Graph, + self, module_data: list[tuple[str, str, str]], manager: build.BuildManager, blocker: bool ) -> None: - # There should be valid cache metadata for each module except - # for those that had an error in themselves or one of their - # dependencies. - error_paths = self.find_error_message_paths(a) - busted_paths = {m.path for id, m in manager.modules.items() if graph[id].transitive_error} - modules = self.find_module_files(manager) - modules.update({module_name: path for module_name, path, text in module_data}) - missing_paths = self.find_missing_cache_files(modules, manager) - # We would like to assert error_paths.issubset(busted_paths) - # but this runs into trouble because while some 'notes' are - # really errors that cause an error to be marked, many are - # just notes attached to other errors. - assert error_paths or not busted_paths, "Some modules reported error despite no errors" - if not missing_paths == busted_paths: - raise AssertionError(f"cache data discrepancy {missing_paths} != {busted_paths}") + if not blocker: + # There should be valid cache metadata for each module except + # in case of a blocking error in themselves or one of their + # dependencies. + modules = self.find_module_files(manager) + modules.update({module_name: path for module_name, path, text in module_data}) + missing_paths = self.find_missing_cache_files(modules, manager) + if missing_paths: + raise AssertionError(f"cache data missing for {missing_paths}") assert os.path.isfile(os.path.join(manager.options.cache_dir, ".gitignore")) cachedir_tag = os.path.join(manager.options.cache_dir, "CACHEDIR.TAG") assert os.path.isfile(cachedir_tag) with open(cachedir_tag) as f: assert f.read().startswith("Signature: 8a477f597d28d172789f06886806bc55") - def find_error_message_paths(self, a: list[str]) -> set[str]: - hits = set() - for line in a: - m = re.match(r"([^\s:]+):(\d+:)?(\d+:)? (error|warning|note):", line) - if m: - p = m.group(1) - hits.add(p) - return hits - def find_module_files(self, manager: build.BuildManager) -> dict[str, str]: return {id: module.path for id, module in manager.modules.items()} diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 06f228721a868..8e05f922be174 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -124,7 +124,7 @@ class A: pass def func1() -> A: pass [rechecked mod1] -[stale] +[stale mod1] [out2] tmp/mod1.py:1: error: Name "A" is not defined @@ -428,7 +428,7 @@ class CustomType: def foo(self) -> str: return "a" [rechecked mod1, mod2, mod2.mod3] -[stale mod2, mod2.mod3] +[stale mod1, mod2, mod2.mod3] [builtins fixtures/module.pyi] [out1] [out2] @@ -466,7 +466,7 @@ class CustomType: def foo(self) -> str: return "a" [rechecked mod1, mod2, mod2.mod3] -[stale mod2.mod3] +[stale mod1, mod2.mod3] [builtins fixtures/module.pyi] [out1] [out2] @@ -541,7 +541,7 @@ def func2() -> str: return "foo" [rechecked mod0, mod1, mod2] -[stale mod2] +[stale mod0, mod2] [out2] tmp/mod1.py:4: error: Incompatible return value type (got "str", expected "int") @@ -952,7 +952,7 @@ reveal_type(b.x) [file parent/b.py.2] x = 10 -[stale parent.b] +[stale parent.a, parent.b] [rechecked parent.a, parent.b] [out2] tmp/parent/a.py:2: note: Revealed type is "builtins.int" @@ -1080,7 +1080,7 @@ class Class: pass [builtins fixtures/args.pyi] [rechecked collections, main, package.subpackage.mod1] -[stale collections, package.subpackage.mod1] +[stale collections, main, package.subpackage.mod1] [out2] tmp/main.py:4: error: "Class" has no attribute "some_attribute" @@ -1120,7 +1120,7 @@ if int(): [builtins fixtures/module_all.pyi] [rechecked main, c, c.submodule] -[stale c] +[stale main, c, c.submodule] [out2] tmp/c/submodule.py:3: error: Incompatible types in assignment (expression has type "str", variable has type "int") tmp/main.py:7: error: "C" has no attribute "foo" @@ -1174,7 +1174,7 @@ reveal_type(foo) foo = 3.14 reveal_type(foo) [rechecked m, n] -[stale] +[stale n] [out1] tmp/n.py:2: note: Revealed type is "builtins.str" tmp/m.py:3: error: Argument 1 to "accept_int" has incompatible type "str"; expected "int" @@ -1204,7 +1204,7 @@ from bad import foo foo(3) [rechecked client] -[stale] +[stale client] [out2] tmp/client.py:4: error: Argument 1 to "foo" has incompatible type "int"; expected "str" @@ -1316,7 +1316,7 @@ reveal_type(bar) bar = "str" [rechecked main] -[stale] +[stale main] [out1] tmp/main.py:3: error: Argument 1 to "accept_int" has incompatible type "str"; expected "int" tmp/main.py:4: note: Revealed type is "builtins.str" @@ -1354,8 +1354,8 @@ class B: class C: def foo(self) -> int: return 1 -[rechecked mod3, mod2, mod1] -[stale mod3, mod2] +[rechecked mod3] +[stale] [out1] tmp/mod3.py:6: error: Incompatible types in assignment (expression has type "str", variable has type "int") tmp/mod1.py:3: note: Revealed type is "builtins.int" @@ -1393,8 +1393,8 @@ class C: class C: def foo(self) -> str: return 'a' -[rechecked mod4, mod3, mod2, mod1] -[stale mod4] +[rechecked mod4, mod3, mod1] +[stale mod1, mod4] [out1] tmp/mod3.py:6: error: Incompatible types in assignment (expression has type "str", variable has type "int") tmp/mod1.py:3: note: Revealed type is "builtins.int" @@ -1438,8 +1438,8 @@ class C: class C: def foo(self) -> str: return 'a' -[rechecked mod4, mod3, mod2, mod1] -[stale mod4, mod3, mod2] +[rechecked mod4, mod3, mod1] +[stale mod1, mod4] [out1] tmp/mod3.py:6: error: Incompatible types in assignment (expression has type "str", variable has type "int") tmp/mod1.py:3: note: Revealed type is "builtins.int" @@ -2173,7 +2173,7 @@ import m x = 1 [delete m.py.2] [rechecked n] -[stale] +[stale n] [out2] tmp/n.py:1: error: Cannot find implementation or library stub for module named "m" tmp/n.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports @@ -2224,7 +2224,7 @@ def foo() -> int: [rechecked m] [stale m] [rechecked2 m] -[stale2] +[stale2 m] [out3] tmp/m.py:2: error: Incompatible return value type (got "str", expected "int") @@ -2243,7 +2243,7 @@ def foo() -> str: def foo(x) -> int: pass [rechecked m, n] -[stale m] +[stale m, n] [rechecked2 m, n] [stale2 m, n] [out2] @@ -2894,7 +2894,7 @@ extra = 1 import m.a # Depends on module with error [file m/c.py] import m # No error here -[rechecked m.a, m.b] +[rechecked] [out1] tmp/m/a.py:1: error: Unsupported operand types for + ("int" and "str") [out2] @@ -3091,10 +3091,10 @@ class A: [stale] [out2] main:2: note: Revealed type is "def (a: builtins.int) -> a.A" -main:3: note: Revealed type is "def [_AT] (self: _AT`1, other: _AT`1) -> builtins.bool" -main:4: note: Revealed type is "def [_AT] (self: _AT`2, other: _AT`2) -> builtins.bool" -main:5: note: Revealed type is "def [_AT] (self: _AT`3, other: _AT`3) -> builtins.bool" -main:6: note: Revealed type is "def [_AT] (self: _AT`4, other: _AT`4) -> builtins.bool" +main:3: note: Revealed type is "def [_AT] (self: _AT`3, other: _AT`3) -> builtins.bool" +main:4: note: Revealed type is "def [_AT] (self: _AT`4, other: _AT`4) -> builtins.bool" +main:5: note: Revealed type is "def [_AT] (self: _AT`5, other: _AT`5) -> builtins.bool" +main:6: note: Revealed type is "def [_AT] (self: _AT`6, other: _AT`6) -> builtins.bool" main:15: error: Unsupported operand types for < ("A" and "int") main:16: error: Unsupported operand types for <= ("A" and "int") main:17: error: Unsupported operand types for > ("A" and "int") @@ -7237,3 +7237,49 @@ bar: int = foo [out2] [out3] tmp/bar.py:2: error: Incompatible types in assignment (expression has type "None", variable has type "int") + +[case testIncrementalBlockingErrorRepeatAndUndo] +import m +[file m.py] +import f +reveal_type(f.x) +[file m.py.3] +import f +reveal_type(f.x) +# touch +[file f.py] +x = 1 +[file f.py.2] +no way +[file f.py.4] +x = 1 +[out] +tmp/m.py:2: note: Revealed type is "builtins.int" +[out2] +tmp/f.py:1: error: Invalid syntax +[out3] +tmp/f.py:1: error: Invalid syntax +[out4] +tmp/m.py:2: note: Revealed type is "builtins.int" + +[case testIncrementalSameErrorOrder] +import m +[file m.py] +import n +def accept_int(x: int) -> None: pass +accept_int(n.foo) +[file n.py] +import other +foo = "hello" +reveal_type(foo) +[file other.py] +[file other.py.2] +# touch +[rechecked other] +[stale] +[out] +tmp/n.py:3: note: Revealed type is "builtins.str" +tmp/m.py:3: error: Argument 1 to "accept_int" has incompatible type "str"; expected "int" +[out2] +tmp/n.py:3: note: Revealed type is "builtins.str" +tmp/m.py:3: error: Argument 1 to "accept_int" has incompatible type "str"; expected "int" diff --git a/test-data/unit/check-serialize.test b/test-data/unit/check-serialize.test index 03c185a5694b3..1498c8d82826c 100644 --- a/test-data/unit/check-serialize.test +++ b/test-data/unit/check-serialize.test @@ -31,7 +31,7 @@ x = '' -- We only do the following two sections once here to avoid repetition. -- Most other test cases are similar. [rechecked a] -[stale] +[stale a] [out2] tmp/a.py:2: error: Incompatible types in assignment (expression has type "str", variable has type "int") From 530bdc5063f2309702ec08797388d635cad4b634 Mon Sep 17 00:00:00 2001 From: Stephen Morton Date: Sat, 13 Sep 2025 10:56:02 -0700 Subject: [PATCH 260/424] stubtest: additional guidance on errors when runtime is object.__init__ (#19733) Fixes #19732 This is a simple check to point users in the right direction when they get errors because their class uses `__new__` but they wrote stubs for `__init__`. I don't feel strongly about the exact wording used here. I also considered "Maybe you meant to define `__new__` instead of `__init__`?". --- mypy/stubtest.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index d4f96a3d9389f..4126f3959ee15 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -1053,7 +1053,10 @@ def get_kind(arg_name: str) -> nodes.ArgKind: def _verify_signature( - stub: Signature[nodes.Argument], runtime: Signature[inspect.Parameter], function_name: str + stub: Signature[nodes.Argument], + runtime: Signature[inspect.Parameter], + function_name: str, + warn_runtime_is_object_init: bool = False, ) -> Iterator[str]: # Check positional arguments match up for stub_arg, runtime_arg in zip(stub.pos, runtime.pos): @@ -1098,6 +1101,8 @@ def _verify_signature( msg = f'runtime does not have parameter "{stub_arg.variable.name}"' if runtime.varkw is not None: msg += ". Maybe you forgot to make it keyword-only in the stub?" + elif warn_runtime_is_object_init: + msg += ". You may need to write stubs for __new__ instead of __init__." yield msg else: yield f'stub parameter "{stub_arg.variable.name}" is not keyword-only' @@ -1137,7 +1142,11 @@ def _verify_signature( if arg not in {runtime_arg.name for runtime_arg in runtime.pos[len(stub.pos) :]}: yield f'runtime parameter "{arg}" is not keyword-only' else: - yield f'runtime does not have parameter "{arg}"' + msg = f'runtime does not have parameter "{arg}"' + if warn_runtime_is_object_init: + msg += ". You may need to write stubs for __new__ instead of __init__." + yield msg + for arg in sorted(set(runtime.kwonly) - set(stub.kwonly)): if arg in {stub_arg.variable.name for stub_arg in stub.pos}: # Don't report this if we've reported it before @@ -1223,7 +1232,12 @@ def verify_funcitem( if not signature: return - for message in _verify_signature(stub_sig, runtime_sig, function_name=stub.name): + for message in _verify_signature( + stub_sig, + runtime_sig, + function_name=stub.name, + warn_runtime_is_object_init=runtime is object.__init__, + ): yield Error( object_path, "is inconsistent, " + message, @@ -1333,7 +1347,12 @@ def verify_overloadedfuncdef( stub_sig = Signature.from_overloadedfuncdef(stub) runtime_sig = Signature.from_inspect_signature(signature) - for message in _verify_signature(stub_sig, runtime_sig, function_name=stub.name): + for message in _verify_signature( + stub_sig, + runtime_sig, + function_name=stub.name, + warn_runtime_is_object_init=runtime is object.__init__, + ): # TODO: This is a little hacky, but the addition here is super useful if "has a default value of type" in message: message += ( From 6cc96f48ab6a8250598012062fe572a2a9e46838 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Sun, 14 Sep 2025 06:05:07 +0700 Subject: [PATCH 261/424] Update docs.yml: add mypy/main.py (#19829) Part of the documentation is automatically generated from the options definitions in mypy/main.py, so we need to run the docs CI when that file is modified. This follows up on https://github.com/python/mypy/pull/19727, which itself follows up on https://github.com/python/mypy/pull/19062 --- .github/workflows/docs.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 3e78bf51913ed..66e7c997f4fad 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -12,6 +12,9 @@ on: # so it's important to do the docs build on all PRs touching mypy/errorcodes.py # in case somebody's adding a new error code without any docs - 'mypy/errorcodes.py' + # Part of the documentation is automatically generated from the options + # definitions in mypy/main.py + - 'mypy/main.py' - 'mypyc/doc/**' - '**/*.rst' - '**/*.md' From 8412d1dd19c45628159ae37ce1822b7d49e66567 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Sun, 14 Sep 2025 06:14:50 +0700 Subject: [PATCH 262/424] Refactor/nit main.py: rename the variable other_group to misc_group (#19832) This better reflects its external name, "Miscellaneous". The current CI suffices to check that this code is correct. --- mypy/main.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index 150d388af84c1..b543cd33fe44e 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -1163,22 +1163,22 @@ def add_invertible_flag( "--skip-c-gen", dest="mypyc_skip_c_generation", action="store_true", help=argparse.SUPPRESS ) - other_group = parser.add_argument_group(title="Miscellaneous") - other_group.add_argument("--quickstart-file", help=argparse.SUPPRESS) - other_group.add_argument("--junit-xml", help="Write junit.xml to the given file") + misc_group = parser.add_argument_group(title="Miscellaneous") + misc_group.add_argument("--quickstart-file", help=argparse.SUPPRESS) + misc_group.add_argument("--junit-xml", help="Write junit.xml to the given file") imports_group.add_argument( "--junit-format", choices=["global", "per_file"], default="global", help="If --junit-xml is set, specifies format. global: single test with all errors; per_file: one test entry per file with failures", ) - other_group.add_argument( + misc_group.add_argument( "--find-occurrences", metavar="CLASS.MEMBER", dest="special-opts:find_occurrences", help="Print out all usages of a class member (experimental)", ) - other_group.add_argument( + misc_group.add_argument( "--scripts-are-modules", action="store_true", help="Script x becomes module x instead of __main__", @@ -1189,7 +1189,7 @@ def add_invertible_flag( default=False, strict_flag=False, help="Install detected missing library stub packages using pip", - group=other_group, + group=misc_group, ) add_invertible_flag( "--non-interactive", @@ -1199,12 +1199,12 @@ def add_invertible_flag( "Install stubs without asking for confirmation and hide " + "errors, with --install-types" ), - group=other_group, + group=misc_group, inverse="--interactive", ) if server_options: - other_group.add_argument( + misc_group.add_argument( "--use-fine-grained-cache", action="store_true", help="Use the cache in fine-grained incremental mode", From 647ea8cf07b93c0ffa8c480143dfedc449e6f2e2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 14 Sep 2025 17:39:09 -0700 Subject: [PATCH 263/424] Sync typeshed (#19848) Sync typeshed Source commit: https://github.com/python/typeshed/commit/0d100b9110f1b30529ba4d1be26d1eb09ae5d42c Note that you will need to close and re-open the PR in order to trigger CI. --------- Co-authored-by: mypybot <> Co-authored-by: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Co-authored-by: AlexWaygood --- mypy/typeshed/stdlib/asyncio/events.pyi | 32 +++++++++++++++---------- mypy/typeshed/stdlib/builtins.pyi | 7 ++---- mypy/typeshed/stdlib/turtle.pyi | 14 +++++++++-- mypy/typeshed/stdlib/unittest/mock.pyi | 3 ++- 4 files changed, 35 insertions(+), 21 deletions(-) diff --git a/mypy/typeshed/stdlib/asyncio/events.pyi b/mypy/typeshed/stdlib/asyncio/events.pyi index 14c4c0bf3d5ac..5dc698bc5e15c 100644 --- a/mypy/typeshed/stdlib/asyncio/events.pyi +++ b/mypy/typeshed/stdlib/asyncio/events.pyi @@ -602,18 +602,25 @@ class AbstractEventLoop: @abstractmethod async def shutdown_default_executor(self) -> None: ... -# This class does not exist at runtime, but stubtest complains if it's marked as -# @type_check_only because it has an alias that does exist at runtime. See mypy#19568. -# @type_check_only -class _AbstractEventLoopPolicy: - @abstractmethod - def get_event_loop(self) -> AbstractEventLoop: ... - @abstractmethod - def set_event_loop(self, loop: AbstractEventLoop | None) -> None: ... - @abstractmethod - def new_event_loop(self) -> AbstractEventLoop: ... - # Child processes handling (Unix only). - if sys.version_info < (3, 14): +if sys.version_info >= (3, 14): + class _AbstractEventLoopPolicy: + @abstractmethod + def get_event_loop(self) -> AbstractEventLoop: ... + @abstractmethod + def set_event_loop(self, loop: AbstractEventLoop | None) -> None: ... + @abstractmethod + def new_event_loop(self) -> AbstractEventLoop: ... + +else: + @type_check_only + class _AbstractEventLoopPolicy: + @abstractmethod + def get_event_loop(self) -> AbstractEventLoop: ... + @abstractmethod + def set_event_loop(self, loop: AbstractEventLoop | None) -> None: ... + @abstractmethod + def new_event_loop(self) -> AbstractEventLoop: ... + # Child processes handling (Unix only). if sys.version_info >= (3, 12): @abstractmethod @deprecated("Deprecated since Python 3.12; removed in Python 3.14.") @@ -627,7 +634,6 @@ class _AbstractEventLoopPolicy: @abstractmethod def set_child_watcher(self, watcher: AbstractChildWatcher) -> None: ... -if sys.version_info < (3, 14): AbstractEventLoopPolicy = _AbstractEventLoopPolicy if sys.version_info >= (3, 14): diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi index ca8d56cb42970..ef6c712e00053 100644 --- a/mypy/typeshed/stdlib/builtins.pyi +++ b/mypy/typeshed/stdlib/builtins.pyi @@ -923,8 +923,7 @@ class slice(Generic[_StartT_co, _StopT_co, _StepT_co]): def indices(self, len: SupportsIndex, /) -> tuple[int, int, int]: ... -# Making this a disjoint_base upsets pyright -# @disjoint_base +@disjoint_base class tuple(Sequence[_T_co]): def __new__(cls, iterable: Iterable[_T_co] = ..., /) -> Self: ... def __len__(self) -> int: ... @@ -1266,10 +1265,8 @@ class property: def __set__(self, instance: Any, value: Any, /) -> None: ... def __delete__(self, instance: Any, /) -> None: ... -# This class does not exist at runtime, but stubtest complains if it's marked as -# @type_check_only because it has an alias that does exist at runtime. See mypy#19568. -# @type_check_only @final +@type_check_only class _NotImplementedType(Any): __call__: None diff --git a/mypy/typeshed/stdlib/turtle.pyi b/mypy/typeshed/stdlib/turtle.pyi index 0b93429904c53..39a995de26124 100644 --- a/mypy/typeshed/stdlib/turtle.pyi +++ b/mypy/typeshed/stdlib/turtle.pyi @@ -463,7 +463,12 @@ class RawTurtle(TPen, TNavigator): # type: ignore[misc] # Conflicting methods def begin_fill(self) -> None: ... def end_fill(self) -> None: ... - def dot(self, size: int | None = None, *color: _Color) -> None: ... + @overload + def dot(self, size: int | _Color | None = None) -> None: ... + @overload + def dot(self, size: int | None, color: _Color, /) -> None: ... + @overload + def dot(self, size: int | None, r: float, g: float, b: float, /) -> None: ... def write( self, arg: object, move: bool = False, align: str = "left", font: tuple[str, int, str] = ("Arial", 8, "normal") ) -> None: ... @@ -747,7 +752,12 @@ if sys.version_info >= (3, 14): def begin_fill() -> None: ... def end_fill() -> None: ... -def dot(size: int | None = None, *color: _Color) -> None: ... +@overload +def dot(size: int | _Color | None = None) -> None: ... +@overload +def dot(size: int | None, color: _Color, /) -> None: ... +@overload +def dot(size: int | None, r: float, g: float, b: float, /) -> None: ... def write(arg: object, move: bool = False, align: str = "left", font: tuple[str, int, str] = ("Arial", 8, "normal")) -> None: ... if sys.version_info >= (3, 14): diff --git a/mypy/typeshed/stdlib/unittest/mock.pyi b/mypy/typeshed/stdlib/unittest/mock.pyi index f4b59e7cab906..f3e58bcd1c009 100644 --- a/mypy/typeshed/stdlib/unittest/mock.pyi +++ b/mypy/typeshed/stdlib/unittest/mock.pyi @@ -508,7 +508,8 @@ class MagicProxy(Base): def create_mock(self) -> Any: ... def __get__(self, obj: Any, _type: Any | None = None) -> Any: ... -class _ANY: +# See https://github.com/python/typeshed/issues/14701 +class _ANY(Any): def __eq__(self, other: object) -> Literal[True]: ... def __ne__(self, other: object) -> Literal[False]: ... __hash__: ClassVar[None] # type: ignore[assignment] From 73affc0c60aa8d9a7fdc43c8d57fd65c9ea870f1 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 15 Sep 2025 17:41:36 +0100 Subject: [PATCH 264/424] Incremental regression test for recursive aliases (#19853) See original PR https://github.com/python/mypy/pull/19845 --- test-data/unit/check-incremental.test | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 8e05f922be174..d9d78715b396a 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -2577,6 +2577,13 @@ C(1)[0] [builtins fixtures/list.pyi] [out] +[case testSerializeRecursiveAlias] +from typing import Callable, Union + +Node = Union[str, int, Callable[[], "Node"]] +n: Node +[out] + [case testSerializeRecursiveAliases1] from typing import Type, Callable, Union From dce8e1c407ccaa9effebbb1ed09fbf0e7070636d Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Mon, 15 Sep 2025 21:27:43 -0400 Subject: [PATCH 265/424] [mypyc] fix: inappropriate `None`s in f-strings (#19846) if a variable is Final but the value is not yet known at compile-time, and that variable is used as an input to an f-string, the f-string will incorrectly contain "None" Fixes [mypyc#1140](https://github.com/mypyc/mypyc/issues/1140) --- mypyc/irbuild/specialize.py | 4 +++- mypyc/test-data/run-strings.test | 9 +++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/mypyc/irbuild/specialize.py b/mypyc/irbuild/specialize.py index 0880c62bc7a58..576b7a7ebffd8 100644 --- a/mypyc/irbuild/specialize.py +++ b/mypyc/irbuild/specialize.py @@ -719,7 +719,9 @@ def get_literal_str(expr: Expression) -> str | None: if isinstance(expr, StrExpr): return expr.value elif isinstance(expr, RefExpr) and isinstance(expr.node, Var) and expr.node.is_final: - return str(expr.node.final_value) + final_value = expr.node.final_value + if final_value is not None: + return str(final_value) return None for i in range(len(exprs) - 1): diff --git a/mypyc/test-data/run-strings.test b/mypyc/test-data/run-strings.test index 6960b0a043038..6a62db6ee3ee0 100644 --- a/mypyc/test-data/run-strings.test +++ b/mypyc/test-data/run-strings.test @@ -412,9 +412,16 @@ def test_basics() -> None: [case testFStrings] import decimal from datetime import datetime +from typing import Final var = 'mypyc' num = 20 +final_known_at_compile_time: Final = 'hello' + +def final_value_setter() -> str: + return 'goodbye' + +final_unknown_at_compile_time: Final = final_value_setter() def test_fstring_basics() -> None: assert f'Hello {var}, this is a test' == "Hello mypyc, this is a test" @@ -451,6 +458,8 @@ def test_fstring_basics() -> None: inf_num = float('inf') assert f'{nan_num}, {inf_num}' == 'nan, inf' + assert f'{final_known_at_compile_time} {final_unknown_at_compile_time}' == 'hello goodbye' + # F-strings would be translated into ''.join[string literals, format method call, ...] in mypy AST. # Currently we are using a str.join specializer for f-string speed up. We might not cover all cases # and the rest ones should fall back to a normal str.join method call. From d27b43b0cb622a8a8a894fbb989032f92ea68eab Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Wed, 17 Sep 2025 20:28:44 +0200 Subject: [PATCH 266/424] Sync variance of typing classes in fixtures with typeshed (#19778) Originally discovered in #19777. Our test fixtures use definitions that are very far from their real counterparts, but at least generics should match if possible. Let's see if it kills more than one testcase (expected from #19777). --- test-data/unit/fixtures/typing-async.pyi | 8 ++-- test-data/unit/fixtures/typing-full.pyi | 46 +++++++++++-------- test-data/unit/fixtures/typing-medium.pyi | 14 +++--- test-data/unit/fixtures/typing-namedtuple.pyi | 4 +- test-data/unit/fixtures/typing-override.pyi | 4 +- test-data/unit/fixtures/typing-typeddict.pyi | 2 +- test-data/unit/lib-stub/typing.pyi | 14 +++--- 7 files changed, 48 insertions(+), 44 deletions(-) diff --git a/test-data/unit/fixtures/typing-async.pyi b/test-data/unit/fixtures/typing-async.pyi index 7ce2821d29168..66509a91b82b7 100644 --- a/test-data/unit/fixtures/typing-async.pyi +++ b/test-data/unit/fixtures/typing-async.pyi @@ -123,13 +123,13 @@ class Mapping(Iterable[T], Generic[T, T_co], metaclass=ABCMeta): @overload def get(self, k: T, default: Union[T_co, V]) -> Union[T_co, V]: pass -class ContextManager(Generic[T]): - def __enter__(self) -> T: pass +class ContextManager(Generic[T_co]): + def __enter__(self) -> T_co: pass # Use Any because not all the precise types are in the fixtures. def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> Any: pass -class AsyncContextManager(Generic[T]): - def __aenter__(self) -> Awaitable[T]: pass +class AsyncContextManager(Generic[T_co]): + def __aenter__(self) -> Awaitable[T_co]: pass # Use Any because not all the precise types are in the fixtures. def __aexit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> Awaitable[Any]: pass diff --git a/test-data/unit/fixtures/typing-full.pyi b/test-data/unit/fixtures/typing-full.pyi index 8e0116aab1c29..3757e868552e1 100644 --- a/test-data/unit/fixtures/typing-full.pyi +++ b/test-data/unit/fixtures/typing-full.pyi @@ -44,7 +44,8 @@ Literal: _SpecialForm T = TypeVar('T') T_co = TypeVar('T_co', covariant=True) -T_contra = TypeVar('T_contra', contravariant=True) +R_co = TypeVar('R_co', covariant=True) +S_contra = TypeVar('S_contra', contravariant=True) U = TypeVar('U') V = TypeVar('V') S = TypeVar('S') @@ -82,9 +83,9 @@ class Iterator(Iterable[T_co], Protocol): @abstractmethod def __next__(self) -> T_co: pass -class Generator(Iterator[T], Generic[T, U, V]): +class Generator(Iterator[T_co], Generic[T_co, S_contra, R_co]): @abstractmethod - def send(self, value: U) -> T: pass + def send(self, value: S_contra) -> T_co: pass @abstractmethod def throw(self, typ: Any, val: Any=None, tb: Any=None) -> None: pass @@ -93,35 +94,40 @@ class Generator(Iterator[T], Generic[T, U, V]): def close(self) -> None: pass @abstractmethod - def __iter__(self) -> 'Generator[T, U, V]': pass + def __iter__(self) -> 'Generator[T_co, S_contra, R_co]': pass -class AsyncGenerator(AsyncIterator[T], Generic[T, U]): +class AsyncGenerator(AsyncIterator[T_co], Generic[T_co, S_contra]): @abstractmethod - def __anext__(self) -> Awaitable[T]: pass + def __anext__(self) -> Awaitable[T_co]: pass @abstractmethod - def asend(self, value: U) -> Awaitable[T]: pass + def asend(self, value: S_contra) -> Awaitable[T_co]: pass @abstractmethod - def athrow(self, typ: Any, val: Any=None, tb: Any=None) -> Awaitable[T]: pass + def athrow(self, typ: Any, val: Any=None, tb: Any=None) -> Awaitable[T_co]: pass @abstractmethod - def aclose(self) -> Awaitable[T]: pass + def aclose(self) -> Awaitable[T_co]: pass @abstractmethod - def __aiter__(self) -> 'AsyncGenerator[T, U]': pass + def __aiter__(self) -> 'AsyncGenerator[T_co, S_contra]': pass @runtime_checkable -class Awaitable(Protocol[T]): +class Awaitable(Protocol[T_co]): @abstractmethod - def __await__(self) -> Generator[Any, Any, T]: pass + def __await__(self) -> Generator[Any, Any, T_co]: pass -class AwaitableGenerator(Generator[T, U, V], Awaitable[V], Generic[T, U, V, S], metaclass=ABCMeta): +class AwaitableGenerator( + Awaitable[R_co], + Generator[T_co, S_contra, R_co], + Generic[T_co, S_contra, R_co, S], + metaclass=ABCMeta +): pass -class Coroutine(Awaitable[V], Generic[T, U, V]): +class Coroutine(Awaitable[R_co], Generic[T_co, S_contra, R_co]): @abstractmethod - def send(self, value: U) -> T: pass + def send(self, value: S_contra) -> T_co: pass @abstractmethod def throw(self, typ: Any, val: Any=None, tb: Any=None) -> None: pass @@ -130,15 +136,15 @@ class Coroutine(Awaitable[V], Generic[T, U, V]): def close(self) -> None: pass @runtime_checkable -class AsyncIterable(Protocol[T]): +class AsyncIterable(Protocol[T_co]): @abstractmethod - def __aiter__(self) -> 'AsyncIterator[T]': pass + def __aiter__(self) -> 'AsyncIterator[T_co]': pass @runtime_checkable -class AsyncIterator(AsyncIterable[T], Protocol): - def __aiter__(self) -> 'AsyncIterator[T]': return self +class AsyncIterator(AsyncIterable[T_co], Protocol): + def __aiter__(self) -> 'AsyncIterator[T_co]': return self @abstractmethod - def __anext__(self) -> Awaitable[T]: pass + def __anext__(self) -> Awaitable[T_co]: pass class Sequence(Iterable[T_co], Container[T_co]): @abstractmethod diff --git a/test-data/unit/fixtures/typing-medium.pyi b/test-data/unit/fixtures/typing-medium.pyi index c722a9ddb12c8..077d4eebf7d39 100644 --- a/test-data/unit/fixtures/typing-medium.pyi +++ b/test-data/unit/fixtures/typing-medium.pyi @@ -32,10 +32,8 @@ Self = 0 T = TypeVar('T') T_co = TypeVar('T_co', covariant=True) -T_contra = TypeVar('T_contra', contravariant=True) -U = TypeVar('U') -V = TypeVar('V') -S = TypeVar('S') +R_co = TypeVar('R_co', covariant=True) +S_contra = TypeVar('S_contra', contravariant=True) # Note: definitions below are different from typeshed, variances are declared # to silence the protocol variance checks. Maybe it is better to use type: ignore? @@ -49,8 +47,8 @@ class Iterable(Protocol[T_co]): class Iterator(Iterable[T_co], Protocol): def __next__(self) -> T_co: pass -class Generator(Iterator[T], Generic[T, U, V]): - def __iter__(self) -> 'Generator[T, U, V]': pass +class Generator(Iterator[T_co], Generic[T_co, S_contra, R_co]): + def __iter__(self) -> 'Generator[T_co, S_contra, R_co]': pass class Sequence(Iterable[T_co]): def __getitem__(self, n: Any) -> T_co: pass @@ -65,8 +63,8 @@ class SupportsInt(Protocol): class SupportsFloat(Protocol): def __float__(self) -> float: pass -class ContextManager(Generic[T]): - def __enter__(self) -> T: pass +class ContextManager(Generic[T_co]): + def __enter__(self) -> T_co: pass # Use Any because not all the precise types are in the fixtures. def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> Any: pass diff --git a/test-data/unit/fixtures/typing-namedtuple.pyi b/test-data/unit/fixtures/typing-namedtuple.pyi index fbb4e43b62e65..5b0ef0845dadf 100644 --- a/test-data/unit/fixtures/typing-namedtuple.pyi +++ b/test-data/unit/fixtures/typing-namedtuple.pyi @@ -18,8 +18,8 @@ class Iterable(Generic[T_co]): pass class Iterator(Iterable[T_co]): pass class Sequence(Iterable[T_co]): pass class Mapping(Iterable[KT], Generic[KT, T_co]): - def keys(self) -> Iterable[T]: pass # Approximate return type - def __getitem__(self, key: T) -> T_co: pass + def keys(self) -> Iterable[KT]: pass # Approximate return type + def __getitem__(self, key: KT) -> T_co: pass class NamedTuple(tuple[Any, ...]): _fields: ClassVar[tuple[str, ...]] diff --git a/test-data/unit/fixtures/typing-override.pyi b/test-data/unit/fixtures/typing-override.pyi index e9d2dfcf55c45..a0287524c84a8 100644 --- a/test-data/unit/fixtures/typing-override.pyi +++ b/test-data/unit/fixtures/typing-override.pyi @@ -18,8 +18,8 @@ class Iterable(Generic[T_co]): pass class Iterator(Iterable[T_co]): pass class Sequence(Iterable[T_co]): pass class Mapping(Iterable[KT], Generic[KT, T_co]): - def keys(self) -> Iterable[T]: pass # Approximate return type - def __getitem__(self, key: T) -> T_co: pass + def keys(self) -> Iterable[KT]: pass # Approximate return type + def __getitem__(self, key: KT) -> T_co: pass def override(__arg: T) -> T: ... diff --git a/test-data/unit/fixtures/typing-typeddict.pyi b/test-data/unit/fixtures/typing-typeddict.pyi index f841a9aae6e78..16658c82528b8 100644 --- a/test-data/unit/fixtures/typing-typeddict.pyi +++ b/test-data/unit/fixtures/typing-typeddict.pyi @@ -61,7 +61,7 @@ class Mapping(Iterable[T], Generic[T, T_co], metaclass=ABCMeta): def __len__(self) -> int: ... def __contains__(self, arg: object) -> int: pass -class MutableMapping(Mapping[T, T_co], Generic[T, T_co], metaclass=ABCMeta): +class MutableMapping(Mapping[T, V], Generic[T, V], metaclass=ABCMeta): # Other methods are not used in tests. def clear(self) -> None: ... diff --git a/test-data/unit/lib-stub/typing.pyi b/test-data/unit/lib-stub/typing.pyi index 86d542a918eef..00fce56920b75 100644 --- a/test-data/unit/lib-stub/typing.pyi +++ b/test-data/unit/lib-stub/typing.pyi @@ -35,8 +35,8 @@ TYPE_CHECKING = 0 T = TypeVar('T') T_co = TypeVar('T_co', covariant=True) -U = TypeVar('U') -V = TypeVar('V') +S_contra = TypeVar('S_contra', contravariant=True) +R_co = TypeVar('R_co', covariant=True) class Iterable(Protocol[T_co]): def __iter__(self) -> Iterator[T_co]: pass @@ -44,8 +44,8 @@ class Iterable(Protocol[T_co]): class Iterator(Iterable[T_co], Protocol): def __next__(self) -> T_co: pass -class Generator(Iterator[T], Generic[T, U, V]): - def __iter__(self) -> Generator[T, U, V]: pass +class Generator(Iterator[T_co], Generic[T_co, S_contra, R_co]): + def __iter__(self) -> Generator[T_co, S_contra, R_co]: pass class Sequence(Iterable[T_co]): def __getitem__(self, n: Any) -> T_co: pass @@ -56,10 +56,10 @@ class Mapping(Iterable[T], Generic[T, T_co]): def keys(self) -> Iterable[T]: pass # Approximate return type def __getitem__(self, key: T) -> T_co: pass -class Awaitable(Protocol[T]): - def __await__(self) -> Generator[Any, Any, T]: pass +class Awaitable(Protocol[T_co]): + def __await__(self) -> Generator[Any, Any, T_co]: pass -class Coroutine(Awaitable[V], Generic[T, U, V]): pass +class Coroutine(Awaitable[R_co], Generic[T_co, S_contra, R_co]): pass def final(meth: T) -> T: pass From 4301be16747910ad00b4360dcc20152a7e377e3a Mon Sep 17 00:00:00 2001 From: Kevin Kannammalil Date: Thu, 18 Sep 2025 13:02:52 -0400 Subject: [PATCH 267/424] Update changelog for 1.18.2 (#19873) Changelog update for 1.18.2 Also updated the changelog to reflect the initial release being 1.18.1, since we had to bump the version due to wheels failing. This adds the cherry picked PRs mentioned in https://github.com/python/mypy/issues/19764#issuecomment-3293411266 --- CHANGELOG.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e6f8c2cac38b..134d251d90b14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,9 @@ ## Next Release -## Mypy 1.18 +## Mypy 1.18.1 -We’ve just uploaded mypy 1.18 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). +We’ve just uploaded mypy 1.18.1 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). Mypy is a static type checker for Python. This release includes new features, performance improvements and bug fixes. You can install it as follows: @@ -14,7 +14,7 @@ You can read the full documentation for this release on [Read the Docs](http://m ### Mypy Performance Improvements -Mypy 1.18 includes numerous performance improvements, resulting in about 40% speedup +Mypy 1.18.1 includes numerous performance improvements, resulting in about 40% speedup compared to 1.17 when type checking mypy itself. In extreme cases, the improvement can be 10x or higher. The list below is an overview of the various mypy optimizations. Many mypyc improvements (discussed in a separate section below) also improve performance. @@ -283,6 +283,12 @@ Related PRs: Please see [git log](https://github.com/python/typeshed/commits/main?after=2480d7e7c74493a024eaf254c5d2c6f452c80ee2+0&branch=main&path=stdlib) for full list of standard library typeshed stub changes. +### Mypy 1.18.2 + +- Fix crash on recursive alias (Ivan Levkivskyi, PR [19845](https://github.com/python/mypy/pull/19845)) +- Add additional guidance for stubtest errors when runtime is `object.__init__` (Stephen Morton, PR [19733](https://github.com/python/mypy/pull/19733)) +- Fix handling of None values in f-string expressions in mypyc (BobTheBuidler, PR [19846](https://github.com/python/mypy/pull/19846)) + ### Acknowledgements Thanks to all mypy contributors who contributed to this release: From f955623ad845ef8f066fe9822c0bc458ced49271 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Sun, 21 Sep 2025 03:46:46 +0700 Subject: [PATCH 268/424] [refactor] Update errorcodes.py: harmonize all of the type annotations (#19880) This completes an in-line todo from 3 years ago relating to a weakness that mypy no longer has (edit: I did not read carefully enough. But I did eventually stumble upon a sufficient workaround for https://github.com/mypyc/mypyc/issues/1142), and removes code with no downside. --- mypy/errorcodes.py | 55 ++++++++++++++++++---------------------- mypy/message_registry.py | 3 ++- 2 files changed, 26 insertions(+), 32 deletions(-) diff --git a/mypy/errorcodes.py b/mypy/errorcodes.py index bcfdbf6edc2bf..a96f5f723a7d2 100644 --- a/mypy/errorcodes.py +++ b/mypy/errorcodes.py @@ -52,32 +52,28 @@ def __hash__(self) -> int: ATTR_DEFINED: Final = ErrorCode("attr-defined", "Check that attribute exists", "General") NAME_DEFINED: Final = ErrorCode("name-defined", "Check that name is defined", "General") -CALL_ARG: Final[ErrorCode] = ErrorCode( +CALL_ARG: Final = ErrorCode( "call-arg", "Check number, names and kinds of arguments in calls", "General" ) ARG_TYPE: Final = ErrorCode("arg-type", "Check argument types in calls", "General") CALL_OVERLOAD: Final = ErrorCode( "call-overload", "Check that an overload variant matches arguments", "General" ) -VALID_TYPE: Final[ErrorCode] = ErrorCode( - "valid-type", "Check that type (annotation) is valid", "General" -) +VALID_TYPE: Final = ErrorCode("valid-type", "Check that type (annotation) is valid", "General") VAR_ANNOTATED: Final = ErrorCode( "var-annotated", "Require variable annotation if type can't be inferred", "General" ) OVERRIDE: Final = ErrorCode( "override", "Check that method override is compatible with base class", "General" ) -RETURN: Final[ErrorCode] = ErrorCode( - "return", "Check that function always returns a value", "General" -) -RETURN_VALUE: Final[ErrorCode] = ErrorCode( +RETURN: Final = ErrorCode("return", "Check that function always returns a value", "General") +RETURN_VALUE: Final = ErrorCode( "return-value", "Check that return value is compatible with signature", "General" ) -ASSIGNMENT: Final[ErrorCode] = ErrorCode( +ASSIGNMENT: Final = ErrorCode( "assignment", "Check that assigned value is compatible with target", "General" ) -METHOD_ASSIGN: Final[ErrorCode] = ErrorCode( +METHOD_ASSIGN: Final = ErrorCode( "method-assign", "Check that assignment target is not a method", "General", @@ -143,9 +139,7 @@ def __hash__(self) -> int: UNUSED_COROUTINE: Final = ErrorCode( "unused-coroutine", "Ensure that all coroutines are used", "General" ) -# TODO: why do we need the explicit type here? Without it mypyc CI builds fail with -# mypy/message_registry.py:37: error: Cannot determine type of "EMPTY_BODY" [has-type] -EMPTY_BODY: Final[ErrorCode] = ErrorCode( +EMPTY_BODY: Final = ErrorCode( "empty-body", "A dedicated error code to opt out return errors for empty/trivial bodies", "General", @@ -160,7 +154,7 @@ def __hash__(self) -> int: "await-not-async", 'Warn about "await" outside coroutine ("async def")', "General" ) # These error codes aren't enabled by default. -NO_UNTYPED_DEF: Final[ErrorCode] = ErrorCode( +NO_UNTYPED_DEF: Final = ErrorCode( "no-untyped-def", "Check that every function has an annotation", "General" ) NO_UNTYPED_CALL: Final = ErrorCode( @@ -186,13 +180,13 @@ def __hash__(self) -> int: UNREACHABLE: Final = ErrorCode( "unreachable", "Warn about unreachable statements or expressions", "General" ) -ANNOTATION_UNCHECKED = ErrorCode( +ANNOTATION_UNCHECKED: Final = ErrorCode( "annotation-unchecked", "Notify about type annotations in unchecked functions", "General" ) -TYPEDDICT_READONLY_MUTATED = ErrorCode( +TYPEDDICT_READONLY_MUTATED: Final = ErrorCode( "typeddict-readonly-mutated", "TypedDict's ReadOnly key is mutated", "General" ) -POSSIBLY_UNDEFINED: Final[ErrorCode] = ErrorCode( +POSSIBLY_UNDEFINED: Final = ErrorCode( "possibly-undefined", "Warn about variables that are defined only in some execution paths", "General", @@ -201,18 +195,18 @@ def __hash__(self) -> int: REDUNDANT_EXPR: Final = ErrorCode( "redundant-expr", "Warn about redundant expressions", "General", default_enabled=False ) -TRUTHY_BOOL: Final[ErrorCode] = ErrorCode( +TRUTHY_BOOL: Final = ErrorCode( "truthy-bool", "Warn about expressions that could always evaluate to true in boolean contexts", "General", default_enabled=False, ) -TRUTHY_FUNCTION: Final[ErrorCode] = ErrorCode( +TRUTHY_FUNCTION: Final = ErrorCode( "truthy-function", "Warn about function that always evaluate to true in boolean contexts", "General", ) -TRUTHY_ITERABLE: Final[ErrorCode] = ErrorCode( +TRUTHY_ITERABLE: Final = ErrorCode( "truthy-iterable", "Warn about Iterable expressions that could always evaluate to true in boolean contexts", "General", @@ -238,13 +232,13 @@ def __hash__(self) -> int: "General", default_enabled=False, ) -REDUNDANT_SELF_TYPE = ErrorCode( +REDUNDANT_SELF_TYPE: Final = ErrorCode( "redundant-self", "Warn about redundant Self type annotations on method first argument", "General", default_enabled=False, ) -USED_BEFORE_DEF: Final[ErrorCode] = ErrorCode( +USED_BEFORE_DEF: Final = ErrorCode( "used-before-def", "Warn about variables that are used before they are defined", "General" ) UNUSED_IGNORE: Final = ErrorCode( @@ -262,7 +256,7 @@ def __hash__(self) -> int: "General", default_enabled=False, ) -MUTABLE_OVERRIDE: Final[ErrorCode] = ErrorCode( +MUTABLE_OVERRIDE: Final = ErrorCode( "mutable-override", "Reject covariant overrides for mutable attributes", "General", @@ -274,10 +268,10 @@ def __hash__(self) -> int: "General", default_enabled=False, ) -METACLASS: Final[ErrorCode] = ErrorCode("metaclass", "Ensure that metaclass is valid", "General") +METACLASS: Final = ErrorCode("metaclass", "Ensure that metaclass is valid", "General") # Syntax errors are often blocking. -SYNTAX: Final[ErrorCode] = ErrorCode("syntax", "Report syntax errors", "General") +SYNTAX: Final = ErrorCode("syntax", "Report syntax errors", "General") # This is an internal marker code for a whole-file ignore. It is not intended to # be user-visible. @@ -285,31 +279,30 @@ def __hash__(self) -> int: del error_codes[FILE.code] # This is a catch-all for remaining uncategorized errors. -MISC: Final[ErrorCode] = ErrorCode("misc", "Miscellaneous other checks", "General") +MISC: Final = ErrorCode("misc", "Miscellaneous other checks", "General") -OVERLOAD_CANNOT_MATCH: Final[ErrorCode] = ErrorCode( +OVERLOAD_CANNOT_MATCH: Final = ErrorCode( "overload-cannot-match", "Warn if an @overload signature can never be matched", "General", sub_code_of=MISC, ) - -OVERLOAD_OVERLAP: Final[ErrorCode] = ErrorCode( +OVERLOAD_OVERLAP: Final = ErrorCode( "overload-overlap", "Warn if multiple @overload variants overlap in unsafe ways", "General", sub_code_of=MISC, ) -PROPERTY_DECORATOR = ErrorCode( +PROPERTY_DECORATOR: Final = ErrorCode( "prop-decorator", "Decorators on top of @property are not supported", "General", sub_code_of=MISC, ) -NARROWED_TYPE_NOT_SUBTYPE: Final[ErrorCode] = ErrorCode( +NARROWED_TYPE_NOT_SUBTYPE: Final = ErrorCode( "narrowed-type-not-subtype", "Warn if a TypeIs function's narrowed type is not a subtype of the original type", "General", diff --git a/mypy/message_registry.py b/mypy/message_registry.py index 09004322aee9f..b0f9ed1b0dfee 100644 --- a/mypy/message_registry.py +++ b/mypy/message_registry.py @@ -11,11 +11,12 @@ from typing import Final, NamedTuple from mypy import errorcodes as codes +from mypy.errorcodes import ErrorCode class ErrorMessage(NamedTuple): value: str - code: codes.ErrorCode | None = None + code: ErrorCode | None = None def format(self, *args: object, **kwargs: object) -> ErrorMessage: return ErrorMessage(self.value.format(*args, **kwargs), code=self.code) From e1aada828c2dc41e448382434bf04b0cb091cc42 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Sun, 21 Sep 2025 03:48:39 +0700 Subject: [PATCH 269/424] Enable warn_unreachable = True for `mypyc` and all other files as well (#19050) Achieves a todo, enabling warn_unreachable = True for `mypyc` and all other files as well --- misc/analyze_cache.py | 6 +++--- misc/profile_check.py | 32 ++++++++++++++++---------------- mypy_self_check.ini | 3 --- 3 files changed, 19 insertions(+), 22 deletions(-) diff --git a/misc/analyze_cache.py b/misc/analyze_cache.py index 0a05493b77a31..f911522f5c648 100644 --- a/misc/analyze_cache.py +++ b/misc/analyze_cache.py @@ -41,7 +41,7 @@ def extract(chunks: Iterable[JsonDict]) -> Iterable[JsonDict]: if isinstance(chunk, dict): yield chunk yield from extract(chunk.values()) - elif isinstance(chunk, list): + elif isinstance(chunk, list): # type: ignore[unreachable] #TODO: is this actually unreachable, or are our types wrong? yield from extract(chunk) yield from extract([chunk.data for chunk in chunks]) @@ -93,7 +93,7 @@ def compress(chunk: JsonDict) -> JsonDict: def helper(chunk: JsonDict) -> JsonDict: nonlocal counter if not isinstance(chunk, dict): - return chunk + return chunk # type: ignore[unreachable] #TODO: is this actually unreachable, or are our types wrong? if len(chunk) <= 2: return chunk @@ -124,7 +124,7 @@ def decompress(chunk: JsonDict) -> JsonDict: def helper(chunk: JsonDict) -> JsonDict: if not isinstance(chunk, dict): - return chunk + return chunk # type: ignore[unreachable] #TODO: is this actually unreachable, or are our types wrong? if ".id" in chunk: return cache[chunk[".id"]] diff --git a/misc/profile_check.py b/misc/profile_check.py index b29535020f0a7..6bd23b09b2d5b 100644 --- a/misc/profile_check.py +++ b/misc/profile_check.py @@ -78,22 +78,22 @@ def check_requirements() -> None: if sys.platform != "linux": # TODO: How to make this work on other platforms? sys.exit("error: Only Linux is supported") - - try: - subprocess.run(["perf", "-h"], capture_output=True) - except (subprocess.CalledProcessError, FileNotFoundError): - print("error: The 'perf' profiler is not installed") - sys.exit(1) - - try: - subprocess.run(["clang", "--version"], capture_output=True) - except (subprocess.CalledProcessError, FileNotFoundError): - print("error: The clang compiler is not installed") - sys.exit(1) - - if not os.path.isfile("mypy_self_check.ini"): - print("error: Run this in the mypy repository root") - sys.exit(1) + else: # fun fact/todo: we have to use else here, because of https://github.com/python/mypy/issues/10773 + try: + subprocess.run(["perf", "-h"], capture_output=True) + except (subprocess.CalledProcessError, FileNotFoundError): + print("error: The 'perf' profiler is not installed") + sys.exit(1) + + try: + subprocess.run(["clang", "--version"], capture_output=True) + except (subprocess.CalledProcessError, FileNotFoundError): + print("error: The clang compiler is not installed") + sys.exit(1) + + if not os.path.isfile("mypy_self_check.ini"): + print("error: Run this in the mypy repository root") + sys.exit(1) def main() -> None: diff --git a/mypy_self_check.ini b/mypy_self_check.ini index 8bf7a514f481f..67b65381cfd02 100644 --- a/mypy_self_check.ini +++ b/mypy_self_check.ini @@ -12,7 +12,4 @@ exclude = mypy/typeshed/|mypyc/test-data/|mypyc/lib-rt/ enable_error_code = ignore-without-code,redundant-expr enable_incomplete_feature = PreciseTupleTypes show_error_code_links = True - -[mypy-mypy.*] -# TODO: enable for `mypyc` and other files as well warn_unreachable = True From feeb3f00a63d31cf5c7369885c4dd4a294133f59 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Sun, 21 Sep 2025 03:50:05 +0700 Subject: [PATCH 270/424] [docs] main.py: junit documentation elaboration (#19867) Elaborate on the information given in --help and command_line.rst for junit, to make it more correct and comprehensive. I manually examined the generated results and found them satisfactory. Note that this also puts --junit-format into misc group not import group; the inclusion into imports group seems to have been a mistake in #16388 although one could perhaps argue it is tangentially related to imports in some way. But it does not influence import discovery, unlike the other options. Putting it into import group also makes it display in a completely different place, which is not as helpful as right next to its related option. --- docs/source/command_line.rst | 8 +++++++- docs/source/config_file.rst | 9 +++++++++ mypy/main.py | 10 +++++++--- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index c1b757a00ef20..270125e96cb61 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -1255,12 +1255,18 @@ Miscellaneous stub packages were found, they are installed and then another run is performed. -.. option:: --junit-xml JUNIT_XML +.. option:: --junit-xml JUNIT_XML_OUTPUT_FILE Causes mypy to generate a JUnit XML test result document with type checking results. This can make it easier to integrate mypy with continuous integration (CI) tools. +.. option:: --junit-format {global,per_file} + + If --junit-xml is set, specifies format. + global (default): single test with all errors; + per_file: one test entry per file with failures. + .. option:: --find-occurrences CLASS.MEMBER This flag will make mypy print out all usages of a class member diff --git a/docs/source/config_file.rst b/docs/source/config_file.rst index 934e465a7c237..7abd1f02db68b 100644 --- a/docs/source/config_file.rst +++ b/docs/source/config_file.rst @@ -1153,6 +1153,15 @@ These options may only be set in the global section (``[mypy]``). type checking results. This can make it easier to integrate mypy with continuous integration (CI) tools. +.. confval:: junit_format + + :type: string + :default: ``global`` + + If junit_xml is set, specifies format. + global (default): single test with all errors; + per_file: one test entry per file with failures. + .. confval:: scripts_are_modules :type: boolean diff --git a/mypy/main.py b/mypy/main.py index b543cd33fe44e..9ebbf78ded09d 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -1165,12 +1165,16 @@ def add_invertible_flag( misc_group = parser.add_argument_group(title="Miscellaneous") misc_group.add_argument("--quickstart-file", help=argparse.SUPPRESS) - misc_group.add_argument("--junit-xml", help="Write junit.xml to the given file") - imports_group.add_argument( + misc_group.add_argument( + "--junit-xml", + metavar="JUNIT_XML_OUTPUT_FILE", + help="Write a JUnit XML test result document with type checking results to the given file", + ) + misc_group.add_argument( "--junit-format", choices=["global", "per_file"], default="global", - help="If --junit-xml is set, specifies format. global: single test with all errors; per_file: one test entry per file with failures", + help="If --junit-xml is set, specifies format. global (default): single test with all errors; per_file: one test entry per file with failures", ) misc_group.add_argument( "--find-occurrences", From fa3566a87d3466e8465d5bafea5b0f0bde3b4eaf Mon Sep 17 00:00:00 2001 From: Piotr Sawicki Date: Mon, 22 Sep 2025 21:13:45 +0200 Subject: [PATCH 271/424] [mypyc] Transform object.__new__ inside __new__ (#19866) #19739 introduced support for compiling `__new__` methods in native classes. Native classes differ internally from regular python types which results in `TypeError`s being raised when `object.__new__(cls)` is called when `cls` is a native class. To avoid making this call, `super().__new__(cls)` is transformed into a call to an internal setup function. I forgot to replicate this for calls to equivalent `object.__new__(cls)` calls so this PR fixes that. This introduced a regression because before my changes, `__new__` methods with `object.__new__(cls)` were effectively ignored at runtime, and after my changes they started raising `TypeError`s. Note that these calls are left as-is outside of `__new__` methods so it's still possible to trigger the `TypeError` but that is not a regression as this was the case before. For example this code: ``` class Test: pass t = object.__new__(Test) ``` results in `TypeError: object.__new__(Test) is not safe, use Test.__new__()`. This differs from interpreted python but the error message is actually correct in that using `Test.__new__(Test)` instead works. --- mypyc/irbuild/builder.py | 4 + mypyc/irbuild/expression.py | 35 +-- mypyc/irbuild/specialize.py | 44 ++++ mypyc/irbuild/statement.py | 34 +++ mypyc/test-data/irbuild-classes.test | 334 ++++++++++++++++++++++++++- mypyc/test-data/run-classes.test | 197 ++++++++++++++++ 6 files changed, 620 insertions(+), 28 deletions(-) diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index 4f2f539118d74..12b5bc7f8f824 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -1437,6 +1437,10 @@ def add_function(self, func_ir: FuncIR, line: int) -> None: self.function_names.add(name) self.functions.append(func_ir) + def get_current_class_ir(self) -> ClassIR | None: + type_info = self.fn_info.fitem.info + return self.mapper.type_to_ir.get(type_info) + def gen_arg_defaults(builder: IRBuilder) -> None: """Generate blocks for arguments that have default values. diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index 4409b1acff265..1f39b09c09952 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -57,7 +57,6 @@ from mypyc.ir.ops import ( Assign, BasicBlock, - Call, ComparisonOp, Integer, LoadAddress, @@ -98,7 +97,11 @@ join_formatted_strings, tokenizer_printf_style, ) -from mypyc.irbuild.specialize import apply_function_specialization, apply_method_specialization +from mypyc.irbuild.specialize import ( + apply_function_specialization, + apply_method_specialization, + translate_object_new, +) from mypyc.primitives.bytes_ops import bytes_slice_op from mypyc.primitives.dict_ops import dict_get_item_op, dict_new_op, exact_dict_set_item_op from mypyc.primitives.generic_ops import iter_op, name_op @@ -473,35 +476,15 @@ def translate_super_method_call(builder: IRBuilder, expr: CallExpr, callee: Supe if callee.name in base.method_decls: break else: + if callee.name == "__new__": + result = translate_object_new(builder, expr, MemberExpr(callee.call, "__new__")) + if result: + return result if ir.is_ext_class and ir.builtin_base is None and not ir.inherits_python: if callee.name == "__init__" and len(expr.args) == 0: # Call translates to object.__init__(self), which is a # no-op, so omit the call. return builder.none() - elif callee.name == "__new__": - # object.__new__(cls) - assert ( - len(expr.args) == 1 - ), f"Expected object.__new__() call to have exactly 1 argument, got {len(expr.args)}" - typ_arg = expr.args[0] - method_args = builder.fn_info.fitem.arg_names - if ( - isinstance(typ_arg, NameExpr) - and len(method_args) > 0 - and method_args[0] == typ_arg.name - ): - subtype = builder.accept(expr.args[0]) - return builder.add(Call(ir.setup, [subtype], expr.line)) - - if callee.name == "__new__": - call = "super().__new__()" - if not ir.is_ext_class: - builder.error(f"{call} not supported for non-extension classes", expr.line) - if ir.inherits_python: - builder.error( - f"{call} not supported for classes inheriting from non-native classes", - expr.line, - ) return translate_call(builder, expr, callee) decl = base.method_decl(callee.name) diff --git a/mypyc/irbuild/specialize.py b/mypyc/irbuild/specialize.py index 576b7a7ebffd8..29820787d10cb 100644 --- a/mypyc/irbuild/specialize.py +++ b/mypyc/irbuild/specialize.py @@ -30,12 +30,14 @@ NameExpr, RefExpr, StrExpr, + SuperExpr, TupleExpr, Var, ) from mypy.types import AnyType, TypeOfAny from mypyc.ir.ops import ( BasicBlock, + Call, Extend, Integer, RaiseStandardError, @@ -68,6 +70,7 @@ is_list_rprimitive, is_uint8_rprimitive, list_rprimitive, + object_rprimitive, set_rprimitive, str_rprimitive, uint8_rprimitive, @@ -1002,3 +1005,44 @@ def translate_ord(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value if isinstance(arg, (StrExpr, BytesExpr)) and len(arg.value) == 1: return Integer(ord(arg.value)) return None + + +@specialize_function("__new__", object_rprimitive) +def translate_object_new(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: + fn = builder.fn_info + if fn.name != "__new__": + return None + + is_super_new = isinstance(expr.callee, SuperExpr) + is_object_new = ( + isinstance(callee, MemberExpr) + and isinstance(callee.expr, NameExpr) + and callee.expr.fullname == "builtins.object" + ) + if not (is_super_new or is_object_new): + return None + + ir = builder.get_current_class_ir() + if ir is None: + return None + + call = '"object.__new__()"' + if not ir.is_ext_class: + builder.error(f"{call} not supported for non-extension classes", expr.line) + return None + if ir.inherits_python: + builder.error( + f"{call} not supported for classes inheriting from non-native classes", expr.line + ) + return None + if len(expr.args) != 1: + builder.error(f"{call} supported only with 1 argument, got {len(expr.args)}", expr.line) + return None + + typ_arg = expr.args[0] + method_args = fn.fitem.arg_names + if isinstance(typ_arg, NameExpr) and len(method_args) > 0 and method_args[0] == typ_arg.name: + subtype = builder.accept(expr.args[0]) + return builder.add(Call(ir.setup, [subtype], expr.line)) + + return None diff --git a/mypyc/irbuild/statement.py b/mypyc/irbuild/statement.py index eeeb40ac672fa..c83c5550d059b 100644 --- a/mypyc/irbuild/statement.py +++ b/mypyc/irbuild/statement.py @@ -33,6 +33,7 @@ ListExpr, Lvalue, MatchStmt, + NameExpr, OperatorAssignmentStmt, RaiseStmt, ReturnStmt, @@ -170,10 +171,43 @@ def transform_return_stmt(builder: IRBuilder, stmt: ReturnStmt) -> None: builder.nonlocal_control[-1].gen_return(builder, retval, stmt.line) +def check_unsupported_cls_assignment(builder: IRBuilder, stmt: AssignmentStmt) -> None: + fn = builder.fn_info + method_args = fn.fitem.arg_names + if fn.name != "__new__" or len(method_args) == 0: + return + + ir = builder.get_current_class_ir() + if ir is None or ir.inherits_python or not ir.is_ext_class: + return + + cls_arg = method_args[0] + + def flatten(lvalues: list[Expression]) -> list[Expression]: + flat = [] + for lvalue in lvalues: + if isinstance(lvalue, (TupleExpr, ListExpr)): + flat += flatten(lvalue.items) + else: + flat.append(lvalue) + return flat + + lvalues = flatten(stmt.lvalues) + + for lvalue in lvalues: + if isinstance(lvalue, NameExpr) and lvalue.name == cls_arg: + # Disallowed because it could break the transformation of object.__new__ calls + # inside __new__ methods. + builder.error( + f'Assignment to argument "{cls_arg}" in "__new__" method unsupported', stmt.line + ) + + def transform_assignment_stmt(builder: IRBuilder, stmt: AssignmentStmt) -> None: lvalues = stmt.lvalues assert lvalues builder.disallow_class_assignments(lvalues, stmt.line) + check_unsupported_cls_assignment(builder, stmt) first_lvalue = lvalues[0] if stmt.type and isinstance(stmt.rvalue, TempNode): # This is actually a variable annotation without initializer. Don't generate diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index 78ca7b68cefbc..92857f525ccae 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -1662,6 +1662,7 @@ L0: [case testDunderNew] from __future__ import annotations +from typing import Any class Test: val: int @@ -1686,6 +1687,169 @@ class NewClassMethod: def fn2() -> NewClassMethod: return NewClassMethod.__new__(42) +class NotTransformed: + def __new__(cls, val: int) -> Any: + return super().__new__(str) + + def factory(cls: Any, val: int) -> Any: + cls = str + return super().__new__(cls) + +[out] +def Test.__new__(cls, val): + cls :: object + val :: int + r0, obj :: __main__.Test + r1 :: bool +L0: + r0 = __mypyc__Test_setup(cls) + obj = r0 + obj.val = val; r1 = is_error + return obj +def fn(): + r0 :: object + r1 :: __main__.Test +L0: + r0 = __main__.Test :: type + r1 = Test.__new__(r0, 84) + return r1 +def NewClassMethod.__new__(cls, val): + cls :: object + val :: int + r0, obj :: __main__.NewClassMethod + r1 :: bool +L0: + r0 = __mypyc__NewClassMethod_setup(cls) + obj = r0 + obj.val = val; r1 = is_error + return obj +def fn2(): + r0 :: object + r1 :: __main__.NewClassMethod +L0: + r0 = __main__.NewClassMethod :: type + r1 = NewClassMethod.__new__(r0, 84) + return r1 +def NotTransformed.__new__(cls, val): + cls :: object + val :: int + r0 :: object + r1 :: str + r2, r3 :: object + r4 :: object[2] + r5 :: object_ptr + r6 :: object + r7 :: str + r8, r9 :: object + r10 :: object[1] + r11 :: object_ptr + r12 :: object + r13 :: str +L0: + r0 = builtins :: module + r1 = 'super' + r2 = CPyObject_GetAttr(r0, r1) + r3 = __main__.NotTransformed :: type + r4 = [r3, cls] + r5 = load_address r4 + r6 = PyObject_Vectorcall(r2, r5, 2, 0) + keep_alive r3, cls + r7 = '__new__' + r8 = CPyObject_GetAttr(r6, r7) + r9 = load_address PyUnicode_Type + r10 = [r9] + r11 = load_address r10 + r12 = PyObject_Vectorcall(r8, r11, 1, 0) + keep_alive r9 + r13 = cast(str, r12) + return r13 +def NotTransformed.factory(cls, val): + cls :: object + val :: int + r0, r1 :: object + r2 :: str + r3, r4 :: object + r5 :: object[2] + r6 :: object_ptr + r7 :: object + r8 :: str + r9 :: object + r10 :: object[1] + r11 :: object_ptr + r12 :: object +L0: + r0 = load_address PyUnicode_Type + cls = r0 + r1 = builtins :: module + r2 = 'super' + r3 = CPyObject_GetAttr(r1, r2) + r4 = __main__.NotTransformed :: type + r5 = [r4, cls] + r6 = load_address r5 + r7 = PyObject_Vectorcall(r3, r6, 2, 0) + keep_alive r4, cls + r8 = '__new__' + r9 = CPyObject_GetAttr(r7, r8) + r10 = [cls] + r11 = load_address r10 + r12 = PyObject_Vectorcall(r9, r11, 1, 0) + keep_alive cls + return r12 + +[case testObjectDunderNew_64bit] +from __future__ import annotations +from mypy_extensions import mypyc_attr +from typing import Any + +class Test: + val: int + + def __new__(cls, val: int) -> Test: + obj = object.__new__(cls) + obj.val = val + return obj + +def fn() -> Test: + return Test.__new__(Test, 42) + +class NewClassMethod: + val: int + + @classmethod + def __new__(cls, val: int) -> NewClassMethod: + obj = object.__new__(cls) + obj.val = val + return obj + +def fn2() -> NewClassMethod: + return NewClassMethod.__new__(42) + +class NotTransformed: + def __new__(cls, val: int) -> Any: + return object.__new__(str) + + def factory(cls: Any, val: int) -> Any: + cls = str + return object.__new__(cls) + +@mypyc_attr(native_class=False) +class NonNative: + def __new__(cls: Any) -> Any: + cls = str + return cls("str") + +class InheritsPython(dict): + def __new__(cls: Any) -> Any: + cls = dict + return cls({}) + +class ObjectNewOutsideDunderNew: + def __init__(self) -> None: + object.__new__(ObjectNewOutsideDunderNew) + +def object_new_outside_class() -> None: + object.__new__(Test) + [out] def Test.__new__(cls, val): cls :: object @@ -1721,19 +1885,185 @@ L0: r0 = __main__.NewClassMethod :: type r1 = NewClassMethod.__new__(r0, 84) return r1 +def NotTransformed.__new__(cls, val): + cls :: object + val :: int + r0 :: object + r1 :: str + r2, r3 :: object + r4 :: str + r5 :: object[2] + r6 :: object_ptr + r7 :: object + r8 :: str +L0: + r0 = builtins :: module + r1 = 'object' + r2 = CPyObject_GetAttr(r0, r1) + r3 = load_address PyUnicode_Type + r4 = '__new__' + r5 = [r2, r3] + r6 = load_address r5 + r7 = PyObject_VectorcallMethod(r4, r6, 9223372036854775810, 0) + keep_alive r2, r3 + r8 = cast(str, r7) + return r8 +def NotTransformed.factory(cls, val): + cls :: object + val :: int + r0, r1 :: object + r2 :: str + r3 :: object + r4 :: str + r5 :: object[2] + r6 :: object_ptr + r7 :: object +L0: + r0 = load_address PyUnicode_Type + cls = r0 + r1 = builtins :: module + r2 = 'object' + r3 = CPyObject_GetAttr(r1, r2) + r4 = '__new__' + r5 = [r3, cls] + r6 = load_address r5 + r7 = PyObject_VectorcallMethod(r4, r6, 9223372036854775810, 0) + keep_alive r3, cls + return r7 +def __new___NonNative_obj.__get__(__mypyc_self__, instance, owner): + __mypyc_self__, instance, owner, r0 :: object + r1 :: bit + r2 :: object +L0: + r0 = load_address _Py_NoneStruct + r1 = instance == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return __mypyc_self__ +L2: + r2 = PyMethod_New(__mypyc_self__, instance) + return r2 +def __new___NonNative_obj.__call__(__mypyc_self__, cls): + __mypyc_self__ :: __main__.__new___NonNative_obj + cls, r0 :: object + r1 :: str + r2 :: object[1] + r3 :: object_ptr + r4 :: object +L0: + r0 = load_address PyUnicode_Type + cls = r0 + r1 = 'str' + r2 = [r1] + r3 = load_address r2 + r4 = PyObject_Vectorcall(cls, r3, 1, 0) + keep_alive r1 + return r4 +def InheritsPython.__new__(cls): + cls, r0 :: object + r1 :: dict + r2 :: object[1] + r3 :: object_ptr + r4 :: object +L0: + r0 = load_address PyDict_Type + cls = r0 + r1 = PyDict_New() + r2 = [r1] + r3 = load_address r2 + r4 = PyObject_Vectorcall(cls, r3, 1, 0) + keep_alive r1 + return r4 +def ObjectNewOutsideDunderNew.__init__(self): + self :: __main__.ObjectNewOutsideDunderNew + r0 :: object + r1 :: str + r2, r3 :: object + r4 :: str + r5 :: object[2] + r6 :: object_ptr + r7 :: object +L0: + r0 = builtins :: module + r1 = 'object' + r2 = CPyObject_GetAttr(r0, r1) + r3 = __main__.ObjectNewOutsideDunderNew :: type + r4 = '__new__' + r5 = [r2, r3] + r6 = load_address r5 + r7 = PyObject_VectorcallMethod(r4, r6, 9223372036854775810, 0) + keep_alive r2, r3 + return 1 +def object_new_outside_class(): + r0 :: object + r1 :: str + r2, r3 :: object + r4 :: str + r5 :: object[2] + r6 :: object_ptr + r7 :: object +L0: + r0 = builtins :: module + r1 = 'object' + r2 = CPyObject_GetAttr(r0, r1) + r3 = __main__.Test :: type + r4 = '__new__' + r5 = [r2, r3] + r6 = load_address r5 + r7 = PyObject_VectorcallMethod(r4, r6, 9223372036854775810, 0) + keep_alive r2, r3 + return 1 [case testUnsupportedDunderNew] from __future__ import annotations from mypy_extensions import mypyc_attr +from typing import Any @mypyc_attr(native_class=False) class NonNative: def __new__(cls) -> NonNative: - return super().__new__(cls) # E: super().__new__() not supported for non-extension classes + return super().__new__(cls) # E: "object.__new__()" not supported for non-extension classes class InheritsPython(dict): def __new__(cls) -> InheritsPython: - return super().__new__(cls) # E: super().__new__() not supported for classes inheriting from non-native classes + return super().__new__(cls) # E: "object.__new__()" not supported for classes inheriting from non-native classes + +@mypyc_attr(native_class=False) +class NonNativeObjectNew: + def __new__(cls) -> NonNativeObjectNew: + return object.__new__(cls) # E: "object.__new__()" not supported for non-extension classes + +class InheritsPythonObjectNew(dict): + def __new__(cls) -> InheritsPythonObjectNew: + return object.__new__(cls) # E: "object.__new__()" not supported for classes inheriting from non-native classes + +class ClsAssignment: + def __new__(cls: Any) -> Any: + cls = str # E: Assignment to argument "cls" in "__new__" method unsupported + return super().__new__(cls) + +class ClsTupleAssignment: + def __new__(class_i_want: Any, val: int) -> Any: + class_i_want, val = dict, 1 # E: Assignment to argument "class_i_want" in "__new__" method unsupported + return object.__new__(class_i_want) + +class ClsListAssignment: + def __new__(cls: Any, val: str) -> Any: + [cls, val] = [object, "object"] # E: Assignment to argument "cls" in "__new__" method unsupported + return object.__new__(cls) + +class ClsNestedAssignment: + def __new__(cls: Any, val1: str, val2: int) -> Any: + [val1, [val2, cls]] = ["val1", [2, int]] # E: Assignment to argument "cls" in "__new__" method unsupported + return object.__new__(cls) + +class WrongNumberOfArgs: + def __new__(cls): + return super().__new__() # E: "object.__new__()" supported only with 1 argument, got 0 + +class WrongNumberOfArgsObjectNew: + def __new__(cls): + return object.__new__(cls, 1) # E: "object.__new__()" supported only with 1 argument, got 2 [case testClassWithFreeList] from mypy_extensions import mypyc_attr, trait diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 6c4ddc03887ab..3d0250cd24ee5 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -3493,6 +3493,156 @@ Add(0, 5)=5 running __new__ with 1 and 0 Add(1, 0)=1 +[case testObjectDunderNew] +from __future__ import annotations +from typing import Any, Union + +from testutil import assertRaises + +class Add: + l: IntLike + r: IntLike + + def __new__(cls, l: IntLike, r: IntLike) -> Any: + return ( + l if r == 0 else + r if l == 0 else + object.__new__(cls) + ) + + def __init__(self, l: IntLike, r: IntLike): + self.l = l + self.r = r + +IntLike = Union[int, Add] + +class RaisesException: + def __new__(cls, val: int) -> RaisesException: + if val == 0: + raise RuntimeError("Invalid value!") + return object.__new__(cls) + + def __init__(self, val: int) -> None: + self.val = val + +class ClsArgNotPassed: + def __new__(cls) -> Any: + return object.__new__(str) + +class SkipsBase(Add): + def __new__(cls) -> Any: + obj = object.__new__(cls) + obj.l = 0 + obj.r = 0 + return obj + +def test_dunder_new() -> None: + add_instance: Any = Add(1, 5) + assert type(add_instance) == Add + assert add_instance.l == 1 + assert add_instance.r == 5 + + # TODO: explicit types should not be needed but mypy does not use + # the return type of __new__ which makes mypyc add casts to Add. + right_int: Any = Add(0, 5) + assert type(right_int) == int + assert right_int == 5 + + left_int: Any = Add(1, 0) + assert type(left_int) == int + assert left_int == 1 + + with assertRaises(RuntimeError, "Invalid value!"): + _ = RaisesException(0) + + not_raised = RaisesException(1) + assert not_raised.val == 1 + + with assertRaises(TypeError, "object.__new__(str) is not safe, use str.__new__()"): + _ = ClsArgNotPassed() + + skip = SkipsBase.__new__(SkipsBase) + assert type(skip) == SkipsBase + assert skip.l == 0 + assert skip.r == 0 + +[case testObjectDunderNewInInterpreted] +from __future__ import annotations +from typing import Any, Union + +class Add: + l: IntLike + r: IntLike + + def __new__(cls, l: IntLike, r: IntLike) -> Any: + print(f'running __new__ with {l} and {r}') + + return ( + l if r == 0 else + r if l == 0 else + object.__new__(cls) + ) + + def __init__(self, l: IntLike, r: IntLike): + self.l = l + self.r = r + + def __repr__(self) -> str: + return f'({self.l} + {self.r})' + +IntLike = Union[int, Add] + +class RaisesException: + def __new__(cls, val: int) -> RaisesException: + if val == 0: + raise RuntimeError("Invalid value!") + return object.__new__(cls) + + def __init__(self, val: int) -> None: + self.val = val + +class ClsArgNotPassed: + def __new__(cls) -> Any: + return object.__new__(str) + +class SkipsBase(Add): + def __new__(cls) -> Any: + obj = object.__new__(cls) + obj.l = 0 + obj.r = 0 + return obj + +[file driver.py] +from native import Add, ClsArgNotPassed, RaisesException, SkipsBase + +from testutil import assertRaises + +print(f'{Add(1, 5)=}') +print(f'{Add(0, 5)=}') +print(f'{Add(1, 0)=}') + +with assertRaises(RuntimeError, "Invalid value!"): + raised = RaisesException(0) + +not_raised = RaisesException(1) +assert not_raised.val == 1 + +with assertRaises(TypeError, "object.__new__(str) is not safe, use str.__new__()"): + str_as_cls = ClsArgNotPassed() + +skip = SkipsBase.__new__(SkipsBase) +assert type(skip) == SkipsBase +assert skip.l == 0 +assert skip.r == 0 + +[out] +running __new__ with 1 and 5 +Add(1, 5)=(1 + 5) +running __new__ with 0 and 5 +Add(0, 5)=5 +running __new__ with 1 and 0 +Add(1, 0)=1 + [case testInheritedDunderNew] from __future__ import annotations from mypy_extensions import mypyc_attr @@ -3795,6 +3945,53 @@ assert t.generic == "{}" assert t.bitfield == 0x0C assert t.default == 10 +[case testUntransformedDunderNewCalls] +from testutil import assertRaises +from typing import Any + +class TestStrCls: + def __new__(cls): + return str.__new__(cls) + + @classmethod + def factory(cls): + return str.__new__(cls) + +class TestStrStr: + def __new__(cls): + return str.__new__(str) + + @classmethod + def factory(cls): + return str.__new__(str) + +class TestStrInt: + def __new__(cls): + return str.__new__(int) + + @classmethod + def factory(cls): + return str.__new__(int) + +def test_untransformed_dunder_new() -> None: + with assertRaises(TypeError, "str.__new__(TestStrCls): TestStrCls is not a subtype of str"): + i = TestStrCls() + + j: Any = TestStrStr() + assert j == "" + + with assertRaises(TypeError, "str.__new__(int): int is not a subtype of str"): + k = TestStrInt() + + with assertRaises(TypeError, "str.__new__(TestStrCls): TestStrCls is not a subtype of str"): + i = TestStrCls.factory() + + j = TestStrStr.factory() + assert j == "" + + with assertRaises(TypeError, "str.__new__(int): int is not a subtype of str"): + k = TestStrInt.factory() + [case testPerTypeFreeList] from __future__ import annotations From d96b9dcb1dcd0c642e920e46e9f077ca9c7e88c2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 23 Sep 2025 14:12:14 +0100 Subject: [PATCH 272/424] Fix generation of incorrect indirect deps from locals (#19906) Don't generate an indirect dependency to module `bar` if a local variable has name `bar`. This aims to fix the root cause of the issue #19903 tries to solve. --- mypy/semanal.py | 18 ++++++++++-------- test-data/unit/check-incremental.test | 21 +++++++++++++++++++++ 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index b3fd1b98bfd20..d78df2e199b8b 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -5928,8 +5928,7 @@ def visit_member_expr(self, expr: MemberExpr) -> None: if isinstance(sym.node, PlaceholderNode): self.process_placeholder(expr.name, "attribute", expr) return - if sym.node is not None: - self.record_imported_symbol(sym.node) + self.record_imported_symbol(sym) expr.kind = sym.kind expr.fullname = sym.fullname or "" expr.node = sym.node @@ -5960,7 +5959,7 @@ def visit_member_expr(self, expr: MemberExpr) -> None: if type_info: n = type_info.names.get(expr.name) if n is not None and isinstance(n.node, (MypyFile, TypeInfo, TypeAlias)): - self.record_imported_symbol(n.node) + self.record_imported_symbol(n) expr.kind = n.kind expr.fullname = n.fullname or "" expr.node = n.node @@ -6282,14 +6281,17 @@ def lookup( self, name: str, ctx: Context, suppress_errors: bool = False ) -> SymbolTableNode | None: node = self._lookup(name, ctx, suppress_errors) - if node is not None and node.node is not None: + if node is not None: # This call is unfortunate from performance point of view, but # needed for rare cases like e.g. testIncrementalChangingAlias. - self.record_imported_symbol(node.node) + self.record_imported_symbol(node) return node - def record_imported_symbol(self, node: SymbolNode) -> None: + def record_imported_symbol(self, sym: SymbolTableNode) -> None: """If the symbol was not defined in current module, add its module to module_refs.""" + if sym.kind == LDEF or sym.node is None: + return + node = sym.node if not node.fullname: return if isinstance(node, MypyFile): @@ -6519,8 +6521,8 @@ def lookup_qualified( self.name_not_defined(name, ctx, namespace=namespace) return None sym = nextsym - if sym is not None and sym.node is not None: - self.record_imported_symbol(sym.node) + if sym is not None: + self.record_imported_symbol(sym) return sym def lookup_type_node(self, expr: Expression) -> SymbolTableNode | None: diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index d9d78715b396a..e91b8778e9868 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -7290,3 +7290,24 @@ tmp/m.py:3: error: Argument 1 to "accept_int" has incompatible type "str"; expec [out2] tmp/n.py:3: note: Revealed type is "builtins.str" tmp/m.py:3: error: Argument 1 to "accept_int" has incompatible type "str"; expected "int" + +[case testIncrementalNoIndirectDepFromLocal] +import foo +import bar + +[file foo.py] +# Having a local named 'bar' shouldn't generate a dependency on module 'bar' +def f(bar: int) -> int: + return bar + +[file bar.py] +import foo +x = 1 + +[file bar.py.2] +import foo +x = 2 + +[out] +[rechecked bar] +[stale] From 354bea6352ee7a38b05e2f42c874e7d1f7bf557a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 23 Sep 2025 14:15:08 +0100 Subject: [PATCH 273/424] Don't consider indirect dependencies when calculating SCCs (#19903) SCC construction should only need to consider import dependencies, since indirect dependencies are not available during non-incremental runs, and we want SCCs to be identical in incremental and non-incremental runs. This may improve performance slightly and will make mypy more robust in case there are extra indirect dependencies (see #19906). --- mypy/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/build.py b/mypy/build.py index 2d3296a4713ee..ad25b811ff7cd 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -3488,7 +3488,7 @@ def process_stale_scc(graph: Graph, scc: list[str], manager: BuildManager) -> No def sorted_components( - graph: Graph, vertices: AbstractSet[str] | None = None, pri_max: int = PRI_ALL + graph: Graph, vertices: AbstractSet[str] | None = None, pri_max: int = PRI_INDIRECT ) -> list[AbstractSet[str]]: """Return the graph's SCCs, topologically sorted by dependencies. From 00f2b29ecceaaad0d69700f651adf9b7678f9907 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 25 Sep 2025 17:00:15 +0100 Subject: [PATCH 274/424] Improve record_imported_symbol() (#19924) This has four improvements: * Skip placeholders, they are not guaranteed to have a correct `fullname`. * Skip nodes from `builtins`/`typing`, these don't add anything. * Don't use `rsplit(".")` on variables/functions recursively, always use enclosing class. * (Most importantly) add missing `maxsplit=1`. --- mypy/semanal.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index d78df2e199b8b..17dc9bfadc1f2 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -6292,22 +6292,37 @@ def record_imported_symbol(self, sym: SymbolTableNode) -> None: if sym.kind == LDEF or sym.node is None: return node = sym.node - if not node.fullname: + if isinstance(node, PlaceholderNode) or not node.fullname: + # This node is not ready yet. return + if node.fullname.startswith(("builtins.", "typing.")): + # Skip dependencies on builtins/typing. + return + # Modules, classes, and type aliases store defining module directly. if isinstance(node, MypyFile): fullname = node.fullname elif isinstance(node, TypeInfo): fullname = node.module_name elif isinstance(node, TypeAlias): fullname = node.module - elif isinstance(node, (Var, FuncDef, OverloadedFuncDef)) and node.info: - fullname = node.info.module_name + elif isinstance(node, (Var, FuncDef, OverloadedFuncDef, Decorator)): + # For functions/variables infer defining module from enclosing class. + info = node.var.info if isinstance(node, Decorator) else node.info + if info: + fullname = info.module_name + else: + # global function/variable + fullname = node.fullname.rsplit(".", maxsplit=1)[0] else: - fullname = node.fullname.rsplit(".")[0] + # Some nodes (currently only TypeVarLikeExpr subclasses) don't store + # module fullname explicitly, infer it from the node fullname iteratively. + # TODO: this is not 100% robust for type variables nested within a class + # with a name that matches name of a submodule. + fullname = node.fullname.rsplit(".", maxsplit=1)[0] if fullname == self.cur_mod_id: return while "." in fullname and fullname not in self.modules: - fullname = fullname.rsplit(".")[0] + fullname = fullname.rsplit(".", maxsplit=1)[0] if fullname != self.cur_mod_id: self.cur_mod_node.module_refs.add(fullname) From c058b09f544271c582416e873348c63868eeae50 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 25 Sep 2025 17:41:13 +0100 Subject: [PATCH 275/424] Install librt in mypy_primer (#19925) Fixes https://github.com/python/mypy/issues/19844 --- .github/workflows/mypy_primer.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/mypy_primer.yml b/.github/workflows/mypy_primer.yml index 1ff984247fb65..8a04c75b0e2e5 100644 --- a/.github/workflows/mypy_primer.yml +++ b/.github/workflows/mypy_primer.yml @@ -67,6 +67,7 @@ jobs: --debug \ --additional-flags="--debug-serialize" \ --output concise \ + --mypy-install-librt \ | tee diff_${{ matrix.shard-index }}.txt ) || [ $? -eq 1 ] - if: ${{ matrix.shard-index == 0 }} From a936e3008d86b12b575978da1069e76fa1e89c3b Mon Sep 17 00:00:00 2001 From: Piotr Sawicki Date: Thu, 25 Sep 2025 18:55:26 +0200 Subject: [PATCH 276/424] [mypyc] Generate __getattr__ wrapper (#19909) Fixes https://github.com/mypyc/mypyc/issues/887 Generate a wrapper function for `__getattr__` in classes where it's defined and put it into the `tp_getattro` slot. At runtime, this wrapper function is called for every attribute access, so to match behavior of interpreted python it needs to first check if the given name is present in the type dictionary and only call the user-defined `__getattr__` when it's not present. Checking the type dictionary is implemented using a cpython function `_PyObject_GenericGetAttrWithDict` which is also used in the default attribute access handler in [cpython](https://github.com/python/cpython/blob/dd45179fa0f5ad2fd169cdd35065df2c3bce85bc/Objects/typeobject.c#L10676). In compiled code, the wrapper will only be called when the attribute name cannot be statically resolved. When it can be resolved, the attribute will be accessed directly in the underlying C struct, or the generated function will be directly called in case of resolving method names. No change from existing behavior. When the name cannot be statically resolved, mypyc generates calls to `PyObject_GetAttr` which internally calls `tp_getattro`. In interpreted code that uses compiled classes, attribute access will always result in calls to `PyObject_GetAttr` so there's always a dict look-up in the wrapper to find the attribute. But that's the case already, the dict look-up happens in the default attribute access handler in cpython. So the wrapper should not bring any negative performance impact. --- mypyc/codegen/emitclass.py | 7 + mypyc/ir/ops.py | 6 +- mypyc/irbuild/builder.py | 3 +- mypyc/irbuild/function.py | 56 +++- mypyc/irbuild/ll_builder.py | 2 + mypyc/lib-rt/CPy.h | 4 + mypyc/primitives/generic_ops.py | 9 + mypyc/primitives/registry.py | 3 + mypyc/test-data/irbuild-classes.test | 125 ++++++++ mypyc/test-data/run-classes.test | 432 +++++++++++++++++++++++++++ mypyc/transform/refcount.py | 5 +- 11 files changed, 648 insertions(+), 4 deletions(-) diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 94f32b3224a98..9e8f9c74bc6d9 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -39,6 +39,12 @@ def native_slot(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: return f"{NATIVE_PREFIX}{fn.cname(emitter.names)}" +def dunder_attr_slot(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: + wrapper_fn = cl.get_method(fn.name + "__wrapper") + assert wrapper_fn + return f"{NATIVE_PREFIX}{wrapper_fn.cname(emitter.names)}" + + # We maintain a table from dunder function names to struct slots they # correspond to and functions that generate a wrapper (if necessary) # and return the function name to stick in the slot. @@ -55,6 +61,7 @@ def native_slot(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: "__iter__": ("tp_iter", native_slot), "__hash__": ("tp_hash", generate_hash_wrapper), "__get__": ("tp_descr_get", generate_get_wrapper), + "__getattr__": ("tp_getattro", dunder_attr_slot), } AS_MAPPING_SLOT_DEFS: SlotTable = { diff --git a/mypyc/ir/ops.py b/mypyc/ir/ops.py index 4b3b5eb3c8cae..76c1e07a79d5a 100644 --- a/mypyc/ir/ops.py +++ b/mypyc/ir/ops.py @@ -1221,6 +1221,7 @@ def __init__( var_arg_idx: int = -1, *, is_pure: bool = False, + returns_null: bool = False, ) -> None: self.error_kind = error_kind super().__init__(line) @@ -1235,7 +1236,10 @@ def __init__( # and all the arguments are immutable. Pure functions support # additional optimizations. Pure functions never fail. self.is_pure = is_pure - if is_pure: + # The function might return a null value that does not indicate + # an error. + self.returns_null = returns_null + if is_pure or returns_null: assert error_kind == ERR_NEVER def sources(self) -> list[Value]: diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index 12b5bc7f8f824..f4ee4371b9bf4 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -1241,6 +1241,7 @@ def enter_method( ret_type: RType, fn_info: FuncInfo | str = "", self_type: RType | None = None, + internal: bool = False, ) -> Iterator[None]: """Generate IR for a method. @@ -1268,7 +1269,7 @@ def enter_method( sig = FuncSignature(args, ret_type) name = self.function_name_stack.pop() class_ir = self.class_ir_stack.pop() - decl = FuncDecl(name, class_ir.name, self.module_name, sig) + decl = FuncDecl(name, class_ir.name, self.module_name, sig, internal=internal) ir = FuncIR(decl, arg_regs, blocks) class_ir.methods[name] = ir class_ir.method_decls[name] = ir.decl diff --git a/mypyc/irbuild/function.py b/mypyc/irbuild/function.py index f0fc424aea540..a9a098d25ddef 100644 --- a/mypyc/irbuild/function.py +++ b/mypyc/irbuild/function.py @@ -42,6 +42,7 @@ ) from mypyc.ir.ops import ( BasicBlock, + ComparisonOp, GetAttr, Integer, LoadAddress, @@ -81,7 +82,7 @@ dict_new_op, exact_dict_set_item_op, ) -from mypyc.primitives.generic_ops import py_setattr_op +from mypyc.primitives.generic_ops import generic_getattr, py_setattr_op from mypyc.primitives.misc_ops import register_function from mypyc.primitives.registry import builtin_names from mypyc.sametype import is_same_method_signature, is_same_type @@ -364,6 +365,56 @@ def gen_func_ir( return (func_ir, func_reg) +def generate_getattr_wrapper(builder: IRBuilder, cdef: ClassDef, getattr: FuncDef) -> None: + """ + Generate a wrapper function for __getattr__ that can be put into the tp_getattro slot. + The wrapper takes one argument besides self which is the attribute name. + It first checks if the name matches any of the attributes of this class. + If it does, it returns that attribute. If none match, it calls __getattr__. + + __getattr__ is not supported in classes that allow interpreted subclasses because the + tp_getattro slot is inherited by subclasses and if the subclass overrides __getattr__, + the override would be ignored in our wrapper. TODO: To support this, the wrapper would + have to check type of self and if it's not the compiled class, resolve "__getattr__" against + the type at runtime and call the returned method, like _Py_slot_tp_getattr_hook in cpython. + + __getattr__ is not supported in classes which inherit from non-native classes because those + have __dict__ which currently has some strange interactions when class attributes and + variables are assigned through __dict__ vs. through regular attribute access. Allowing + __getattr__ on top of that could be problematic. + """ + name = getattr.name + "__wrapper" + ir = builder.mapper.type_to_ir[cdef.info] + line = getattr.line + + error_base = f'"__getattr__" not supported in class "{cdef.name}" because ' + if ir.allow_interpreted_subclasses: + builder.error(error_base + "it allows interpreted subclasses", line) + if ir.inherits_python: + builder.error(error_base + "it inherits from a non-native class", line) + + with builder.enter_method(ir, name, object_rprimitive, internal=True): + attr_arg = builder.add_argument("attr", object_rprimitive) + generic_getattr_result = builder.call_c(generic_getattr, [builder.self(), attr_arg], line) + + return_generic, call_getattr = BasicBlock(), BasicBlock() + null = Integer(0, object_rprimitive, line) + got_generic = builder.add( + ComparisonOp(generic_getattr_result, null, ComparisonOp.NEQ, line) + ) + builder.add_bool_branch(got_generic, return_generic, call_getattr) + + builder.activate_block(return_generic) + builder.add(Return(generic_getattr_result, line)) + + builder.activate_block(call_getattr) + # No attribute matched so call user-provided __getattr__. + getattr_result = builder.gen_method_call( + builder.self(), getattr.name, [attr_arg], object_rprimitive, line + ) + builder.add(Return(getattr_result, line)) + + def handle_ext_method(builder: IRBuilder, cdef: ClassDef, fdef: FuncDef) -> None: # Perform the function of visit_method for methods inside extension classes. name = fdef.name @@ -430,6 +481,9 @@ def handle_ext_method(builder: IRBuilder, cdef: ClassDef, fdef: FuncDef) -> None class_ir.glue_methods[(class_ir, name)] = f builder.functions.append(f) + if fdef.name == "__getattr__": + generate_getattr_wrapper(builder, cdef, fdef) + def handle_non_ext_method( builder: IRBuilder, non_ext: NonExtClassInfo, cdef: ClassDef, fdef: FuncDef diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index 4b85c13892c1d..37f2add4abbd9 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -2048,6 +2048,7 @@ def call_c( line, var_arg_idx, is_pure=desc.is_pure, + returns_null=desc.returns_null, ) ) if desc.is_borrowed: @@ -2131,6 +2132,7 @@ def primitive_op( desc.extra_int_constants, desc.priority, is_pure=desc.is_pure, + returns_null=False, ) return self.call_c(c_desc, args, line, result_type=result_type) diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index 5dec7509ac7b9..b9cecb9280f36 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -949,6 +949,10 @@ PyObject *CPy_GetANext(PyObject *aiter); void CPy_SetTypeAliasTypeComputeFunction(PyObject *alias, PyObject *compute_value); void CPyTrace_LogEvent(const char *location, const char *line, const char *op, const char *details); +static inline PyObject *CPyObject_GenericGetAttr(PyObject *self, PyObject *name) { + return _PyObject_GenericGetAttrWithDict(self, name, NULL, 1); +} + #if CPY_3_11_FEATURES PyObject *CPy_GetName(PyObject *obj); #endif diff --git a/mypyc/primitives/generic_ops.py b/mypyc/primitives/generic_ops.py index 8a4ddc3702808..ff978b7c8c3bb 100644 --- a/mypyc/primitives/generic_ops.py +++ b/mypyc/primitives/generic_ops.py @@ -401,3 +401,12 @@ c_function_name="CPy_GetName", error_kind=ERR_MAGIC, ) + +# look-up name in tp_dict but don't raise AttributeError on failure +generic_getattr = custom_op( + arg_types=[object_rprimitive, object_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyObject_GenericGetAttr", + error_kind=ERR_NEVER, + returns_null=True, +) diff --git a/mypyc/primitives/registry.py b/mypyc/primitives/registry.py index 07546663d08ed..3188bc322809f 100644 --- a/mypyc/primitives/registry.py +++ b/mypyc/primitives/registry.py @@ -61,6 +61,7 @@ class CFunctionDescription(NamedTuple): extra_int_constants: list[tuple[int, RType]] priority: int is_pure: bool + returns_null: bool # A description for C load operations including LoadGlobal and LoadAddress @@ -253,6 +254,7 @@ def custom_op( is_borrowed: bool = False, *, is_pure: bool = False, + returns_null: bool = False, ) -> CFunctionDescription: """Create a one-off CallC op that can't be automatically generated from the AST. @@ -274,6 +276,7 @@ def custom_op( extra_int_constants, 0, is_pure=is_pure, + returns_null=returns_null, ) diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index 92857f525ccae..76e28711c5e37 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -2088,3 +2088,128 @@ class NonNative: @mypyc_attr(free_list_len=1, allow_interpreted_subclasses=True) # E: "free_list_len" can't be used in a class that allows interpreted subclasses class InterpSub: pass + +[case testUnsupportedGetAttr] +from mypy_extensions import mypyc_attr + +@mypyc_attr(allow_interpreted_subclasses=True) +class AllowsInterpreted: + def __getattr__(self, attr: str) -> object: # E: "__getattr__" not supported in class "AllowsInterpreted" because it allows interpreted subclasses + return 0 + +class InheritsInterpreted(dict): + def __getattr__(self, attr: str) -> object: # E: "__getattr__" not supported in class "InheritsInterpreted" because it inherits from a non-native class + return 0 + +@mypyc_attr(native_class=False) +class NonNative: + pass + +class InheritsNonNative(NonNative): + def __getattr__(self, attr: str) -> object: # E: "__getattr__" not supported in class "InheritsNonNative" because it inherits from a non-native class + return 0 + +[case testGetAttr] +from typing import ClassVar + +class GetAttr: + class_var = "x" + class_var_annotated: ClassVar[int] = 99 + + def __init__(self, regular_attr: int): + self.regular_attr = regular_attr + + def __getattr__(self, attr: str) -> object: + return attr + + def method(self) -> int: + return 0 + +def test_getattr() -> list[object]: + i = GetAttr(42) + one = i.one + two = i.regular_attr + three = i.class_var + four = i.class_var_annotated + five = i.method() + return [one, two, three, four, five] + +[typing fixtures/typing-full.pyi] +[out] +def GetAttr.__init__(self, regular_attr): + self :: __main__.GetAttr + regular_attr :: int +L0: + self.regular_attr = regular_attr + return 1 +def GetAttr.__getattr__(self, attr): + self :: __main__.GetAttr + attr :: str +L0: + return attr +def GetAttr.__getattr____wrapper(__mypyc_self__, attr): + __mypyc_self__ :: __main__.GetAttr + attr, r0 :: object + r1 :: bit + r2 :: str + r3 :: object +L0: + r0 = CPyObject_GenericGetAttr(__mypyc_self__, attr) + r1 = r0 != 0 + if r1 goto L1 else goto L2 :: bool +L1: + return r0 +L2: + r2 = cast(str, attr) + r3 = __mypyc_self__.__getattr__(r2) + return r3 +def GetAttr.method(self): + self :: __main__.GetAttr +L0: + return 0 +def GetAttr.__mypyc_defaults_setup(__mypyc_self__): + __mypyc_self__ :: __main__.GetAttr + r0 :: str +L0: + r0 = 'x' + __mypyc_self__.class_var = r0 + return 1 +def test_getattr(): + r0, i :: __main__.GetAttr + r1 :: str + r2, one :: object + r3, two :: int + r4, three, r5 :: str + r6 :: object + r7, four, r8, five :: int + r9 :: list + r10, r11, r12 :: object + r13 :: ptr +L0: + r0 = GetAttr(84) + i = r0 + r1 = 'one' + r2 = CPyObject_GetAttr(i, r1) + one = r2 + r3 = i.regular_attr + two = r3 + r4 = i.class_var + three = r4 + r5 = 'class_var_annotated' + r6 = CPyObject_GetAttr(i, r5) + r7 = unbox(int, r6) + four = r7 + r8 = i.method() + five = r8 + r9 = PyList_New(5) + r10 = box(int, two) + r11 = box(int, four) + r12 = box(int, five) + r13 = list_items r9 + buf_init_item r13, 0, one + buf_init_item r13, 1, r10 + buf_init_item r13, 2, three + buf_init_item r13, 3, r11 + buf_init_item r13, 4, r12 + keep_alive r9 + return r9 diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 3d0250cd24ee5..b2f1a088585d7 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -4088,3 +4088,435 @@ def test_inheritance_2() -> None: x = None y = None d = None + +[case testDunderGetAttr] +from mypy_extensions import mypyc_attr +from typing import ClassVar + +class GetAttr: + class_var = "x" + + def __init__(self, extra_attrs: dict[str, object], regular_attr: int): + self.extra_attrs = extra_attrs + self.regular_attr = regular_attr + + def __getattr__(self, attr: str) -> object: + return self.extra_attrs.get(attr) + +class GetAttrDefault: + class_var: ClassVar[str] = "x" + + def __init__(self, extra_attrs: dict[str, object], regular_attr: int): + self.extra_attrs = extra_attrs + self.regular_attr = regular_attr + + def __getattr__(self, attr: str, default: int = 8, mult: int = 1) -> object: + return self.extra_attrs.get(attr, default * mult) + +class GetAttrInherited(GetAttr): + subclass_var = "y" + + def __init__(self, extra_attrs: dict[str, object], regular_attr: int, sub_attr: int): + super().__init__(extra_attrs, regular_attr) + self.sub_attr = sub_attr + +class GetAttrOverridden(GetAttr): + subclass_var: ClassVar[str] = "y" + + def __init__(self, extra_attrs: dict[str, object], regular_attr: int, sub_attr: int): + super().__init__(extra_attrs, regular_attr) + self.sub_attr = sub_attr + + def __getattr__(self, attr: str) -> str: + return attr + +@mypyc_attr(native_class=False) +class GetAttrNonNative: + class_var = "x" + + def __init__(self, extra_attrs: dict[str, object], regular_attr: int): + self.extra_attrs = extra_attrs + self.regular_attr = regular_attr + + def __getattr__(self, attr: str) -> object: + return self.extra_attrs.get(attr) + +def test_getattr() -> None: + i = GetAttr({"one": 1, "two": "two", "three": 3.14}, 42) + assert i.__getattr__("one") == 1 + assert i.__getattr__("regular_attr") == None + assert i.__getattr__("class_var") == None + assert i.__getattr__("four") == None + + assert getattr(i, "two") == "two" + assert getattr(i, "regular_attr") == 42 + assert getattr(i, "class_var") == "x" + assert getattr(i, "four") == None + + assert i.three == 3.14 + assert i.regular_attr == 42 + assert i.class_var == "x" + assert i.four == None + + assert i.__class__ == GetAttr + + i.extra_attrs["regular_attr"] = (4, 4, 4) + assert i.__getattr__("regular_attr") == (4, 4, 4) + assert getattr(i, "regular_attr") == 42 + assert i.regular_attr == 42 + +def test_getattr_default() -> None: + i = GetAttrDefault({"one": 1, "two": "two", "three": 3.14}, 42) + assert i.__getattr__("one") == 1 + assert i.__getattr__("regular_attr") == 8 + assert i.__getattr__("class_var") == 8 + assert i.__getattr__("four", 4, 3) == 12 + + assert getattr(i, "two") == "two" + assert getattr(i, "regular_attr") == 42 + assert getattr(i, "class_var") == "x" + assert getattr(i, "four") == 8 + + assert i.three == 3.14 + assert i.regular_attr == 42 + assert i.class_var == "x" + assert i.four == 8 + + assert i.__class__ == GetAttrDefault + + i.extra_attrs["class_var"] = (4, 4, 4) + assert i.__getattr__("class_var") == (4, 4, 4) + assert getattr(i, "class_var") == "x" + assert i.class_var == "x" + +def test_getattr_inherited() -> None: + i = GetAttrInherited({"one": 1, "two": "two", "three": 3.14}, 42, 24) + assert i.__getattr__("one") == 1 + assert i.__getattr__("regular_attr") == None + assert i.__getattr__("sub_attr") == None + assert i.__getattr__("class_var") == None + assert i.__getattr__("subclass_var") == None + assert i.__getattr__("four") == None + + assert getattr(i, "two") == "two" + assert getattr(i, "regular_attr") == 42 + assert getattr(i, "sub_attr") == 24 + assert getattr(i, "class_var") == "x" + assert getattr(i, "subclass_var") == "y" + assert getattr(i, "four") == None + + assert i.three == 3.14 + assert i.regular_attr == 42 + assert i.sub_attr == 24 + assert i.class_var == "x" + assert i.subclass_var == "y" + assert i.four == None + + assert i.__class__ == GetAttrInherited + + i.extra_attrs["sub_attr"] = (4, 4, 4) + assert i.__getattr__("sub_attr") == (4, 4, 4) + assert getattr(i, "sub_attr") == 24 + assert i.sub_attr == 24 + + base_ref: GetAttr = i + assert getattr(base_ref, "sub_attr") == 24 + assert base_ref.sub_attr == 24 + + assert getattr(base_ref, "subclass_var") == "y" + assert base_ref.subclass_var == "y" + + assert getattr(base_ref, "new") == None + assert base_ref.new == None + + assert base_ref.__class__ == GetAttrInherited + + +def test_getattr_overridden() -> None: + i = GetAttrOverridden({"one": 1, "two": "two", "three": 3.14}, 42, 24) + assert i.__getattr__("one") == "one" + assert i.__getattr__("regular_attr") == "regular_attr" + assert i.__getattr__("sub_attr") == "sub_attr" + assert i.__getattr__("class_var") == "class_var" + assert i.__getattr__("subclass_var") == "subclass_var" + assert i.__getattr__("four") == "four" + + assert getattr(i, "two") == "two" + assert getattr(i, "regular_attr") == 42 + assert getattr(i, "sub_attr") == 24 + assert getattr(i, "class_var") == "x" + assert getattr(i, "subclass_var") == "y" + assert getattr(i, "four") == "four" + + assert i.three == "three" + assert i.regular_attr == 42 + assert i.sub_attr == 24 + assert i.class_var == "x" + assert i.subclass_var == "y" + assert i.four == "four" + + assert i.__class__ == GetAttrOverridden + + i.extra_attrs["subclass_var"] = (4, 4, 4) + assert i.__getattr__("subclass_var") == "subclass_var" + assert getattr(i, "subclass_var") == "y" + assert i.subclass_var == "y" + + base_ref: GetAttr = i + assert getattr(base_ref, "sub_attr") == 24 + assert base_ref.sub_attr == 24 + + assert getattr(base_ref, "subclass_var") == "y" + assert base_ref.subclass_var == "y" + + assert getattr(base_ref, "new") == "new" + assert base_ref.new == "new" + + assert base_ref.__class__ == GetAttrOverridden + +def test_getattr_nonnative() -> None: + i = GetAttr({"one": 1, "two": "two", "three": 3.14}, 42) + assert i.__getattr__("one") == 1 + assert i.__getattr__("regular_attr") == None + assert i.__getattr__("class_var") == None + assert i.__getattr__("four") == None + + assert getattr(i, "two") == "two" + assert getattr(i, "regular_attr") == 42 + assert getattr(i, "class_var") == "x" + assert getattr(i, "four") == None + + assert i.three == 3.14 + assert i.regular_attr == 42 + assert i.class_var == "x" + assert i.four == None + + assert i.__class__ == GetAttr + + i.extra_attrs["regular_attr"] = (4, 4, 4) + assert i.__getattr__("regular_attr") == (4, 4, 4) + assert getattr(i, "regular_attr") == 42 + assert i.regular_attr == 42 + +[typing fixtures/typing-full.pyi] + +[case testDunderGetAttrInterpreted] +from mypy_extensions import mypyc_attr +from typing import ClassVar + +class GetAttr: + class_var = "x" + + def __init__(self, extra_attrs: dict[str, object], regular_attr: int): + self.extra_attrs = extra_attrs + self.regular_attr = regular_attr + + def __getattr__(self, attr: str) -> object: + return self.extra_attrs.get(attr) + +class GetAttrDefault: + class_var: ClassVar[str] = "x" + + def __init__(self, extra_attrs: dict[str, object], regular_attr: int): + self.extra_attrs = extra_attrs + self.regular_attr = regular_attr + + def __getattr__(self, attr: str, default: int = 8, mult: int = 1) -> object: + return self.extra_attrs.get(attr, default * mult) + +class GetAttrInherited(GetAttr): + subclass_var = "y" + + def __init__(self, extra_attrs: dict[str, object], regular_attr: int, sub_attr: int): + super().__init__(extra_attrs, regular_attr) + self.sub_attr = sub_attr + +class GetAttrOverridden(GetAttr): + subclass_var: ClassVar[str] = "y" + + def __init__(self, extra_attrs: dict[str, object], regular_attr: int, sub_attr: int): + super().__init__(extra_attrs, regular_attr) + self.sub_attr = sub_attr + + def __getattr__(self, attr: str) -> str: + return attr + +@mypyc_attr(native_class=False) +class GetAttrNonNative: + class_var = "x" + + def __init__(self, extra_attrs: dict[str, object], regular_attr: int): + self.extra_attrs = extra_attrs + self.regular_attr = regular_attr + + def __getattr__(self, attr: str) -> object: + return self.extra_attrs.get(attr) + +[file driver.py] +from native import GetAttr, GetAttrDefault, GetAttrInherited, GetAttrOverridden, GetAttrNonNative + +def test_getattr() -> None: + i = GetAttr({"one": 1, "two": "two", "three": 3.14}, 42) + assert i.__getattr__("one") == 1 + assert i.__getattr__("regular_attr") == None + assert i.__getattr__("class_var") == None + assert i.__getattr__("four") == None + + assert getattr(i, "two") == "two" + assert getattr(i, "regular_attr") == 42 + assert getattr(i, "class_var") == "x" + assert getattr(i, "four") == None + + assert i.three == 3.14 + assert i.regular_attr == 42 + assert i.class_var == "x" + assert i.four == None + + assert i.__class__ == GetAttr + + i.extra_attrs["regular_attr"] = (4, 4, 4) + assert i.__getattr__("regular_attr") == (4, 4, 4) + assert getattr(i, "regular_attr") == 42 + assert i.regular_attr == 42 + +def test_getattr_default() -> None: + i = GetAttrDefault({"one": 1, "two": "two", "three": 3.14}, 42) + assert i.__getattr__("one") == 1 + assert i.__getattr__("regular_attr") == 8 + assert i.__getattr__("class_var") == 8 + assert i.__getattr__("four", 4, 3) == 12 + + assert getattr(i, "two") == "two" + assert getattr(i, "regular_attr") == 42 + assert getattr(i, "class_var") == "x" + assert getattr(i, "four") == 8 + + assert i.three == 3.14 + assert i.regular_attr == 42 + assert i.class_var == "x" + assert i.four == 8 + + assert i.__class__ == GetAttrDefault + + i.extra_attrs["class_var"] = (4, 4, 4) + assert i.__getattr__("class_var") == (4, 4, 4) + assert getattr(i, "class_var") == "x" + assert i.class_var == "x" + +def test_getattr_inherited() -> None: + i = GetAttrInherited({"one": 1, "two": "two", "three": 3.14}, 42, 24) + assert i.__getattr__("one") == 1 + assert i.__getattr__("regular_attr") == None + assert i.__getattr__("sub_attr") == None + assert i.__getattr__("class_var") == None + assert i.__getattr__("subclass_var") == None + assert i.__getattr__("four") == None + + assert getattr(i, "two") == "two" + assert getattr(i, "regular_attr") == 42 + assert getattr(i, "sub_attr") == 24 + assert getattr(i, "class_var") == "x" + assert getattr(i, "subclass_var") == "y" + assert getattr(i, "four") == None + + assert i.three == 3.14 + assert i.regular_attr == 42 + assert i.sub_attr == 24 + assert i.class_var == "x" + assert i.subclass_var == "y" + assert i.four == None + + assert i.__class__ == GetAttrInherited + + i.extra_attrs["sub_attr"] = (4, 4, 4) + assert i.__getattr__("sub_attr") == (4, 4, 4) + assert getattr(i, "sub_attr") == 24 + assert i.sub_attr == 24 + + base_ref: GetAttr = i + assert getattr(base_ref, "sub_attr") == 24 + assert base_ref.sub_attr == 24 + + assert getattr(base_ref, "subclass_var") == "y" + assert base_ref.subclass_var == "y" + + assert getattr(base_ref, "new") == None + assert base_ref.new == None + + assert base_ref.__class__ == GetAttrInherited + + +def test_getattr_overridden() -> None: + i = GetAttrOverridden({"one": 1, "two": "two", "three": 3.14}, 42, 24) + assert i.__getattr__("one") == "one" + assert i.__getattr__("regular_attr") == "regular_attr" + assert i.__getattr__("sub_attr") == "sub_attr" + assert i.__getattr__("class_var") == "class_var" + assert i.__getattr__("subclass_var") == "subclass_var" + assert i.__getattr__("four") == "four" + + assert getattr(i, "two") == "two" + assert getattr(i, "regular_attr") == 42 + assert getattr(i, "sub_attr") == 24 + assert getattr(i, "class_var") == "x" + assert getattr(i, "subclass_var") == "y" + assert getattr(i, "four") == "four" + + assert i.three == "three" + assert i.regular_attr == 42 + assert i.sub_attr == 24 + assert i.class_var == "x" + assert i.subclass_var == "y" + assert i.four == "four" + + assert i.__class__ == GetAttrOverridden + + i.extra_attrs["subclass_var"] = (4, 4, 4) + assert i.__getattr__("subclass_var") == "subclass_var" + assert getattr(i, "subclass_var") == "y" + assert i.subclass_var == "y" + + base_ref: GetAttr = i + assert getattr(base_ref, "sub_attr") == 24 + assert base_ref.sub_attr == 24 + + assert getattr(base_ref, "subclass_var") == "y" + assert base_ref.subclass_var == "y" + + assert getattr(base_ref, "new") == "new" + assert base_ref.new == "new" + + assert base_ref.__class__ == GetAttrOverridden + +def test_getattr_nonnative() -> None: + i = GetAttr({"one": 1, "two": "two", "three": 3.14}, 42) + assert i.__getattr__("one") == 1 + assert i.__getattr__("regular_attr") == None + assert i.__getattr__("class_var") == None + assert i.__getattr__("four") == None + + assert getattr(i, "two") == "two" + assert getattr(i, "regular_attr") == 42 + assert getattr(i, "class_var") == "x" + assert getattr(i, "four") == None + + assert i.three == 3.14 + assert i.regular_attr == 42 + assert i.class_var == "x" + assert i.four == None + + assert i.__class__ == GetAttr + + i.extra_attrs["regular_attr"] = (4, 4, 4) + assert i.__getattr__("regular_attr") == (4, 4, 4) + assert getattr(i, "regular_attr") == 42 + assert i.regular_attr == 42 + +test_getattr() +test_getattr_default() +test_getattr_inherited() +test_getattr_overridden() +test_getattr_nonnative() + + +[typing fixtures/typing-full.pyi] diff --git a/mypyc/transform/refcount.py b/mypyc/transform/refcount.py index 60daebc415fd6..beacb409edfbf 100644 --- a/mypyc/transform/refcount.py +++ b/mypyc/transform/refcount.py @@ -33,6 +33,7 @@ Assign, BasicBlock, Branch, + CallC, ControlOp, DecRef, Goto, @@ -89,7 +90,9 @@ def insert_ref_count_opcodes(ir: FuncIR) -> None: def is_maybe_undefined(post_must_defined: set[Value], src: Value) -> bool: - return isinstance(src, Register) and src not in post_must_defined + return (isinstance(src, Register) and src not in post_must_defined) or ( + isinstance(src, CallC) and src.returns_null + ) def maybe_append_dec_ref( From 19697af9051707b4db55bc2d5301436d872f452c Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Thu, 25 Sep 2025 15:06:30 -0400 Subject: [PATCH 277/424] fix: something Shantanu found (#19859) to be honest I have no idea what this code does, but I know we need to fix it per advice from @hauntsaninja in #19846 --- mypy/strconv.py | 4 +++- test-data/unit/semanal-basic.test | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/mypy/strconv.py b/mypy/strconv.py index 3e9d37586f725..d1595139572ac 100644 --- a/mypy/strconv.py +++ b/mypy/strconv.py @@ -388,7 +388,9 @@ def visit_name_expr(self, o: mypy.nodes.NameExpr) -> str: o.name, o.kind, o.fullname, o.is_inferred_def or o.is_special_form, o.node ) if isinstance(o.node, mypy.nodes.Var) and o.node.is_final: - pretty += f" = {o.node.final_value}" + final_value = o.node.final_value + if final_value is not None: + pretty += f" = {o.node.final_value}" return short_type(o) + "(" + pretty + ")" def pretty_name( diff --git a/test-data/unit/semanal-basic.test b/test-data/unit/semanal-basic.test index 1f03ed22648d6..773092cab6238 100644 --- a/test-data/unit/semanal-basic.test +++ b/test-data/unit/semanal-basic.test @@ -521,7 +521,7 @@ MypyFile:1( NameExpr(True [builtins.True]) Literal[True]?) AssignmentStmt:8( - NameExpr(n* [__main__.n] = None) + NameExpr(n* [__main__.n]) CallExpr:8( NameExpr(func [__main__.func]) Args()))) From 89f7223b57db2a7a6188767988481dd81b58e14c Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Sat, 27 Sep 2025 02:19:41 -0400 Subject: [PATCH 278/424] [docs] Fix a common and harmless grammatical error, 'allows to' (#19900) This is a grammatical error (at least in American English, which I think this project is written in) because in the relevant sense the verb "allows" is transitive and needs to apply to a noun, not a verb. More info, if desired: https://english.stackexchange.com/questions/60271/grammatical-complements-for-allow https://english.stackexchange.com/questions/85069/is-the-construction-it-allows-to-proper-english (I don't know of any weighty publications who have taken this question on, so it's just a bunch of speakers, such as myself, opining.) --- docs/source/command_line.rst | 4 ++-- docs/source/error_code_list.rst | 4 ++-- docs/source/mypy_daemon.rst | 6 +++--- mypy/traverser.py | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index 270125e96cb61..d667fa0ff7277 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -372,7 +372,7 @@ definitions or calls. .. option:: --untyped-calls-exclude - This flag allows to selectively disable :option:`--disallow-untyped-calls` + This flag allows one to selectively disable :option:`--disallow-untyped-calls` for functions and methods defined in specific packages, modules, or classes. Note that each exclude entry acts as a prefix. For example (assuming there are no type annotations for ``third_party_lib`` available): @@ -562,7 +562,7 @@ potentially problematic or redundant in some way. .. option:: --deprecated-calls-exclude - This flag allows to selectively disable :ref:`deprecated` warnings + This flag allows one to selectively disable :ref:`deprecated` warnings for functions and methods defined in specific packages, modules, or classes. Note that each exclude entry acts as a prefix. For example (assuming ``foo.A.func`` is deprecated): diff --git a/docs/source/error_code_list.rst b/docs/source/error_code_list.rst index 6deed549c2f17..229230eda4caa 100644 --- a/docs/source/error_code_list.rst +++ b/docs/source/error_code_list.rst @@ -1032,8 +1032,8 @@ Warn about top level await expressions [top-level-await] This error code is separate from the general ``[syntax]`` errors, because in some environments (e.g. IPython) a top level ``await`` is allowed. In such environments a user may want to use ``--disable-error-code=top-level-await``, -that allows to still have errors for other improper uses of ``await``, for -example: +which allows one to still have errors for other improper uses of ``await``, +for example: .. code-block:: python diff --git a/docs/source/mypy_daemon.rst b/docs/source/mypy_daemon.rst index 6c511e14eb953..e0fc8129a0b82 100644 --- a/docs/source/mypy_daemon.rst +++ b/docs/source/mypy_daemon.rst @@ -252,16 +252,16 @@ command. Statically inspect expressions ****************************** -The daemon allows to get declared or inferred type of an expression (or other +The daemon allows one to get the declared or inferred type of an expression (or other information about an expression, such as known attributes or definition location) -using ``dmypy inspect LOCATION`` command. The location of the expression should be +using the ``dmypy inspect LOCATION`` command. The location of the expression should be specified in the format ``path/to/file.py:line:column[:end_line:end_column]``. Both line and column are 1-based. Both start and end position are inclusive. These rules match how mypy prints the error location in error messages. If a span is given (i.e. all 4 numbers), then only an exactly matching expression is inspected. If only a position is given (i.e. 2 numbers, line and column), mypy -will inspect all *expressions*, that include this position, starting from the +will inspect all expressions that include this position, starting from the innermost one. Consider this Python code snippet: diff --git a/mypy/traverser.py b/mypy/traverser.py index 7d7794822396e..3c249391bdf87 100644 --- a/mypy/traverser.py +++ b/mypy/traverser.py @@ -504,7 +504,7 @@ class ExtendedTraverserVisitor(TraverserVisitor): In addition to the base traverser it: * has visit_ methods for leaf nodes * has common method that is called for all nodes - * allows to skip recursing into a node + * allows skipping recursing into a node Note that this traverser still doesn't visit some internal mypy constructs like _promote expression and Var. From 16cd4c5215aa2301b50341c22048ffb8a4737a63 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 27 Sep 2025 11:48:05 +0100 Subject: [PATCH 279/424] Traverse type alias args in collect visitor (#19943) Fixes https://github.com/python/mypy/issues/19941 Fix is trivial (and a bit embarrassing, LOL). --- mypy/type_visitor.py | 2 +- mypy/types.py | 8 ++++++-- test-data/unit/check-recursive-types.test | 9 +++++++++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/mypy/type_visitor.py b/mypy/type_visitor.py index 86ef6ade84718..35846e7c3ddd3 100644 --- a/mypy/type_visitor.py +++ b/mypy/type_visitor.py @@ -51,7 +51,7 @@ get_proper_type, ) -T = TypeVar("T") +T = TypeVar("T", covariant=True) @trait diff --git a/mypy/types.py b/mypy/types.py index e0e897e04cadf..38c17e240ccf2 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -3938,8 +3938,12 @@ def visit_type_alias_type(self, t: TypeAliasType, /) -> list[mypy.nodes.TypeAlia assert t.alias is not None if t.alias not in self.seen_alias_nodes: self.seen_alias_nodes.add(t.alias) - return [t.alias] + t.alias.target.accept(self) - return [] + res = [t.alias] + t.alias.target.accept(self) + else: + res = [] + for arg in t.args: + res.extend(arg.accept(self)) + return res def is_named_instance(t: Type, fullnames: str | tuple[str, ...]) -> TypeGuard[Instance]: diff --git a/test-data/unit/check-recursive-types.test b/test-data/unit/check-recursive-types.test index 86e9f02b52636..4f451aa062d69 100644 --- a/test-data/unit/check-recursive-types.test +++ b/test-data/unit/check-recursive-types.test @@ -1014,3 +1014,12 @@ from bogus import Foo # type: ignore A = Callable[[Foo, "B"], Foo] # E: Type alias target becomes "Callable[[Any, B], Any]" due to an unfollowed import B = Callable[[Foo, A], Foo] # E: Type alias target becomes "Callable[[Any, A], Any]" due to an unfollowed import + +[case testRecursiveAliasOnArgumentDetected] +from typing import TypeVar + +T = TypeVar("T") +L = list[T] + +A = L[A] +a: A = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "A") From df4ae6a6b64d49f9f930bbdaf82ed5294dd0f2bc Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 29 Sep 2025 14:56:54 +0100 Subject: [PATCH 280/424] [mypyc] Fix broken exception/cancellation handling in async def (#19951) Fix an unexpected undefined attribute error in a generator/async def. The intent was to explicitly check if an attribute was undefined, but the attribute read implicitly raised `AttributeError`, so the check was never reached. Fixed by allowing error values to be read without raising `AttributeError`. The error this fixes would look like this (with no line number associated with it): ``` AttributeError("attribute '__mypyc_temp__12' of 'wait_Condition_gen' undefined") ``` --- mypyc/irbuild/builder.py | 17 ++++++++- mypyc/irbuild/statement.py | 9 ++++- mypyc/test-data/fixtures/testutil.py | 2 +- mypyc/test-data/run-async.test | 57 ++++++++++++++++++++++++++++ 4 files changed, 81 insertions(+), 4 deletions(-) diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index f4ee4371b9bf4..fa0e605d6776e 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -700,7 +700,12 @@ def get_assignment_target( assert False, "Unsupported lvalue: %r" % lvalue def read( - self, target: Value | AssignmentTarget, line: int = -1, can_borrow: bool = False + self, + target: Value | AssignmentTarget, + line: int = -1, + *, + can_borrow: bool = False, + allow_error_value: bool = False, ) -> Value: if isinstance(target, Value): return target @@ -716,7 +721,15 @@ def read( if isinstance(target, AssignmentTargetAttr): if isinstance(target.obj.type, RInstance) and target.obj.type.class_ir.is_ext_class: borrow = can_borrow and target.can_borrow - return self.add(GetAttr(target.obj, target.attr, line, borrow=borrow)) + return self.add( + GetAttr( + target.obj, + target.attr, + line, + borrow=borrow, + allow_error_value=allow_error_value, + ) + ) else: return self.py_get_attr(target.obj, target.attr, line) diff --git a/mypyc/irbuild/statement.py b/mypyc/irbuild/statement.py index c83c5550d059b..fdcf4f7771538 100644 --- a/mypyc/irbuild/statement.py +++ b/mypyc/irbuild/statement.py @@ -829,7 +829,14 @@ def transform_try_finally_stmt_async( # Check if we have a return value if ret_reg: return_block, check_old_exc = BasicBlock(), BasicBlock() - builder.add(Branch(builder.read(ret_reg), check_old_exc, return_block, Branch.IS_ERROR)) + builder.add( + Branch( + builder.read(ret_reg, allow_error_value=True), + check_old_exc, + return_block, + Branch.IS_ERROR, + ) + ) builder.activate_block(return_block) builder.nonlocal_control[-1].gen_return(builder, builder.read(ret_reg), -1) diff --git a/mypyc/test-data/fixtures/testutil.py b/mypyc/test-data/fixtures/testutil.py index 36ec41c8f38b2..b2700f731ddd6 100644 --- a/mypyc/test-data/fixtures/testutil.py +++ b/mypyc/test-data/fixtures/testutil.py @@ -44,7 +44,7 @@ def assertRaises(typ: type, msg: str = '') -> Iterator[None]: try: yield - except Exception as e: + except BaseException as e: assert type(e) is typ, f"{e!r} is not a {typ.__name__}" assert msg in str(e), f'Message "{e}" does not match "{msg}"' else: diff --git a/mypyc/test-data/run-async.test b/mypyc/test-data/run-async.test index 55cde4ab44f1f..94a1cd2e97c5f 100644 --- a/mypyc/test-data/run-async.test +++ b/mypyc/test-data/run-async.test @@ -1234,3 +1234,60 @@ def test_callable_arg_same_name_as_helper() -> None: [file asyncio/__init__.pyi] def run(x: object) -> object: ... + +[case testRunAsyncCancelFinallySpecialCase] +import asyncio + +from testutil import assertRaises + +# Greatly simplified from asyncio.Condition +class Condition: + async def acquire(self) -> None: pass + + async def wait(self) -> bool: + l = asyncio.get_running_loop() + fut = l.create_future() + a = [] + try: + try: + a.append(fut) + try: + await fut + return True + finally: + a.pop() + finally: + err = None + while True: + try: + await self.acquire() + break + except asyncio.CancelledError as e: + err = e + + if err is not None: + try: + raise err + finally: + err = None + except BaseException: + raise + +async def do_cancel() -> None: + cond = Condition() + wait = asyncio.create_task(cond.wait()) + asyncio.get_running_loop().call_soon(wait.cancel) + with assertRaises(asyncio.CancelledError): + await wait + +def test_cancel_special_case() -> None: + asyncio.run(do_cancel()) + +[file asyncio/__init__.pyi] +from typing import Any + +class CancelledError(Exception): ... + +def run(x: object) -> object: ... +def get_running_loop() -> Any: ... +def create_task(x: object) -> Any: ... From 8392e1a74a8af9a0d8e4c85e0c1a8247c08b8e0b Mon Sep 17 00:00:00 2001 From: Piotr Sawicki Date: Mon, 29 Sep 2025 16:18:52 +0200 Subject: [PATCH 281/424] [mypyc] Generate __setattr__ wrapper (#19937) Generate wrapper function for `__setattr__` and set it as the `tp_setattro` slot. The wrapper doesn't have to do anything because interpreted python runs the defined `__setattr__` on every attribute assignment. So the wrapper only reports errors on unsupported uses of `__setattr__` and makes the function signature match with the slot. Since `__setattr__` should run on every attribute assignment, native classes with `__setattr__` generate calls to this function on attribute assignment instead of direct assignment to the underlying C struct. Native classes generally don't have `__dict__` which makes implementing dynamic attributes more challenging. The native class has to manage its own dictionary of names to values and handle assignment to regular attributes specially. With `__dict__`, assigning values to regular and dynamic attributes can be done by simply assigning the value in `__dict__`, ie. `self.__dict__[name] = value` or `object.__setattr__(self, name, value)`. With a custom attribute dictionary, assigning with `name` being a regular attribute doesn't work because it would only update the value in the custom dictionary, not the actual attribute. On the other hand, the `object.__setattr__` call doesn't work for dynamic attributes and raises an `AttributeError` without `__dict__`. So something like this has to be implemented as a work-around: ``` def __setattr__(self, name: str, val: object) -> None: if name == "regular_attribute": object.__setattr__(self, "regular_attribute", val) else: self._attribute_dict[name] = val ``` To make this efficient in native classes, calls to `object.__setattr__` or equivalent `super().__setattr__` are transformed to direct C struct assignments when the name literal matches an attribute name. --- mypyc/codegen/emitclass.py | 1 + mypyc/irbuild/builder.py | 12 +- mypyc/irbuild/expression.py | 7 + mypyc/irbuild/function.py | 31 ++ mypyc/irbuild/specialize.py | 52 ++- mypyc/lib-rt/CPy.h | 3 + mypyc/primitives/generic_ops.py | 7 + mypyc/test-data/fixtures/ir.py | 1 + mypyc/test-data/irbuild-classes.test | 473 ++++++++++++++++++++++ mypyc/test-data/run-classes.test | 585 +++++++++++++++++++++++++++ 10 files changed, 1161 insertions(+), 11 deletions(-) diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 9e8f9c74bc6d9..122f62a0d5826 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -62,6 +62,7 @@ def dunder_attr_slot(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: "__hash__": ("tp_hash", generate_hash_wrapper), "__get__": ("tp_descr_get", generate_get_wrapper), "__getattr__": ("tp_getattro", dunder_attr_slot), + "__setattr__": ("tp_setattro", dunder_attr_slot), } AS_MAPPING_SLOT_DEFS: SlotTable = { diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index fa0e605d6776e..1253821459917 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -76,6 +76,7 @@ Integer, IntOp, LoadStatic, + MethodCall, Op, PrimitiveDescription, RaiseStandardError, @@ -748,8 +749,15 @@ def assign(self, target: Register | AssignmentTarget, rvalue_reg: Value, line: i self.add(Assign(target.register, rvalue_reg)) elif isinstance(target, AssignmentTargetAttr): if isinstance(target.obj_type, RInstance): - rvalue_reg = self.coerce_rvalue(rvalue_reg, target.type, line) - self.add(SetAttr(target.obj, target.attr, rvalue_reg, line)) + setattr = target.obj_type.class_ir.get_method("__setattr__") + if setattr: + key = self.load_str(target.attr) + boxed_reg = self.builder.box(rvalue_reg) + call = MethodCall(target.obj, setattr.name, [key, boxed_reg], line) + self.add(call) + else: + rvalue_reg = self.coerce_rvalue(rvalue_reg, target.type, line) + self.add(SetAttr(target.obj, target.attr, rvalue_reg, line)) else: key = self.load_str(target.attr) boxed_reg = self.builder.box(rvalue_reg) diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index 1f39b09c09952..54a101bc4961b 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -101,6 +101,7 @@ apply_function_specialization, apply_method_specialization, translate_object_new, + translate_object_setattr, ) from mypyc.primitives.bytes_ops import bytes_slice_op from mypyc.primitives.dict_ops import dict_get_item_op, dict_new_op, exact_dict_set_item_op @@ -480,6 +481,12 @@ def translate_super_method_call(builder: IRBuilder, expr: CallExpr, callee: Supe result = translate_object_new(builder, expr, MemberExpr(callee.call, "__new__")) if result: return result + elif callee.name == "__setattr__": + result = translate_object_setattr( + builder, expr, MemberExpr(callee.call, "__setattr__") + ) + if result: + return result if ir.is_ext_class and ir.builtin_base is None and not ir.inherits_python: if callee.name == "__init__" and len(expr.args) == 0: # Call translates to object.__init__(self), which is a diff --git a/mypyc/irbuild/function.py b/mypyc/irbuild/function.py index a9a098d25ddef..51bdc76495f2f 100644 --- a/mypyc/irbuild/function.py +++ b/mypyc/irbuild/function.py @@ -57,6 +57,7 @@ from mypyc.ir.rtypes import ( RInstance, bool_rprimitive, + c_int_rprimitive, dict_rprimitive, int_rprimitive, object_rprimitive, @@ -415,6 +416,34 @@ def generate_getattr_wrapper(builder: IRBuilder, cdef: ClassDef, getattr: FuncDe builder.add(Return(getattr_result, line)) +def generate_setattr_wrapper(builder: IRBuilder, cdef: ClassDef, setattr: FuncDef) -> None: + """ + Generate a wrapper function for __setattr__ that can be put into the tp_setattro slot. + The wrapper takes two arguments besides self - attribute name and the new value. + Returns 0 on success and -1 on failure. Restrictions are similar to the __getattr__ + wrapper above. + + This one is simpler because to match interpreted python semantics it's enough to always + call the user-provided function, including for names matching regular attributes. + """ + name = setattr.name + "__wrapper" + ir = builder.mapper.type_to_ir[cdef.info] + line = setattr.line + + error_base = f'"__setattr__" not supported in class "{cdef.name}" because ' + if ir.allow_interpreted_subclasses: + builder.error(error_base + "it allows interpreted subclasses", line) + if ir.inherits_python: + builder.error(error_base + "it inherits from a non-native class", line) + + with builder.enter_method(ir, name, c_int_rprimitive, internal=True): + attr_arg = builder.add_argument("attr", object_rprimitive) + value_arg = builder.add_argument("value", object_rprimitive) + + builder.gen_method_call(builder.self(), setattr.name, [attr_arg, value_arg], None, line) + builder.add(Return(Integer(0, c_int_rprimitive), line)) + + def handle_ext_method(builder: IRBuilder, cdef: ClassDef, fdef: FuncDef) -> None: # Perform the function of visit_method for methods inside extension classes. name = fdef.name @@ -483,6 +512,8 @@ def handle_ext_method(builder: IRBuilder, cdef: ClassDef, fdef: FuncDef) -> None if fdef.name == "__getattr__": generate_getattr_wrapper(builder, cdef, fdef) + elif fdef.name == "__setattr__": + generate_setattr_wrapper(builder, cdef, fdef) def handle_non_ext_method( diff --git a/mypyc/irbuild/specialize.py b/mypyc/irbuild/specialize.py index 29820787d10cb..42b7710c98da6 100644 --- a/mypyc/irbuild/specialize.py +++ b/mypyc/irbuild/specialize.py @@ -42,6 +42,7 @@ Integer, RaiseStandardError, Register, + SetAttr, Truncate, Unreachable, Value, @@ -97,6 +98,7 @@ isinstance_dict, ) from mypyc.primitives.float_ops import isinstance_float +from mypyc.primitives.generic_ops import generic_setattr from mypyc.primitives.int_ops import isinstance_int from mypyc.primitives.list_ops import isinstance_list, new_list_set_item_op from mypyc.primitives.misc_ops import isinstance_bool @@ -1007,19 +1009,24 @@ def translate_ord(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value return None -@specialize_function("__new__", object_rprimitive) -def translate_object_new(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: - fn = builder.fn_info - if fn.name != "__new__": - return None - - is_super_new = isinstance(expr.callee, SuperExpr) - is_object_new = ( +def is_object(callee: RefExpr) -> bool: + """Returns True for object. calls.""" + return ( isinstance(callee, MemberExpr) and isinstance(callee.expr, NameExpr) and callee.expr.fullname == "builtins.object" ) - if not (is_super_new or is_object_new): + + +def is_super_or_object(expr: CallExpr, callee: RefExpr) -> bool: + """Returns True for super(). or object. calls.""" + return isinstance(expr.callee, SuperExpr) or is_object(callee) + + +@specialize_function("__new__", object_rprimitive) +def translate_object_new(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: + fn = builder.fn_info + if fn.name != "__new__" or not is_super_or_object(expr, callee): return None ir = builder.get_current_class_ir() @@ -1046,3 +1053,30 @@ def translate_object_new(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> return builder.add(Call(ir.setup, [subtype], expr.line)) return None + + +@specialize_function("__setattr__", object_rprimitive) +def translate_object_setattr(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: + is_super = isinstance(expr.callee, SuperExpr) + is_object_callee = is_object(callee) + if not ((is_super and len(expr.args) >= 2) or (is_object_callee and len(expr.args) >= 3)): + return None + + self_reg = builder.accept(expr.args[0]) if is_object_callee else builder.self() + ir = builder.get_current_class_ir() + if ir and (not ir.is_ext_class or ir.builtin_base or ir.inherits_python): + return None + # Need to offset by 1 for super().__setattr__ calls because there is no self arg in this case. + name_idx = 0 if is_super else 1 + value_idx = 1 if is_super else 2 + attr_name = expr.args[name_idx] + attr_value = expr.args[value_idx] + value = builder.accept(attr_value) + + if isinstance(attr_name, StrExpr) and ir and ir.has_attr(attr_name.value): + name = attr_name.value + value = builder.coerce(value, ir.attributes[name], expr.line) + return builder.add(SetAttr(self_reg, name, value, expr.line)) + + name_reg = builder.accept(attr_name) + return builder.call_c(generic_setattr, [self_reg, name_reg, value], expr.line) diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index b9cecb9280f36..e9dfd8de36834 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -952,6 +952,9 @@ void CPyTrace_LogEvent(const char *location, const char *line, const char *op, c static inline PyObject *CPyObject_GenericGetAttr(PyObject *self, PyObject *name) { return _PyObject_GenericGetAttrWithDict(self, name, NULL, 1); } +static inline int CPyObject_GenericSetAttr(PyObject *self, PyObject *name, PyObject *value) { + return _PyObject_GenericSetAttrWithDict(self, name, value, NULL); +} #if CPY_3_11_FEATURES PyObject *CPy_GetName(PyObject *obj); diff --git a/mypyc/primitives/generic_ops.py b/mypyc/primitives/generic_ops.py index ff978b7c8c3bb..16bd074396d2b 100644 --- a/mypyc/primitives/generic_ops.py +++ b/mypyc/primitives/generic_ops.py @@ -410,3 +410,10 @@ error_kind=ERR_NEVER, returns_null=True, ) + +generic_setattr = custom_op( + arg_types=[object_rprimitive, object_rprimitive, object_rprimitive], + return_type=c_int_rprimitive, + c_function_name="CPyObject_GenericSetAttr", + error_kind=ERR_NEG_INT, +) diff --git a/mypyc/test-data/fixtures/ir.py b/mypyc/test-data/fixtures/ir.py index a4b4f3ce2b1fc..22a6a5986cbd9 100644 --- a/mypyc/test-data/fixtures/ir.py +++ b/mypyc/test-data/fixtures/ir.py @@ -45,6 +45,7 @@ def __init__(self) -> None: pass def __eq__(self, x: object) -> bool: pass def __ne__(self, x: object) -> bool: pass def __str__(self) -> str: pass + def __setattr__(self, k: str, v: object) -> None: pass class type: def __init__(self, o: object) -> None: ... diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index 76e28711c5e37..a98b3a7d3dcf9 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -2213,3 +2213,476 @@ L0: buf_init_item r13, 4, r12 keep_alive r9 return r9 + +[case testUnsupportedSetAttr] +from mypy_extensions import mypyc_attr + +@mypyc_attr(allow_interpreted_subclasses=True) +class AllowsInterpreted: + def __setattr__(self, attr: str, val: object) -> None: # E: "__setattr__" not supported in class "AllowsInterpreted" because it allows interpreted subclasses + pass + +class InheritsInterpreted(dict): + def __setattr__(self, attr: str, val: object) -> None: # E: "__setattr__" not supported in class "InheritsInterpreted" because it inherits from a non-native class + pass + +@mypyc_attr(native_class=False) +class NonNative: + pass + +class InheritsNonNative(NonNative): + def __setattr__(self, attr: str, val: object) -> None: # E: "__setattr__" not supported in class "InheritsNonNative" because it inherits from a non-native class + pass + +[case testSetAttr] +from typing import ClassVar +class SetAttr: + _attributes: dict[str, object] + regular_attr: int + class_var: ClassVar[str] = "x" + + def __init__(self, regular_attr: int, extra_attrs: dict[str, object], new_attr: str, new_val: object) -> None: + super().__setattr__("_attributes", extra_attrs) + object.__setattr__(self, "regular_attr", regular_attr) + + super().__setattr__(new_attr, new_val) + object.__setattr__(self, new_attr, new_val) + + def __setattr__(self, key: str, val: object) -> None: + if key == "regular_attr": + super().__setattr__("regular_attr", val) + elif key == "class_var": + raise AttributeError() + else: + self._attributes[key] = val + +def test(attr: str, val: object) -> None: + i = SetAttr(99, {}, attr, val) + i.regular_attr = 100 + i.new_attr = 101 + + object.__setattr__(i, "regular_attr", 11) + object.__setattr__(i, attr, val) + +[typing fixtures/typing-full.pyi] +[out] +def SetAttr.__init__(self, regular_attr, extra_attrs, new_attr, new_val): + self :: __main__.SetAttr + regular_attr :: int + extra_attrs :: dict + new_attr :: str + new_val :: object + r0 :: i32 + r1 :: bit + r2 :: i32 + r3 :: bit +L0: + self._attributes = extra_attrs + self.regular_attr = regular_attr + r0 = CPyObject_GenericSetAttr(self, new_attr, new_val) + r1 = r0 >= 0 :: signed + r2 = CPyObject_GenericSetAttr(self, new_attr, new_val) + r3 = r2 >= 0 :: signed + return 1 +def SetAttr.__setattr__(self, key, val): + self :: __main__.SetAttr + key :: str + val :: object + r0 :: str + r1 :: bool + r2 :: int + r3 :: bool + r4 :: str + r5 :: bool + r6 :: object + r7 :: str + r8, r9 :: object + r10 :: dict + r11 :: i32 + r12 :: bit +L0: + r0 = 'regular_attr' + r1 = CPyStr_Equal(key, r0) + if r1 goto L1 else goto L2 :: bool +L1: + r2 = unbox(int, val) + self.regular_attr = r2; r3 = is_error + goto L6 +L2: + r4 = 'class_var' + r5 = CPyStr_Equal(key, r4) + if r5 goto L3 else goto L4 :: bool +L3: + r6 = builtins :: module + r7 = 'AttributeError' + r8 = CPyObject_GetAttr(r6, r7) + r9 = PyObject_Vectorcall(r8, 0, 0, 0) + CPy_Raise(r9) + unreachable +L4: + r10 = self._attributes + r11 = CPyDict_SetItem(r10, key, val) + r12 = r11 >= 0 :: signed +L5: +L6: + return 1 +def SetAttr.__setattr____wrapper(__mypyc_self__, attr, value): + __mypyc_self__ :: __main__.SetAttr + attr, value :: object + r0 :: str + r1 :: None +L0: + r0 = cast(str, attr) + r1 = __mypyc_self__.__setattr__(r0, value) + return 0 +def test(attr, val): + attr :: str + val :: object + r0 :: dict + r1, i :: __main__.SetAttr + r2 :: str + r3 :: object + r4 :: None + r5 :: str + r6 :: object + r7 :: i32 + r8 :: bit + r9 :: str + r10 :: object + r11 :: i32 + r12 :: bit + r13 :: i32 + r14 :: bit +L0: + r0 = PyDict_New() + r1 = SetAttr(198, r0, attr, val) + i = r1 + r2 = 'regular_attr' + r3 = object 100 + r4 = i.__setattr__(r2, r3) + r5 = 'new_attr' + r6 = object 101 + r7 = PyObject_SetAttr(i, r5, r6) + r8 = r7 >= 0 :: signed + r9 = 'regular_attr' + r10 = object 11 + r11 = CPyObject_GenericSetAttr(i, r9, r10) + r12 = r11 >= 0 :: signed + r13 = CPyObject_GenericSetAttr(i, attr, val) + r14 = r13 >= 0 :: signed + return 1 + +[case testUntransformedSetAttr_64bit] +from mypy_extensions import mypyc_attr + +class SetAttr: + def super_missing_args(self): + super().__setattr__() + super().__setattr__("attr") + + def object_missing_args(self): + object.__setattr__() + object.__setattr__(self) + object.__setattr__(self, "attr") + +@mypyc_attr(native_class=False) +class NonNative: + def super_setattr(self, key: str, val: object) -> None: + super().__setattr__(key, val) + + def object_setattr(self, key: str, val: object) -> None: + object.__setattr__(self, key, val) + +class InheritsPython(NonNative): + def super_setattr(self, key: str, val: object) -> None: + super().__setattr__(key, val) + + def object_setattr(self, key: str, val: object) -> None: + object.__setattr__(self, key, val) + +class BuiltInBase(dict): + def super_setattr(self, key: str, val: object) -> None: + super().__setattr__(key, val) + + def object_setattr(self, key: str, val: object) -> None: + object.__setattr__(self, key, val) + +[typing fixtures/typing-full.pyi] +[out] +def SetAttr.super_missing_args(self): + self :: __main__.SetAttr + r0 :: object + r1 :: str + r2, r3 :: object + r4 :: object[2] + r5 :: object_ptr + r6 :: object + r7 :: str + r8, r9, r10 :: object + r11 :: str + r12, r13 :: object + r14 :: object[2] + r15 :: object_ptr + r16 :: object + r17 :: str + r18 :: object + r19 :: str + r20 :: object[1] + r21 :: object_ptr + r22, r23 :: object +L0: + r0 = builtins :: module + r1 = 'super' + r2 = CPyObject_GetAttr(r0, r1) + r3 = __main__.SetAttr :: type + r4 = [r3, self] + r5 = load_address r4 + r6 = PyObject_Vectorcall(r2, r5, 2, 0) + keep_alive r3, self + r7 = '__setattr__' + r8 = CPyObject_GetAttr(r6, r7) + r9 = PyObject_Vectorcall(r8, 0, 0, 0) + r10 = builtins :: module + r11 = 'super' + r12 = CPyObject_GetAttr(r10, r11) + r13 = __main__.SetAttr :: type + r14 = [r13, self] + r15 = load_address r14 + r16 = PyObject_Vectorcall(r12, r15, 2, 0) + keep_alive r13, self + r17 = '__setattr__' + r18 = CPyObject_GetAttr(r16, r17) + r19 = 'attr' + r20 = [r19] + r21 = load_address r20 + r22 = PyObject_Vectorcall(r18, r21, 1, 0) + keep_alive r19 + r23 = box(None, 1) + return r23 +def SetAttr.object_missing_args(self): + self :: __main__.SetAttr + r0 :: object + r1 :: str + r2 :: object + r3 :: str + r4 :: object[1] + r5 :: object_ptr + r6, r7 :: object + r8 :: str + r9 :: object + r10 :: str + r11 :: object[2] + r12 :: object_ptr + r13, r14 :: object + r15 :: str + r16 :: object + r17, r18 :: str + r19 :: object[3] + r20 :: object_ptr + r21, r22 :: object +L0: + r0 = builtins :: module + r1 = 'object' + r2 = CPyObject_GetAttr(r0, r1) + r3 = '__setattr__' + r4 = [r2] + r5 = load_address r4 + r6 = PyObject_VectorcallMethod(r3, r5, 9223372036854775809, 0) + keep_alive r2 + r7 = builtins :: module + r8 = 'object' + r9 = CPyObject_GetAttr(r7, r8) + r10 = '__setattr__' + r11 = [r9, self] + r12 = load_address r11 + r13 = PyObject_VectorcallMethod(r10, r12, 9223372036854775810, 0) + keep_alive r9, self + r14 = builtins :: module + r15 = 'object' + r16 = CPyObject_GetAttr(r14, r15) + r17 = 'attr' + r18 = '__setattr__' + r19 = [r16, self, r17] + r20 = load_address r19 + r21 = PyObject_VectorcallMethod(r18, r20, 9223372036854775811, 0) + keep_alive r16, self, r17 + r22 = box(None, 1) + return r22 +def super_setattr_NonNative_obj.__get__(__mypyc_self__, instance, owner): + __mypyc_self__, instance, owner, r0 :: object + r1 :: bit + r2 :: object +L0: + r0 = load_address _Py_NoneStruct + r1 = instance == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return __mypyc_self__ +L2: + r2 = PyMethod_New(__mypyc_self__, instance) + return r2 +def super_setattr_NonNative_obj.__call__(__mypyc_self__, self, key, val): + __mypyc_self__ :: __main__.super_setattr_NonNative_obj + self :: __main__.NonNative + key :: str + val, r0 :: object + r1 :: str + r2, r3 :: object + r4 :: object[2] + r5 :: object_ptr + r6 :: object + r7 :: str + r8 :: object + r9 :: object[2] + r10 :: object_ptr + r11 :: object +L0: + r0 = builtins :: module + r1 = 'super' + r2 = CPyObject_GetAttr(r0, r1) + r3 = __main__.NonNative :: type + r4 = [r3, self] + r5 = load_address r4 + r6 = PyObject_Vectorcall(r2, r5, 2, 0) + keep_alive r3, self + r7 = '__setattr__' + r8 = CPyObject_GetAttr(r6, r7) + r9 = [key, val] + r10 = load_address r9 + r11 = PyObject_Vectorcall(r8, r10, 2, 0) + keep_alive key, val + return 1 +def object_setattr_NonNative_obj.__get__(__mypyc_self__, instance, owner): + __mypyc_self__, instance, owner, r0 :: object + r1 :: bit + r2 :: object +L0: + r0 = load_address _Py_NoneStruct + r1 = instance == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return __mypyc_self__ +L2: + r2 = PyMethod_New(__mypyc_self__, instance) + return r2 +def object_setattr_NonNative_obj.__call__(__mypyc_self__, self, key, val): + __mypyc_self__ :: __main__.object_setattr_NonNative_obj + self :: __main__.NonNative + key :: str + val, r0 :: object + r1 :: str + r2 :: object + r3 :: str + r4 :: object[4] + r5 :: object_ptr + r6 :: object +L0: + r0 = builtins :: module + r1 = 'object' + r2 = CPyObject_GetAttr(r0, r1) + r3 = '__setattr__' + r4 = [r2, self, key, val] + r5 = load_address r4 + r6 = PyObject_VectorcallMethod(r3, r5, 9223372036854775812, 0) + keep_alive r2, self, key, val + return 1 +def InheritsPython.super_setattr(self, key, val): + self :: __main__.InheritsPython + key :: str + val, r0 :: object + r1 :: str + r2, r3 :: object + r4 :: object[2] + r5 :: object_ptr + r6 :: object + r7 :: str + r8 :: object + r9 :: object[2] + r10 :: object_ptr + r11 :: object +L0: + r0 = builtins :: module + r1 = 'super' + r2 = CPyObject_GetAttr(r0, r1) + r3 = __main__.InheritsPython :: type + r4 = [r3, self] + r5 = load_address r4 + r6 = PyObject_Vectorcall(r2, r5, 2, 0) + keep_alive r3, self + r7 = '__setattr__' + r8 = CPyObject_GetAttr(r6, r7) + r9 = [key, val] + r10 = load_address r9 + r11 = PyObject_Vectorcall(r8, r10, 2, 0) + keep_alive key, val + return 1 +def InheritsPython.object_setattr(self, key, val): + self :: __main__.InheritsPython + key :: str + val, r0 :: object + r1 :: str + r2 :: object + r3 :: str + r4 :: object[4] + r5 :: object_ptr + r6 :: object +L0: + r0 = builtins :: module + r1 = 'object' + r2 = CPyObject_GetAttr(r0, r1) + r3 = '__setattr__' + r4 = [r2, self, key, val] + r5 = load_address r4 + r6 = PyObject_VectorcallMethod(r3, r5, 9223372036854775812, 0) + keep_alive r2, self, key, val + return 1 +def BuiltInBase.super_setattr(self, key, val): + self :: dict + key :: str + val, r0 :: object + r1 :: str + r2, r3 :: object + r4 :: object[2] + r5 :: object_ptr + r6 :: object + r7 :: str + r8 :: object + r9 :: object[2] + r10 :: object_ptr + r11 :: object +L0: + r0 = builtins :: module + r1 = 'super' + r2 = CPyObject_GetAttr(r0, r1) + r3 = __main__.BuiltInBase :: type + r4 = [r3, self] + r5 = load_address r4 + r6 = PyObject_Vectorcall(r2, r5, 2, 0) + keep_alive r3, self + r7 = '__setattr__' + r8 = CPyObject_GetAttr(r6, r7) + r9 = [key, val] + r10 = load_address r9 + r11 = PyObject_Vectorcall(r8, r10, 2, 0) + keep_alive key, val + return 1 +def BuiltInBase.object_setattr(self, key, val): + self :: dict + key :: str + val, r0 :: object + r1 :: str + r2 :: object + r3 :: str + r4 :: object[4] + r5 :: object_ptr + r6 :: object +L0: + r0 = builtins :: module + r1 = 'object' + r2 = CPyObject_GetAttr(r0, r1) + r3 = '__setattr__' + r4 = [r2, self, key, val] + r5 = load_address r4 + r6 = PyObject_VectorcallMethod(r3, r5, 9223372036854775812, 0) + keep_alive r2, self, key, val + return 1 diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index b2f1a088585d7..d10f7b19067cb 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -4518,5 +4518,590 @@ test_getattr_inherited() test_getattr_overridden() test_getattr_nonnative() +[typing fixtures/typing-full.pyi] + +[case testDunderSetAttr] +from mypy_extensions import mypyc_attr +from testutil import assertRaises +from typing import ClassVar + +class SetAttr: + _attributes: dict[str, object] + regular_attr: int + class_var: ClassVar[str] = "x" + const: int = 42 + + def __init__(self, regular_attr: int, extra_attrs: dict[str, object]) -> None: + super().__setattr__("_attributes", extra_attrs) + super().__setattr__("regular_attr", regular_attr) + + def __setattr__(self, key: str, val: object) -> None: + if key == "regular_attr": + super().__setattr__("regular_attr", val) + elif key == "class_var" or key == "const": + raise AttributeError() + else: + self._attributes[key] = val + + def __getattr__(self, key: str) -> object: + return self._attributes.get(key) + +class SetAttrInherited(SetAttr): + def __init__(self, regular_attr: int, extra_attrs: dict[str, object]) -> None: + super().__init__(regular_attr, extra_attrs) + +class SetAttrOverridden(SetAttr): + sub_attr: int + subclass_var: ClassVar[str] = "y" + + def __init__(self, regular_attr: int, sub_attr: int, extra_attrs: dict[str, object]) -> None: + super().__init__(regular_attr, extra_attrs) + object.__setattr__(self, "sub_attr", sub_attr) + + def __setattr__(self, key: str, val: object) -> None: + if key == "sub_attr": + object.__setattr__(self, "sub_attr", val) + elif key == "subclass_var": + raise AttributeError() + else: + super().__setattr__(key, val) + +@mypyc_attr(native_class=False) +class SetAttrNonNative: + _attributes: dict[str, object] + regular_attr: int + class_var: ClassVar[str] = "x" + const: int = 42 + + def __init__(self, regular_attr: int, extra_attrs: dict[str, object]) -> None: + super().__setattr__("_attributes", extra_attrs) + super().__setattr__("regular_attr", regular_attr) + + def __setattr__(self, key: str, val: object) -> None: + if key == "regular_attr": + super().__setattr__("regular_attr", val) + elif key == "class_var" or key == "const": + raise AttributeError() + else: + self._attributes[key] = val + + def __getattr__(self, key: str) -> object: + return self._attributes.get(key) + +class NoSetAttr: + def __init__(self, attr: int) -> None: + self.attr = attr + + def object_setattr(self, attr: str, val: object) -> None: + object.__setattr__(self, attr, val) + + def super_setattr(self, attr: str, val: object) -> None: + super().__setattr__(attr, val) + +@mypyc_attr(native_class=False) +class NoSetAttrNonNative: + def __init__(self, attr: int) -> None: + self.attr = attr + + def object_setattr(self, attr: str, val: object) -> None: + object.__setattr__(self, attr, val) + + def super_setattr(self, attr: str, val: object) -> None: + super().__setattr__(attr, val) + + def __getattr__(self, attr: str) -> object: + pass + +def test_setattr() -> None: + i = SetAttr(99, {"one": 1}) + assert i.class_var == "x" + assert i.regular_attr == 99 + assert i.one == 1 + assert i.two == None + assert i.const == 42 + + i.__setattr__("two", "2") + assert i.two == "2" + i.__setattr__("regular_attr", 101) + assert i.regular_attr == 101 + with assertRaises(AttributeError): + i.__setattr__("class_var", "y") + with assertRaises(AttributeError): + i.__setattr__("const", 43) + + setattr(i, "three", (3,3,3)) + assert i.three == (3,3,3) + setattr(i, "regular_attr", 102) + assert i.regular_attr == 102 + with assertRaises(AttributeError): + setattr(i, "class_var", "z") + with assertRaises(AttributeError): + setattr(i, "const", 44) + + i.four = [4,4] + assert i.four == [4,4] + i.regular_attr = 103 + assert i.regular_attr == 103 + with assertRaises(AttributeError): + i.const = 45 + +def test_setattr_inherited() -> None: + i = SetAttrInherited(99, {"one": 1}) + assert i.class_var == "x" + assert i.regular_attr == 99 + assert i.one == 1 + assert i.two == None + assert i.const == 42 + + i.__setattr__("two", "2") + assert i.two == "2" + i.__setattr__("regular_attr", 101) + assert i.regular_attr == 101 + with assertRaises(AttributeError): + i.__setattr__("class_var", "y") + with assertRaises(AttributeError): + i.__setattr__("const", 43) + + setattr(i, "three", (3,3,3)) + assert i.three == (3,3,3) + setattr(i, "regular_attr", 102) + assert i.regular_attr == 102 + with assertRaises(AttributeError): + setattr(i, "class_var", "z") + with assertRaises(AttributeError): + setattr(i, "const", 44) + + i.four = [4,4] + assert i.four == [4,4] + i.regular_attr = 103 + assert i.regular_attr == 103 + with assertRaises(AttributeError): + i.const = 45 + +def test_setattr_overridden() -> None: + i = SetAttrOverridden(99, 1, {"one": 1}) + assert i.class_var == "x" + assert i.subclass_var == "y" + assert i.regular_attr == 99 + assert i.sub_attr == 1 + assert i.one == 1 + assert i.two == None + assert i.const == 42 + + i.__setattr__("two", "2") + assert i.two == "2" + i.__setattr__("regular_attr", 101) + assert i.regular_attr == 101 + i.__setattr__("sub_attr", 2) + assert i.sub_attr == 2 + with assertRaises(AttributeError): + i.__setattr__("class_var", "y") + with assertRaises(AttributeError): + i.__setattr__("subclass_var", "a") + with assertRaises(AttributeError): + i.__setattr__("const", 43) + + setattr(i, "three", (3,3,3)) + assert i.three == (3,3,3) + setattr(i, "regular_attr", 102) + assert i.regular_attr == 102 + setattr(i, "sub_attr", 3) + assert i.sub_attr == 3 + with assertRaises(AttributeError): + setattr(i, "class_var", "z") + with assertRaises(AttributeError): + setattr(i, "subclass_var", "b") + with assertRaises(AttributeError): + setattr(i, "const", 44) + + i.four = [4,4] + assert i.four == [4,4] + i.regular_attr = 103 + assert i.regular_attr == 103 + i.sub_attr = 4 + assert i.sub_attr == 4 + with assertRaises(AttributeError): + i.const = 45 + + base_ref: SetAttr = i + setattr(base_ref, "sub_attr", 5) + assert base_ref.sub_attr == 5 + + base_ref.sub_attr = 6 + assert base_ref.sub_attr == 6 + + with assertRaises(AttributeError): + setattr(base_ref, "subclass_var", "c") + +def test_setattr_nonnative() -> None: + i = SetAttrNonNative(99, {"one": 1}) + assert i.class_var == "x" + assert i.regular_attr == 99 + assert i.one == 1 + assert i.two == None + assert i.const == 42 + + i.__setattr__("two", "2") + assert i.two == "2" + i.__setattr__("regular_attr", 101) + assert i.regular_attr == 101 + with assertRaises(AttributeError): + i.__setattr__("class_var", "y") + with assertRaises(AttributeError): + i.__setattr__("const", 43) + + setattr(i, "three", (3,3,3)) + assert i.three == (3,3,3) + setattr(i, "regular_attr", 102) + assert i.regular_attr == 102 + with assertRaises(AttributeError): + setattr(i, "class_var", "z") + with assertRaises(AttributeError): + setattr(i, "const", 44) + + i.four = [4,4] + assert i.four == [4,4] + i.regular_attr = 103 + assert i.regular_attr == 103 + with assertRaises(AttributeError): + i.const = 45 + +def test_no_setattr() -> None: + i = NoSetAttr(99) + i.super_setattr("attr", 100) + assert i.attr == 100 + + i.object_setattr("attr", 101) + assert i.attr == 101 + + object.__setattr__(i, "attr", 102) + assert i.attr == 102 + + with assertRaises(AttributeError): + i.super_setattr("not_attr", 100) + + with assertRaises(AttributeError): + i.object_setattr("not_attr", 101) + + with assertRaises(AttributeError): + object.__setattr__(i, "not_attr", 102) + +def test_no_setattr_nonnative() -> None: + i = NoSetAttrNonNative(99) + i.super_setattr("attr", 100) + assert i.attr == 100 + + i.object_setattr("attr", 101) + assert i.attr == 101 + + object.__setattr__(i, "attr", 102) + assert i.attr == 102 + + i.super_setattr("one", 100) + assert i.one == 100 + + i.object_setattr("two", 101) + assert i.two == 101 + + object.__setattr__(i, "three", 102) + assert i.three == 102 + +[typing fixtures/typing-full.pyi] + +[case testDunderSetAttrInterpreted] +from mypy_extensions import mypyc_attr +from typing import ClassVar + +class SetAttr: + _attributes: dict[str, object] + regular_attr: int + class_var: ClassVar[str] = "x" + const: int = 42 + + def __init__(self, regular_attr: int, extra_attrs: dict[str, object]) -> None: + super().__setattr__("_attributes", extra_attrs) + super().__setattr__("regular_attr", regular_attr) + + def __setattr__(self, key: str, val: object) -> None: + if key == "regular_attr": + super().__setattr__("regular_attr", val) + elif key == "class_var" or key == "const": + raise AttributeError() + else: + self._attributes[key] = val + + def __getattr__(self, key: str) -> object: + return self._attributes.get(key) + +class SetAttrInherited(SetAttr): + def __init__(self, regular_attr: int, extra_attrs: dict[str, object]) -> None: + super().__init__(regular_attr, extra_attrs) + +class SetAttrOverridden(SetAttr): + sub_attr: int + subclass_var: ClassVar[str] = "y" + + def __init__(self, regular_attr: int, sub_attr: int, extra_attrs: dict[str, object]) -> None: + super().__init__(regular_attr, extra_attrs) + object.__setattr__(self, "sub_attr", sub_attr) + + def __setattr__(self, key: str, val: object) -> None: + if key == "sub_attr": + object.__setattr__(self, "sub_attr", val) + elif key == "subclass_var": + raise AttributeError() + else: + super().__setattr__(key, val) + +@mypyc_attr(native_class=False) +class SetAttrNonNative: + _attributes: dict[str, object] + regular_attr: int + class_var: ClassVar[str] = "x" + const: int = 42 + + def __init__(self, regular_attr: int, extra_attrs: dict[str, object]) -> None: + super().__setattr__("_attributes", extra_attrs) + super().__setattr__("regular_attr", regular_attr) + + def __setattr__(self, key: str, val: object) -> None: + if key == "regular_attr": + super().__setattr__("regular_attr", val) + elif key == "class_var" or key == "const": + raise AttributeError() + else: + self._attributes[key] = val + + def __getattr__(self, key: str) -> object: + return self._attributes.get(key) + +class NoSetAttr: + def __init__(self, attr: int) -> None: + self.attr = attr + + def object_setattr(self, attr: str, val: object) -> None: + object.__setattr__(self, attr, val) + + def super_setattr(self, attr: str, val: object) -> None: + super().__setattr__(attr, val) + +@mypyc_attr(native_class=False) +class NoSetAttrNonNative: + def __init__(self, attr: int) -> None: + self.attr = attr + + def object_setattr(self, attr: str, val: object) -> None: + object.__setattr__(self, attr, val) + + def super_setattr(self, attr: str, val: object) -> None: + super().__setattr__(attr, val) + + def __getattr__(self, attr: str) -> object: + pass + +[file driver.py] +from native import SetAttr, SetAttrInherited, SetAttrOverridden, SetAttrNonNative, NoSetAttr, NoSetAttrNonNative +from testutil import assertRaises + +def test_setattr() -> None: + i = SetAttr(99, {"one": 1}) + assert i.class_var == "x" + assert i.regular_attr == 99 + assert i.one == 1 + assert i.two == None + assert i.const == 42 + + i.__setattr__("two", "2") + assert i.two == "2" + i.__setattr__("regular_attr", 101) + assert i.regular_attr == 101 + with assertRaises(AttributeError): + i.__setattr__("class_var", "y") + with assertRaises(AttributeError): + i.__setattr__("const", 43) + + setattr(i, "three", (3,3,3)) + assert i.three == (3,3,3) + setattr(i, "regular_attr", 102) + assert i.regular_attr == 102 + with assertRaises(AttributeError): + setattr(i, "class_var", "z") + with assertRaises(AttributeError): + setattr(i, "const", 44) + + i.four = [4,4] + assert i.four == [4,4] + i.regular_attr = 103 + assert i.regular_attr == 103 + with assertRaises(AttributeError): + i.const = 45 + +def test_setattr_inherited() -> None: + i = SetAttrInherited(99, {"one": 1}) + assert i.class_var == "x" + assert i.regular_attr == 99 + assert i.one == 1 + assert i.two == None + assert i.const == 42 + + i.__setattr__("two", "2") + assert i.two == "2" + i.__setattr__("regular_attr", 101) + assert i.regular_attr == 101 + with assertRaises(AttributeError): + i.__setattr__("class_var", "y") + with assertRaises(AttributeError): + i.__setattr__("const", 43) + + setattr(i, "three", (3,3,3)) + assert i.three == (3,3,3) + setattr(i, "regular_attr", 102) + assert i.regular_attr == 102 + with assertRaises(AttributeError): + setattr(i, "class_var", "z") + with assertRaises(AttributeError): + setattr(i, "const", 44) + + i.four = [4,4] + assert i.four == [4,4] + i.regular_attr = 103 + assert i.regular_attr == 103 + with assertRaises(AttributeError): + i.const = 45 + +def test_setattr_overridden() -> None: + i = SetAttrOverridden(99, 1, {"one": 1}) + assert i.class_var == "x" + assert i.subclass_var == "y" + assert i.regular_attr == 99 + assert i.sub_attr == 1 + assert i.one == 1 + assert i.two == None + assert i.const == 42 + + i.__setattr__("two", "2") + assert i.two == "2" + i.__setattr__("regular_attr", 101) + assert i.regular_attr == 101 + i.__setattr__("sub_attr", 2) + assert i.sub_attr == 2 + with assertRaises(AttributeError): + i.__setattr__("class_var", "y") + with assertRaises(AttributeError): + i.__setattr__("subclass_var", "a") + with assertRaises(AttributeError): + i.__setattr__("const", 43) + + setattr(i, "three", (3,3,3)) + assert i.three == (3,3,3) + setattr(i, "regular_attr", 102) + assert i.regular_attr == 102 + setattr(i, "sub_attr", 3) + assert i.sub_attr == 3 + with assertRaises(AttributeError): + setattr(i, "class_var", "z") + with assertRaises(AttributeError): + setattr(i, "subclass_var", "b") + with assertRaises(AttributeError): + setattr(i, "const", 44) + + i.four = [4,4] + assert i.four == [4,4] + i.regular_attr = 103 + assert i.regular_attr == 103 + i.sub_attr = 4 + assert i.sub_attr == 4 + with assertRaises(AttributeError): + i.const = 45 + + base_ref: SetAttr = i + setattr(base_ref, "sub_attr", 5) + assert base_ref.sub_attr == 5 + + base_ref.sub_attr = 6 + assert base_ref.sub_attr == 6 + + with assertRaises(AttributeError): + setattr(base_ref, "subclass_var", "c") + +def test_setattr_nonnative() -> None: + i = SetAttrNonNative(99, {"one": 1}) + assert i.class_var == "x" + assert i.regular_attr == 99 + assert i.one == 1 + assert i.two == None + assert i.const == 42 + + i.__setattr__("two", "2") + assert i.two == "2" + i.__setattr__("regular_attr", 101) + assert i.regular_attr == 101 + with assertRaises(AttributeError): + i.__setattr__("class_var", "y") + with assertRaises(AttributeError): + i.__setattr__("const", 43) + + setattr(i, "three", (3,3,3)) + assert i.three == (3,3,3) + setattr(i, "regular_attr", 102) + assert i.regular_attr == 102 + with assertRaises(AttributeError): + setattr(i, "class_var", "z") + with assertRaises(AttributeError): + setattr(i, "const", 44) + + i.four = [4,4] + assert i.four == [4,4] + i.regular_attr = 103 + assert i.regular_attr == 103 + with assertRaises(AttributeError): + i.const = 45 + +def test_no_setattr() -> None: + i = NoSetAttr(99) + i.super_setattr("attr", 100) + assert i.attr == 100 + + i.object_setattr("attr", 101) + assert i.attr == 101 + + object.__setattr__(i, "attr", 102) + assert i.attr == 102 + + with assertRaises(AttributeError): + i.super_setattr("not_attr", 100) + + with assertRaises(AttributeError): + i.object_setattr("not_attr", 101) + + with assertRaises(AttributeError): + object.__setattr__(i, "not_attr", 102) + +def test_no_setattr_nonnative() -> None: + i = NoSetAttrNonNative(99) + i.super_setattr("attr", 100) + assert i.attr == 100 + + i.object_setattr("attr", 101) + assert i.attr == 101 + + object.__setattr__(i, "attr", 102) + assert i.attr == 102 + + i.super_setattr("one", 100) + assert i.one == 100 + + i.object_setattr("two", 101) + assert i.two == 101 + + object.__setattr__(i, "three", 102) + assert i.three == 102 + +test_setattr() +test_setattr_inherited() +test_setattr_overridden() +test_setattr_nonnative() +test_no_setattr() +test_no_setattr_nonnative() [typing fixtures/typing-full.pyi] From 44bbb18db02efd86446e039fbbfa0353f7db537c Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Mon, 29 Sep 2025 10:25:02 -0400 Subject: [PATCH 282/424] [mypyc] feat: optimize away first index check in for loops if length > 1 (#19933) It is not necessary to enter the gen_condition block on the first iteration if the length is known to be positive, we can safely skip it. This won't be particularly impactful but its a low-hanging fruit. --- mypyc/irbuild/for_helpers.py | 7 ++++++- mypyc/test-data/irbuild-lists.test | 4 ++++ mypyc/test-data/irbuild-tuple.test | 4 ++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index 5edee6cb4df40..db986f3fd9a77 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -183,7 +183,12 @@ def for_loop_helper_with_index( builder.push_loop_stack(step_block, exit_block) - builder.goto_and_activate(condition_block) + if isinstance(length, Integer) and length.value > 0: + builder.goto(body_block) + builder.activate_block(condition_block) + else: + builder.goto_and_activate(condition_block) + for_gen.gen_condition() builder.activate_block(body_block) diff --git a/mypyc/test-data/irbuild-lists.test b/mypyc/test-data/irbuild-lists.test index 2f5b3b39319e7..d864bfd19df26 100644 --- a/mypyc/test-data/irbuild-lists.test +++ b/mypyc/test-data/irbuild-lists.test @@ -645,6 +645,7 @@ L0: r0 = 'abc' r1 = PyList_New(3) r2 = 0 + goto L2 L1: r3 = r2 < 3 :: signed if r3 goto L2 else goto L4 :: bool @@ -690,6 +691,7 @@ L0: r0 = 'abc' r1 = PyList_New(3) r2 = 0 + goto L2 L1: r3 = r2 < 3 :: signed if r3 goto L2 else goto L4 :: bool @@ -798,6 +800,7 @@ L0: r0 = b'abc' r1 = PyList_New(3) r2 = 0 + goto L2 L1: r3 = r2 < 3 :: signed if r3 goto L2 else goto L8 :: bool @@ -952,6 +955,7 @@ L0: r18 = CPyList_Extend(r10, r17) r19 = PyList_New(13) r20 = 0 + goto L2 L1: r21 = var_object_size r10 r22 = r20 < r21 :: signed diff --git a/mypyc/test-data/irbuild-tuple.test b/mypyc/test-data/irbuild-tuple.test index 081cc1b174c9b..7507b62557401 100644 --- a/mypyc/test-data/irbuild-tuple.test +++ b/mypyc/test-data/irbuild-tuple.test @@ -417,6 +417,7 @@ L0: r0 = 'abc' r1 = PyTuple_New(3) r2 = 0 + goto L2 L1: r3 = r2 < 3 :: signed if r3 goto L2 else goto L4 :: bool @@ -462,6 +463,7 @@ L0: r0 = 'abc' r1 = PyTuple_New(3) r2 = 0 + goto L2 L1: r3 = r2 < 3 :: signed if r3 goto L2 else goto L4 :: bool @@ -570,6 +572,7 @@ L0: r0 = b'abc' r1 = PyTuple_New(3) r2 = 0 + goto L2 L1: r3 = r2 < 3 :: signed if r3 goto L2 else goto L8 :: bool @@ -879,6 +882,7 @@ L0: r18 = CPyList_Extend(r10, r17) r19 = PyTuple_New(13) r20 = 0 + goto L2 L1: r21 = var_object_size r10 r22 = r20 < r21 :: signed From e0090d0ab74b4933088e94abaad0b2409e9cd50a Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 30 Sep 2025 09:48:31 +0100 Subject: [PATCH 283/424] Use published version of librt (#19945) Ref https://github.com/mypyc/mypyc/issues/1128 Another step towards making fixed format cache the default one. --- mypy-requirements.txt | 1 + mypy_self_check.ini | 1 + mypyc/lib-rt/setup.py | 4 +--- pyproject.toml | 1 + setup.py | 2 -- test-requirements.txt | 2 ++ 6 files changed, 6 insertions(+), 5 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index 8965a70c13b70..6927ddd25d81e 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -4,3 +4,4 @@ typing_extensions>=4.6.0 mypy_extensions>=1.0.0 pathspec>=0.9.0 tomli>=1.1.0; python_version<'3.11' +librt>=0.1.0 diff --git a/mypy_self_check.ini b/mypy_self_check.ini index 67b65381cfd02..0b49b3de862bc 100644 --- a/mypy_self_check.ini +++ b/mypy_self_check.ini @@ -13,3 +13,4 @@ enable_error_code = ignore-without-code,redundant-expr enable_incomplete_feature = PreciseTupleTypes show_error_code_links = True warn_unreachable = True +fixed_format_cache = True diff --git a/mypyc/lib-rt/setup.py b/mypyc/lib-rt/setup.py index 3a5976cf88b28..36b55e44dcd1d 100644 --- a/mypyc/lib-rt/setup.py +++ b/mypyc/lib-rt/setup.py @@ -73,8 +73,6 @@ def run(self) -> None: # TODO: we need a way to share our preferred C flags and get_extension() logic with # mypyc/build.py without code duplication. setup( - name="mypy-native", - version="0.0.1", ext_modules=[ Extension( "native_internal", @@ -88,5 +86,5 @@ def run(self) -> None: ], include_dirs=["."], ) - ], + ] ) diff --git a/pyproject.toml b/pyproject.toml index 032bfcb609e78..46593af0ab72c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,7 @@ requires = [ "mypy_extensions>=1.0.0", "pathspec>=0.9.0", "tomli>=1.1.0; python_version<'3.11'", + "librt>=0.1.0", # the following is from build-requirements.txt "types-psutil", "types-setuptools", diff --git a/setup.py b/setup.py index 798ff4f6c7101..3f53d96dbd85c 100644 --- a/setup.py +++ b/setup.py @@ -156,8 +156,6 @@ def run(self) -> None: log_trace=log_trace, # Mypy itself is allowed to use native_internal extension. depends_on_native_internal=True, - # TODO: temporary, remove this after we publish mypy-native on PyPI. - install_native_libs=True, ) else: diff --git a/test-requirements.txt b/test-requirements.txt index 521208c5aa27d..5402adcb682ca 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -22,6 +22,8 @@ identify==2.6.13 # via pre-commit iniconfig==2.1.0 # via pytest +librt==0.1.0 + # via -r mypy-requirements.txt lxml==6.0.1 ; python_version < "3.15" # via -r test-requirements.in mypy-extensions==1.1.0 From a12565c8253f6b0996a9b11162776c9083aaceb8 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 30 Sep 2025 12:16:23 +0100 Subject: [PATCH 284/424] Do not update states after writing cache (#19936) This is a little preparation for https://github.com/python/mypy/issues/933. We need to minimize the number of actions after writing cache, since these are things workers will need to communicate back to coordinator. Previously we updated `State.meta` after writing cache, but this looks not needed anymore, and would complicate parallel checking. Also delete couple unused attributes on `State`. --- mypy/build.py | 28 +++++++--------------------- 1 file changed, 7 insertions(+), 21 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index ad25b811ff7cd..234dd6843292e 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -1546,7 +1546,7 @@ def write_cache( source_hash: str, ignore_all: bool, manager: BuildManager, -) -> tuple[str, tuple[dict[str, Any], str, str] | None]: +) -> tuple[str, tuple[dict[str, Any], str] | None]: """Write cache files for a module. Note that this mypy's behavior is still correct when any given @@ -1568,7 +1568,7 @@ def write_cache( Returns: A tuple containing the interface hash and inner tuple with cache meta JSON - that should be written and paths to cache files (inner tuple may be None, + that should be written and path to cache file (inner tuple may be None, if the cache data could not be written). """ metastore = manager.metastore @@ -1662,12 +1662,10 @@ def write_cache( "ignore_all": ignore_all, "plugin_data": plugin_data, } - return interface_hash, (meta, meta_json, data_json) + return interface_hash, (meta, meta_json) -def write_cache_meta( - meta: dict[str, Any], manager: BuildManager, meta_json: str, data_json: str -) -> CacheMeta: +def write_cache_meta(meta: dict[str, Any], manager: BuildManager, meta_json: str) -> None: # Write meta cache file metastore = manager.metastore meta_str = json_dumps(meta, manager.options.debug_cache) @@ -1677,8 +1675,6 @@ def write_cache_meta( # The next run will simply find the cache entry out of date. manager.log(f"Error writing meta JSON file {meta_json}") - return cache_meta_from_dict(meta, data_json) - """Dependency manager. @@ -1864,9 +1860,6 @@ class State: # List of (path, line number) tuples giving context for import import_context: list[tuple[str, int]] - # The State from which this module was imported, if any - caller_state: State | None = None - # If caller_state is set, the line number in the caller where the import occurred caller_line = 0 @@ -1917,7 +1910,6 @@ def __init__( self.manager = manager State.order_counter += 1 self.order = State.order_counter - self.caller_state = caller_state self.caller_line = caller_line if caller_state: self.import_context = caller_state.import_context.copy() @@ -2008,11 +2000,6 @@ def __init__( self.parse_file(temporary=temporary) self.compute_dependencies() - @property - def xmeta(self) -> CacheMeta: - assert self.meta, "missing meta on allegedly fresh module" - return self.meta - def add_ancestors(self) -> None: if self.path is not None: _, name = os.path.split(self.path) @@ -2479,7 +2466,7 @@ def valid_references(self) -> set[str]: return valid_refs - def write_cache(self) -> tuple[dict[str, Any], str, str] | None: + def write_cache(self) -> tuple[dict[str, Any], str] | None: assert self.tree is not None, "Internal error: method must be called on parsed file only" # We don't support writing cache files in fine-grained incremental mode. if ( @@ -3477,14 +3464,13 @@ def process_stale_scc(graph: Graph, scc: list[str], manager: BuildManager) -> No for id in stale: meta_tuple = meta_tuples[id] if meta_tuple is None: - graph[id].meta = None continue - meta, meta_json, data_json = meta_tuple + meta, meta_json = meta_tuple meta["dep_hashes"] = { dep: graph[dep].interface_hash for dep in graph[id].dependencies if dep in graph } meta["error_lines"] = errors_by_id.get(id, []) - graph[id].meta = write_cache_meta(meta, manager, meta_json, data_json) + write_cache_meta(meta, manager, meta_json) def sorted_components( From bcef9f4c96b2a41283ff20963ced3b9ecb119435 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Tue, 30 Sep 2025 09:42:54 -0400 Subject: [PATCH 285/424] [mypyc] feat: support RTuple in `sequence_from_generator_preallocate_helper` (#19931) This PR extends `sequence_from_generator_preallocate_helper` to work with RTuple types. This is ready for review. --- mypyc/irbuild/for_helpers.py | 66 +++++++++------ mypyc/test-data/irbuild-tuple.test | 128 ++++++++++++++++++----------- 2 files changed, 120 insertions(+), 74 deletions(-) diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index db986f3fd9a77..20440d4a26f49 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -28,6 +28,7 @@ TypeAlias, Var, ) +from mypy.types import LiteralType, TupleType, get_proper_type, get_proper_types from mypyc.ir.ops import ( ERR_NEVER, BasicBlock, @@ -36,6 +37,7 @@ IntOp, LoadAddress, LoadErrorValue, + LoadLiteral, LoadMem, MethodCall, RaiseStandardError, @@ -170,7 +172,7 @@ def for_loop_helper_with_index( body_insts: a function that generates the body of the loop. It needs a index as parameter. """ - assert is_sequence_rprimitive(expr_reg.type) + assert is_sequence_rprimitive(expr_reg.type), (expr_reg, expr_reg.type) target_type = builder.get_sequence_type(expr) body_block = BasicBlock() @@ -217,10 +219,9 @@ def sequence_from_generator_preallocate_helper( there is no condition list in the generator and only one original sequence with one index is allowed. - e.g. (1) tuple(f(x) for x in a_list/a_tuple/a_str/a_bytes) - (2) list(f(x) for x in a_list/a_tuple/a_str/a_bytes) - (3) [f(x) for x in a_list/a_tuple/a_str/a_bytes] - RTuple as an original sequence is not supported yet. + e.g. (1) tuple(f(x) for x in a_list/a_tuple/a_str/a_bytes/an_rtuple) + (2) list(f(x) for x in a_list/a_tuple/a_str/a_bytes/an_rtuple) + (3) [f(x) for x in a_list/a_tuple/a_str/a_bytes/an_rtuple] Args: empty_op_llbuilder: A function that can generate an empty sequence op when @@ -235,23 +236,41 @@ def sequence_from_generator_preallocate_helper( implementation. """ if len(gen.sequences) == 1 and len(gen.indices) == 1 and len(gen.condlists[0]) == 0: - rtype = builder.node_type(gen.sequences[0]) - if is_sequence_rprimitive(rtype): - sequence = builder.accept(gen.sequences[0]) - length = get_expr_length_value( - builder, gen.sequences[0], sequence, gen.line, use_pyssize_t=True - ) - target_op = empty_op_llbuilder(length, gen.line) - - def set_item(item_index: Value) -> None: - e = builder.accept(gen.left_expr) - builder.call_c(set_item_op, [target_op, item_index, e], gen.line) - - for_loop_helper_with_index( - builder, gen.indices[0], gen.sequences[0], sequence, set_item, gen.line, length - ) + line = gen.line + sequence_expr = gen.sequences[0] + rtype = builder.node_type(sequence_expr) + if not (is_sequence_rprimitive(rtype) or isinstance(rtype, RTuple)): + return None + sequence = builder.accept(sequence_expr) + length = get_expr_length_value(builder, sequence_expr, sequence, line, use_pyssize_t=True) + if isinstance(rtype, RTuple): + # If input is RTuple, box it to tuple_rprimitive for generic iteration + # TODO: this can be optimized a bit better with an unrolled ForRTuple helper + proper_type = get_proper_type(builder.types[sequence_expr]) + assert isinstance(proper_type, TupleType), proper_type + + get_item_ops = [ + ( + LoadLiteral(typ.value, object_rprimitive) + if isinstance(typ, LiteralType) + else TupleGet(sequence, i, line) + ) + for i, typ in enumerate(get_proper_types(proper_type.items)) + ] + items = list(map(builder.add, get_item_ops)) + sequence = builder.new_tuple(items, line) + + target_op = empty_op_llbuilder(length, line) + + def set_item(item_index: Value) -> None: + e = builder.accept(gen.left_expr) + builder.call_c(set_item_op, [target_op, item_index, e], line) + + for_loop_helper_with_index( + builder, gen.indices[0], sequence_expr, sequence, set_item, line, length + ) - return target_op + return target_op return None @@ -804,7 +823,7 @@ class ForSequence(ForGenerator): def init( self, expr_reg: Value, target_type: RType, reverse: bool, length: Value | None = None ) -> None: - assert is_sequence_rprimitive(expr_reg.type), expr_reg + assert is_sequence_rprimitive(expr_reg.type), (expr_reg, expr_reg.type) builder = self.builder # Record a Value indicating the length of the sequence, if known at compile time. self.length = length @@ -834,7 +853,6 @@ def init( def gen_condition(self) -> None: builder = self.builder line = self.line - # TODO: Don't reload the length each time when iterating an immutable sequence? if self.reverse: # If we are iterating in reverse order, we obviously need # to check that the index is still positive. Somewhat less @@ -1216,7 +1234,7 @@ def get_expr_length_value( builder: IRBuilder, expr: Expression, expr_reg: Value, line: int, use_pyssize_t: bool ) -> Value: rtype = builder.node_type(expr) - assert is_sequence_rprimitive(rtype), rtype + assert is_sequence_rprimitive(rtype) or isinstance(rtype, RTuple), rtype length = get_expr_length(expr) if length is None: # We cannot compute the length at compile time, so we will fetch it. diff --git a/mypyc/test-data/irbuild-tuple.test b/mypyc/test-data/irbuild-tuple.test index 7507b62557401..3613c5f0101d7 100644 --- a/mypyc/test-data/irbuild-tuple.test +++ b/mypyc/test-data/irbuild-tuple.test @@ -694,36 +694,50 @@ L0: return r1 def test(): r0, source :: tuple[int, int, int] - r1 :: list - r2, r3, r4 :: object - r5, x :: int - r6 :: bool - r7 :: object - r8 :: i32 - r9, r10 :: bit - r11, a :: tuple + r1 :: object + r2 :: native_int + r3 :: bit + r4, r5, r6 :: int + r7, r8, r9 :: object + r10, r11 :: tuple + r12 :: native_int + r13 :: bit + r14 :: object + r15, x :: int + r16 :: bool + r17 :: object + r18 :: native_int + a :: tuple L0: r0 = (2, 4, 6) source = r0 - r1 = PyList_New(0) - r2 = box(tuple[int, int, int], source) - r3 = PyObject_GetIter(r2) + r1 = box(tuple[int, int, int], source) + r2 = PyObject_Size(r1) + r3 = r2 >= 0 :: signed + r4 = source[0] + r5 = source[1] + r6 = source[2] + r7 = box(int, r4) + r8 = box(int, r5) + r9 = box(int, r6) + r10 = PyTuple_Pack(3, r7, r8, r9) + r11 = PyTuple_New(r2) + r12 = 0 L1: - r4 = PyIter_Next(r3) - if is_error(r4) goto L4 else goto L2 + r13 = r12 < r2 :: signed + if r13 goto L2 else goto L4 :: bool L2: - r5 = unbox(int, r4) - x = r5 - r6 = f(x) - r7 = box(bool, r6) - r8 = PyList_Append(r1, r7) - r9 = r8 >= 0 :: signed + r14 = CPySequenceTuple_GetItemUnsafe(r10, r12) + r15 = unbox(int, r14) + x = r15 + r16 = f(x) + r17 = box(bool, r16) + CPySequenceTuple_SetItemUnsafe(r11, r12, r17) L3: + r18 = r12 + 1 + r12 = r18 goto L1 L4: - r10 = CPy_NoErrOccurred() -L5: - r11 = PyList_AsTuple(r1) a = r11 return 1 @@ -746,42 +760,56 @@ L0: r1 = int_eq r0, 0 return r1 def test(): - r0 :: list - r1 :: tuple[int, int, int] - r2 :: bool - r3, r4, r5 :: object - r6, x :: int - r7 :: bool - r8 :: object - r9 :: i32 - r10, r11 :: bit - r12, a :: tuple -L0: - r0 = PyList_New(0) - r1 = __main__.source :: static - if is_error(r1) goto L1 else goto L2 + r0 :: tuple[int, int, int] + r1 :: bool + r2 :: object + r3 :: native_int + r4 :: bit + r5, r6, r7 :: int + r8, r9, r10 :: object + r11, r12 :: tuple + r13 :: native_int + r14 :: bit + r15 :: object + r16, x :: int + r17 :: bool + r18 :: object + r19 :: native_int + a :: tuple +L0: + r0 = __main__.source :: static + if is_error(r0) goto L1 else goto L2 L1: - r2 = raise NameError('value for final name "source" was not set') + r1 = raise NameError('value for final name "source" was not set') unreachable L2: - r3 = box(tuple[int, int, int], r1) - r4 = PyObject_GetIter(r3) + r2 = box(tuple[int, int, int], r0) + r3 = PyObject_Size(r2) + r4 = r3 >= 0 :: signed + r5 = r0[0] + r6 = r0[1] + r7 = r0[2] + r8 = box(int, r5) + r9 = box(int, r6) + r10 = box(int, r7) + r11 = PyTuple_Pack(3, r8, r9, r10) + r12 = PyTuple_New(r3) + r13 = 0 L3: - r5 = PyIter_Next(r4) - if is_error(r5) goto L6 else goto L4 + r14 = r13 < r3 :: signed + if r14 goto L4 else goto L6 :: bool L4: - r6 = unbox(int, r5) - x = r6 - r7 = f(x) - r8 = box(bool, r7) - r9 = PyList_Append(r0, r8) - r10 = r9 >= 0 :: signed + r15 = CPySequenceTuple_GetItemUnsafe(r11, r13) + r16 = unbox(int, r15) + x = r16 + r17 = f(x) + r18 = box(bool, r17) + CPySequenceTuple_SetItemUnsafe(r12, r13, r18) L5: + r19 = r13 + 1 + r13 = r19 goto L3 L6: - r11 = CPy_NoErrOccurred() -L7: - r12 = PyList_AsTuple(r0) a = r12 return 1 From 6feecce803fc7ffcb6b68405205703e04874cb99 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Tue, 30 Sep 2025 09:47:20 -0400 Subject: [PATCH 286/424] [mypyc] feat: specialize isinstance for tuple of primitive types (#19949) This PR specializes isinstance calls where the type argument is a tuple of primitive types. We can skip tuple creation and the associated refcounting, and daisy-chain the primitive checks with an early exit option at each step. --- mypyc/irbuild/specialize.py | 72 ++++++++++++++++++++++--- mypyc/test-data/irbuild-isinstance.test | 28 ++++++++++ mypyc/test-data/run-misc.test | 23 ++++++++ 3 files changed, 115 insertions(+), 8 deletions(-) diff --git a/mypyc/irbuild/specialize.py b/mypyc/irbuild/specialize.py index 42b7710c98da6..84807a7fdb535 100644 --- a/mypyc/irbuild/specialize.py +++ b/mypyc/irbuild/specialize.py @@ -14,7 +14,7 @@ from __future__ import annotations -from typing import Callable, Final, Optional +from typing import Callable, Final, Optional, cast from mypy.nodes import ( ARG_NAMED, @@ -40,6 +40,7 @@ Call, Extend, Integer, + PrimitiveDescription, RaiseStandardError, Register, SetAttr, @@ -589,26 +590,81 @@ def translate_isinstance(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> if not (len(expr.args) == 2 and expr.arg_kinds == [ARG_POS, ARG_POS]): return None - if isinstance(expr.args[1], (RefExpr, TupleExpr)): - builder.types[expr.args[0]] = AnyType(TypeOfAny.from_error) + obj_expr = expr.args[0] + type_expr = expr.args[1] - irs = builder.flatten_classes(expr.args[1]) + if isinstance(type_expr, TupleExpr) and not type_expr.items: + # we can compile this case to a noop + return builder.false() + + if isinstance(type_expr, (RefExpr, TupleExpr)): + builder.types[obj_expr] = AnyType(TypeOfAny.from_error) + + irs = builder.flatten_classes(type_expr) if irs is not None: can_borrow = all( ir.is_ext_class and not ir.inherits_python and not ir.allow_interpreted_subclasses for ir in irs ) - obj = builder.accept(expr.args[0], can_borrow=can_borrow) + obj = builder.accept(obj_expr, can_borrow=can_borrow) return builder.builder.isinstance_helper(obj, irs, expr.line) - if isinstance(expr.args[1], RefExpr): - node = expr.args[1].node + if isinstance(type_expr, RefExpr): + node = type_expr.node if node: desc = isinstance_primitives.get(node.fullname) if desc: - obj = builder.accept(expr.args[0]) + obj = builder.accept(obj_expr) return builder.primitive_op(desc, [obj], expr.line) + elif isinstance(type_expr, TupleExpr): + node_names: list[str] = [] + for item in type_expr.items: + if not isinstance(item, RefExpr): + return None + if item.node is None: + return None + if item.node.fullname not in node_names: + node_names.append(item.node.fullname) + + descs = [isinstance_primitives.get(fullname) for fullname in node_names] + if None in descs: + # not all types are primitive types, abort + return None + + obj = builder.accept(obj_expr) + + retval = Register(bool_rprimitive) + pass_block = BasicBlock() + fail_block = BasicBlock() + exit_block = BasicBlock() + + # Chain the checks: if any succeed, jump to pass_block; else, continue + for i, desc in enumerate(descs): + is_last = i == len(descs) - 1 + next_block = fail_block if is_last else BasicBlock() + builder.add_bool_branch( + builder.primitive_op(cast(PrimitiveDescription, desc), [obj], expr.line), + pass_block, + next_block, + ) + if not is_last: + builder.activate_block(next_block) + + # If any check passed + builder.activate_block(pass_block) + builder.assign(retval, builder.true(), expr.line) + builder.goto(exit_block) + + # If all checks failed + builder.activate_block(fail_block) + builder.assign(retval, builder.false(), expr.line) + builder.goto(exit_block) + + # Return the result + builder.activate_block(exit_block) + return retval + return None diff --git a/mypyc/test-data/irbuild-isinstance.test b/mypyc/test-data/irbuild-isinstance.test index 0df9448b819f2..36a9300350bd3 100644 --- a/mypyc/test-data/irbuild-isinstance.test +++ b/mypyc/test-data/irbuild-isinstance.test @@ -189,3 +189,31 @@ def is_tuple(x): L0: r0 = PyTuple_Check(x) return r0 + +[case testTupleOfPrimitives] +from typing import Any + +def is_instance(x: Any) -> bool: + return isinstance(x, (str, int, bytes)) + +[out] +def is_instance(x): + x :: object + r0, r1, r2 :: bit + r3 :: bool +L0: + r0 = PyUnicode_Check(x) + if r0 goto L3 else goto L1 :: bool +L1: + r1 = PyLong_Check(x) + if r1 goto L3 else goto L2 :: bool +L2: + r2 = PyBytes_Check(x) + if r2 goto L3 else goto L4 :: bool +L3: + r3 = 1 + goto L5 +L4: + r3 = 0 +L5: + return r3 diff --git a/mypyc/test-data/run-misc.test b/mypyc/test-data/run-misc.test index 129946a4c3300..1074906357eed 100644 --- a/mypyc/test-data/run-misc.test +++ b/mypyc/test-data/run-misc.test @@ -1173,3 +1173,26 @@ def test_dummy_context() -> None: with c: assert c.c == 1 assert c.c == 0 + +[case testIsInstanceTuple] +from typing import Any + +def isinstance_empty(x: Any) -> bool: + return isinstance(x, ()) +def isinstance_single(x: Any) -> bool: + return isinstance(x, (str,)) +def isinstance_multi(x: Any) -> bool: + return isinstance(x, (str, int)) + +def test_isinstance_empty() -> None: + assert isinstance_empty("a") is False + assert isinstance_empty(1) is False + assert isinstance_empty(None) is False +def test_isinstance_single() -> None: + assert isinstance_single("a") is True + assert isinstance_single(1) is False + assert isinstance_single(None) is False +def test_isinstance_multi() -> None: + assert isinstance_multi("a") is True + assert isinstance_multi(1) is True + assert isinstance_multi(None) is False From 4936099b126f4c982c68b25208c434959d1602da Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 30 Sep 2025 22:49:45 -0700 Subject: [PATCH 287/424] Sync typeshed (#19959) Source commit: https://github.com/python/typeshed/commit/91055c730ffcda6311654cf32d663858ece69bad --- mypy/typeshed/stdlib/_collections_abc.pyi | 9 +- mypy/typeshed/stdlib/_frozen_importlib.pyi | 2 +- mypy/typeshed/stdlib/_tkinter.pyi | 40 +- mypy/typeshed/stdlib/asyncio/tools.pyi | 5 + mypy/typeshed/stdlib/builtins.pyi | 2 +- .../concurrent/interpreters/_queues.pyi | 34 +- mypy/typeshed/stdlib/configparser.pyi | 6 +- mypy/typeshed/stdlib/copy.pyi | 12 +- mypy/typeshed/stdlib/curses/__init__.pyi | 5 +- mypy/typeshed/stdlib/dbm/sqlite3.pyi | 2 +- mypy/typeshed/stdlib/email/headerregistry.pyi | 2 +- mypy/typeshed/stdlib/errno.pyi | 1 + mypy/typeshed/stdlib/faulthandler.pyi | 8 +- .../stdlib/importlib/resources/_common.pyi | 2 +- .../stdlib/multiprocessing/shared_memory.pyi | 2 +- mypy/typeshed/stdlib/tkinter/__init__.pyi | 1372 ++++++++--------- mypy/typeshed/stdlib/tkinter/ttk.pyi | 326 ++-- mypy/typeshed/stdlib/types.pyi | 36 +- mypy/typeshed/stdlib/typing.pyi | 9 +- 19 files changed, 948 insertions(+), 927 deletions(-) diff --git a/mypy/typeshed/stdlib/_collections_abc.pyi b/mypy/typeshed/stdlib/_collections_abc.pyi index c63606a13ca99..319577c9284bc 100644 --- a/mypy/typeshed/stdlib/_collections_abc.pyi +++ b/mypy/typeshed/stdlib/_collections_abc.pyi @@ -1,12 +1,13 @@ import sys from abc import abstractmethod from types import MappingProxyType -from typing import ( # noqa: Y022,Y038,UP035 +from typing import ( # noqa: Y022,Y038,UP035,Y057 AbstractSet as Set, AsyncGenerator as AsyncGenerator, AsyncIterable as AsyncIterable, AsyncIterator as AsyncIterator, Awaitable as Awaitable, + ByteString as ByteString, Callable as Callable, ClassVar, Collection as Collection, @@ -59,12 +60,8 @@ __all__ = [ "ValuesView", "Sequence", "MutableSequence", + "ByteString", ] -if sys.version_info < (3, 14): - from typing import ByteString as ByteString # noqa: Y057,UP035 - - __all__ += ["ByteString"] - if sys.version_info >= (3, 12): __all__ += ["Buffer"] diff --git a/mypy/typeshed/stdlib/_frozen_importlib.pyi b/mypy/typeshed/stdlib/_frozen_importlib.pyi index 93aaed82e2e1c..58db64a016f34 100644 --- a/mypy/typeshed/stdlib/_frozen_importlib.pyi +++ b/mypy/typeshed/stdlib/_frozen_importlib.pyi @@ -13,7 +13,7 @@ def __import__( name: str, globals: Mapping[str, object] | None = None, locals: Mapping[str, object] | None = None, - fromlist: Sequence[str] = (), + fromlist: Sequence[str] | None = (), level: int = 0, ) -> ModuleType: ... def spec_from_loader( diff --git a/mypy/typeshed/stdlib/_tkinter.pyi b/mypy/typeshed/stdlib/_tkinter.pyi index 46366ccc17405..a3868f467c6ca 100644 --- a/mypy/typeshed/stdlib/_tkinter.pyi +++ b/mypy/typeshed/stdlib/_tkinter.pyi @@ -54,34 +54,34 @@ _TkinterTraceFunc: TypeAlias = Callable[[tuple[str, ...]], object] @final class TkappType: # Please keep in sync with tkinter.Tk - def adderrorinfo(self, msg, /): ... + def adderrorinfo(self, msg: str, /): ... def call(self, command: Any, /, *args: Any) -> Any: ... - def createcommand(self, name, func, /): ... + def createcommand(self, name: str, func, /): ... if sys.platform != "win32": - def createfilehandler(self, file, mask, func, /): ... - def deletefilehandler(self, file, /): ... + def createfilehandler(self, file, mask: int, func, /): ... + def deletefilehandler(self, file, /) -> None: ... - def createtimerhandler(self, milliseconds, func, /): ... - def deletecommand(self, name, /): ... + def createtimerhandler(self, milliseconds: int, func, /): ... + def deletecommand(self, name: str, /): ... def dooneevent(self, flags: int = 0, /): ... def eval(self, script: str, /) -> str: ... - def evalfile(self, fileName, /): ... - def exprboolean(self, s, /): ... - def exprdouble(self, s, /): ... - def exprlong(self, s, /): ... - def exprstring(self, s, /): ... - def getboolean(self, arg, /): ... - def getdouble(self, arg, /): ... - def getint(self, arg, /): ... + def evalfile(self, fileName: str, /): ... + def exprboolean(self, s: str, /): ... + def exprdouble(self, s: str, /): ... + def exprlong(self, s: str, /): ... + def exprstring(self, s: str, /): ... + def getboolean(self, arg, /) -> bool: ... + def getdouble(self, arg, /) -> float: ... + def getint(self, arg, /) -> int: ... def getvar(self, *args, **kwargs): ... def globalgetvar(self, *args, **kwargs): ... def globalsetvar(self, *args, **kwargs): ... def globalunsetvar(self, *args, **kwargs): ... def interpaddr(self) -> int: ... def loadtk(self) -> None: ... - def mainloop(self, threshold: int = 0, /): ... - def quit(self): ... - def record(self, script, /): ... + def mainloop(self, threshold: int = 0, /) -> None: ... + def quit(self) -> None: ... + def record(self, script: str, /): ... def setvar(self, *ags, **kwargs): ... if sys.version_info < (3, 11): @deprecated("Deprecated since Python 3.9; removed in Python 3.11. Use `splitlist()` instead.") @@ -90,7 +90,7 @@ class TkappType: def splitlist(self, arg, /): ... def unsetvar(self, *args, **kwargs): ... def wantobjects(self, *args, **kwargs): ... - def willdispatch(self): ... + def willdispatch(self) -> None: ... if sys.version_info >= (3, 12): def gettrace(self, /) -> _TkinterTraceFunc | None: ... def settrace(self, func: _TkinterTraceFunc | None, /) -> None: ... @@ -140,5 +140,5 @@ else: /, ): ... -def getbusywaitinterval(): ... -def setbusywaitinterval(new_val, /): ... +def getbusywaitinterval() -> int: ... +def setbusywaitinterval(new_val: int, /) -> None: ... diff --git a/mypy/typeshed/stdlib/asyncio/tools.pyi b/mypy/typeshed/stdlib/asyncio/tools.pyi index 65c7f27e0b85e..bc8b809b9c055 100644 --- a/mypy/typeshed/stdlib/asyncio/tools.pyi +++ b/mypy/typeshed/stdlib/asyncio/tools.pyi @@ -1,3 +1,4 @@ +import sys from collections.abc import Iterable from enum import Enum from typing import NamedTuple, SupportsIndex, type_check_only @@ -37,5 +38,9 @@ class CycleFoundException(Exception): def get_all_awaited_by(pid: SupportsIndex) -> list[_AwaitedInfo]: ... def build_async_tree(result: Iterable[_AwaitedInfo], task_emoji: str = "(T)", cor_emoji: str = "") -> list[list[str]]: ... def build_task_table(result: Iterable[_AwaitedInfo]) -> list[list[int | str]]: ... + +if sys.version_info >= (3, 14): + def exit_with_permission_help_text() -> None: ... + def display_awaited_by_tasks_table(pid: SupportsIndex) -> None: ... def display_awaited_by_tasks_tree(pid: SupportsIndex) -> None: ... diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi index ef6c712e00053..e276441523c8c 100644 --- a/mypy/typeshed/stdlib/builtins.pyi +++ b/mypy/typeshed/stdlib/builtins.pyi @@ -1917,7 +1917,7 @@ def __import__( name: str, globals: Mapping[str, object] | None = None, locals: Mapping[str, object] | None = None, - fromlist: Sequence[str] = (), + fromlist: Sequence[str] | None = (), level: int = 0, ) -> types.ModuleType: ... def __build_class__(func: Callable[[], CellType | Any], name: str, /, *bases: Any, metaclass: Any = ..., **kwds: Any) -> Any: ... diff --git a/mypy/typeshed/stdlib/concurrent/interpreters/_queues.pyi b/mypy/typeshed/stdlib/concurrent/interpreters/_queues.pyi index 7493f87809c82..bdf08d93d1e00 100644 --- a/mypy/typeshed/stdlib/concurrent/interpreters/_queues.pyi +++ b/mypy/typeshed/stdlib/concurrent/interpreters/_queues.pyi @@ -45,14 +45,30 @@ if sys.version_info >= (3, 13): # needed to satisfy pyright checks for Python < def empty(self) -> bool: ... def full(self) -> bool: ... def qsize(self) -> int: ... - def put( - self, - obj: object, - timeout: SupportsIndex | None = None, - *, - unbounditems: _AnyUnbound | None = None, - _delay: float = 0.01, - ) -> None: ... + if sys.version_info >= (3, 14): + def put( + self, + obj: object, + block: bool = True, + timeout: SupportsIndex | None = None, + *, + unbounditems: _AnyUnbound | None = None, + _delay: float = 0.01, + ) -> None: ... + else: + def put( + self, + obj: object, + timeout: SupportsIndex | None = None, + *, + unbounditems: _AnyUnbound | None = None, + _delay: float = 0.01, + ) -> None: ... + def put_nowait(self, obj: object, *, unbounditems: _AnyUnbound | None = None) -> None: ... - def get(self, timeout: SupportsIndex | None = None, *, _delay: float = 0.01) -> object: ... + if sys.version_info >= (3, 14): + def get(self, block: bool = True, timeout: SupportsIndex | None = None, *, _delay: float = 0.01) -> object: ... + else: + def get(self, timeout: SupportsIndex | None = None, *, _delay: float = 0.01) -> object: ... + def get_nowait(self) -> object: ... diff --git a/mypy/typeshed/stdlib/configparser.pyi b/mypy/typeshed/stdlib/configparser.pyi index 764a8a965ea26..1909d80e3d189 100644 --- a/mypy/typeshed/stdlib/configparser.pyi +++ b/mypy/typeshed/stdlib/configparser.pyi @@ -258,9 +258,9 @@ class RawConfigParser(_Parser): ) -> None: ... def __len__(self) -> int: ... - def __getitem__(self, key: str) -> SectionProxy: ... - def __setitem__(self, key: str, value: _Section) -> None: ... - def __delitem__(self, key: str) -> None: ... + def __getitem__(self, key: _SectionName) -> SectionProxy: ... + def __setitem__(self, key: _SectionName, value: _Section) -> None: ... + def __delitem__(self, key: _SectionName) -> None: ... def __iter__(self) -> Iterator[str]: ... def __contains__(self, key: object) -> bool: ... def defaults(self) -> _Section: ... diff --git a/mypy/typeshed/stdlib/copy.pyi b/mypy/typeshed/stdlib/copy.pyi index 10d2f0ae37103..373899ea2635f 100644 --- a/mypy/typeshed/stdlib/copy.pyi +++ b/mypy/typeshed/stdlib/copy.pyi @@ -1,16 +1,15 @@ import sys from typing import Any, Protocol, TypeVar, type_check_only -from typing_extensions import Self __all__ = ["Error", "copy", "deepcopy"] _T = TypeVar("_T") -_SR = TypeVar("_SR", bound=_SupportsReplace) +_RT_co = TypeVar("_RT_co", covariant=True) @type_check_only -class _SupportsReplace(Protocol): - # In reality doesn't support args, but there's no other great way to express this. - def __replace__(self, *args: Any, **kwargs: Any) -> Self: ... +class _SupportsReplace(Protocol[_RT_co]): + # In reality doesn't support args, but there's no great way to express this. + def __replace__(self, /, *_: Any, **changes: Any) -> _RT_co: ... # None in CPython but non-None in Jython PyStringMap: Any @@ -21,7 +20,8 @@ def copy(x: _T) -> _T: ... if sys.version_info >= (3, 13): __all__ += ["replace"] - def replace(obj: _SR, /, **changes: Any) -> _SR: ... + # The types accepted by `**changes` match those of `obj.__replace__`. + def replace(obj: _SupportsReplace[_RT_co], /, **changes: Any) -> _RT_co: ... class Error(Exception): ... diff --git a/mypy/typeshed/stdlib/curses/__init__.pyi b/mypy/typeshed/stdlib/curses/__init__.pyi index 2c0231c13087e..3e32487ad99f2 100644 --- a/mypy/typeshed/stdlib/curses/__init__.pyi +++ b/mypy/typeshed/stdlib/curses/__init__.pyi @@ -14,8 +14,9 @@ _T = TypeVar("_T") _P = ParamSpec("_P") # available after calling `curses.initscr()` -LINES: Final[int] -COLS: Final[int] +# not `Final` as it can change during the terminal resize: +LINES: int +COLS: int # available after calling `curses.start_color()` COLORS: Final[int] diff --git a/mypy/typeshed/stdlib/dbm/sqlite3.pyi b/mypy/typeshed/stdlib/dbm/sqlite3.pyi index 446a0cf155fa7..e2fba93b20017 100644 --- a/mypy/typeshed/stdlib/dbm/sqlite3.pyi +++ b/mypy/typeshed/stdlib/dbm/sqlite3.pyi @@ -26,4 +26,4 @@ class _Database(MutableMapping[bytes, bytes]): def __enter__(self) -> Self: ... def __exit__(self, *args: Unused) -> None: ... -def open(filename: StrOrBytesPath, /, flag: Literal["r", "w,", "c", "n"] = "r", mode: int = 0o666) -> _Database: ... +def open(filename: StrOrBytesPath, /, flag: Literal["r", "w", "c", "n"] = "r", mode: int = 0o666) -> _Database: ... diff --git a/mypy/typeshed/stdlib/email/headerregistry.pyi b/mypy/typeshed/stdlib/email/headerregistry.pyi index dff9593b731f5..bea68307e0091 100644 --- a/mypy/typeshed/stdlib/email/headerregistry.pyi +++ b/mypy/typeshed/stdlib/email/headerregistry.pyi @@ -41,7 +41,7 @@ class DateHeader: max_count: ClassVar[Literal[1] | None] def init(self, name: str, *, parse_tree: TokenList, defects: Iterable[MessageDefect], datetime: _datetime) -> None: ... @property - def datetime(self) -> _datetime: ... + def datetime(self) -> _datetime | None: ... @staticmethod def value_parser(value: str) -> UnstructuredTokenList: ... @classmethod diff --git a/mypy/typeshed/stdlib/errno.pyi b/mypy/typeshed/stdlib/errno.pyi index 4f19b5aee87e4..e025e1fd13b9b 100644 --- a/mypy/typeshed/stdlib/errno.pyi +++ b/mypy/typeshed/stdlib/errno.pyi @@ -121,6 +121,7 @@ if sys.platform == "darwin": ESHLIBVERS: Final[int] if sys.version_info >= (3, 11): EQFULL: Final[int] + ENOTCAPABLE: Final[int] # available starting with 3.11.1 if sys.platform != "darwin": EDEADLOCK: Final[int] diff --git a/mypy/typeshed/stdlib/faulthandler.pyi b/mypy/typeshed/stdlib/faulthandler.pyi index 8f93222c9936e..33d08995eb759 100644 --- a/mypy/typeshed/stdlib/faulthandler.pyi +++ b/mypy/typeshed/stdlib/faulthandler.pyi @@ -9,7 +9,13 @@ if sys.version_info >= (3, 14): def dump_c_stack(file: FileDescriptorLike = ...) -> None: ... def dump_traceback_later(timeout: float, repeat: bool = ..., file: FileDescriptorLike = ..., exit: bool = ...) -> None: ... -def enable(file: FileDescriptorLike = ..., all_threads: bool = ...) -> None: ... + +if sys.version_info >= (3, 14): + def enable(file: FileDescriptorLike = ..., all_threads: bool = ..., c_stack: bool = True) -> None: ... + +else: + def enable(file: FileDescriptorLike = ..., all_threads: bool = ...) -> None: ... + def is_enabled() -> bool: ... if sys.platform != "win32": diff --git a/mypy/typeshed/stdlib/importlib/resources/_common.pyi b/mypy/typeshed/stdlib/importlib/resources/_common.pyi index 3dd961bb657b1..11a93ca82d8df 100644 --- a/mypy/typeshed/stdlib/importlib/resources/_common.pyi +++ b/mypy/typeshed/stdlib/importlib/resources/_common.pyi @@ -21,7 +21,7 @@ if sys.version_info >= (3, 11): @overload def files(anchor: Anchor | None = None) -> Traversable: ... @overload - @deprecated("First parameter to files is renamed to 'anchor'") + @deprecated("Deprecated since Python 3.12; will be removed in Python 3.15. Use `anchor` parameter instead.") def files(package: Anchor | None = None) -> Traversable: ... else: diff --git a/mypy/typeshed/stdlib/multiprocessing/shared_memory.pyi b/mypy/typeshed/stdlib/multiprocessing/shared_memory.pyi index 1a12812c27e4d..f75a372a69a2d 100644 --- a/mypy/typeshed/stdlib/multiprocessing/shared_memory.pyi +++ b/mypy/typeshed/stdlib/multiprocessing/shared_memory.pyi @@ -15,7 +15,7 @@ class SharedMemory: def __init__(self, name: str | None = None, create: bool = False, size: int = 0) -> None: ... @property - def buf(self) -> memoryview: ... + def buf(self) -> memoryview | None: ... @property def name(self) -> str: ... @property diff --git a/mypy/typeshed/stdlib/tkinter/__init__.pyi b/mypy/typeshed/stdlib/tkinter/__init__.pyi index 54dd70baf1996..b653545f1d9ca 100644 --- a/mypy/typeshed/stdlib/tkinter/__init__.pyi +++ b/mypy/typeshed/stdlib/tkinter/__init__.pyi @@ -173,21 +173,8 @@ EXCEPTION: Final = _tkinter.EXCEPTION # # You can also read the manual pages online: https://www.tcl.tk/doc/ -# Some widgets have an option named -compound that accepts different values -# than the _Compound defined here. Many other options have similar things. -_Anchor: TypeAlias = Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] # manual page: Tk_GetAnchor -_ButtonCommand: TypeAlias = str | Callable[[], Any] # accepts string of tcl code, return value is returned from Button.invoke() -_Compound: TypeAlias = Literal["top", "left", "center", "right", "bottom", "none"] # -compound in manual page named 'options' # manual page: Tk_GetCursor _Cursor: TypeAlias = str | tuple[str] | tuple[str, str] | tuple[str, str, str] | tuple[str, str, str, str] -# example when it's sequence: entry['invalidcommand'] = [entry.register(print), '%P'] -_EntryValidateCommand: TypeAlias = str | list[str] | tuple[str, ...] | Callable[[], bool] -_ImageSpec: TypeAlias = _Image | str # str can be from e.g. tkinter.image_names() -_Relief: TypeAlias = Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] # manual page: Tk_GetRelief -_ScreenUnits: TypeAlias = str | float # Often the right type instead of int. Manual page: Tk_GetPixels -# -xscrollcommand and -yscrollcommand in 'options' manual page -_XYScrollCommand: TypeAlias = str | Callable[[float, float], object] -_TakeFocusValue: TypeAlias = bool | Literal[0, 1, ""] | Callable[[str], bool | None] # -takefocus in manual page named 'options' if sys.version_info >= (3, 11): @type_check_only @@ -333,12 +320,12 @@ class Variable: @deprecated("Deprecated since Python 3.14. Use `trace_remove()` instead.") def trace_vdelete(self, mode, cbname) -> None: ... @deprecated("Deprecated since Python 3.14. Use `trace_info()` instead.") - def trace_vinfo(self): ... + def trace_vinfo(self) -> list[Incomplete]: ... else: def trace(self, mode, callback) -> str: ... def trace_variable(self, mode, callback) -> str: ... def trace_vdelete(self, mode, cbname) -> None: ... - def trace_vinfo(self): ... + def trace_vinfo(self) -> list[Incomplete]: ... def __eq__(self, other: object) -> bool: ... def __del__(self) -> None: ... @@ -373,14 +360,14 @@ def mainloop(n: int = 0) -> None: ... getint = int getdouble = float -def getboolean(s): ... +def getboolean(s) -> bool: ... _Ts = TypeVarTuple("_Ts") @type_check_only class _GridIndexInfo(TypedDict, total=False): - minsize: _ScreenUnits - pad: _ScreenUnits + minsize: float | str + pad: float | str uniform: str | None weight: int @@ -403,9 +390,9 @@ class Misc: def wait_visibility(self, window: Misc | None = None) -> None: ... def setvar(self, name: str = "PY_VAR", value: str = "1") -> None: ... def getvar(self, name: str = "PY_VAR"): ... - def getint(self, s): ... - def getdouble(self, s): ... - def getboolean(self, s): ... + def getint(self, s) -> int: ... + def getdouble(self, s) -> float: ... + def getboolean(self, s) -> bool: ... def focus_set(self) -> None: ... focus = focus_set def focus_force(self) -> None: ... @@ -473,13 +460,13 @@ class Misc: def winfo_atom(self, name: str, displayof: Literal[0] | Misc | None = 0) -> int: ... def winfo_atomname(self, id: int, displayof: Literal[0] | Misc | None = 0) -> str: ... def winfo_cells(self) -> int: ... - def winfo_children(self) -> list[Widget]: ... # Widget because it can't be Toplevel or Tk + def winfo_children(self) -> list[Widget | Toplevel]: ... def winfo_class(self) -> str: ... def winfo_colormapfull(self) -> bool: ... def winfo_containing(self, rootX: int, rootY: int, displayof: Literal[0] | Misc | None = 0) -> Misc | None: ... def winfo_depth(self) -> int: ... def winfo_exists(self) -> bool: ... - def winfo_fpixels(self, number: _ScreenUnits) -> float: ... + def winfo_fpixels(self, number: float | str) -> float: ... def winfo_geometry(self) -> str: ... def winfo_height(self) -> int: ... def winfo_id(self) -> int: ... @@ -489,7 +476,7 @@ class Misc: def winfo_name(self) -> str: ... def winfo_parent(self) -> str: ... # return value needs nametowidget() def winfo_pathname(self, id: int, displayof: Literal[0] | Misc | None = 0): ... - def winfo_pixels(self, number: _ScreenUnits) -> int: ... + def winfo_pixels(self, number: float | str) -> int: ... def winfo_pointerx(self) -> int: ... def winfo_pointerxy(self) -> tuple[int, int]: ... def winfo_pointery(self) -> int: ... @@ -580,7 +567,7 @@ class Misc: @overload def pack_propagate(self) -> None: ... propagate = pack_propagate - def grid_anchor(self, anchor: _Anchor | None = None) -> None: ... + def grid_anchor(self, anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] | None = None) -> None: ... anchor = grid_anchor @overload def grid_bbox( @@ -596,8 +583,8 @@ class Misc: index: int | str | list[int] | tuple[int, ...], cnf: _GridIndexInfo = {}, *, - minsize: _ScreenUnits = ..., - pad: _ScreenUnits = ..., + minsize: float | str = ..., + pad: float | str = ..., uniform: str = ..., weight: int = ..., ) -> _GridIndexInfo | MaybeNone: ... # can be None but annoying to check @@ -606,14 +593,14 @@ class Misc: index: int | str | list[int] | tuple[int, ...], cnf: _GridIndexInfo = {}, *, - minsize: _ScreenUnits = ..., - pad: _ScreenUnits = ..., + minsize: float | str = ..., + pad: float | str = ..., uniform: str = ..., weight: int = ..., ) -> _GridIndexInfo | MaybeNone: ... # can be None but annoying to check columnconfigure = grid_columnconfigure rowconfigure = grid_rowconfigure - def grid_location(self, x: _ScreenUnits, y: _ScreenUnits) -> tuple[int, int]: ... + def grid_location(self, x: float | str, y: float | str) -> tuple[int, int]: ... @overload def grid_propagate(self, flag: bool) -> None: ... @overload @@ -632,32 +619,32 @@ class Misc: sequence: str, *, above: Misc | int = ..., - borderwidth: _ScreenUnits = ..., + borderwidth: float | str = ..., button: int = ..., count: int = ..., data: Any = ..., # anything with usable str() value delta: int = ..., detail: str = ..., focus: bool = ..., - height: _ScreenUnits = ..., + height: float | str = ..., keycode: int = ..., keysym: str = ..., mode: str = ..., override: bool = ..., place: Literal["PlaceOnTop", "PlaceOnBottom"] = ..., root: Misc | int = ..., - rootx: _ScreenUnits = ..., - rooty: _ScreenUnits = ..., + rootx: float | str = ..., + rooty: float | str = ..., sendevent: bool = ..., serial: int = ..., state: int | str = ..., subwindow: Misc | int = ..., time: int = ..., warp: bool = ..., - width: _ScreenUnits = ..., + width: float | str = ..., when: Literal["now", "tail", "head", "mark"] = ..., - x: _ScreenUnits = ..., - y: _ScreenUnits = ..., + x: float | str = ..., + y: float | str = ..., ) -> None: ... def event_info(self, virtual: str | None = None) -> tuple[str, ...]: ... def image_names(self) -> tuple[str, ...]: ... @@ -681,23 +668,23 @@ class XView: @overload def xview(self) -> tuple[float, float]: ... @overload - def xview(self, *args): ... + def xview(self, *args) -> None: ... def xview_moveto(self, fraction: float) -> None: ... @overload def xview_scroll(self, number: int, what: Literal["units", "pages"]) -> None: ... @overload - def xview_scroll(self, number: _ScreenUnits, what: Literal["pixels"]) -> None: ... + def xview_scroll(self, number: float | str, what: Literal["pixels"]) -> None: ... class YView: @overload def yview(self) -> tuple[float, float]: ... @overload - def yview(self, *args): ... + def yview(self, *args) -> None: ... def yview_moveto(self, fraction: float) -> None: ... @overload def yview_scroll(self, number: int, what: Literal["units", "pages"]) -> None: ... @overload - def yview_scroll(self, number: _ScreenUnits, what: Literal["pixels"]) -> None: ... + def yview_scroll(self, number: float | str, what: Literal["pixels"]) -> None: ... if sys.platform == "darwin": @type_check_only @@ -993,21 +980,21 @@ class Tk(Misc, Wm): cnf: dict[str, Any] | None = None, *, background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., cursor: _Cursor = ..., - height: _ScreenUnits = ..., + height: float | str = ..., highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., + highlightthickness: float | str = ..., menu: Menu = ..., - padx: _ScreenUnits = ..., - pady: _ScreenUnits = ..., - relief: _Relief = ..., - takefocus: _TakeFocusValue = ..., - width: _ScreenUnits = ..., + padx: float | str = ..., + pady: float | str = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., + width: float | str = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -1018,27 +1005,27 @@ class Tk(Misc, Wm): # Tk has __getattr__ so that tk_instance.foo falls back to tk_instance.tk.foo # Please keep in sync with _tkinter.TkappType. # Some methods are intentionally missing because they are inherited from Misc instead. - def adderrorinfo(self, msg, /): ... + def adderrorinfo(self, msg: str, /): ... def call(self, command: Any, /, *args: Any) -> Any: ... - def createcommand(self, name, func, /): ... + def createcommand(self, name: str, func, /): ... if sys.platform != "win32": - def createfilehandler(self, file, mask, func, /): ... - def deletefilehandler(self, file, /): ... + def createfilehandler(self, file, mask: int, func, /): ... + def deletefilehandler(self, file, /) -> None: ... - def createtimerhandler(self, milliseconds, func, /): ... - def dooneevent(self, flags: int = ..., /): ... + def createtimerhandler(self, milliseconds: int, func, /): ... + def dooneevent(self, flags: int = 0, /): ... def eval(self, script: str, /) -> str: ... - def evalfile(self, fileName, /): ... - def exprboolean(self, s, /): ... - def exprdouble(self, s, /): ... - def exprlong(self, s, /): ... - def exprstring(self, s, /): ... + def evalfile(self, fileName: str, /): ... + def exprboolean(self, s: str, /): ... + def exprdouble(self, s: str, /): ... + def exprlong(self, s: str, /): ... + def exprstring(self, s: str, /): ... def globalgetvar(self, *args, **kwargs): ... def globalsetvar(self, *args, **kwargs): ... def globalunsetvar(self, *args, **kwargs): ... def interpaddr(self) -> int: ... def loadtk(self) -> None: ... - def record(self, script, /): ... + def record(self, script: str, /): ... if sys.version_info < (3, 11): @deprecated("Deprecated since Python 3.9; removed in Python 3.11. Use `splitlist()` instead.") def split(self, arg, /): ... @@ -1046,7 +1033,7 @@ class Tk(Misc, Wm): def splitlist(self, arg, /): ... def unsetvar(self, *args, **kwargs): ... def wantobjects(self, *args, **kwargs): ... - def willdispatch(self): ... + def willdispatch(self) -> None: ... def Tcl(screenName: str | None = None, baseName: str | None = None, className: str = "Tk", useTk: bool = False) -> Tk: ... @@ -1056,11 +1043,11 @@ _InMiscNonTotal = TypedDict("_InMiscNonTotal", {"in": Misc}, total=False) @type_check_only class _PackInfo(_InMiscTotal): # 'before' and 'after' never appear in _PackInfo - anchor: _Anchor + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] expand: bool fill: Literal["none", "x", "y", "both"] side: Literal["left", "right", "top", "bottom"] - # Paddings come out as int or tuple of int, even though any _ScreenUnits + # Paddings come out as int or tuple of int, even though any screen units # can be specified in pack(). ipadx: int ipady: int @@ -1069,7 +1056,7 @@ class _PackInfo(_InMiscTotal): class Pack: # _PackInfo is not the valid type for cnf because pad stuff accepts any - # _ScreenUnits instead of int only. I didn't bother to create another + # screen units instead of int only. I didn't bother to create another # TypedDict for cnf because it appears to be a legacy thing that was # replaced by **kwargs. def pack_configure( @@ -1077,15 +1064,15 @@ class Pack: cnf: Mapping[str, Any] | None = {}, *, after: Misc = ..., - anchor: _Anchor = ..., + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = ..., before: Misc = ..., expand: bool | Literal[0, 1] = 0, fill: Literal["none", "x", "y", "both"] = ..., side: Literal["left", "right", "top", "bottom"] = ..., - ipadx: _ScreenUnits = ..., - ipady: _ScreenUnits = ..., - padx: _ScreenUnits | tuple[_ScreenUnits, _ScreenUnits] = ..., - pady: _ScreenUnits | tuple[_ScreenUnits, _ScreenUnits] = ..., + ipadx: float | str = ..., + ipady: float | str = ..., + padx: float | str | tuple[float | str, float | str] = ..., + pady: float | str | tuple[float | str, float | str] = ..., in_: Misc = ..., **kw: Any, # allow keyword argument named 'in', see #4836 ) -> None: ... @@ -1097,7 +1084,7 @@ class Pack: @type_check_only class _PlaceInfo(_InMiscNonTotal): # empty dict if widget hasn't been placed - anchor: _Anchor + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] bordermode: Literal["inside", "outside", "ignore"] width: str # can be int()ed (even after e.g. widget.place(height='2.3c') or similar) height: str # can be int()ed @@ -1113,12 +1100,12 @@ class Place: self, cnf: Mapping[str, Any] | None = {}, *, - anchor: _Anchor = ..., + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = ..., bordermode: Literal["inside", "outside", "ignore"] = ..., - width: _ScreenUnits = ..., - height: _ScreenUnits = ..., - x: _ScreenUnits = ..., - y: _ScreenUnits = ..., + width: float | str = ..., + height: float | str = ..., + x: float | str = ..., + y: float | str = ..., # str allowed for compatibility with place_info() relheight: str | float = ..., relwidth: str | float = ..., @@ -1153,10 +1140,10 @@ class Grid: columnspan: int = ..., row: int = ..., rowspan: int = ..., - ipadx: _ScreenUnits = ..., - ipady: _ScreenUnits = ..., - padx: _ScreenUnits | tuple[_ScreenUnits, _ScreenUnits] = ..., - pady: _ScreenUnits | tuple[_ScreenUnits, _ScreenUnits] = ..., + ipadx: float | str = ..., + ipady: float | str = ..., + padx: float | str | tuple[float | str, float | str] = ..., + pady: float | str | tuple[float | str, float | str] = ..., sticky: str = ..., # consists of letters 'n', 's', 'w', 'e', may contain repeats, may be empty in_: Misc = ..., **kw: Any, # allow keyword argument named 'in', see #4836 @@ -1170,8 +1157,8 @@ class Grid: class BaseWidget(Misc): master: Misc - widgetName: Incomplete - def __init__(self, master, widgetName, cnf={}, kw={}, extra=()) -> None: ... + widgetName: str + def __init__(self, master, widgetName: str, cnf={}, kw={}, extra=()) -> None: ... def destroy(self) -> None: ... # This class represents any widget except Toplevel or Tk. @@ -1201,28 +1188,28 @@ class Toplevel(BaseWidget, Wm): cnf: dict[str, Any] | None = {}, *, background: str = ..., - bd: _ScreenUnits = 0, + bd: float | str = 0, bg: str = ..., - border: _ScreenUnits = 0, - borderwidth: _ScreenUnits = 0, + border: float | str = 0, + borderwidth: float | str = 0, class_: str = "Toplevel", colormap: Literal["new", ""] | Misc = "", container: bool = False, cursor: _Cursor = "", - height: _ScreenUnits = 0, + height: float | str = 0, highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = 0, + highlightthickness: float | str = 0, menu: Menu = ..., name: str = ..., - padx: _ScreenUnits = 0, - pady: _ScreenUnits = 0, - relief: _Relief = "flat", + padx: float | str = 0, + pady: float | str = 0, + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = "flat", screen: str = "", # can't be changed after creating widget - takefocus: _TakeFocusValue = 0, + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = 0, use: int = ..., visual: str | tuple[str, int] = "", - width: _ScreenUnits = 0, + width: float | str = 0, ) -> None: ... @overload def configure( @@ -1230,21 +1217,21 @@ class Toplevel(BaseWidget, Wm): cnf: dict[str, Any] | None = None, *, background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., cursor: _Cursor = ..., - height: _ScreenUnits = ..., + height: float | str = ..., highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., + highlightthickness: float | str = ..., menu: Menu = ..., - padx: _ScreenUnits = ..., - pady: _ScreenUnits = ..., - relief: _Relief = ..., - takefocus: _TakeFocusValue = ..., - width: _ScreenUnits = ..., + padx: float | str = ..., + pady: float | str = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., + width: float | str = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -1258,15 +1245,15 @@ class Button(Widget): *, activebackground: str = ..., activeforeground: str = ..., - anchor: _Anchor = "center", + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = "center", background: str = ..., - bd: _ScreenUnits = ..., # same as borderwidth + bd: float | str = ..., # same as borderwidth bg: str = ..., # same as background bitmap: str = "", - border: _ScreenUnits = ..., # same as borderwidth - borderwidth: _ScreenUnits = ..., - command: _ButtonCommand = "", - compound: _Compound = "none", + border: float | str = ..., # same as borderwidth + borderwidth: float | str = ..., + command: str | Callable[[], Any] = "", + compound: Literal["top", "left", "center", "right", "bottom", "none"] = "none", cursor: _Cursor = "", default: Literal["normal", "active", "disabled"] = "disabled", disabledforeground: str = ..., @@ -1274,30 +1261,30 @@ class Button(Widget): font: _FontDescription = "TkDefaultFont", foreground: str = ..., # width and height must be int for buttons containing just text, but - # ints are also valid _ScreenUnits - height: _ScreenUnits = 0, + # buttons with an image accept any screen units. + height: float | str = 0, highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = 1, - image: _ImageSpec = "", + highlightthickness: float | str = 1, + image: _Image | str = "", justify: Literal["left", "center", "right"] = "center", name: str = ..., - overrelief: _Relief | Literal[""] = "", - padx: _ScreenUnits = ..., - pady: _ScreenUnits = ..., - relief: _Relief = ..., + overrelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove", ""] = "", + padx: float | str = ..., + pady: float | str = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., repeatdelay: int = ..., repeatinterval: int = ..., state: Literal["normal", "active", "disabled"] = "normal", - takefocus: _TakeFocusValue = "", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = "", text: float | str = "", # We allow the textvariable to be any Variable, not necessarily # StringVar. This is useful for e.g. a button that displays the value # of an IntVar. textvariable: Variable = ..., underline: int = -1, - width: _ScreenUnits = 0, - wraplength: _ScreenUnits = 0, + width: float | str = 0, + wraplength: float | str = 0, ) -> None: ... @overload def configure( @@ -1306,40 +1293,40 @@ class Button(Widget): *, activebackground: str = ..., activeforeground: str = ..., - anchor: _Anchor = ..., + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = ..., background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., bitmap: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., - command: _ButtonCommand = ..., - compound: _Compound = ..., + border: float | str = ..., + borderwidth: float | str = ..., + command: str | Callable[[], Any] = ..., + compound: Literal["top", "left", "center", "right", "bottom", "none"] = ..., cursor: _Cursor = ..., default: Literal["normal", "active", "disabled"] = ..., disabledforeground: str = ..., fg: str = ..., font: _FontDescription = ..., foreground: str = ..., - height: _ScreenUnits = ..., + height: float | str = ..., highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., - image: _ImageSpec = ..., + highlightthickness: float | str = ..., + image: _Image | str = ..., justify: Literal["left", "center", "right"] = ..., - overrelief: _Relief | Literal[""] = ..., - padx: _ScreenUnits = ..., - pady: _ScreenUnits = ..., - relief: _Relief = ..., + overrelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove", ""] = ..., + padx: float | str = ..., + pady: float | str = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., repeatdelay: int = ..., repeatinterval: int = ..., state: Literal["normal", "active", "disabled"] = ..., - takefocus: _TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., text: float | str = ..., textvariable: Variable = ..., underline: int = ..., - width: _ScreenUnits = ..., - wraplength: _ScreenUnits = ..., + width: float | str = ..., + wraplength: float | str = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -1354,41 +1341,39 @@ class Canvas(Widget, XView, YView): cnf: dict[str, Any] | None = {}, *, background: str = ..., - bd: _ScreenUnits = 0, + bd: float | str = 0, bg: str = ..., - border: _ScreenUnits = 0, - borderwidth: _ScreenUnits = 0, + border: float | str = 0, + borderwidth: float | str = 0, closeenough: float = 1.0, confine: bool = True, cursor: _Cursor = "", - # canvas manual page has a section named COORDINATES, and the first - # part of it describes _ScreenUnits. - height: _ScreenUnits = ..., + height: float | str = ..., # see COORDINATES in canvas manual page highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., + highlightthickness: float | str = ..., insertbackground: str = ..., - insertborderwidth: _ScreenUnits = 0, + insertborderwidth: float | str = 0, insertofftime: int = 300, insertontime: int = 600, - insertwidth: _ScreenUnits = 2, + insertwidth: float | str = 2, name: str = ..., offset=..., # undocumented - relief: _Relief = "flat", + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = "flat", # Setting scrollregion to None doesn't reset it back to empty, # but setting it to () does. - scrollregion: tuple[_ScreenUnits, _ScreenUnits, _ScreenUnits, _ScreenUnits] | tuple[()] = (), + scrollregion: tuple[float | str, float | str, float | str, float | str] | tuple[()] = (), selectbackground: str = ..., - selectborderwidth: _ScreenUnits = 1, + selectborderwidth: float | str = 1, selectforeground: str = ..., # man page says that state can be 'hidden', but it can't state: Literal["normal", "disabled"] = "normal", - takefocus: _TakeFocusValue = "", - width: _ScreenUnits = ..., - xscrollcommand: _XYScrollCommand = "", - xscrollincrement: _ScreenUnits = 0, - yscrollcommand: _XYScrollCommand = "", - yscrollincrement: _ScreenUnits = 0, + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = "", + width: float | str = ..., + xscrollcommand: str | Callable[[float, float], object] = "", + xscrollincrement: float | str = 0, + yscrollcommand: str | Callable[[float, float], object] = "", + yscrollincrement: float | str = 0, ) -> None: ... @overload def configure( @@ -1396,35 +1381,35 @@ class Canvas(Widget, XView, YView): cnf: dict[str, Any] | None = None, *, background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., closeenough: float = ..., confine: bool = ..., cursor: _Cursor = ..., - height: _ScreenUnits = ..., + height: float | str = ..., highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., + highlightthickness: float | str = ..., insertbackground: str = ..., - insertborderwidth: _ScreenUnits = ..., + insertborderwidth: float | str = ..., insertofftime: int = ..., insertontime: int = ..., - insertwidth: _ScreenUnits = ..., + insertwidth: float | str = ..., offset=..., # undocumented - relief: _Relief = ..., - scrollregion: tuple[_ScreenUnits, _ScreenUnits, _ScreenUnits, _ScreenUnits] | tuple[()] = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., + scrollregion: tuple[float | str, float | str, float | str, float | str] | tuple[()] = ..., selectbackground: str = ..., - selectborderwidth: _ScreenUnits = ..., + selectborderwidth: float | str = ..., selectforeground: str = ..., state: Literal["normal", "disabled"] = ..., - takefocus: _TakeFocusValue = ..., - width: _ScreenUnits = ..., - xscrollcommand: _XYScrollCommand = ..., - xscrollincrement: _ScreenUnits = ..., - yscrollcommand: _XYScrollCommand = ..., - yscrollincrement: _ScreenUnits = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., + width: float | str = ..., + xscrollcommand: str | Callable[[float, float], object] = ..., + xscrollincrement: float | str = ..., + yscrollcommand: str | Callable[[float, float], object] = ..., + yscrollincrement: float | str = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -1434,20 +1419,20 @@ class Canvas(Widget, XView, YView): def addtag_all(self, newtag: str) -> None: ... def addtag_below(self, newtag: str, tagOrId: str | int) -> None: ... def addtag_closest( - self, newtag: str, x: _ScreenUnits, y: _ScreenUnits, halo: _ScreenUnits | None = None, start: str | int | None = None + self, newtag: str, x: float | str, y: float | str, halo: float | str | None = None, start: str | int | None = None ) -> None: ... - def addtag_enclosed(self, newtag: str, x1: _ScreenUnits, y1: _ScreenUnits, x2: _ScreenUnits, y2: _ScreenUnits) -> None: ... - def addtag_overlapping(self, newtag: str, x1: _ScreenUnits, y1: _ScreenUnits, x2: _ScreenUnits, y2: _ScreenUnits) -> None: ... + def addtag_enclosed(self, newtag: str, x1: float | str, y1: float | str, x2: float | str, y2: float | str) -> None: ... + def addtag_overlapping(self, newtag: str, x1: float | str, y1: float | str, x2: float | str, y2: float | str) -> None: ... def addtag_withtag(self, newtag: str, tagOrId: str | int) -> None: ... def find(self, *args): ... # internal method def find_above(self, tagOrId: str | int) -> tuple[int, ...]: ... def find_all(self) -> tuple[int, ...]: ... def find_below(self, tagOrId: str | int) -> tuple[int, ...]: ... def find_closest( - self, x: _ScreenUnits, y: _ScreenUnits, halo: _ScreenUnits | None = None, start: str | int | None = None + self, x: float | str, y: float | str, halo: float | str | None = None, start: str | int | None = None ) -> tuple[int, ...]: ... - def find_enclosed(self, x1: _ScreenUnits, y1: _ScreenUnits, x2: _ScreenUnits, y2: _ScreenUnits) -> tuple[int, ...]: ... - def find_overlapping(self, x1: _ScreenUnits, y1: _ScreenUnits, x2: _ScreenUnits, y2: float) -> tuple[int, ...]: ... + def find_enclosed(self, x1: float | str, y1: float | str, x2: float | str, y2: float | str) -> tuple[int, ...]: ... + def find_overlapping(self, x1: float | str, y1: float | str, x2: float | str, y2: float) -> tuple[int, ...]: ... def find_withtag(self, tagOrId: str | int) -> tuple[int, ...]: ... # Incompatible with Misc.bbox(), tkinter violates LSP def bbox(self, *args: str | int) -> tuple[int, int, int, int]: ... # type: ignore[override] @@ -1492,25 +1477,25 @@ class Canvas(Widget, XView, YView): activedash: str | int | list[int] | tuple[int, ...] = ..., activefill: str = ..., activestipple: str = ..., - activewidth: _ScreenUnits = ..., + activewidth: float | str = ..., arrow: Literal["first", "last", "both"] = ..., arrowshape: tuple[float, float, float] = ..., capstyle: Literal["round", "projecting", "butt"] = ..., dash: str | int | list[int] | tuple[int, ...] = ..., - dashoffset: _ScreenUnits = ..., + dashoffset: float | str = ..., disableddash: str | int | list[int] | tuple[int, ...] = ..., disabledfill: str = ..., disabledstipple: str = ..., - disabledwidth: _ScreenUnits = ..., + disabledwidth: float | str = ..., fill: str = ..., joinstyle: Literal["round", "bevel", "miter"] = ..., - offset: _ScreenUnits = ..., + offset: float | str = ..., smooth: bool = ..., splinesteps: float = ..., state: Literal["normal", "hidden", "disabled"] = ..., stipple: str = ..., tags: str | list[str] | tuple[str, ...] = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> int: ... @overload def create_line( @@ -1522,25 +1507,25 @@ class Canvas(Widget, XView, YView): activedash: str | int | list[int] | tuple[int, ...] = ..., activefill: str = ..., activestipple: str = ..., - activewidth: _ScreenUnits = ..., + activewidth: float | str = ..., arrow: Literal["first", "last", "both"] = ..., arrowshape: tuple[float, float, float] = ..., capstyle: Literal["round", "projecting", "butt"] = ..., dash: str | int | list[int] | tuple[int, ...] = ..., - dashoffset: _ScreenUnits = ..., + dashoffset: float | str = ..., disableddash: str | int | list[int] | tuple[int, ...] = ..., disabledfill: str = ..., disabledstipple: str = ..., - disabledwidth: _ScreenUnits = ..., + disabledwidth: float | str = ..., fill: str = ..., joinstyle: Literal["round", "bevel", "miter"] = ..., - offset: _ScreenUnits = ..., + offset: float | str = ..., smooth: bool = ..., splinesteps: float = ..., state: Literal["normal", "hidden", "disabled"] = ..., stipple: str = ..., tags: str | list[str] | tuple[str, ...] = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> int: ... @overload def create_line( @@ -1558,25 +1543,25 @@ class Canvas(Widget, XView, YView): activedash: str | int | list[int] | tuple[int, ...] = ..., activefill: str = ..., activestipple: str = ..., - activewidth: _ScreenUnits = ..., + activewidth: float | str = ..., arrow: Literal["first", "last", "both"] = ..., arrowshape: tuple[float, float, float] = ..., capstyle: Literal["round", "projecting", "butt"] = ..., dash: str | int | list[int] | tuple[int, ...] = ..., - dashoffset: _ScreenUnits = ..., + dashoffset: float | str = ..., disableddash: str | int | list[int] | tuple[int, ...] = ..., disabledfill: str = ..., disabledstipple: str = ..., - disabledwidth: _ScreenUnits = ..., + disabledwidth: float | str = ..., fill: str = ..., joinstyle: Literal["round", "bevel", "miter"] = ..., - offset: _ScreenUnits = ..., + offset: float | str = ..., smooth: bool = ..., splinesteps: float = ..., state: Literal["normal", "hidden", "disabled"] = ..., stipple: str = ..., tags: str | list[str] | tuple[str, ...] = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> int: ... @overload def create_oval( @@ -1592,24 +1577,24 @@ class Canvas(Widget, XView, YView): activeoutline: str = ..., activeoutlinestipple: str = ..., activestipple: str = ..., - activewidth: _ScreenUnits = ..., + activewidth: float | str = ..., dash: str | int | list[int] | tuple[int, ...] = ..., - dashoffset: _ScreenUnits = ..., + dashoffset: float | str = ..., disableddash: str | int | list[int] | tuple[int, ...] = ..., disabledfill: str = ..., disabledoutline: str = ..., disabledoutlinestipple: str = ..., disabledstipple: str = ..., - disabledwidth: _ScreenUnits = ..., + disabledwidth: float | str = ..., fill: str = ..., - offset: _ScreenUnits = ..., + offset: float | str = ..., outline: str = ..., - outlineoffset: _ScreenUnits = ..., + outlineoffset: float | str = ..., outlinestipple: str = ..., state: Literal["normal", "hidden", "disabled"] = ..., stipple: str = ..., tags: str | list[str] | tuple[str, ...] = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> int: ... @overload def create_oval( @@ -1623,24 +1608,24 @@ class Canvas(Widget, XView, YView): activeoutline: str = ..., activeoutlinestipple: str = ..., activestipple: str = ..., - activewidth: _ScreenUnits = ..., + activewidth: float | str = ..., dash: str | int | list[int] | tuple[int, ...] = ..., - dashoffset: _ScreenUnits = ..., + dashoffset: float | str = ..., disableddash: str | int | list[int] | tuple[int, ...] = ..., disabledfill: str = ..., disabledoutline: str = ..., disabledoutlinestipple: str = ..., disabledstipple: str = ..., - disabledwidth: _ScreenUnits = ..., + disabledwidth: float | str = ..., fill: str = ..., - offset: _ScreenUnits = ..., + offset: float | str = ..., outline: str = ..., - outlineoffset: _ScreenUnits = ..., + outlineoffset: float | str = ..., outlinestipple: str = ..., state: Literal["normal", "hidden", "disabled"] = ..., stipple: str = ..., tags: str | list[str] | tuple[str, ...] = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> int: ... @overload def create_oval( @@ -1660,24 +1645,24 @@ class Canvas(Widget, XView, YView): activeoutline: str = ..., activeoutlinestipple: str = ..., activestipple: str = ..., - activewidth: _ScreenUnits = ..., + activewidth: float | str = ..., dash: str | int | list[int] | tuple[int, ...] = ..., - dashoffset: _ScreenUnits = ..., + dashoffset: float | str = ..., disableddash: str | int | list[int] | tuple[int, ...] = ..., disabledfill: str = ..., disabledoutline: str = ..., disabledoutlinestipple: str = ..., disabledstipple: str = ..., - disabledwidth: _ScreenUnits = ..., + disabledwidth: float | str = ..., fill: str = ..., - offset: _ScreenUnits = ..., + offset: float | str = ..., outline: str = ..., - outlineoffset: _ScreenUnits = ..., + outlineoffset: float | str = ..., outlinestipple: str = ..., state: Literal["normal", "hidden", "disabled"] = ..., stipple: str = ..., tags: str | list[str] | tuple[str, ...] = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> int: ... @overload def create_polygon( @@ -1693,27 +1678,27 @@ class Canvas(Widget, XView, YView): activeoutline: str = ..., activeoutlinestipple: str = ..., activestipple: str = ..., - activewidth: _ScreenUnits = ..., + activewidth: float | str = ..., dash: str | int | list[int] | tuple[int, ...] = ..., - dashoffset: _ScreenUnits = ..., + dashoffset: float | str = ..., disableddash: str | int | list[int] | tuple[int, ...] = ..., disabledfill: str = ..., disabledoutline: str = ..., disabledoutlinestipple: str = ..., disabledstipple: str = ..., - disabledwidth: _ScreenUnits = ..., + disabledwidth: float | str = ..., fill: str = ..., joinstyle: Literal["round", "bevel", "miter"] = ..., - offset: _ScreenUnits = ..., + offset: float | str = ..., outline: str = ..., - outlineoffset: _ScreenUnits = ..., + outlineoffset: float | str = ..., outlinestipple: str = ..., smooth: bool = ..., splinesteps: float = ..., state: Literal["normal", "hidden", "disabled"] = ..., stipple: str = ..., tags: str | list[str] | tuple[str, ...] = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> int: ... @overload def create_polygon( @@ -1727,27 +1712,27 @@ class Canvas(Widget, XView, YView): activeoutline: str = ..., activeoutlinestipple: str = ..., activestipple: str = ..., - activewidth: _ScreenUnits = ..., + activewidth: float | str = ..., dash: str | int | list[int] | tuple[int, ...] = ..., - dashoffset: _ScreenUnits = ..., + dashoffset: float | str = ..., disableddash: str | int | list[int] | tuple[int, ...] = ..., disabledfill: str = ..., disabledoutline: str = ..., disabledoutlinestipple: str = ..., disabledstipple: str = ..., - disabledwidth: _ScreenUnits = ..., + disabledwidth: float | str = ..., fill: str = ..., joinstyle: Literal["round", "bevel", "miter"] = ..., - offset: _ScreenUnits = ..., + offset: float | str = ..., outline: str = ..., - outlineoffset: _ScreenUnits = ..., + outlineoffset: float | str = ..., outlinestipple: str = ..., smooth: bool = ..., splinesteps: float = ..., state: Literal["normal", "hidden", "disabled"] = ..., stipple: str = ..., tags: str | list[str] | tuple[str, ...] = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> int: ... @overload def create_polygon( @@ -1767,27 +1752,27 @@ class Canvas(Widget, XView, YView): activeoutline: str = ..., activeoutlinestipple: str = ..., activestipple: str = ..., - activewidth: _ScreenUnits = ..., + activewidth: float | str = ..., dash: str | int | list[int] | tuple[int, ...] = ..., - dashoffset: _ScreenUnits = ..., + dashoffset: float | str = ..., disableddash: str | int | list[int] | tuple[int, ...] = ..., disabledfill: str = ..., disabledoutline: str = ..., disabledoutlinestipple: str = ..., disabledstipple: str = ..., - disabledwidth: _ScreenUnits = ..., + disabledwidth: float | str = ..., fill: str = ..., joinstyle: Literal["round", "bevel", "miter"] = ..., - offset: _ScreenUnits = ..., + offset: float | str = ..., outline: str = ..., - outlineoffset: _ScreenUnits = ..., + outlineoffset: float | str = ..., outlinestipple: str = ..., smooth: bool = ..., splinesteps: float = ..., state: Literal["normal", "hidden", "disabled"] = ..., stipple: str = ..., tags: str | list[str] | tuple[str, ...] = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> int: ... @overload def create_rectangle( @@ -1803,24 +1788,24 @@ class Canvas(Widget, XView, YView): activeoutline: str = ..., activeoutlinestipple: str = ..., activestipple: str = ..., - activewidth: _ScreenUnits = ..., + activewidth: float | str = ..., dash: str | int | list[int] | tuple[int, ...] = ..., - dashoffset: _ScreenUnits = ..., + dashoffset: float | str = ..., disableddash: str | int | list[int] | tuple[int, ...] = ..., disabledfill: str = ..., disabledoutline: str = ..., disabledoutlinestipple: str = ..., disabledstipple: str = ..., - disabledwidth: _ScreenUnits = ..., + disabledwidth: float | str = ..., fill: str = ..., - offset: _ScreenUnits = ..., + offset: float | str = ..., outline: str = ..., - outlineoffset: _ScreenUnits = ..., + outlineoffset: float | str = ..., outlinestipple: str = ..., state: Literal["normal", "hidden", "disabled"] = ..., stipple: str = ..., tags: str | list[str] | tuple[str, ...] = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> int: ... @overload def create_rectangle( @@ -1834,24 +1819,24 @@ class Canvas(Widget, XView, YView): activeoutline: str = ..., activeoutlinestipple: str = ..., activestipple: str = ..., - activewidth: _ScreenUnits = ..., + activewidth: float | str = ..., dash: str | int | list[int] | tuple[int, ...] = ..., - dashoffset: _ScreenUnits = ..., + dashoffset: float | str = ..., disableddash: str | int | list[int] | tuple[int, ...] = ..., disabledfill: str = ..., disabledoutline: str = ..., disabledoutlinestipple: str = ..., disabledstipple: str = ..., - disabledwidth: _ScreenUnits = ..., + disabledwidth: float | str = ..., fill: str = ..., - offset: _ScreenUnits = ..., + offset: float | str = ..., outline: str = ..., - outlineoffset: _ScreenUnits = ..., + outlineoffset: float | str = ..., outlinestipple: str = ..., state: Literal["normal", "hidden", "disabled"] = ..., stipple: str = ..., tags: str | list[str] | tuple[str, ...] = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> int: ... @overload def create_rectangle( @@ -1871,24 +1856,24 @@ class Canvas(Widget, XView, YView): activeoutline: str = ..., activeoutlinestipple: str = ..., activestipple: str = ..., - activewidth: _ScreenUnits = ..., + activewidth: float | str = ..., dash: str | int | list[int] | tuple[int, ...] = ..., - dashoffset: _ScreenUnits = ..., + dashoffset: float | str = ..., disableddash: str | int | list[int] | tuple[int, ...] = ..., disabledfill: str = ..., disabledoutline: str = ..., disabledoutlinestipple: str = ..., disabledstipple: str = ..., - disabledwidth: _ScreenUnits = ..., + disabledwidth: float | str = ..., fill: str = ..., - offset: _ScreenUnits = ..., + offset: float | str = ..., outline: str = ..., - outlineoffset: _ScreenUnits = ..., + outlineoffset: float | str = ..., outlinestipple: str = ..., state: Literal["normal", "hidden", "disabled"] = ..., stipple: str = ..., tags: str | list[str] | tuple[str, ...] = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> int: ... @overload def create_text( @@ -1899,19 +1884,19 @@ class Canvas(Widget, XView, YView): *, activefill: str = ..., activestipple: str = ..., - anchor: _Anchor = ..., + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = ..., angle: float | str = ..., disabledfill: str = ..., disabledstipple: str = ..., fill: str = ..., font: _FontDescription = ..., justify: Literal["left", "center", "right"] = ..., - offset: _ScreenUnits = ..., + offset: float | str = ..., state: Literal["normal", "hidden", "disabled"] = ..., stipple: str = ..., tags: str | list[str] | tuple[str, ...] = ..., text: float | str = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> int: ... @overload def create_text( @@ -1921,19 +1906,19 @@ class Canvas(Widget, XView, YView): *, activefill: str = ..., activestipple: str = ..., - anchor: _Anchor = ..., + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = ..., angle: float | str = ..., disabledfill: str = ..., disabledstipple: str = ..., fill: str = ..., font: _FontDescription = ..., justify: Literal["left", "center", "right"] = ..., - offset: _ScreenUnits = ..., + offset: float | str = ..., state: Literal["normal", "hidden", "disabled"] = ..., stipple: str = ..., tags: str | list[str] | tuple[str, ...] = ..., text: float | str = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> int: ... @overload def create_window( @@ -1942,11 +1927,11 @@ class Canvas(Widget, XView, YView): y: float, /, *, - anchor: _Anchor = ..., - height: _ScreenUnits = ..., + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = ..., + height: float | str = ..., state: Literal["normal", "hidden", "disabled"] = ..., tags: str | list[str] | tuple[str, ...] = ..., - width: _ScreenUnits = ..., + width: float | str = ..., window: Widget = ..., ) -> int: ... @overload @@ -1955,11 +1940,11 @@ class Canvas(Widget, XView, YView): coords: tuple[float, float] | list[int] | list[float], /, *, - anchor: _Anchor = ..., - height: _ScreenUnits = ..., + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = ..., + height: float | str = ..., state: Literal["normal", "hidden", "disabled"] = ..., tags: str | list[str] | tuple[str, ...] = ..., - width: _ScreenUnits = ..., + width: float | str = ..., window: Widget = ..., ) -> int: ... def dchars(self, *args) -> None: ... @@ -1992,9 +1977,7 @@ class Canvas(Widget, XView, YView): def tag_raise(self, first: str | int, second: str | int | None = ..., /) -> None: ... def tkraise(self, first: str | int, second: str | int | None = ..., /) -> None: ... # type: ignore[override] def lift(self, first: str | int, second: str | int | None = ..., /) -> None: ... # type: ignore[override] - def scale( - self, tagOrId: str | int, xOrigin: _ScreenUnits, yOrigin: _ScreenUnits, xScale: float, yScale: float, / - ) -> None: ... + def scale(self, tagOrId: str | int, xOrigin: float | str, yOrigin: float | str, xScale: float, yScale: float, /) -> None: ... def scan_mark(self, x, y) -> None: ... def scan_dragto(self, x, y, gain: int = 10) -> None: ... def select_adjust(self, tagOrId, index) -> None: ... @@ -2012,29 +1995,29 @@ class Checkbutton(Widget): *, activebackground: str = ..., activeforeground: str = ..., - anchor: _Anchor = "center", + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = "center", background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., bitmap: str = "", - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., - command: _ButtonCommand = "", - compound: _Compound = "none", + border: float | str = ..., + borderwidth: float | str = ..., + command: str | Callable[[], Any] = "", + compound: Literal["top", "left", "center", "right", "bottom", "none"] = "none", cursor: _Cursor = "", disabledforeground: str = ..., fg: str = ..., font: _FontDescription = "TkDefaultFont", foreground: str = ..., - height: _ScreenUnits = 0, + height: float | str = 0, highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = 1, - image: _ImageSpec = "", + highlightthickness: float | str = 1, + image: _Image | str = "", indicatoron: bool = True, justify: Literal["left", "center", "right"] = "center", name: str = ..., - offrelief: _Relief = ..., + offrelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., # The checkbutton puts a value to its variable when it's checked or # unchecked. We don't restrict the type of that value here, so # Any-typing is fine. @@ -2047,22 +2030,22 @@ class Checkbutton(Widget): # done by setting variable to empty string (the default). offvalue: Any = 0, onvalue: Any = 1, - overrelief: _Relief | Literal[""] = "", - padx: _ScreenUnits = 1, - pady: _ScreenUnits = 1, - relief: _Relief = "flat", + overrelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove", ""] = "", + padx: float | str = 1, + pady: float | str = 1, + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = "flat", selectcolor: str = ..., - selectimage: _ImageSpec = "", + selectimage: _Image | str = "", state: Literal["normal", "active", "disabled"] = "normal", - takefocus: _TakeFocusValue = "", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = "", text: float | str = "", textvariable: Variable = ..., - tristateimage: _ImageSpec = "", + tristateimage: _Image | str = "", tristatevalue: Any = "", underline: int = -1, variable: Variable | Literal[""] = ..., - width: _ScreenUnits = 0, - wraplength: _ScreenUnits = 0, + width: float | str = 0, + wraplength: float | str = 0, ) -> None: ... @overload def configure( @@ -2071,46 +2054,46 @@ class Checkbutton(Widget): *, activebackground: str = ..., activeforeground: str = ..., - anchor: _Anchor = ..., + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = ..., background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., bitmap: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., - command: _ButtonCommand = ..., - compound: _Compound = ..., + border: float | str = ..., + borderwidth: float | str = ..., + command: str | Callable[[], Any] = ..., + compound: Literal["top", "left", "center", "right", "bottom", "none"] = ..., cursor: _Cursor = ..., disabledforeground: str = ..., fg: str = ..., font: _FontDescription = ..., foreground: str = ..., - height: _ScreenUnits = ..., + height: float | str = ..., highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., - image: _ImageSpec = ..., + highlightthickness: float | str = ..., + image: _Image | str = ..., indicatoron: bool = ..., justify: Literal["left", "center", "right"] = ..., - offrelief: _Relief = ..., + offrelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., offvalue: Any = ..., onvalue: Any = ..., - overrelief: _Relief | Literal[""] = ..., - padx: _ScreenUnits = ..., - pady: _ScreenUnits = ..., - relief: _Relief = ..., + overrelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove", ""] = ..., + padx: float | str = ..., + pady: float | str = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., selectcolor: str = ..., - selectimage: _ImageSpec = ..., + selectimage: _Image | str = ..., state: Literal["normal", "active", "disabled"] = ..., - takefocus: _TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., text: float | str = ..., textvariable: Variable = ..., - tristateimage: _ImageSpec = ..., + tristateimage: _Image | str = ..., tristatevalue: Any = ..., underline: int = ..., variable: Variable | Literal[""] = ..., - width: _ScreenUnits = ..., - wraplength: _ScreenUnits = ..., + width: float | str = ..., + wraplength: float | str = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -2128,10 +2111,10 @@ class Entry(Widget, XView): cnf: dict[str, Any] | None = {}, *, background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., cursor: _Cursor = "xterm", disabledbackground: str = ..., disabledforeground: str = ..., @@ -2141,30 +2124,30 @@ class Entry(Widget, XView): foreground: str = ..., highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., + highlightthickness: float | str = ..., insertbackground: str = ..., - insertborderwidth: _ScreenUnits = 0, + insertborderwidth: float | str = 0, insertofftime: int = 300, insertontime: int = 600, - insertwidth: _ScreenUnits = ..., - invalidcommand: _EntryValidateCommand = "", - invcmd: _EntryValidateCommand = "", # same as invalidcommand + insertwidth: float | str = ..., + invalidcommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = "", + invcmd: str | list[str] | tuple[str, ...] | Callable[[], bool] = "", # same as invalidcommand justify: Literal["left", "center", "right"] = "left", name: str = ..., readonlybackground: str = ..., - relief: _Relief = "sunken", + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = "sunken", selectbackground: str = ..., - selectborderwidth: _ScreenUnits = ..., + selectborderwidth: float | str = ..., selectforeground: str = ..., show: str = "", state: Literal["normal", "disabled", "readonly"] = "normal", - takefocus: _TakeFocusValue = "", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = "", textvariable: Variable = ..., validate: Literal["none", "focus", "focusin", "focusout", "key", "all"] = "none", - validatecommand: _EntryValidateCommand = "", - vcmd: _EntryValidateCommand = "", # same as validatecommand + validatecommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = "", + vcmd: str | list[str] | tuple[str, ...] | Callable[[], bool] = "", # same as validatecommand width: int = 20, - xscrollcommand: _XYScrollCommand = "", + xscrollcommand: str | Callable[[float, float], object] = "", ) -> None: ... @overload def configure( @@ -2172,10 +2155,10 @@ class Entry(Widget, XView): cnf: dict[str, Any] | None = None, *, background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., cursor: _Cursor = ..., disabledbackground: str = ..., disabledforeground: str = ..., @@ -2185,29 +2168,29 @@ class Entry(Widget, XView): foreground: str = ..., highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., + highlightthickness: float | str = ..., insertbackground: str = ..., - insertborderwidth: _ScreenUnits = ..., + insertborderwidth: float | str = ..., insertofftime: int = ..., insertontime: int = ..., - insertwidth: _ScreenUnits = ..., - invalidcommand: _EntryValidateCommand = ..., - invcmd: _EntryValidateCommand = ..., + insertwidth: float | str = ..., + invalidcommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., + invcmd: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., justify: Literal["left", "center", "right"] = ..., readonlybackground: str = ..., - relief: _Relief = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., selectbackground: str = ..., - selectborderwidth: _ScreenUnits = ..., + selectborderwidth: float | str = ..., selectforeground: str = ..., show: str = ..., state: Literal["normal", "disabled", "readonly"] = ..., - takefocus: _TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., textvariable: Variable = ..., validate: Literal["none", "focus", "focusin", "focusout", "key", "all"] = ..., - validatecommand: _EntryValidateCommand = ..., - vcmd: _EntryValidateCommand = ..., + validatecommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., + vcmd: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., width: int = ..., - xscrollcommand: _XYScrollCommand = ..., + xscrollcommand: str | Callable[[float, float], object] = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -2239,25 +2222,25 @@ class Frame(Widget): cnf: dict[str, Any] | None = {}, *, background: str = ..., - bd: _ScreenUnits = 0, + bd: float | str = 0, bg: str = ..., - border: _ScreenUnits = 0, - borderwidth: _ScreenUnits = 0, + border: float | str = 0, + borderwidth: float | str = 0, class_: str = "Frame", # can't be changed with configure() colormap: Literal["new", ""] | Misc = "", # can't be changed with configure() container: bool = False, # can't be changed with configure() cursor: _Cursor = "", - height: _ScreenUnits = 0, + height: float | str = 0, highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = 0, + highlightthickness: float | str = 0, name: str = ..., - padx: _ScreenUnits = 0, - pady: _ScreenUnits = 0, - relief: _Relief = "flat", - takefocus: _TakeFocusValue = 0, + padx: float | str = 0, + pady: float | str = 0, + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = "flat", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = 0, visual: str | tuple[str, int] = "", # can't be changed with configure() - width: _ScreenUnits = 0, + width: float | str = 0, ) -> None: ... @overload def configure( @@ -2265,20 +2248,20 @@ class Frame(Widget): cnf: dict[str, Any] | None = None, *, background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., cursor: _Cursor = ..., - height: _ScreenUnits = ..., + height: float | str = ..., highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., - padx: _ScreenUnits = ..., - pady: _ScreenUnits = ..., - relief: _Relief = ..., - takefocus: _TakeFocusValue = ..., - width: _ScreenUnits = ..., + highlightthickness: float | str = ..., + padx: float | str = ..., + pady: float | str = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., + width: float | str = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -2292,36 +2275,36 @@ class Label(Widget): *, activebackground: str = ..., activeforeground: str = ..., - anchor: _Anchor = "center", + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = "center", background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., bitmap: str = "", - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., - compound: _Compound = "none", + border: float | str = ..., + borderwidth: float | str = ..., + compound: Literal["top", "left", "center", "right", "bottom", "none"] = "none", cursor: _Cursor = "", disabledforeground: str = ..., fg: str = ..., font: _FontDescription = "TkDefaultFont", foreground: str = ..., - height: _ScreenUnits = 0, + height: float | str = 0, highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = 0, - image: _ImageSpec = "", + highlightthickness: float | str = 0, + image: _Image | str = "", justify: Literal["left", "center", "right"] = "center", name: str = ..., - padx: _ScreenUnits = 1, - pady: _ScreenUnits = 1, - relief: _Relief = "flat", + padx: float | str = 1, + pady: float | str = 1, + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = "flat", state: Literal["normal", "active", "disabled"] = "normal", - takefocus: _TakeFocusValue = 0, + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = 0, text: float | str = "", textvariable: Variable = ..., underline: int = -1, - width: _ScreenUnits = 0, - wraplength: _ScreenUnits = 0, + width: float | str = 0, + wraplength: float | str = 0, ) -> None: ... @overload def configure( @@ -2330,35 +2313,35 @@ class Label(Widget): *, activebackground: str = ..., activeforeground: str = ..., - anchor: _Anchor = ..., + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = ..., background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., bitmap: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., - compound: _Compound = ..., + border: float | str = ..., + borderwidth: float | str = ..., + compound: Literal["top", "left", "center", "right", "bottom", "none"] = ..., cursor: _Cursor = ..., disabledforeground: str = ..., fg: str = ..., font: _FontDescription = ..., foreground: str = ..., - height: _ScreenUnits = ..., + height: float | str = ..., highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., - image: _ImageSpec = ..., + highlightthickness: float | str = ..., + image: _Image | str = ..., justify: Literal["left", "center", "right"] = ..., - padx: _ScreenUnits = ..., - pady: _ScreenUnits = ..., - relief: _Relief = ..., + padx: float | str = ..., + pady: float | str = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., state: Literal["normal", "active", "disabled"] = ..., - takefocus: _TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., text: float | str = ..., textvariable: Variable = ..., underline: int = ..., - width: _ScreenUnits = ..., - wraplength: _ScreenUnits = ..., + width: float | str = ..., + wraplength: float | str = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -2372,10 +2355,10 @@ class Listbox(Widget, XView, YView): *, activestyle: Literal["dotbox", "none", "underline"] = ..., background: str = ..., - bd: _ScreenUnits = 1, + bd: float | str = 1, bg: str = ..., - border: _ScreenUnits = 1, - borderwidth: _ScreenUnits = 1, + border: float | str = 1, + borderwidth: float | str = 1, cursor: _Cursor = "", disabledforeground: str = ..., exportselection: bool | Literal[0, 1] = 1, @@ -2385,7 +2368,7 @@ class Listbox(Widget, XView, YView): height: int = 10, highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., + highlightthickness: float | str = ..., justify: Literal["left", "center", "right"] = "left", # There's no tkinter.ListVar, but seems like bare tkinter.Variable # actually works for this: @@ -2398,9 +2381,9 @@ class Listbox(Widget, XView, YView): # ('foo', 'bar', 'baz') listvariable: Variable = ..., name: str = ..., - relief: _Relief = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., selectbackground: str = ..., - selectborderwidth: _ScreenUnits = 0, + selectborderwidth: float | str = 0, selectforeground: str = ..., # from listbox man page: "The value of the [selectmode] option may be # arbitrary, but the default bindings expect it to be either single, @@ -2411,10 +2394,10 @@ class Listbox(Widget, XView, YView): selectmode: str | Literal["single", "browse", "multiple", "extended"] = "browse", # noqa: Y051 setgrid: bool = False, state: Literal["normal", "disabled"] = "normal", - takefocus: _TakeFocusValue = "", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = "", width: int = 20, - xscrollcommand: _XYScrollCommand = "", - yscrollcommand: _XYScrollCommand = "", + xscrollcommand: str | Callable[[float, float], object] = "", + yscrollcommand: str | Callable[[float, float], object] = "", ) -> None: ... @overload def configure( @@ -2423,10 +2406,10 @@ class Listbox(Widget, XView, YView): *, activestyle: Literal["dotbox", "none", "underline"] = ..., background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., cursor: _Cursor = ..., disabledforeground: str = ..., exportselection: bool = ..., @@ -2436,20 +2419,20 @@ class Listbox(Widget, XView, YView): height: int = ..., highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., + highlightthickness: float | str = ..., justify: Literal["left", "center", "right"] = ..., listvariable: Variable = ..., - relief: _Relief = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., selectbackground: str = ..., - selectborderwidth: _ScreenUnits = ..., + selectborderwidth: float | str = ..., selectforeground: str = ..., selectmode: str | Literal["single", "browse", "multiple", "extended"] = ..., # noqa: Y051 setgrid: bool = ..., state: Literal["normal", "disabled"] = ..., - takefocus: _TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., width: int = ..., - xscrollcommand: _XYScrollCommand = ..., - yscrollcommand: _XYScrollCommand = ..., + xscrollcommand: str | Callable[[float, float], object] = ..., + yscrollcommand: str | Callable[[float, float], object] = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -2485,13 +2468,13 @@ class Menu(Widget): cnf: dict[str, Any] | None = {}, *, activebackground: str = ..., - activeborderwidth: _ScreenUnits = ..., + activeborderwidth: float | str = ..., activeforeground: str = ..., background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., cursor: _Cursor = "arrow", disabledforeground: str = ..., fg: str = ..., @@ -2499,9 +2482,9 @@ class Menu(Widget): foreground: str = ..., name: str = ..., postcommand: Callable[[], object] | str = "", - relief: _Relief = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., selectcolor: str = ..., - takefocus: _TakeFocusValue = 0, + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = 0, tearoff: bool | Literal[0, 1] = 1, # I guess tearoffcommand arguments are supposed to be widget objects, # but they are widget name strings. Use nametowidget() to handle the @@ -2516,22 +2499,22 @@ class Menu(Widget): cnf: dict[str, Any] | None = None, *, activebackground: str = ..., - activeborderwidth: _ScreenUnits = ..., + activeborderwidth: float | str = ..., activeforeground: str = ..., background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., cursor: _Cursor = ..., disabledforeground: str = ..., fg: str = ..., font: _FontDescription = ..., foreground: str = ..., postcommand: Callable[[], object] | str = ..., - relief: _Relief = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., selectcolor: str = ..., - takefocus: _TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., tearoff: bool = ..., tearoffcommand: Callable[[str, str], object] | str = ..., title: str = ..., @@ -2555,11 +2538,11 @@ class Menu(Widget): bitmap: str = ..., columnbreak: int = ..., command: Callable[[], object] | str = ..., - compound: _Compound = ..., + compound: Literal["top", "left", "center", "right", "bottom", "none"] = ..., font: _FontDescription = ..., foreground: str = ..., hidemargin: bool = ..., - image: _ImageSpec = ..., + image: _Image | str = ..., label: str = ..., menu: Menu = ..., state: Literal["normal", "active", "disabled"] = ..., @@ -2576,17 +2559,17 @@ class Menu(Widget): bitmap: str = ..., columnbreak: int = ..., command: Callable[[], object] | str = ..., - compound: _Compound = ..., + compound: Literal["top", "left", "center", "right", "bottom", "none"] = ..., font: _FontDescription = ..., foreground: str = ..., hidemargin: bool = ..., - image: _ImageSpec = ..., + image: _Image | str = ..., indicatoron: bool = ..., label: str = ..., offvalue: Any = ..., onvalue: Any = ..., selectcolor: str = ..., - selectimage: _ImageSpec = ..., + selectimage: _Image | str = ..., state: Literal["normal", "active", "disabled"] = ..., underline: int = ..., variable: Variable = ..., @@ -2602,11 +2585,11 @@ class Menu(Widget): bitmap: str = ..., columnbreak: int = ..., command: Callable[[], object] | str = ..., - compound: _Compound = ..., + compound: Literal["top", "left", "center", "right", "bottom", "none"] = ..., font: _FontDescription = ..., foreground: str = ..., hidemargin: bool = ..., - image: _ImageSpec = ..., + image: _Image | str = ..., label: str = ..., state: Literal["normal", "active", "disabled"] = ..., underline: int = ..., @@ -2622,15 +2605,15 @@ class Menu(Widget): bitmap: str = ..., columnbreak: int = ..., command: Callable[[], object] | str = ..., - compound: _Compound = ..., + compound: Literal["top", "left", "center", "right", "bottom", "none"] = ..., font: _FontDescription = ..., foreground: str = ..., hidemargin: bool = ..., - image: _ImageSpec = ..., + image: _Image | str = ..., indicatoron: bool = ..., label: str = ..., selectcolor: str = ..., - selectimage: _ImageSpec = ..., + selectimage: _Image | str = ..., state: Literal["normal", "active", "disabled"] = ..., underline: int = ..., value: Any = ..., @@ -2649,11 +2632,11 @@ class Menu(Widget): bitmap: str = ..., columnbreak: int = ..., command: Callable[[], object] | str = ..., - compound: _Compound = ..., + compound: Literal["top", "left", "center", "right", "bottom", "none"] = ..., font: _FontDescription = ..., foreground: str = ..., hidemargin: bool = ..., - image: _ImageSpec = ..., + image: _Image | str = ..., label: str = ..., menu: Menu = ..., state: Literal["normal", "active", "disabled"] = ..., @@ -2671,17 +2654,17 @@ class Menu(Widget): bitmap: str = ..., columnbreak: int = ..., command: Callable[[], object] | str = ..., - compound: _Compound = ..., + compound: Literal["top", "left", "center", "right", "bottom", "none"] = ..., font: _FontDescription = ..., foreground: str = ..., hidemargin: bool = ..., - image: _ImageSpec = ..., + image: _Image | str = ..., indicatoron: bool = ..., label: str = ..., offvalue: Any = ..., onvalue: Any = ..., selectcolor: str = ..., - selectimage: _ImageSpec = ..., + selectimage: _Image | str = ..., state: Literal["normal", "active", "disabled"] = ..., underline: int = ..., variable: Variable = ..., @@ -2698,11 +2681,11 @@ class Menu(Widget): bitmap: str = ..., columnbreak: int = ..., command: Callable[[], object] | str = ..., - compound: _Compound = ..., + compound: Literal["top", "left", "center", "right", "bottom", "none"] = ..., font: _FontDescription = ..., foreground: str = ..., hidemargin: bool = ..., - image: _ImageSpec = ..., + image: _Image | str = ..., label: str = ..., state: Literal["normal", "active", "disabled"] = ..., underline: int = ..., @@ -2719,15 +2702,15 @@ class Menu(Widget): bitmap: str = ..., columnbreak: int = ..., command: Callable[[], object] | str = ..., - compound: _Compound = ..., + compound: Literal["top", "left", "center", "right", "bottom", "none"] = ..., font: _FontDescription = ..., foreground: str = ..., hidemargin: bool = ..., - image: _ImageSpec = ..., + image: _Image | str = ..., indicatoron: bool = ..., label: str = ..., selectcolor: str = ..., - selectimage: _ImageSpec = ..., + selectimage: _Image | str = ..., state: Literal["normal", "active", "disabled"] = ..., underline: int = ..., value: Any = ..., @@ -2756,39 +2739,39 @@ class Menubutton(Widget): *, activebackground: str = ..., activeforeground: str = ..., - anchor: _Anchor = ..., + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = ..., background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., bitmap: str = "", - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., - compound: _Compound = "none", + border: float | str = ..., + borderwidth: float | str = ..., + compound: Literal["top", "left", "center", "right", "bottom", "none"] = "none", cursor: _Cursor = "", direction: Literal["above", "below", "left", "right", "flush"] = "below", disabledforeground: str = ..., fg: str = ..., font: _FontDescription = "TkDefaultFont", foreground: str = ..., - height: _ScreenUnits = 0, + height: float | str = 0, highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = 0, - image: _ImageSpec = "", + highlightthickness: float | str = 0, + image: _Image | str = "", indicatoron: bool = ..., justify: Literal["left", "center", "right"] = ..., menu: Menu = ..., name: str = ..., - padx: _ScreenUnits = ..., - pady: _ScreenUnits = ..., - relief: _Relief = "flat", + padx: float | str = ..., + pady: float | str = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = "flat", state: Literal["normal", "active", "disabled"] = "normal", - takefocus: _TakeFocusValue = 0, + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = 0, text: float | str = "", textvariable: Variable = ..., underline: int = -1, - width: _ScreenUnits = 0, - wraplength: _ScreenUnits = 0, + width: float | str = 0, + wraplength: float | str = 0, ) -> None: ... @overload def configure( @@ -2797,38 +2780,38 @@ class Menubutton(Widget): *, activebackground: str = ..., activeforeground: str = ..., - anchor: _Anchor = ..., + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = ..., background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., bitmap: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., - compound: _Compound = ..., + border: float | str = ..., + borderwidth: float | str = ..., + compound: Literal["top", "left", "center", "right", "bottom", "none"] = ..., cursor: _Cursor = ..., direction: Literal["above", "below", "left", "right", "flush"] = ..., disabledforeground: str = ..., fg: str = ..., font: _FontDescription = ..., foreground: str = ..., - height: _ScreenUnits = ..., + height: float | str = ..., highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., - image: _ImageSpec = ..., + highlightthickness: float | str = ..., + image: _Image | str = ..., indicatoron: bool = ..., justify: Literal["left", "center", "right"] = ..., menu: Menu = ..., - padx: _ScreenUnits = ..., - pady: _ScreenUnits = ..., - relief: _Relief = ..., + padx: float | str = ..., + pady: float | str = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., state: Literal["normal", "active", "disabled"] = ..., - takefocus: _TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., text: float | str = ..., textvariable: Variable = ..., underline: int = ..., - width: _ScreenUnits = ..., - wraplength: _ScreenUnits = ..., + width: float | str = ..., + wraplength: float | str = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -2840,58 +2823,58 @@ class Message(Widget): master: Misc | None = None, cnf: dict[str, Any] | None = {}, *, - anchor: _Anchor = "center", + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = "center", aspect: int = 150, background: str = ..., - bd: _ScreenUnits = 1, + bd: float | str = 1, bg: str = ..., - border: _ScreenUnits = 1, - borderwidth: _ScreenUnits = 1, + border: float | str = 1, + borderwidth: float | str = 1, cursor: _Cursor = "", fg: str = ..., font: _FontDescription = "TkDefaultFont", foreground: str = ..., highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = 0, + highlightthickness: float | str = 0, justify: Literal["left", "center", "right"] = "left", name: str = ..., - padx: _ScreenUnits = ..., - pady: _ScreenUnits = ..., - relief: _Relief = "flat", - takefocus: _TakeFocusValue = 0, + padx: float | str = ..., + pady: float | str = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = "flat", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = 0, text: float | str = "", textvariable: Variable = ..., # there's width but no height - width: _ScreenUnits = 0, + width: float | str = 0, ) -> None: ... @overload def configure( self, cnf: dict[str, Any] | None = None, *, - anchor: _Anchor = ..., + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = ..., aspect: int = ..., background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., cursor: _Cursor = ..., fg: str = ..., font: _FontDescription = ..., foreground: str = ..., highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., + highlightthickness: float | str = ..., justify: Literal["left", "center", "right"] = ..., - padx: _ScreenUnits = ..., - pady: _ScreenUnits = ..., - relief: _Relief = ..., - takefocus: _TakeFocusValue = ..., + padx: float | str = ..., + pady: float | str = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., text: float | str = ..., textvariable: Variable = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -2905,46 +2888,46 @@ class Radiobutton(Widget): *, activebackground: str = ..., activeforeground: str = ..., - anchor: _Anchor = "center", + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = "center", background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., bitmap: str = "", - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., - command: _ButtonCommand = "", - compound: _Compound = "none", + border: float | str = ..., + borderwidth: float | str = ..., + command: str | Callable[[], Any] = "", + compound: Literal["top", "left", "center", "right", "bottom", "none"] = "none", cursor: _Cursor = "", disabledforeground: str = ..., fg: str = ..., font: _FontDescription = "TkDefaultFont", foreground: str = ..., - height: _ScreenUnits = 0, + height: float | str = 0, highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = 1, - image: _ImageSpec = "", + highlightthickness: float | str = 1, + image: _Image | str = "", indicatoron: bool = True, justify: Literal["left", "center", "right"] = "center", name: str = ..., - offrelief: _Relief = ..., - overrelief: _Relief | Literal[""] = "", - padx: _ScreenUnits = 1, - pady: _ScreenUnits = 1, - relief: _Relief = "flat", + offrelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., + overrelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove", ""] = "", + padx: float | str = 1, + pady: float | str = 1, + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = "flat", selectcolor: str = ..., - selectimage: _ImageSpec = "", + selectimage: _Image | str = "", state: Literal["normal", "active", "disabled"] = "normal", - takefocus: _TakeFocusValue = "", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = "", text: float | str = "", textvariable: Variable = ..., - tristateimage: _ImageSpec = "", + tristateimage: _Image | str = "", tristatevalue: Any = "", underline: int = -1, value: Any = "", variable: Variable | Literal[""] = ..., - width: _ScreenUnits = 0, - wraplength: _ScreenUnits = 0, + width: float | str = 0, + wraplength: float | str = 0, ) -> None: ... @overload def configure( @@ -2953,45 +2936,45 @@ class Radiobutton(Widget): *, activebackground: str = ..., activeforeground: str = ..., - anchor: _Anchor = ..., + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = ..., background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., bitmap: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., - command: _ButtonCommand = ..., - compound: _Compound = ..., + border: float | str = ..., + borderwidth: float | str = ..., + command: str | Callable[[], Any] = ..., + compound: Literal["top", "left", "center", "right", "bottom", "none"] = ..., cursor: _Cursor = ..., disabledforeground: str = ..., fg: str = ..., font: _FontDescription = ..., foreground: str = ..., - height: _ScreenUnits = ..., + height: float | str = ..., highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., - image: _ImageSpec = ..., + highlightthickness: float | str = ..., + image: _Image | str = ..., indicatoron: bool = ..., justify: Literal["left", "center", "right"] = ..., - offrelief: _Relief = ..., - overrelief: _Relief | Literal[""] = ..., - padx: _ScreenUnits = ..., - pady: _ScreenUnits = ..., - relief: _Relief = ..., + offrelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., + overrelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove", ""] = ..., + padx: float | str = ..., + pady: float | str = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., selectcolor: str = ..., - selectimage: _ImageSpec = ..., + selectimage: _Image | str = ..., state: Literal["normal", "active", "disabled"] = ..., - takefocus: _TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., text: float | str = ..., textvariable: Variable = ..., - tristateimage: _ImageSpec = ..., + tristateimage: _Image | str = ..., tristatevalue: Any = ..., underline: int = ..., value: Any = ..., variable: Variable | Literal[""] = ..., - width: _ScreenUnits = ..., - wraplength: _ScreenUnits = ..., + width: float | str = ..., + wraplength: float | str = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -3009,11 +2992,11 @@ class Scale(Widget): *, activebackground: str = ..., background: str = ..., - bd: _ScreenUnits = 1, + bd: float | str = 1, bg: str = ..., bigincrement: float = 0.0, - border: _ScreenUnits = 1, - borderwidth: _ScreenUnits = 1, + border: float | str = 1, + borderwidth: float | str = 1, # don't know why the callback gets string instead of float command: str | Callable[[str], object] = "", cursor: _Cursor = "", @@ -3024,25 +3007,25 @@ class Scale(Widget): from_: float = 0.0, highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., + highlightthickness: float | str = ..., label: str = "", - length: _ScreenUnits = 100, + length: float | str = 100, name: str = ..., orient: Literal["horizontal", "vertical"] = "vertical", - relief: _Relief = "flat", + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = "flat", repeatdelay: int = 300, repeatinterval: int = 100, resolution: float = 1.0, showvalue: bool = True, - sliderlength: _ScreenUnits = 30, - sliderrelief: _Relief = "raised", + sliderlength: float | str = 30, + sliderrelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = "raised", state: Literal["normal", "active", "disabled"] = "normal", - takefocus: _TakeFocusValue = "", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = "", tickinterval: float = 0.0, to: float = 100.0, troughcolor: str = ..., variable: IntVar | DoubleVar = ..., - width: _ScreenUnits = 15, + width: float | str = 15, ) -> None: ... @overload def configure( @@ -3051,11 +3034,11 @@ class Scale(Widget): *, activebackground: str = ..., background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., bigincrement: float = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., command: str | Callable[[str], object] = ..., cursor: _Cursor = ..., digits: int = ..., @@ -3065,24 +3048,24 @@ class Scale(Widget): from_: float = ..., highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., + highlightthickness: float | str = ..., label: str = ..., - length: _ScreenUnits = ..., + length: float | str = ..., orient: Literal["horizontal", "vertical"] = ..., - relief: _Relief = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., repeatdelay: int = ..., repeatinterval: int = ..., resolution: float = ..., showvalue: bool = ..., - sliderlength: _ScreenUnits = ..., - sliderrelief: _Relief = ..., + sliderlength: float | str = ..., + sliderrelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., state: Literal["normal", "active", "disabled"] = ..., - takefocus: _TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., tickinterval: float = ..., to: float = ..., troughcolor: str = ..., variable: IntVar | DoubleVar = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -3099,31 +3082,31 @@ class Scrollbar(Widget): cnf: dict[str, Any] | None = {}, *, activebackground: str = ..., - activerelief: _Relief = "raised", + activerelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = "raised", background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., # There are many ways how the command may get called. Search for # 'SCROLLING COMMANDS' in scrollbar man page. There doesn't seem to # be any way to specify an overloaded callback function, so we say # that it can take any args while it can't in reality. command: Callable[..., tuple[float, float] | None] | str = "", cursor: _Cursor = "", - elementborderwidth: _ScreenUnits = -1, + elementborderwidth: float | str = -1, highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = 0, + highlightthickness: float | str = 0, jump: bool = False, name: str = ..., orient: Literal["horizontal", "vertical"] = "vertical", - relief: _Relief = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., repeatdelay: int = 300, repeatinterval: int = 100, - takefocus: _TakeFocusValue = "", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = "", troughcolor: str = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> None: ... @overload def configure( @@ -3131,26 +3114,26 @@ class Scrollbar(Widget): cnf: dict[str, Any] | None = None, *, activebackground: str = ..., - activerelief: _Relief = ..., + activerelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., command: Callable[..., tuple[float, float] | None] | str = ..., cursor: _Cursor = ..., - elementborderwidth: _ScreenUnits = ..., + elementborderwidth: float | str = ..., highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., + highlightthickness: float | str = ..., jump: bool = ..., orient: Literal["horizontal", "vertical"] = ..., - relief: _Relief = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., repeatdelay: int = ..., repeatinterval: int = ..., - takefocus: _TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., troughcolor: str = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -3175,54 +3158,54 @@ class Text(Widget, XView, YView): *, autoseparators: bool = True, background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., blockcursor: bool = False, - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., cursor: _Cursor = "xterm", endline: int | Literal[""] = "", exportselection: bool = True, fg: str = ..., font: _FontDescription = "TkFixedFont", foreground: str = ..., - # width is always int, but height is allowed to be ScreenUnits. + # width is always int, but height is allowed to be screen units. # This doesn't make any sense to me, and this isn't documented. # The docs seem to say that both should be integers. - height: _ScreenUnits = 24, + height: float | str = 24, highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., + highlightthickness: float | str = ..., inactiveselectbackground: str = ..., insertbackground: str = ..., - insertborderwidth: _ScreenUnits = 0, + insertborderwidth: float | str = 0, insertofftime: int = 300, insertontime: int = 600, insertunfocussed: Literal["none", "hollow", "solid"] = "none", - insertwidth: _ScreenUnits = ..., + insertwidth: float | str = ..., maxundo: int = 0, name: str = ..., - padx: _ScreenUnits = 1, - pady: _ScreenUnits = 1, - relief: _Relief = ..., + padx: float | str = 1, + pady: float | str = 1, + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., selectbackground: str = ..., - selectborderwidth: _ScreenUnits = ..., + selectborderwidth: float | str = ..., selectforeground: str = ..., setgrid: bool = False, - spacing1: _ScreenUnits = 0, - spacing2: _ScreenUnits = 0, - spacing3: _ScreenUnits = 0, + spacing1: float | str = 0, + spacing2: float | str = 0, + spacing3: float | str = 0, startline: int | Literal[""] = "", state: Literal["normal", "disabled"] = "normal", # Literal inside Tuple doesn't actually work - tabs: _ScreenUnits | str | tuple[_ScreenUnits | str, ...] = "", + tabs: float | str | tuple[float | str, ...] = "", tabstyle: Literal["tabular", "wordprocessor"] = "tabular", - takefocus: _TakeFocusValue = "", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = "", undo: bool = False, width: int = 80, wrap: Literal["none", "char", "word"] = "char", - xscrollcommand: _XYScrollCommand = "", - yscrollcommand: _XYScrollCommand = "", + xscrollcommand: str | Callable[[float, float], object] = "", + yscrollcommand: str | Callable[[float, float], object] = "", ) -> None: ... @overload def configure( @@ -3231,49 +3214,49 @@ class Text(Widget, XView, YView): *, autoseparators: bool = ..., background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., blockcursor: bool = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., cursor: _Cursor = ..., endline: int | Literal[""] = ..., exportselection: bool = ..., fg: str = ..., font: _FontDescription = ..., foreground: str = ..., - height: _ScreenUnits = ..., + height: float | str = ..., highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., + highlightthickness: float | str = ..., inactiveselectbackground: str = ..., insertbackground: str = ..., - insertborderwidth: _ScreenUnits = ..., + insertborderwidth: float | str = ..., insertofftime: int = ..., insertontime: int = ..., insertunfocussed: Literal["none", "hollow", "solid"] = ..., - insertwidth: _ScreenUnits = ..., + insertwidth: float | str = ..., maxundo: int = ..., - padx: _ScreenUnits = ..., - pady: _ScreenUnits = ..., - relief: _Relief = ..., + padx: float | str = ..., + pady: float | str = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., selectbackground: str = ..., - selectborderwidth: _ScreenUnits = ..., + selectborderwidth: float | str = ..., selectforeground: str = ..., setgrid: bool = ..., - spacing1: _ScreenUnits = ..., - spacing2: _ScreenUnits = ..., - spacing3: _ScreenUnits = ..., + spacing1: float | str = ..., + spacing2: float | str = ..., + spacing3: float | str = ..., startline: int | Literal[""] = ..., state: Literal["normal", "disabled"] = ..., - tabs: _ScreenUnits | str | tuple[_ScreenUnits | str, ...] = ..., + tabs: float | str | tuple[float | str, ...] = ..., tabstyle: Literal["tabular", "wordprocessor"] = ..., - takefocus: _TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., undo: bool = ..., width: int = ..., wrap: Literal["none", "char", "word"] = ..., - xscrollcommand: _XYScrollCommand = ..., - yscrollcommand: _XYScrollCommand = ..., + xscrollcommand: str | Callable[[float, float], object] = ..., + yscrollcommand: str | Callable[[float, float], object] = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -3482,10 +3465,10 @@ class Text(Widget, XView, YView): cnf: dict[str, Any] | None = None, *, align: Literal["baseline", "bottom", "center", "top"] = ..., - image: _ImageSpec = ..., + image: _Image | str = ..., name: str = ..., - padx: _ScreenUnits = ..., - pady: _ScreenUnits = ..., + padx: float | str = ..., + pady: float | str = ..., ) -> dict[str, tuple[str, str, str, str, str | int]] | None: ... def image_create( self, @@ -3493,10 +3476,10 @@ class Text(Widget, XView, YView): cnf: dict[str, Any] | None = {}, *, align: Literal["baseline", "bottom", "center", "top"] = ..., - image: _ImageSpec = ..., + image: _Image | str = ..., name: str = ..., - padx: _ScreenUnits = ..., - pady: _ScreenUnits = ..., + padx: float | str = ..., + pady: float | str = ..., ) -> str: ... def image_names(self) -> tuple[str, ...]: ... def index(self, index: _TextIndex) -> str: ... @@ -3553,27 +3536,27 @@ class Text(Widget, XView, YView): *, background: str = ..., bgstipple: str = ..., - borderwidth: _ScreenUnits = ..., - border: _ScreenUnits = ..., # alias for borderwidth + borderwidth: float | str = ..., + border: float | str = ..., # alias for borderwidth elide: bool = ..., fgstipple: str = ..., font: _FontDescription = ..., foreground: str = ..., justify: Literal["left", "right", "center"] = ..., - lmargin1: _ScreenUnits = ..., - lmargin2: _ScreenUnits = ..., + lmargin1: float | str = ..., + lmargin2: float | str = ..., lmargincolor: str = ..., - offset: _ScreenUnits = ..., + offset: float | str = ..., overstrike: bool = ..., overstrikefg: str = ..., - relief: _Relief = ..., - rmargin: _ScreenUnits = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., + rmargin: float | str = ..., rmargincolor: str = ..., selectbackground: str = ..., selectforeground: str = ..., - spacing1: _ScreenUnits = ..., - spacing2: _ScreenUnits = ..., - spacing3: _ScreenUnits = ..., + spacing1: float | str = ..., + spacing2: float | str = ..., + spacing3: float | str = ..., tabs: Any = ..., # the exact type is kind of complicated, see manual page tabstyle: Literal["tabular", "wordprocessor"] = ..., underline: bool = ..., @@ -3616,8 +3599,8 @@ class Text(Widget, XView, YView): *, align: Literal["baseline", "bottom", "center", "top"] = ..., create: str = ..., - padx: _ScreenUnits = ..., - pady: _ScreenUnits = ..., + padx: float | str = ..., + pady: float | str = ..., stretch: bool | Literal[0, 1] = ..., window: Misc | str = ..., ) -> dict[str, tuple[str, str, str, str, str | int]] | None: ... @@ -3629,8 +3612,8 @@ class Text(Widget, XView, YView): *, align: Literal["baseline", "bottom", "center", "top"] = ..., create: str = ..., - padx: _ScreenUnits = ..., - pady: _ScreenUnits = ..., + padx: float | str = ..., + pady: float | str = ..., stretch: bool | Literal[0, 1] = ..., window: Misc | str = ..., ) -> None: ... @@ -3643,7 +3626,6 @@ class _setit: # manual page: tk_optionMenu class OptionMenu(Menubutton): - widgetName: Incomplete menuname: Incomplete def __init__( # differs from other widgets @@ -3825,14 +3807,14 @@ class Spinbox(Widget, XView): *, activebackground: str = ..., background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., buttonbackground: str = ..., buttoncursor: _Cursor = "", - buttondownrelief: _Relief = ..., - buttonuprelief: _Relief = ..., + buttondownrelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., + buttonuprelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., # percent substitutions don't seem to be supported, it's similar to Entry's validation stuff command: Callable[[], object] | str | list[str] | tuple[str, ...] = "", cursor: _Cursor = "xterm", @@ -3846,35 +3828,35 @@ class Spinbox(Widget, XView): from_: float = 0.0, highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., + highlightthickness: float | str = ..., increment: float = 1.0, insertbackground: str = ..., - insertborderwidth: _ScreenUnits = 0, + insertborderwidth: float | str = 0, insertofftime: int = 300, insertontime: int = 600, - insertwidth: _ScreenUnits = ..., - invalidcommand: _EntryValidateCommand = "", - invcmd: _EntryValidateCommand = "", + insertwidth: float | str = ..., + invalidcommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = "", + invcmd: str | list[str] | tuple[str, ...] | Callable[[], bool] = "", justify: Literal["left", "center", "right"] = "left", name: str = ..., readonlybackground: str = ..., - relief: _Relief = "sunken", + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = "sunken", repeatdelay: int = 400, repeatinterval: int = 100, selectbackground: str = ..., - selectborderwidth: _ScreenUnits = ..., + selectborderwidth: float | str = ..., selectforeground: str = ..., state: Literal["normal", "disabled", "readonly"] = "normal", - takefocus: _TakeFocusValue = "", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = "", textvariable: Variable = ..., to: float = 0.0, validate: Literal["none", "focus", "focusin", "focusout", "key", "all"] = "none", - validatecommand: _EntryValidateCommand = "", - vcmd: _EntryValidateCommand = "", + validatecommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = "", + vcmd: str | list[str] | tuple[str, ...] | Callable[[], bool] = "", values: list[str] | tuple[str, ...] = ..., width: int = 20, wrap: bool = False, - xscrollcommand: _XYScrollCommand = "", + xscrollcommand: str | Callable[[float, float], object] = "", ) -> None: ... @overload def configure( @@ -3883,14 +3865,14 @@ class Spinbox(Widget, XView): *, activebackground: str = ..., background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., buttonbackground: str = ..., buttoncursor: _Cursor = ..., - buttondownrelief: _Relief = ..., - buttonuprelief: _Relief = ..., + buttondownrelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., + buttonuprelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., command: Callable[[], object] | str | list[str] | tuple[str, ...] = ..., cursor: _Cursor = ..., disabledbackground: str = ..., @@ -3903,34 +3885,34 @@ class Spinbox(Widget, XView): from_: float = ..., highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., + highlightthickness: float | str = ..., increment: float = ..., insertbackground: str = ..., - insertborderwidth: _ScreenUnits = ..., + insertborderwidth: float | str = ..., insertofftime: int = ..., insertontime: int = ..., - insertwidth: _ScreenUnits = ..., - invalidcommand: _EntryValidateCommand = ..., - invcmd: _EntryValidateCommand = ..., + insertwidth: float | str = ..., + invalidcommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., + invcmd: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., justify: Literal["left", "center", "right"] = ..., readonlybackground: str = ..., - relief: _Relief = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., repeatdelay: int = ..., repeatinterval: int = ..., selectbackground: str = ..., - selectborderwidth: _ScreenUnits = ..., + selectborderwidth: float | str = ..., selectforeground: str = ..., state: Literal["normal", "disabled", "readonly"] = ..., - takefocus: _TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., textvariable: Variable = ..., to: float = ..., validate: Literal["none", "focus", "focusin", "focusout", "key", "all"] = ..., - validatecommand: _EntryValidateCommand = ..., - vcmd: _EntryValidateCommand = ..., + validatecommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., + vcmd: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., values: list[str] | tuple[str, ...] = ..., width: int = ..., wrap: bool = ..., - xscrollcommand: _XYScrollCommand = ..., + xscrollcommand: str | Callable[[float, float], object] = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -3963,10 +3945,10 @@ class LabelFrame(Widget): cnf: dict[str, Any] | None = {}, *, background: str = ..., - bd: _ScreenUnits = 2, + bd: float | str = 2, bg: str = ..., - border: _ScreenUnits = 2, - borderwidth: _ScreenUnits = 2, + border: float | str = 2, + borderwidth: float | str = 2, class_: str = "Labelframe", # can't be changed with configure() colormap: Literal["new", ""] | Misc = "", # can't be changed with configure() container: bool = False, # undocumented, can't be changed with configure() @@ -3974,21 +3956,21 @@ class LabelFrame(Widget): fg: str = ..., font: _FontDescription = "TkDefaultFont", foreground: str = ..., - height: _ScreenUnits = 0, + height: float | str = 0, highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = 0, + highlightthickness: float | str = 0, # 'ne' and 'en' are valid labelanchors, but only 'ne' is a valid _Anchor. labelanchor: Literal["nw", "n", "ne", "en", "e", "es", "se", "s", "sw", "ws", "w", "wn"] = "nw", labelwidget: Misc = ..., name: str = ..., - padx: _ScreenUnits = 0, - pady: _ScreenUnits = 0, - relief: _Relief = "groove", - takefocus: _TakeFocusValue = 0, + padx: float | str = 0, + pady: float | str = 0, + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = "groove", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = 0, text: float | str = "", visual: str | tuple[str, int] = "", # can't be changed with configure() - width: _ScreenUnits = 0, + width: float | str = 0, ) -> None: ... @overload def configure( @@ -3996,26 +3978,26 @@ class LabelFrame(Widget): cnf: dict[str, Any] | None = None, *, background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., cursor: _Cursor = ..., fg: str = ..., font: _FontDescription = ..., foreground: str = ..., - height: _ScreenUnits = ..., + height: float | str = ..., highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., + highlightthickness: float | str = ..., labelanchor: Literal["nw", "n", "ne", "en", "e", "es", "se", "s", "sw", "ws", "w", "wn"] = ..., labelwidget: Misc = ..., - padx: _ScreenUnits = ..., - pady: _ScreenUnits = ..., - relief: _Relief = ..., - takefocus: _TakeFocusValue = ..., + padx: float | str = ..., + pady: float | str = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., text: float | str = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -4028,27 +4010,27 @@ class PanedWindow(Widget): cnf: dict[str, Any] | None = {}, *, background: str = ..., - bd: _ScreenUnits = 1, + bd: float | str = 1, bg: str = ..., - border: _ScreenUnits = 1, - borderwidth: _ScreenUnits = 1, + border: float | str = 1, + borderwidth: float | str = 1, cursor: _Cursor = "", - handlepad: _ScreenUnits = 8, - handlesize: _ScreenUnits = 8, - height: _ScreenUnits = "", + handlepad: float | str = 8, + handlesize: float | str = 8, + height: float | str = "", name: str = ..., opaqueresize: bool = True, orient: Literal["horizontal", "vertical"] = "horizontal", proxybackground: str = "", - proxyborderwidth: _ScreenUnits = 2, - proxyrelief: _Relief = "flat", - relief: _Relief = "flat", + proxyborderwidth: float | str = 2, + proxyrelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = "flat", + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = "flat", sashcursor: _Cursor = "", - sashpad: _ScreenUnits = 0, - sashrelief: _Relief = "flat", - sashwidth: _ScreenUnits = 3, + sashpad: float | str = 0, + sashrelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = "flat", + sashwidth: float | str = 3, showhandle: bool = False, - width: _ScreenUnits = "", + width: float | str = "", ) -> None: ... @overload def configure( @@ -4056,45 +4038,45 @@ class PanedWindow(Widget): cnf: dict[str, Any] | None = None, *, background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., cursor: _Cursor = ..., - handlepad: _ScreenUnits = ..., - handlesize: _ScreenUnits = ..., - height: _ScreenUnits = ..., + handlepad: float | str = ..., + handlesize: float | str = ..., + height: float | str = ..., opaqueresize: bool = ..., orient: Literal["horizontal", "vertical"] = ..., proxybackground: str = ..., - proxyborderwidth: _ScreenUnits = ..., - proxyrelief: _Relief = ..., - relief: _Relief = ..., + proxyborderwidth: float | str = ..., + proxyrelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., sashcursor: _Cursor = ..., - sashpad: _ScreenUnits = ..., - sashrelief: _Relief = ..., - sashwidth: _ScreenUnits = ..., + sashpad: float | str = ..., + sashrelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., + sashwidth: float | str = ..., showhandle: bool = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... config = configure def add(self, child: Widget, **kw) -> None: ... def remove(self, child) -> None: ... - forget: Incomplete + forget = remove # type: ignore[assignment] def identify(self, x: int, y: int): ... - def proxy(self, *args): ... - def proxy_coord(self): ... - def proxy_forget(self): ... - def proxy_place(self, x, y): ... - def sash(self, *args): ... - def sash_coord(self, index): ... - def sash_mark(self, index): ... - def sash_place(self, index, x, y): ... + def proxy(self, *args) -> tuple[Incomplete, ...]: ... + def proxy_coord(self) -> tuple[Incomplete, ...]: ... + def proxy_forget(self) -> tuple[Incomplete, ...]: ... + def proxy_place(self, x, y) -> tuple[Incomplete, ...]: ... + def sash(self, *args) -> tuple[Incomplete, ...]: ... + def sash_coord(self, index) -> tuple[Incomplete, ...]: ... + def sash_mark(self, index) -> tuple[Incomplete, ...]: ... + def sash_place(self, index, x, y) -> tuple[Incomplete, ...]: ... def panecget(self, child, option): ... def paneconfigure(self, tagOrId, cnf=None, **kw): ... - paneconfig: Incomplete + paneconfig = paneconfigure def panes(self): ... def _test() -> None: ... diff --git a/mypy/typeshed/stdlib/tkinter/ttk.pyi b/mypy/typeshed/stdlib/tkinter/ttk.pyi index 86c55eba7006d..1d72acd995126 100644 --- a/mypy/typeshed/stdlib/tkinter/ttk.pyi +++ b/mypy/typeshed/stdlib/tkinter/ttk.pyi @@ -39,21 +39,19 @@ def tclobjs_to_py(adict: dict[Any, Any]) -> dict[Any, Any]: ... def setup_master(master: tkinter.Misc | None = None): ... _Padding: TypeAlias = ( - tkinter._ScreenUnits - | tuple[tkinter._ScreenUnits] - | tuple[tkinter._ScreenUnits, tkinter._ScreenUnits] - | tuple[tkinter._ScreenUnits, tkinter._ScreenUnits, tkinter._ScreenUnits] - | tuple[tkinter._ScreenUnits, tkinter._ScreenUnits, tkinter._ScreenUnits, tkinter._ScreenUnits] + float + | str + | tuple[float | str] + | tuple[float | str, float | str] + | tuple[float | str, float | str, float | str] + | tuple[float | str, float | str, float | str, float | str] ) -# from ttk_widget (aka ttk::widget) manual page, differs from tkinter._Compound -_TtkCompound: TypeAlias = Literal["", "text", "image", tkinter._Compound] - # Last item (option value to apply) varies between different options so use Any. # It could also be any iterable with items matching the tuple, but that case # hasn't been added here for consistency with _Padding above. _Statespec: TypeAlias = tuple[Unpack[tuple[str, ...]], Any] -_ImageStatespec: TypeAlias = tuple[Unpack[tuple[str, ...]], tkinter._ImageSpec] +_ImageStatespec: TypeAlias = tuple[Unpack[tuple[str, ...]], tkinter._Image | str] _VsapiStatespec: TypeAlias = tuple[Unpack[tuple[str, ...]], int] class _Layout(TypedDict, total=False): @@ -69,14 +67,14 @@ _LayoutSpec: TypeAlias = list[tuple[str, _Layout | None]] # Keep these in sync with the appropriate methods in Style class _ElementCreateImageKwargs(TypedDict, total=False): border: _Padding - height: tkinter._ScreenUnits + height: float | str padding: _Padding sticky: str - width: tkinter._ScreenUnits + width: float | str _ElementCreateArgsCrossPlatform: TypeAlias = ( # Could be any sequence here but types are not homogenous so just type it as tuple - tuple[Literal["image"], tkinter._ImageSpec, Unpack[tuple[_ImageStatespec, ...]], _ElementCreateImageKwargs] + tuple[Literal["image"], tkinter._Image | str, Unpack[tuple[_ImageStatespec, ...]], _ElementCreateImageKwargs] | tuple[Literal["from"], str, str] | tuple[Literal["from"], str] # (fromelement is optional) ) @@ -88,8 +86,8 @@ if sys.platform == "win32" and sys.version_info >= (3, 13): padding: _Padding class _ElementCreateVsapiKwargsSize(TypedDict): - width: tkinter._ScreenUnits - height: tkinter._ScreenUnits + width: float | str + height: float | str _ElementCreateVsapiKwargsDict: TypeAlias = ( _ElementCreateVsapiKwargsPadding | _ElementCreateVsapiKwargsMargin | _ElementCreateVsapiKwargsSize @@ -139,14 +137,14 @@ class Style: self, elementname: str, etype: Literal["image"], - default_image: tkinter._ImageSpec, + default_image: tkinter._Image | str, /, *imagespec: _ImageStatespec, border: _Padding = ..., - height: tkinter._ScreenUnits = ..., + height: float | str = ..., padding: _Padding = ..., sticky: str = ..., - width: tkinter._ScreenUnits = ..., + width: float | str = ..., ) -> None: ... @overload def element_create(self, elementname: str, etype: Literal["from"], themename: str, fromelement: str = ..., /) -> None: ... @@ -188,8 +186,8 @@ class Style: vs_statespec: _VsapiStatespec = ..., /, *, - width: tkinter._ScreenUnits, - height: tkinter._ScreenUnits, + width: float | str, + height: float | str, ) -> None: ... def element_names(self) -> tuple[str, ...]: ... @@ -214,16 +212,16 @@ class Button(Widget): master: tkinter.Misc | None = None, *, class_: str = "", - command: tkinter._ButtonCommand = "", - compound: _TtkCompound = "", + command: str | Callable[[], Any] = "", + compound: Literal["", "text", "image", "top", "left", "center", "right", "bottom", "none"] = "", cursor: tkinter._Cursor = "", default: Literal["normal", "active", "disabled"] = "normal", - image: tkinter._ImageSpec = "", + image: tkinter._Image | str = "", name: str = ..., padding=..., # undocumented state: str = "normal", style: str = "", - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., text: float | str = "", textvariable: tkinter.Variable = ..., underline: int = -1, @@ -234,15 +232,15 @@ class Button(Widget): self, cnf: dict[str, Any] | None = None, *, - command: tkinter._ButtonCommand = ..., - compound: _TtkCompound = ..., + command: str | Callable[[], Any] = ..., + compound: Literal["", "text", "image", "top", "left", "center", "right", "bottom", "none"] = ..., cursor: tkinter._Cursor = ..., default: Literal["normal", "active", "disabled"] = ..., - image: tkinter._ImageSpec = ..., + image: tkinter._Image | str = ..., padding=..., state: str = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., text: float | str = ..., textvariable: tkinter.Variable = ..., underline: int = ..., @@ -259,17 +257,17 @@ class Checkbutton(Widget): master: tkinter.Misc | None = None, *, class_: str = "", - command: tkinter._ButtonCommand = "", - compound: _TtkCompound = "", + command: str | Callable[[], Any] = "", + compound: Literal["", "text", "image", "top", "left", "center", "right", "bottom", "none"] = "", cursor: tkinter._Cursor = "", - image: tkinter._ImageSpec = "", + image: tkinter._Image | str = "", name: str = ..., offvalue: Any = 0, onvalue: Any = 1, padding=..., # undocumented state: str = "normal", style: str = "", - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., text: float | str = "", textvariable: tkinter.Variable = ..., underline: int = -1, @@ -284,16 +282,16 @@ class Checkbutton(Widget): self, cnf: dict[str, Any] | None = None, *, - command: tkinter._ButtonCommand = ..., - compound: _TtkCompound = ..., + command: str | Callable[[], Any] = ..., + compound: Literal["", "text", "image", "top", "left", "center", "right", "bottom", "none"] = ..., cursor: tkinter._Cursor = ..., - image: tkinter._ImageSpec = ..., + image: tkinter._Image | str = ..., offvalue: Any = ..., onvalue: Any = ..., padding=..., state: str = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., text: float | str = ..., textvariable: tkinter.Variable = ..., underline: int = ..., @@ -317,18 +315,18 @@ class Entry(Widget, tkinter.Entry): exportselection: bool = True, font: _FontDescription = "TkTextFont", foreground: str = "", - invalidcommand: tkinter._EntryValidateCommand = "", + invalidcommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = "", justify: Literal["left", "center", "right"] = "left", name: str = ..., show: str = "", state: str = "normal", style: str = "", - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., textvariable: tkinter.Variable = ..., validate: Literal["none", "focus", "focusin", "focusout", "key", "all"] = "none", - validatecommand: tkinter._EntryValidateCommand = "", + validatecommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = "", width: int = 20, - xscrollcommand: tkinter._XYScrollCommand = "", + xscrollcommand: str | Callable[[float, float], object] = "", ) -> None: ... @overload # type: ignore[override] def configure( @@ -340,17 +338,17 @@ class Entry(Widget, tkinter.Entry): exportselection: bool = ..., font: _FontDescription = ..., foreground: str = ..., - invalidcommand: tkinter._EntryValidateCommand = ..., + invalidcommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., justify: Literal["left", "center", "right"] = ..., show: str = ..., state: str = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., textvariable: tkinter.Variable = ..., validate: Literal["none", "focus", "focusin", "focusout", "key", "all"] = ..., - validatecommand: tkinter._EntryValidateCommand = ..., + validatecommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., width: int = ..., - xscrollcommand: tkinter._XYScrollCommand = ..., + xscrollcommand: str | Callable[[float, float], object] = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -365,17 +363,17 @@ class Entry(Widget, tkinter.Entry): exportselection: bool = ..., font: _FontDescription = ..., foreground: str = ..., - invalidcommand: tkinter._EntryValidateCommand = ..., + invalidcommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., justify: Literal["left", "center", "right"] = ..., show: str = ..., state: str = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., textvariable: tkinter.Variable = ..., validate: Literal["none", "focus", "focusin", "focusout", "key", "all"] = ..., - validatecommand: tkinter._EntryValidateCommand = ..., + validatecommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., width: int = ..., - xscrollcommand: tkinter._XYScrollCommand = ..., + xscrollcommand: str | Callable[[float, float], object] = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def config(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -395,20 +393,20 @@ class Combobox(Entry): font: _FontDescription = ..., # undocumented foreground: str = ..., # undocumented height: int = 10, - invalidcommand: tkinter._EntryValidateCommand = ..., # undocumented + invalidcommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., # undocumented justify: Literal["left", "center", "right"] = "left", name: str = ..., postcommand: Callable[[], object] | str = "", show=..., # undocumented state: str = "normal", style: str = "", - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., textvariable: tkinter.Variable = ..., validate: Literal["none", "focus", "focusin", "focusout", "key", "all"] = ..., # undocumented - validatecommand: tkinter._EntryValidateCommand = ..., # undocumented + validatecommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., # undocumented values: list[str] | tuple[str, ...] = ..., width: int = 20, - xscrollcommand: tkinter._XYScrollCommand = ..., # undocumented + xscrollcommand: str | Callable[[float, float], object] = ..., # undocumented ) -> None: ... @overload # type: ignore[override] def configure( @@ -421,19 +419,19 @@ class Combobox(Entry): font: _FontDescription = ..., foreground: str = ..., height: int = ..., - invalidcommand: tkinter._EntryValidateCommand = ..., + invalidcommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., justify: Literal["left", "center", "right"] = ..., postcommand: Callable[[], object] | str = ..., show=..., state: str = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., textvariable: tkinter.Variable = ..., validate: Literal["none", "focus", "focusin", "focusout", "key", "all"] = ..., - validatecommand: tkinter._EntryValidateCommand = ..., + validatecommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., values: list[str] | tuple[str, ...] = ..., width: int = ..., - xscrollcommand: tkinter._XYScrollCommand = ..., + xscrollcommand: str | Callable[[float, float], object] = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -449,19 +447,19 @@ class Combobox(Entry): font: _FontDescription = ..., foreground: str = ..., height: int = ..., - invalidcommand: tkinter._EntryValidateCommand = ..., + invalidcommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., justify: Literal["left", "center", "right"] = ..., postcommand: Callable[[], object] | str = ..., show=..., state: str = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., textvariable: tkinter.Variable = ..., validate: Literal["none", "focus", "focusin", "focusout", "key", "all"] = ..., - validatecommand: tkinter._EntryValidateCommand = ..., + validatecommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., values: list[str] | tuple[str, ...] = ..., width: int = ..., - xscrollcommand: tkinter._XYScrollCommand = ..., + xscrollcommand: str | Callable[[float, float], object] = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def config(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -475,32 +473,32 @@ class Frame(Widget): self, master: tkinter.Misc | None = None, *, - border: tkinter._ScreenUnits = ..., - borderwidth: tkinter._ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., class_: str = "", cursor: tkinter._Cursor = "", - height: tkinter._ScreenUnits = 0, + height: float | str = 0, name: str = ..., padding: _Padding = ..., - relief: tkinter._Relief = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., style: str = "", - takefocus: tkinter._TakeFocusValue = "", - width: tkinter._ScreenUnits = 0, + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = "", + width: float | str = 0, ) -> None: ... @overload def configure( self, cnf: dict[str, Any] | None = None, *, - border: tkinter._ScreenUnits = ..., - borderwidth: tkinter._ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., cursor: tkinter._Cursor = ..., - height: tkinter._ScreenUnits = ..., + height: float | str = ..., padding: _Padding = ..., - relief: tkinter._Relief = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., - width: tkinter._ScreenUnits = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., + width: float | str = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -511,54 +509,54 @@ class Label(Widget): self, master: tkinter.Misc | None = None, *, - anchor: tkinter._Anchor = ..., + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = ..., background: str = "", - border: tkinter._ScreenUnits = ..., # alias for borderwidth - borderwidth: tkinter._ScreenUnits = ..., # undocumented + border: float | str = ..., # alias for borderwidth + borderwidth: float | str = ..., # undocumented class_: str = "", - compound: _TtkCompound = "", + compound: Literal["", "text", "image", "top", "left", "center", "right", "bottom", "none"] = "", cursor: tkinter._Cursor = "", font: _FontDescription = ..., foreground: str = "", - image: tkinter._ImageSpec = "", + image: tkinter._Image | str = "", justify: Literal["left", "center", "right"] = ..., name: str = ..., padding: _Padding = ..., - relief: tkinter._Relief = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., state: str = "normal", style: str = "", - takefocus: tkinter._TakeFocusValue = "", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = "", text: float | str = "", textvariable: tkinter.Variable = ..., underline: int = -1, width: int | Literal[""] = "", - wraplength: tkinter._ScreenUnits = ..., + wraplength: float | str = ..., ) -> None: ... @overload def configure( self, cnf: dict[str, Any] | None = None, *, - anchor: tkinter._Anchor = ..., + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = ..., background: str = ..., - border: tkinter._ScreenUnits = ..., - borderwidth: tkinter._ScreenUnits = ..., - compound: _TtkCompound = ..., + border: float | str = ..., + borderwidth: float | str = ..., + compound: Literal["", "text", "image", "top", "left", "center", "right", "bottom", "none"] = ..., cursor: tkinter._Cursor = ..., font: _FontDescription = ..., foreground: str = ..., - image: tkinter._ImageSpec = ..., + image: tkinter._Image | str = ..., justify: Literal["left", "center", "right"] = ..., padding: _Padding = ..., - relief: tkinter._Relief = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., state: str = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., text: float | str = ..., textvariable: tkinter.Variable = ..., underline: int = ..., width: int | Literal[""] = ..., - wraplength: tkinter._ScreenUnits = ..., + wraplength: float | str = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -569,40 +567,40 @@ class Labelframe(Widget): self, master: tkinter.Misc | None = None, *, - border: tkinter._ScreenUnits = ..., - borderwidth: tkinter._ScreenUnits = ..., # undocumented + border: float | str = ..., + borderwidth: float | str = ..., # undocumented class_: str = "", cursor: tkinter._Cursor = "", - height: tkinter._ScreenUnits = 0, + height: float | str = 0, labelanchor: Literal["nw", "n", "ne", "en", "e", "es", "se", "s", "sw", "ws", "w", "wn"] = ..., labelwidget: tkinter.Misc = ..., name: str = ..., padding: _Padding = ..., - relief: tkinter._Relief = ..., # undocumented + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., # undocumented style: str = "", - takefocus: tkinter._TakeFocusValue = "", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = "", text: float | str = "", underline: int = -1, - width: tkinter._ScreenUnits = 0, + width: float | str = 0, ) -> None: ... @overload def configure( self, cnf: dict[str, Any] | None = None, *, - border: tkinter._ScreenUnits = ..., - borderwidth: tkinter._ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., cursor: tkinter._Cursor = ..., - height: tkinter._ScreenUnits = ..., + height: float | str = ..., labelanchor: Literal["nw", "n", "ne", "en", "e", "es", "se", "s", "sw", "ws", "w", "wn"] = ..., labelwidget: tkinter.Misc = ..., padding: _Padding = ..., - relief: tkinter._Relief = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., text: float | str = ..., underline: int = ..., - width: tkinter._ScreenUnits = ..., + width: float | str = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -616,16 +614,16 @@ class Menubutton(Widget): master: tkinter.Misc | None = None, *, class_: str = "", - compound: _TtkCompound = "", + compound: Literal["", "text", "image", "top", "left", "center", "right", "bottom", "none"] = "", cursor: tkinter._Cursor = "", direction: Literal["above", "below", "left", "right", "flush"] = "below", - image: tkinter._ImageSpec = "", + image: tkinter._Image | str = "", menu: tkinter.Menu = ..., name: str = ..., padding=..., # undocumented state: str = "normal", style: str = "", - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., text: float | str = "", textvariable: tkinter.Variable = ..., underline: int = -1, @@ -636,15 +634,15 @@ class Menubutton(Widget): self, cnf: dict[str, Any] | None = None, *, - compound: _TtkCompound = ..., + compound: Literal["", "text", "image", "top", "left", "center", "right", "bottom", "none"] = ..., cursor: tkinter._Cursor = ..., direction: Literal["above", "below", "left", "right", "flush"] = ..., - image: tkinter._ImageSpec = ..., + image: tkinter._Image | str = ..., menu: tkinter.Menu = ..., padding=..., state: str = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., text: float | str = ..., textvariable: tkinter.Variable = ..., underline: int = ..., @@ -665,7 +663,7 @@ class Notebook(Widget): name: str = ..., padding: _Padding = ..., style: str = "", - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., width: int = 0, ) -> None: ... @overload @@ -677,7 +675,7 @@ class Notebook(Widget): height: int = ..., padding: _Padding = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., width: int = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload @@ -694,7 +692,7 @@ class Notebook(Widget): # `image` is a sequence of an image name, followed by zero or more # (sequences of one or more state names followed by an image name) image=..., - compound: tkinter._Compound = ..., + compound: Literal["top", "left", "center", "right", "bottom", "none"] = ..., underline: int = ..., ) -> None: ... def forget(self, tab_id) -> None: ... # type: ignore[override] @@ -719,7 +717,7 @@ class Panedwindow(Widget, tkinter.PanedWindow): name: str = ..., orient: Literal["vertical", "horizontal"] = "vertical", # can't be changed with configure() style: str = "", - takefocus: tkinter._TakeFocusValue = "", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = "", width: int = 0, ) -> None: ... def add(self, child: tkinter.Widget, *, weight: int = ..., **kw) -> None: ... @@ -731,7 +729,7 @@ class Panedwindow(Widget, tkinter.PanedWindow): cursor: tkinter._Cursor = ..., height: int = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., width: int = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload @@ -745,7 +743,7 @@ class Panedwindow(Widget, tkinter.PanedWindow): cursor: tkinter._Cursor = ..., height: int = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., width: int = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload @@ -764,14 +762,14 @@ class Progressbar(Widget): *, class_: str = "", cursor: tkinter._Cursor = "", - length: tkinter._ScreenUnits = 100, + length: float | str = 100, maximum: float = 100, mode: Literal["determinate", "indeterminate"] = "determinate", name: str = ..., orient: Literal["horizontal", "vertical"] = "horizontal", phase: int = 0, # docs say read-only but assigning int to this works style: str = "", - takefocus: tkinter._TakeFocusValue = "", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = "", value: float = 0.0, variable: tkinter.IntVar | tkinter.DoubleVar = ..., ) -> None: ... @@ -781,13 +779,13 @@ class Progressbar(Widget): cnf: dict[str, Any] | None = None, *, cursor: tkinter._Cursor = ..., - length: tkinter._ScreenUnits = ..., + length: float | str = ..., maximum: float = ..., mode: Literal["determinate", "indeterminate"] = ..., orient: Literal["horizontal", "vertical"] = ..., phase: int = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., value: float = ..., variable: tkinter.IntVar | tkinter.DoubleVar = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @@ -804,15 +802,15 @@ class Radiobutton(Widget): master: tkinter.Misc | None = None, *, class_: str = "", - command: tkinter._ButtonCommand = "", - compound: _TtkCompound = "", + command: str | Callable[[], Any] = "", + compound: Literal["", "text", "image", "top", "left", "center", "right", "bottom", "none"] = "", cursor: tkinter._Cursor = "", - image: tkinter._ImageSpec = "", + image: tkinter._Image | str = "", name: str = ..., padding=..., # undocumented state: str = "normal", style: str = "", - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., text: float | str = "", textvariable: tkinter.Variable = ..., underline: int = -1, @@ -825,14 +823,14 @@ class Radiobutton(Widget): self, cnf: dict[str, Any] | None = None, *, - command: tkinter._ButtonCommand = ..., - compound: _TtkCompound = ..., + command: str | Callable[[], Any] = ..., + compound: Literal["", "text", "image", "top", "left", "center", "right", "bottom", "none"] = ..., cursor: tkinter._Cursor = ..., - image: tkinter._ImageSpec = ..., + image: tkinter._Image | str = ..., padding=..., state: str = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., text: float | str = ..., textvariable: tkinter.Variable = ..., underline: int = ..., @@ -855,12 +853,12 @@ class Scale(Widget, tkinter.Scale): # type: ignore[misc] command: str | Callable[[str], object] = "", cursor: tkinter._Cursor = "", from_: float = 0, - length: tkinter._ScreenUnits = 100, + length: float | str = 100, name: str = ..., orient: Literal["horizontal", "vertical"] = "horizontal", state: str = ..., # undocumented style: str = "", - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., to: float = 1.0, value: float = 0, variable: tkinter.IntVar | tkinter.DoubleVar = ..., @@ -873,11 +871,11 @@ class Scale(Widget, tkinter.Scale): # type: ignore[misc] command: str | Callable[[str], object] = ..., cursor: tkinter._Cursor = ..., from_: float = ..., - length: tkinter._ScreenUnits = ..., + length: float | str = ..., orient: Literal["horizontal", "vertical"] = ..., state: str = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., to: float = ..., value: float = ..., variable: tkinter.IntVar | tkinter.DoubleVar = ..., @@ -893,11 +891,11 @@ class Scale(Widget, tkinter.Scale): # type: ignore[misc] command: str | Callable[[str], object] = ..., cursor: tkinter._Cursor = ..., from_: float = ..., - length: tkinter._ScreenUnits = ..., + length: float | str = ..., orient: Literal["horizontal", "vertical"] = ..., state: str = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., to: float = ..., value: float = ..., variable: tkinter.IntVar | tkinter.DoubleVar = ..., @@ -918,7 +916,7 @@ class Scrollbar(Widget, tkinter.Scrollbar): # type: ignore[misc] name: str = ..., orient: Literal["horizontal", "vertical"] = "vertical", style: str = "", - takefocus: tkinter._TakeFocusValue = "", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = "", ) -> None: ... @overload # type: ignore[override] def configure( @@ -929,7 +927,7 @@ class Scrollbar(Widget, tkinter.Scrollbar): # type: ignore[misc] cursor: tkinter._Cursor = ..., orient: Literal["horizontal", "vertical"] = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -943,7 +941,7 @@ class Scrollbar(Widget, tkinter.Scrollbar): # type: ignore[misc] cursor: tkinter._Cursor = ..., orient: Literal["horizontal", "vertical"] = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def config(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -958,7 +956,7 @@ class Separator(Widget): name: str = ..., orient: Literal["horizontal", "vertical"] = "horizontal", style: str = "", - takefocus: tkinter._TakeFocusValue = "", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = "", ) -> None: ... @overload def configure( @@ -968,7 +966,7 @@ class Separator(Widget): cursor: tkinter._Cursor = ..., orient: Literal["horizontal", "vertical"] = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -983,7 +981,7 @@ class Sizegrip(Widget): cursor: tkinter._Cursor = ..., name: str = ..., style: str = "", - takefocus: tkinter._TakeFocusValue = "", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = "", ) -> None: ... @overload def configure( @@ -992,7 +990,7 @@ class Sizegrip(Widget): *, cursor: tkinter._Cursor = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -1013,21 +1011,21 @@ class Spinbox(Entry): format: str = "", from_: float = 0, increment: float = 1, - invalidcommand: tkinter._EntryValidateCommand = ..., # undocumented + invalidcommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., # undocumented justify: Literal["left", "center", "right"] = ..., # undocumented name: str = ..., show=..., # undocumented state: str = "normal", style: str = "", - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., textvariable: tkinter.Variable = ..., # undocumented to: float = 0, validate: Literal["none", "focus", "focusin", "focusout", "key", "all"] = "none", - validatecommand: tkinter._EntryValidateCommand = "", + validatecommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = "", values: list[str] | tuple[str, ...] = ..., width: int = ..., # undocumented wrap: bool = False, - xscrollcommand: tkinter._XYScrollCommand = "", + xscrollcommand: str | Callable[[float, float], object] = "", ) -> None: ... @overload # type: ignore[override] def configure( @@ -1043,20 +1041,20 @@ class Spinbox(Entry): format: str = ..., from_: float = ..., increment: float = ..., - invalidcommand: tkinter._EntryValidateCommand = ..., + invalidcommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., justify: Literal["left", "center", "right"] = ..., show=..., state: str = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., textvariable: tkinter.Variable = ..., to: float = ..., validate: Literal["none", "focus", "focusin", "focusout", "key", "all"] = ..., - validatecommand: tkinter._EntryValidateCommand = ..., + validatecommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., values: list[str] | tuple[str, ...] = ..., width: int = ..., wrap: bool = ..., - xscrollcommand: tkinter._XYScrollCommand = ..., + xscrollcommand: str | Callable[[float, float], object] = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -1083,7 +1081,7 @@ class _TreeviewTagDict(TypedDict): class _TreeviewHeaderDict(TypedDict): text: str image: list[str] | Literal[""] - anchor: tkinter._Anchor + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] command: str state: str # Doesn't seem to appear anywhere else than in these dicts @@ -1092,7 +1090,7 @@ class _TreeviewColumnDict(TypedDict): width: int minwidth: int stretch: bool # actually 0 or 1 - anchor: tkinter._Anchor + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] id: str class Treeview(Widget, tkinter.XView, tkinter.YView): @@ -1114,9 +1112,9 @@ class Treeview(Widget, tkinter.XView, tkinter.YView): # surprised if someone is using it. show: Literal["tree", "headings", "tree headings", ""] | list[str] | tuple[str, ...] = ("tree", "headings"), style: str = "", - takefocus: tkinter._TakeFocusValue = ..., - xscrollcommand: tkinter._XYScrollCommand = "", - yscrollcommand: tkinter._XYScrollCommand = "", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., + xscrollcommand: str | Callable[[float, float], object] = "", + yscrollcommand: str | Callable[[float, float], object] = "", ) -> None: ... @overload def configure( @@ -1131,9 +1129,9 @@ class Treeview(Widget, tkinter.XView, tkinter.YView): selectmode: Literal["extended", "browse", "none"] = ..., show: Literal["tree", "headings", "tree headings", ""] | list[str] | tuple[str, ...] = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., - xscrollcommand: tkinter._XYScrollCommand = ..., - yscrollcommand: tkinter._XYScrollCommand = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., + xscrollcommand: str | Callable[[float, float], object] = ..., + yscrollcommand: str | Callable[[float, float], object] = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -1160,7 +1158,7 @@ class Treeview(Widget, tkinter.XView, tkinter.YView): width: int = ..., minwidth: int = ..., stretch: bool = ..., - anchor: tkinter._Anchor = ..., + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = ..., # id is read-only ) -> _TreeviewColumnDict | None: ... def delete(self, *items: str | int) -> None: ... @@ -1189,8 +1187,8 @@ class Treeview(Widget, tkinter.XView, tkinter.YView): option: None = None, *, text: str = ..., - image: tkinter._ImageSpec = ..., - anchor: tkinter._Anchor = ..., + image: tkinter._Image | str = ..., + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = ..., command: str | Callable[[], object] = ..., ) -> None: ... # Internal Method. Leave untyped: @@ -1208,7 +1206,7 @@ class Treeview(Widget, tkinter.XView, tkinter.YView): *, id: str | int = ..., # same as iid text: str = ..., - image: tkinter._ImageSpec = ..., + image: tkinter._Image | str = ..., values: list[Any] | tuple[Any, ...] = ..., open: bool = ..., tags: str | list[str] | tuple[str, ...] = ..., @@ -1234,7 +1232,7 @@ class Treeview(Widget, tkinter.XView, tkinter.YView): option: None = None, *, text: str = ..., - image: tkinter._ImageSpec = ..., + image: tkinter._Image | str = ..., values: list[Any] | tuple[Any, ...] | Literal[""] = ..., open: bool = ..., tags: str | list[str] | tuple[str, ...] = ..., @@ -1294,7 +1292,7 @@ class Treeview(Widget, tkinter.XView, tkinter.YView): foreground: str = ..., background: str = ..., font: _FontDescription = ..., - image: tkinter._ImageSpec = ..., + image: tkinter._Image | str = ..., ) -> _TreeviewTagDict | MaybeNone: ... # can be None but annoying to check @overload def tag_has(self, tagname: str, item: None = None) -> tuple[str, ...]: ... @@ -1313,18 +1311,18 @@ class LabeledScale(Frame): from_: float = 0, to: float = 10, *, - border: tkinter._ScreenUnits = ..., - borderwidth: tkinter._ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., class_: str = "", compound: Literal["top", "bottom"] = "top", cursor: tkinter._Cursor = "", - height: tkinter._ScreenUnits = 0, + height: float | str = 0, name: str = ..., padding: _Padding = ..., - relief: tkinter._Relief = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., style: str = "", - takefocus: tkinter._TakeFocusValue = "", - width: tkinter._ScreenUnits = 0, + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = "", + width: float | str = 0, ) -> None: ... # destroy is overridden, signature does not change value: Any diff --git a/mypy/typeshed/stdlib/types.pyi b/mypy/typeshed/stdlib/types.pyi index 591d5da2360dc..ba343ce9effc0 100644 --- a/mypy/typeshed/stdlib/types.pyi +++ b/mypy/typeshed/stdlib/types.pyi @@ -392,8 +392,8 @@ class CellType: cell_contents: Any _YieldT_co = TypeVar("_YieldT_co", covariant=True) -_SendT_contra = TypeVar("_SendT_contra", contravariant=True) -_ReturnT_co = TypeVar("_ReturnT_co", covariant=True) +_SendT_contra = TypeVar("_SendT_contra", contravariant=True, default=None) +_ReturnT_co = TypeVar("_ReturnT_co", covariant=True, default=None) @final class GeneratorType(Generator[_YieldT_co, _SendT_contra, _ReturnT_co]): @@ -450,16 +450,25 @@ class AsyncGeneratorType(AsyncGenerator[_YieldT_co, _SendT_contra]): def aclose(self) -> Coroutine[Any, Any, None]: ... def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... +# Non-default variations to accommodate coroutines +_SendT_nd_contra = TypeVar("_SendT_nd_contra", contravariant=True) +_ReturnT_nd_co = TypeVar("_ReturnT_nd_co", covariant=True) + @final -class CoroutineType(Coroutine[_YieldT_co, _SendT_contra, _ReturnT_co]): +class CoroutineType(Coroutine[_YieldT_co, _SendT_nd_contra, _ReturnT_nd_co]): __name__: str __qualname__: str @property def cr_await(self) -> Any | None: ... @property def cr_code(self) -> CodeType: ... - @property - def cr_frame(self) -> FrameType: ... + if sys.version_info >= (3, 12): + @property + def cr_frame(self) -> FrameType | None: ... + else: + @property + def cr_frame(self) -> FrameType: ... + @property def cr_running(self) -> bool: ... @property @@ -469,8 +478,8 @@ class CoroutineType(Coroutine[_YieldT_co, _SendT_contra, _ReturnT_co]): def cr_suspended(self) -> bool: ... def close(self) -> None: ... - def __await__(self) -> Generator[Any, None, _ReturnT_co]: ... - def send(self, arg: _SendT_contra, /) -> _YieldT_co: ... + def __await__(self) -> Generator[Any, None, _ReturnT_nd_co]: ... + def send(self, arg: _SendT_nd_contra, /) -> _YieldT_co: ... @overload def throw( self, typ: type[BaseException], val: BaseException | object = ..., tb: TracebackType | None = ..., / @@ -717,10 +726,19 @@ if sys.version_info >= (3, 10): def __args__(self) -> tuple[Any, ...]: ... @property def __parameters__(self) -> tuple[Any, ...]: ... - def __or__(self, value: Any, /) -> UnionType: ... - def __ror__(self, value: Any, /) -> UnionType: ... + # `(int | str) | Literal["foo"]` returns a generic alias to an instance of `_SpecialForm` (`Union`). + # Normally we'd express this using the return type of `_SpecialForm.__ror__`, + # but because `UnionType.__or__` accepts `Any`, type checkers will use + # the return type of `UnionType.__or__` to infer the result of this operation + # rather than `_SpecialForm.__ror__`. To mitigate this, we use `| Any` + # in the return type of `UnionType.__(r)or__`. + def __or__(self, value: Any, /) -> UnionType | Any: ... + def __ror__(self, value: Any, /) -> UnionType | Any: ... def __eq__(self, value: object, /) -> bool: ... def __hash__(self) -> int: ... + # you can only subscript a `UnionType` instance if at least one of the elements + # in the union is a generic alias instance that has a non-empty `__parameters__` + def __getitem__(self, parameters: Any) -> object: ... if sys.version_info >= (3, 13): @final diff --git a/mypy/typeshed/stdlib/typing.pyi b/mypy/typeshed/stdlib/typing.pyi index 15a5864613d1f..ca25c92d5c34a 100644 --- a/mypy/typeshed/stdlib/typing.pyi +++ b/mypy/typeshed/stdlib/typing.pyi @@ -41,6 +41,7 @@ __all__ = [ "AsyncIterator", "Awaitable", "BinaryIO", + "ByteString", "Callable", "ChainMap", "ClassVar", @@ -109,9 +110,6 @@ __all__ = [ "runtime_checkable", ] -if sys.version_info < (3, 14): - __all__ += ["ByteString"] - if sys.version_info >= (3, 14): __all__ += ["evaluate_forward_ref"] @@ -579,7 +577,7 @@ class Awaitable(Protocol[_T_co]): @abstractmethod def __await__(self) -> Generator[Any, Any, _T_co]: ... -# Non-default variations to accommodate couroutines, and `AwaitableGenerator` having a 4th type parameter. +# Non-default variations to accommodate coroutines, and `AwaitableGenerator` having a 4th type parameter. _SendT_nd_contra = TypeVar("_SendT_nd_contra", contravariant=True) _ReturnT_nd_co = TypeVar("_ReturnT_nd_co", covariant=True) @@ -923,8 +921,7 @@ class TextIO(IO[str]): @abstractmethod def __enter__(self) -> TextIO: ... -if sys.version_info < (3, 14): - ByteString: typing_extensions.TypeAlias = bytes | bytearray | memoryview +ByteString: typing_extensions.TypeAlias = bytes | bytearray | memoryview # Functions From 053c0545dfe48126b66af8ece091af3c0c29bddc Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 1 Oct 2025 13:08:05 +0100 Subject: [PATCH 288/424] Fix crash on invalid unpack in base class (#19962) Fixes https://github.com/python/mypy/issues/19960 Fix is trivial, we can't use the "assertion" before type checking phase. I also fix missing line/column for union types. I am surprised this didn't cause problems before. --- mypy/semanal_shared.py | 4 +++- mypy/typeanal.py | 2 +- test-data/unit/check-python312.test | 10 ++++++++++ test-data/unit/check-typevar-tuple.test | 24 ++++++++++++++++++++++++ 4 files changed, 38 insertions(+), 2 deletions(-) diff --git a/mypy/semanal_shared.py b/mypy/semanal_shared.py index e94604b66381a..c49b13d08f451 100644 --- a/mypy/semanal_shared.py +++ b/mypy/semanal_shared.py @@ -303,7 +303,9 @@ def calculate_tuple_fallback(typ: TupleType) -> None: ): items.append(unpacked_type.args[0]) else: - raise NotImplementedError + # This is called before semanal_typeargs.py fixes broken unpacks, + # where the error should also be generated. + items.append(AnyType(TypeOfAny.from_error)) else: items.append(item) fallback.args = (make_simplified_union(items),) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 81fb87fbf9ee1..d7a07c9f48e32 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -634,7 +634,7 @@ def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Typ ) elif fullname == "typing.Union": items = self.anal_array(t.args) - return UnionType.make_union(items) + return UnionType.make_union(items, line=t.line, column=t.column) elif fullname == "typing.Optional": if len(t.args) != 1: self.fail( diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 382822ced8617..6a72a7e4d5b51 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -2121,3 +2121,13 @@ class A: ... x1: Alias1[A] # ok x2: Alias2[A] # ok + +[case testUndefinedUnpackInPEP696Base] +# Typo below is intentional. +class MyTuple[*Ts](tuple[*TS]): # E: Name "TS" is not defined + ... + +x: MyTuple[int, str] +reveal_type(x[0]) # N: Revealed type is "Any" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index c668f14eaa503..927a4f037a4a0 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -2692,3 +2692,27 @@ tuple(a) (x,) = a (_,) = a [builtins fixtures/tuple.pyi] + +[case testNoCrashOnUndefinedUnpackInBase] +from typing import TypeVarTuple, Generic, Unpack + +Ts = TypeVarTuple("Ts") + +class MyTuple(tuple[Unpack[TsWithTypo]], Generic[Unpack[Ts]]): # E: Name "TsWithTypo" is not defined + ... + +x: MyTuple[int, str] +reveal_type(x[0]) # N: Revealed type is "Any" +[builtins fixtures/tuple.pyi] + +[case testNoCrashOnInvalidUnpackInBase] +from typing import TypeVarTuple, Generic, Unpack, Union + +Ts = TypeVarTuple("Ts") + +class MyTuple(tuple[Unpack[Union[int, str]]], Generic[Unpack[Ts]]): # E: "Union[int, str]" cannot be unpacked (must be tuple or TypeVarTuple) + ... + +x: MyTuple[int, str] +reveal_type(x[0]) # N: Revealed type is "Any" +[builtins fixtures/tuple.pyi] From 4171da0cf394bb0b718eea3019760a8a33de0c74 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 2 Oct 2025 01:37:50 +0100 Subject: [PATCH 289/424] Delete native_internal import fallback (#19966) This should not be needed after we switched to `librt` published on PyPI. --- mypy/cache.py | 71 ++++++++++----------------------------------------- 1 file changed, 14 insertions(+), 57 deletions(-) diff --git a/mypy/cache.py b/mypy/cache.py index 08e3b05d1a753..51deac914efc8 100644 --- a/mypy/cache.py +++ b/mypy/cache.py @@ -1,65 +1,22 @@ from __future__ import annotations from collections.abc import Sequence -from typing import TYPE_CHECKING, Final +from typing import Final from mypy_extensions import u8 - -try: - from native_internal import ( - Buffer as Buffer, - read_bool as read_bool, - read_float as read_float, - read_int as read_int, - read_str as read_str, - read_tag as read_tag, - write_bool as write_bool, - write_float as write_float, - write_int as write_int, - write_str as write_str, - write_tag as write_tag, - ) -except ImportError: - # TODO: temporary, remove this after we publish mypy-native on PyPI. - if not TYPE_CHECKING: - - class Buffer: - def __init__(self, source: bytes = b"") -> None: - raise NotImplementedError - - def getvalue(self) -> bytes: - raise NotImplementedError - - def read_int(data: Buffer) -> int: - raise NotImplementedError - - def write_int(data: Buffer, value: int) -> None: - raise NotImplementedError - - def read_tag(data: Buffer) -> u8: - raise NotImplementedError - - def write_tag(data: Buffer, value: u8) -> None: - raise NotImplementedError - - def read_str(data: Buffer) -> str: - raise NotImplementedError - - def write_str(data: Buffer, value: str) -> None: - raise NotImplementedError - - def read_bool(data: Buffer) -> bool: - raise NotImplementedError - - def write_bool(data: Buffer, value: bool) -> None: - raise NotImplementedError - - def read_float(data: Buffer) -> float: - raise NotImplementedError - - def write_float(data: Buffer, value: float) -> None: - raise NotImplementedError - +from native_internal import ( + Buffer as Buffer, + read_bool as read_bool, + read_float as read_float, + read_int as read_int, + read_str as read_str, + read_tag as read_tag, + write_bool as write_bool, + write_float as write_float, + write_int as write_int, + write_str as write_str, + write_tag as write_tag, +) # Always use this type alias to refer to type tags. Tag = u8 From baabf49b06c8b6e2c4e6d5fcd0e92cf325eef790 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Thu, 2 Oct 2025 02:06:27 -0400 Subject: [PATCH 290/424] feat: support constant folding in `ExpressionChecker.check_str_format_call` [1/1] (#19977) This PR implements constant folding in `ExpressionChecker.check_str_format_call` --- mypy/checkexpr.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 73282c94be4eb..b8f9bf0874678 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -18,6 +18,7 @@ from mypy.checker_shared import ExpressionCheckerSharedApi from mypy.checkmember import analyze_member_access, has_operator from mypy.checkstrformat import StringFormatterChecker +from mypy.constant_fold import constant_fold_expr from mypy.erasetype import erase_type, remove_instance_last_known_values, replace_meta_vars from mypy.errors import ErrorInfo, ErrorWatcher, report_internal_error from mypy.expandtype import ( @@ -656,11 +657,12 @@ def visit_call_expr_inner(self, e: CallExpr, allow_none_return: bool = False) -> return ret_type def check_str_format_call(self, e: CallExpr) -> None: - """More precise type checking for str.format() calls on literals.""" + """More precise type checking for str.format() calls on literals and folded constants.""" assert isinstance(e.callee, MemberExpr) format_value = None - if isinstance(e.callee.expr, StrExpr): - format_value = e.callee.expr.value + folded_callee_expr = constant_fold_expr(e.callee.expr, "") + if isinstance(folded_callee_expr, str): + format_value = folded_callee_expr elif self.chk.has_type(e.callee.expr): typ = get_proper_type(self.chk.lookup_type(e.callee.expr)) if ( From 41efd21f143fdb250bd52d85f700b2d807aa26fd Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Thu, 2 Oct 2025 02:07:05 -0400 Subject: [PATCH 291/424] [mypyc] feat: support constant folding in `try_optimize_int_floor_divide` (#19973) This PR attempts to constant fold the divisor value in `try_optimize_int_floor_divide` I'm not sure any test changes are warranted for a small PR of this nature. --- mypyc/irbuild/expression.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index 54a101bc4961b..59ecc4ac2c5c2 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -558,7 +558,7 @@ def transform_op_expr(builder: IRBuilder, expr: OpExpr) -> Value: # Special case some int ops to allow borrowing operands. if is_int_rprimitive(ltype) and is_int_rprimitive(rtype): if expr.op == "//": - expr = try_optimize_int_floor_divide(expr) + expr = try_optimize_int_floor_divide(builder, expr) if expr.op in int_borrow_friendly_op: borrow_left = is_borrow_friendly_expr(builder, expr.right) borrow_right = True @@ -571,11 +571,11 @@ def transform_op_expr(builder: IRBuilder, expr: OpExpr) -> Value: return builder.binary_op(left, right, expr.op, expr.line) -def try_optimize_int_floor_divide(expr: OpExpr) -> OpExpr: +def try_optimize_int_floor_divide(builder: IRBuilder, expr: OpExpr) -> OpExpr: """Replace // with a power of two with a right shift, if possible.""" - if not isinstance(expr.right, IntExpr): + divisor = constant_fold_expr(builder, expr.right) + if not isinstance(divisor, int): return expr - divisor = expr.right.value shift = divisor.bit_length() - 1 if 0 < shift < 28 and divisor == (1 << shift): return OpExpr(">>", expr.left, IntExpr(shift)) From c71fef017877d84932496f4a7d9e77891479e57f Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Thu, 2 Oct 2025 02:12:25 -0400 Subject: [PATCH 292/424] [mypyc] feat: support constant folding in `translate_str_format` (#19971) This PR adds support for constant folding inside of `translate_str_format` --- mypyc/irbuild/specialize.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/mypyc/irbuild/specialize.py b/mypyc/irbuild/specialize.py index 84807a7fdb535..a099e97390ea6 100644 --- a/mypyc/irbuild/specialize.py +++ b/mypyc/irbuild/specialize.py @@ -78,6 +78,7 @@ uint8_rprimitive, ) from mypyc.irbuild.builder import IRBuilder +from mypyc.irbuild.constant_fold import constant_fold_expr from mypyc.irbuild.for_helpers import ( comprehension_helper, sequence_from_generator_preallocate_helper, @@ -716,21 +717,18 @@ def translate_dict_setdefault(builder: IRBuilder, expr: CallExpr, callee: RefExp @specialize_function("format", str_rprimitive) def translate_str_format(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: - if ( - isinstance(callee, MemberExpr) - and isinstance(callee.expr, StrExpr) - and expr.arg_kinds.count(ARG_POS) == len(expr.arg_kinds) - ): - format_str = callee.expr.value - tokens = tokenizer_format_call(format_str) - if tokens is None: - return None - literals, format_ops = tokens - # Convert variables to strings - substitutions = convert_format_expr_to_str(builder, format_ops, expr.args, expr.line) - if substitutions is None: - return None - return join_formatted_strings(builder, literals, substitutions, expr.line) + if isinstance(callee, MemberExpr): + folded_callee = constant_fold_expr(builder, callee.expr) + if isinstance(folded_callee, str) and expr.arg_kinds.count(ARG_POS) == len(expr.arg_kinds): + tokens = tokenizer_format_call(folded_callee) + if tokens is None: + return None + literals, format_ops = tokens + # Convert variables to strings + substitutions = convert_format_expr_to_str(builder, format_ops, expr.args, expr.line) + if substitutions is None: + return None + return join_formatted_strings(builder, literals, substitutions, expr.line) return None From bf0a60c5a2a3a0ad5f685d9e3260ef9d23822a1e Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Thu, 2 Oct 2025 14:05:45 +0200 Subject: [PATCH 293/424] Add librt as runtime dependency (#19986) `librt` is needed both at build and runtime. It's already part of mypy-requirements.txt. --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 46593af0ab72c..589679113c3b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,6 +54,7 @@ dependencies = [ "mypy_extensions>=1.0.0", "pathspec>=0.9.0", "tomli>=1.1.0; python_version<'3.11'", + "librt>=0.1.0", ] dynamic = ["version"] From 29ee9c27f812057517e092d46750d4f7ce042d84 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Thu, 2 Oct 2025 09:43:04 -0400 Subject: [PATCH 294/424] [mypyc] feat: support constant folding in `IRBuilder.extract_int` [1/1] (#19969) This PR adds support for constant folding inside of `IRBuilder.extract_int` --- mypyc/irbuild/builder.py | 10 +++------- mypyc/irbuild/constant_fold.py | 6 ++++-- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index 1253821459917..63930123135fe 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -40,7 +40,6 @@ TypeAlias, TypeInfo, TypeParam, - UnaryExpr, Var, ) from mypy.types import ( @@ -106,6 +105,7 @@ object_rprimitive, str_rprimitive, ) +from mypyc.irbuild.constant_fold import constant_fold_expr from mypyc.irbuild.context import FuncInfo, ImplicitClass from mypyc.irbuild.ll_builder import LowLevelIRBuilder from mypyc.irbuild.mapper import Mapper @@ -965,12 +965,8 @@ def maybe_spill_assignable(self, value: Value) -> Register | AssignmentTarget: return reg def extract_int(self, e: Expression) -> int | None: - if isinstance(e, IntExpr): - return e.value - elif isinstance(e, UnaryExpr) and e.op == "-" and isinstance(e.expr, IntExpr): - return -e.expr.value - else: - return None + folded = constant_fold_expr(self, e) + return folded if isinstance(folded, int) else None def get_sequence_type(self, expr: Expression) -> RType: return self.get_sequence_type_from_type(self.types[expr]) diff --git a/mypyc/irbuild/constant_fold.py b/mypyc/irbuild/constant_fold.py index 12a4b15dd40c8..b1133f95b18ee 100644 --- a/mypyc/irbuild/constant_fold.py +++ b/mypyc/irbuild/constant_fold.py @@ -10,7 +10,7 @@ from __future__ import annotations -from typing import Final, Union +from typing import TYPE_CHECKING, Final, Union from mypy.constant_fold import constant_fold_binary_op, constant_fold_unary_op from mypy.nodes import ( @@ -26,9 +26,11 @@ UnaryExpr, Var, ) -from mypyc.irbuild.builder import IRBuilder from mypyc.irbuild.util import bytes_from_str +if TYPE_CHECKING: + from mypyc.irbuild.builder import IRBuilder + # All possible result types of constant folding ConstantValue = Union[int, float, complex, str, bytes] CONST_TYPES: Final = (int, float, complex, str, bytes) From 536f3df8a5cddd949bd6acaa9d89c892888afbb9 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 2 Oct 2025 17:20:20 +0100 Subject: [PATCH 295/424] Rename remaining refs to mypy-native to librt (#19989) We agreed that PyPI distribution name is `librt` so use it consistently. --- mypy/modulefinder.py | 8 ++++---- mypy/typeshed/stubs/librt/METADATA.toml | 1 + .../stubs/{mypy-native => librt}/native_internal.pyi | 0 mypy/typeshed/stubs/mypy-native/METADATA.toml | 1 - 4 files changed, 5 insertions(+), 5 deletions(-) create mode 100644 mypy/typeshed/stubs/librt/METADATA.toml rename mypy/typeshed/stubs/{mypy-native => librt}/native_internal.pyi (100%) delete mode 100644 mypy/typeshed/stubs/mypy-native/METADATA.toml diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index d61c9ee3ec3fd..5176b7e1df523 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -796,7 +796,7 @@ def default_lib_path( custom_typeshed_dir = os.path.abspath(custom_typeshed_dir) typeshed_dir = os.path.join(custom_typeshed_dir, "stdlib") mypy_extensions_dir = os.path.join(custom_typeshed_dir, "stubs", "mypy-extensions") - mypy_native_dir = os.path.join(custom_typeshed_dir, "stubs", "mypy-native") + librt_dir = os.path.join(custom_typeshed_dir, "stubs", "librt") versions_file = os.path.join(typeshed_dir, "VERSIONS") if not os.path.isdir(typeshed_dir) or not os.path.isfile(versions_file): print( @@ -812,13 +812,13 @@ def default_lib_path( data_dir = auto typeshed_dir = os.path.join(data_dir, "typeshed", "stdlib") mypy_extensions_dir = os.path.join(data_dir, "typeshed", "stubs", "mypy-extensions") - mypy_native_dir = os.path.join(data_dir, "typeshed", "stubs", "mypy-native") + librt_dir = os.path.join(data_dir, "typeshed", "stubs", "librt") path.append(typeshed_dir) - # Get mypy-extensions and mypy-native stubs from typeshed, since we treat them as + # Get mypy-extensions and librt stubs from typeshed, since we treat them as # "internal" libraries, similar to typing and typing-extensions. path.append(mypy_extensions_dir) - path.append(mypy_native_dir) + path.append(librt_dir) # Add fallback path that can be used if we have a broken installation. if sys.platform != "win32": diff --git a/mypy/typeshed/stubs/librt/METADATA.toml b/mypy/typeshed/stubs/librt/METADATA.toml new file mode 100644 index 0000000000000..37dc09b102d40 --- /dev/null +++ b/mypy/typeshed/stubs/librt/METADATA.toml @@ -0,0 +1 @@ +version = "0.1.*" diff --git a/mypy/typeshed/stubs/mypy-native/native_internal.pyi b/mypy/typeshed/stubs/librt/native_internal.pyi similarity index 100% rename from mypy/typeshed/stubs/mypy-native/native_internal.pyi rename to mypy/typeshed/stubs/librt/native_internal.pyi diff --git a/mypy/typeshed/stubs/mypy-native/METADATA.toml b/mypy/typeshed/stubs/mypy-native/METADATA.toml deleted file mode 100644 index 76574b01cb4b1..0000000000000 --- a/mypy/typeshed/stubs/mypy-native/METADATA.toml +++ /dev/null @@ -1 +0,0 @@ -version = "0.0.*" From a62f273d36c281073326e94894ecb087deff760a Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 2 Oct 2025 17:20:36 +0100 Subject: [PATCH 296/424] Stop calling fixed format cache experimental (#19967) It has been around for some time with no issues so far. We can stop calling it experimental so that more people try it before we make it default. --- mypy/main.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index 9ebbf78ded09d..19cb4f0d0a995 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -1066,11 +1066,7 @@ def add_invertible_flag( incremental_group.add_argument( "--fixed-format-cache", action="store_true", - help=( - "Use experimental fast and compact fixed format cache" - if compilation_status == "yes" - else argparse.SUPPRESS - ), + help="Use new fast and compact fixed format cache", ) incremental_group.add_argument( "--skip-version-check", From 1b8841b665efdc308828ece877cb8e5141954a7b Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Thu, 2 Oct 2025 15:25:55 -0400 Subject: [PATCH 297/424] [mypyc] feat: support constant folding in `translate_ord` [1/1] (#19968) This PR adds support for constant folding inside of `translate_ord` --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- mypyc/irbuild/specialize.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/mypyc/irbuild/specialize.py b/mypyc/irbuild/specialize.py index a099e97390ea6..e810f11bd079c 100644 --- a/mypyc/irbuild/specialize.py +++ b/mypyc/irbuild/specialize.py @@ -19,7 +19,6 @@ from mypy.nodes import ( ARG_NAMED, ARG_POS, - BytesExpr, CallExpr, DictExpr, Expression, @@ -1057,9 +1056,9 @@ def translate_float(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Valu def translate_ord(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: if len(expr.args) != 1 or expr.arg_kinds[0] != ARG_POS: return None - arg = expr.args[0] - if isinstance(arg, (StrExpr, BytesExpr)) and len(arg.value) == 1: - return Integer(ord(arg.value)) + arg = constant_fold_expr(builder, expr.args[0]) + if isinstance(arg, (str, bytes)) and len(arg) == 1: + return Integer(ord(arg)) return None From a35e84b0a6b420459a062557c155af6e22876752 Mon Sep 17 00:00:00 2001 From: James Hilton-Balfe Date: Sat, 4 Oct 2025 02:43:41 +0100 Subject: [PATCH 298/424] Allow returning Literals in __new__ (#15687) Unblocks https://github.com/python/typeshed/pull/10465 --------- Co-authored-by: Ivan Levkivskyi --- mypy/checker.py | 3 ++- mypy/checkmember.py | 2 ++ mypy/typeops.py | 2 +- mypy/types.py | 2 ++ test-data/unit/check-classes.test | 21 ++++++++++++++++++ test-data/unit/fixtures/__new__.pyi | 1 + test-data/unit/fixtures/literal__new__.pyi | 25 ++++++++++++++++++++++ 7 files changed, 54 insertions(+), 2 deletions(-) create mode 100644 test-data/unit/fixtures/literal__new__.pyi diff --git a/mypy/checker.py b/mypy/checker.py index 96b55f321a73c..3bee7b633339d 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -1801,7 +1801,8 @@ def check___new___signature(self, fdef: FuncDef, typ: CallableType) -> None: "but must return a subtype of", ) elif not isinstance( - get_proper_type(bound_type.ret_type), (AnyType, Instance, TupleType, UninhabitedType) + get_proper_type(bound_type.ret_type), + (AnyType, Instance, TupleType, UninhabitedType, LiteralType), ): self.fail( message_registry.NON_INSTANCE_NEW_TYPE.format( diff --git a/mypy/checkmember.py b/mypy/checkmember.py index f19a76ec6a342..719b48b14e07c 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -410,6 +410,8 @@ def analyze_type_callable_member_access(name: str, typ: FunctionLike, mx: Member ret_type = tuple_fallback(ret_type) if isinstance(ret_type, TypedDictType): ret_type = ret_type.fallback + if isinstance(ret_type, LiteralType): + ret_type = ret_type.fallback if isinstance(ret_type, Instance): if not mx.is_operator: # When Python sees an operator (eg `3 == 4`), it automatically translates that diff --git a/mypy/typeops.py b/mypy/typeops.py index 298ad4d16f8c6..d058bb8201d38 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -319,7 +319,7 @@ def class_callable( default_ret_type = fill_typevars(info) explicit_type = init_ret_type if is_new else orig_self_type if ( - isinstance(explicit_type, (Instance, TupleType, UninhabitedType)) + isinstance(explicit_type, (Instance, TupleType, UninhabitedType, LiteralType)) # We have to skip protocols, because it can be a subtype of a return type # by accident. Like `Hashable` is a subtype of `object`. See #11799 and isinstance(default_ret_type, Instance) diff --git a/mypy/types.py b/mypy/types.py index 38c17e240ccf2..426d560c2bf7e 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -2278,6 +2278,8 @@ def type_object(self) -> mypy.nodes.TypeInfo: ret = ret.partial_fallback if isinstance(ret, TypedDictType): ret = ret.fallback + if isinstance(ret, LiteralType): + ret = ret.fallback assert isinstance(ret, Instance) return ret.type diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index 498a2c12b6e8c..c0b1114db5120 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -472,6 +472,27 @@ class B(A): def __new__(cls) -> B: pass +[case testOverride__new__WithLiteralReturnPassing] +from typing import Literal + +class Falsy: + def __bool__(self) -> Literal[False]: pass + +reveal_type(bool(Falsy())) # N: Revealed type is "Literal[False]" +reveal_type(int()) # N: Revealed type is "Literal[0]" + +[builtins fixtures/literal__new__.pyi] +[typing fixtures/typing-medium.pyi] + +[case testOverride__new__WithLiteralReturnFailing] +from typing import Literal + +class Foo: + def __new__(cls) -> Literal[1]: pass # E: Incompatible return type for "__new__" (returns "Literal[1]", but must return a subtype of "Foo") + +[builtins fixtures/__new__.pyi] +[typing fixtures/typing-medium.pyi] + [case testOverride__new__AndCallObject] from typing import TypeVar, Generic diff --git a/test-data/unit/fixtures/__new__.pyi b/test-data/unit/fixtures/__new__.pyi index 401de6fb9cd1b..57d3624ce92cf 100644 --- a/test-data/unit/fixtures/__new__.pyi +++ b/test-data/unit/fixtures/__new__.pyi @@ -12,6 +12,7 @@ class object: class type: def __init__(self, x) -> None: pass +class float: pass class int: pass class bool: pass class str: pass diff --git a/test-data/unit/fixtures/literal__new__.pyi b/test-data/unit/fixtures/literal__new__.pyi new file mode 100644 index 0000000000000..971bc39bfff4b --- /dev/null +++ b/test-data/unit/fixtures/literal__new__.pyi @@ -0,0 +1,25 @@ +from typing import Literal, Protocol, overload + +class object: + def __init__(self) -> None: pass + +class type: + def __init__(self, x) -> None: pass + +class str: pass +class dict: pass +class float: pass +class int: + def __new__(cls) -> Literal[0]: pass + +class _Truthy(Protocol): + def __bool__(self) -> Literal[True]: pass + +class _Falsy(Protocol): + def __bool__(self) -> Literal[False]: pass + +class bool(int): + @overload + def __new__(cls, __o: _Truthy) -> Literal[True]: pass + @overload + def __new__(cls, __o: _Falsy) -> Literal[False]: pass From fe313349ee9895d49e7b63ec096e3c7f76d96f2e Mon Sep 17 00:00:00 2001 From: Sigve Sebastian Farstad Date: Sat, 4 Oct 2025 04:07:21 +0200 Subject: [PATCH 299/424] Support error codes from plugins in options (#19719) Mypy has options for enabling or disabling specific error codes. These work fine, except that it is not possible to enable or disable error codes from plugins, only mypy's original error codes. The crux of the issue is that mypy validates and rejects unknown error codes passed in the options before it loads plugins and learns about the any error codes that might get registered. There are many ways to solve this. This commit tries to find a pragmatic solution where the relevant options parsing is deferred until after plugin loading. Error code validation in the config parser, where plugins are not loaded yet, is also skipped entirely, since the error code options are re-validated later anyway. This means that this commit introduces a small observable change in behavior when running with invalid error codes specified, as shown in the test test_config_file_error_codes_invalid. This fixes https://github.com/python/mypy/issues/12987. --------- Co-authored-by: John Belmonte Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Ivan Levkivskyi --- mypy/build.py | 7 ++++ mypy/config_parser.py | 19 +++-------- mypy/main.py | 1 - mypy/test/teststubtest.py | 35 ++++++++++++++------ test-data/unit/check-plugin-error-codes.test | 32 ++++++++++++++++++ test-data/unit/cmdline.test | 10 ------ 6 files changed, 67 insertions(+), 37 deletions(-) create mode 100644 test-data/unit/check-plugin-error-codes.test diff --git a/mypy/build.py b/mypy/build.py index 234dd6843292e..9f840499fcc23 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -143,6 +143,10 @@ def __init__(self, manager: BuildManager, graph: Graph) -> None: self.errors: list[str] = [] # Filled in by build if desired +def build_error(msg: str) -> NoReturn: + raise CompileError([f"mypy: error: {msg}"]) + + def build( sources: list[BuildSource], options: Options, @@ -241,6 +245,9 @@ def _build( errors = Errors(options, read_source=lambda path: read_py_file(path, cached_read)) plugin, snapshot = load_plugins(options, errors, stdout, extra_plugins) + # Validate error codes after plugins are loaded. + options.process_error_codes(error_callback=build_error) + # Add catch-all .gitignore to cache dir if we created it cache_dir_existed = os.path.isdir(options.cache_dir) diff --git a/mypy/config_parser.py b/mypy/config_parser.py index 5f08f342241ee..2bfd2a1e2eefe 100644 --- a/mypy/config_parser.py +++ b/mypy/config_parser.py @@ -8,8 +8,6 @@ import sys from io import StringIO -from mypy.errorcodes import error_codes - if sys.version_info >= (3, 11): import tomllib else: @@ -87,15 +85,6 @@ def complain(x: object, additional_info: str = "") -> Never: complain(v) -def validate_codes(codes: list[str]) -> list[str]: - invalid_codes = set(codes) - set(error_codes.keys()) - if invalid_codes: - raise argparse.ArgumentTypeError( - f"Invalid error code(s): {', '.join(sorted(invalid_codes))}" - ) - return codes - - def validate_package_allow_list(allow_list: list[str]) -> list[str]: for p in allow_list: msg = f"Invalid allow list entry: {p}" @@ -209,8 +198,8 @@ def split_commas(value: str) -> list[str]: [p.strip() for p in split_commas(s)] ), "enable_incomplete_feature": lambda s: [p.strip() for p in split_commas(s)], - "disable_error_code": lambda s: validate_codes([p.strip() for p in split_commas(s)]), - "enable_error_code": lambda s: validate_codes([p.strip() for p in split_commas(s)]), + "disable_error_code": lambda s: [p.strip() for p in split_commas(s)], + "enable_error_code": lambda s: [p.strip() for p in split_commas(s)], "package_root": lambda s: [p.strip() for p in split_commas(s)], "cache_dir": expand_path, "python_executable": expand_path, @@ -234,8 +223,8 @@ def split_commas(value: str) -> list[str]: "always_false": try_split, "untyped_calls_exclude": lambda s: validate_package_allow_list(try_split(s)), "enable_incomplete_feature": try_split, - "disable_error_code": lambda s: validate_codes(try_split(s)), - "enable_error_code": lambda s: validate_codes(try_split(s)), + "disable_error_code": lambda s: try_split(s), + "enable_error_code": lambda s: try_split(s), "package_root": try_split, "exclude": str_or_array_as_list, "packages": try_split, diff --git a/mypy/main.py b/mypy/main.py index 19cb4f0d0a995..a44632ed96f94 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -1462,7 +1462,6 @@ def set_strict_flags() -> None: validate_package_allow_list(options.untyped_calls_exclude) validate_package_allow_list(options.deprecated_calls_exclude) - options.process_error_codes(error_callback=parser.error) options.process_incomplete_features(error_callback=parser.error, warning_callback=print) # Compute absolute path for custom typeshed (if present). diff --git a/mypy/test/teststubtest.py b/mypy/test/teststubtest.py index 28263e20099d7..800f522d90a01 100644 --- a/mypy/test/teststubtest.py +++ b/mypy/test/teststubtest.py @@ -12,6 +12,8 @@ from collections.abc import Iterator from typing import Any, Callable +from pytest import raises + import mypy.stubtest from mypy import build, nodes from mypy.modulefinder import BuildSource @@ -171,7 +173,12 @@ def build_helper(source: str) -> build.BuildResult: def run_stubtest_with_stderr( - stub: str, runtime: str, options: list[str], config_file: str | None = None + stub: str, + runtime: str, + options: list[str], + config_file: str | None = None, + output: io.StringIO | None = None, + outerr: io.StringIO | None = None, ) -> tuple[str, str]: with use_tmp_dir(TEST_MODULE_NAME) as tmp_dir: with open("builtins.pyi", "w") as f: @@ -188,8 +195,8 @@ def run_stubtest_with_stderr( with open(f"{TEST_MODULE_NAME}_config.ini", "w") as f: f.write(config_file) options = options + ["--mypy-config-file", f"{TEST_MODULE_NAME}_config.ini"] - output = io.StringIO() - outerr = io.StringIO() + output = io.StringIO() if output is None else output + outerr = io.StringIO() if outerr is None else outerr with contextlib.redirect_stdout(output), contextlib.redirect_stderr(outerr): test_stubs(parse_options([TEST_MODULE_NAME] + options), use_builtins_fixtures=True) filtered_output = remove_color_code( @@ -2888,14 +2895,20 @@ def test_config_file_error_codes_invalid(self) -> None: runtime = "temp = 5\n" stub = "temp: int\n" config_file = "[mypy]\ndisable_error_code = not-a-valid-name\n" - output, outerr = run_stubtest_with_stderr( - stub=stub, runtime=runtime, options=[], config_file=config_file - ) - assert output == "Success: no issues found in 1 module\n" - assert outerr == ( - "test_module_config.ini: [mypy]: disable_error_code: " - "Invalid error code(s): not-a-valid-name\n" - ) + output = io.StringIO() + outerr = io.StringIO() + with raises(SystemExit): + run_stubtest_with_stderr( + stub=stub, + runtime=runtime, + options=[], + config_file=config_file, + output=output, + outerr=outerr, + ) + + assert output.getvalue() == "error: Invalid error code(s): not-a-valid-name\n" + assert outerr.getvalue() == "" def test_config_file_wrong_incomplete_feature(self) -> None: runtime = "x = 1\n" diff --git a/test-data/unit/check-plugin-error-codes.test b/test-data/unit/check-plugin-error-codes.test new file mode 100644 index 0000000000000..95789477977e5 --- /dev/null +++ b/test-data/unit/check-plugin-error-codes.test @@ -0,0 +1,32 @@ +[case testCustomErrorCodeFromPluginIsTargetable] +# flags: --config-file tmp/mypy.ini --show-error-codes + +def main() -> None: + return +main() # E: Custom error [custom] + +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/custom_errorcode.py + +[case testCustomErrorCodeCanBeDisabled] +# flags: --config-file tmp/mypy.ini --show-error-codes --disable-error-code=custom + +def main() -> None: + return +main() # no output expected when disabled + +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/custom_errorcode.py + +[case testCustomErrorCodeCanBeReenabled] +# flags: --config-file tmp/mypy.ini --show-error-codes --disable-error-code=custom --enable-error-code=custom + +def main() -> None: + return +main() # E: Custom error [custom] + +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/custom_errorcode.py diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test index ff60c24b72a57..35d7b700b1618 100644 --- a/test-data/unit/cmdline.test +++ b/test-data/unit/cmdline.test @@ -960,8 +960,6 @@ src/foo/bar.py: note: Common resolutions include: a) adding `__init__.py` somewh [file test.py] x = 1 [out] -usage: mypy [-h] [-v] [-V] [more options; see below] - [-m MODULE] [-p PACKAGE] [-c PROGRAM_TEXT] [files ...] mypy: error: Invalid error code(s): YOLO == Return code: 2 @@ -970,8 +968,6 @@ mypy: error: Invalid error code(s): YOLO [file test.py] x = 1 [out] -usage: mypy [-h] [-v] [-V] [more options; see below] - [-m MODULE] [-p PACKAGE] [-c PROGRAM_TEXT] [files ...] mypy: error: Invalid error code(s): YOLO == Return code: 2 @@ -980,8 +976,6 @@ mypy: error: Invalid error code(s): YOLO [file test.py] x = 1 [out] -usage: mypy [-h] [-v] [-V] [more options; see below] - [-m MODULE] [-p PACKAGE] [-c PROGRAM_TEXT] [files ...] mypy: error: Invalid error code(s): YOLO, YOLO2 == Return code: 2 @@ -990,8 +984,6 @@ mypy: error: Invalid error code(s): YOLO, YOLO2 [file test.py] x = 1 [out] -usage: mypy [-h] [-v] [-V] [more options; see below] - [-m MODULE] [-p PACKAGE] [-c PROGRAM_TEXT] [files ...] mypy: error: Invalid error code(s): YOLO == Return code: 2 @@ -1000,8 +992,6 @@ mypy: error: Invalid error code(s): YOLO [file test.py] x = 1 [out] -usage: mypy [-h] [-v] [-V] [more options; see below] - [-m MODULE] [-p PACKAGE] [-c PROGRAM_TEXT] [files ...] mypy: error: Invalid error code(s): YOLO == Return code: 2 From de2f375eb0da1c446e2aab1474c28b8d7707a848 Mon Sep 17 00:00:00 2001 From: "Thiago J. Barbalho" <11036045+gacheiro@users.noreply.github.com> Date: Fri, 3 Oct 2025 23:10:04 -0300 Subject: [PATCH 300/424] =?UTF-8?q?[docs]=20Replace=20`List`=20with=20buil?= =?UTF-8?q?t=E2=80=91in=20`list`=20(PEP=E2=80=AF585)=20(#20000)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Updated the docs to replace typing.List with the built‑in generic list (PEP 585). The example would not work if someone copied and pasted it. --- docs/source/common_issues.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/common_issues.rst b/docs/source/common_issues.rst index aa325dd3b05c4..e4239bd7a8eed 100644 --- a/docs/source/common_issues.rst +++ b/docs/source/common_issues.rst @@ -731,7 +731,7 @@ This example demonstrates both safe and unsafe overrides: class NarrowerReturn(A): # A more specific return type is fine - def test(self, t: Sequence[int]) -> List[str]: # OK + def test(self, t: Sequence[int]) -> list[str]: # OK ... class GeneralizedReturn(A): @@ -746,7 +746,7 @@ not necessary: .. code-block:: python class NarrowerArgument(A): - def test(self, t: List[int]) -> Sequence[str]: # type: ignore[override] + def test(self, t: list[int]) -> Sequence[str]: # type: ignore[override] ... .. _unreachable: From 6dc46987ddcab77ac8f164a3f35cf595180846cb Mon Sep 17 00:00:00 2001 From: sobolevn Date: Sun, 5 Oct 2025 21:36:30 +0300 Subject: [PATCH 301/424] [stubtest] Improve `allowlist` docs with better example (#20007) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Also fix markup in several places. It used to be: Снимок экрана 2025-10-05 в 09 35 17 Notice the `--` problem. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Alex Waygood --- docs/source/stubtest.rst | 81 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 74 insertions(+), 7 deletions(-) diff --git a/docs/source/stubtest.rst b/docs/source/stubtest.rst index 59889252f0569..e7ea69290b78a 100644 --- a/docs/source/stubtest.rst +++ b/docs/source/stubtest.rst @@ -99,8 +99,75 @@ to mypy build errors". In this case, you will need to mitigate those errors before stubtest will run. Despite potential overlap in errors here, stubtest is not intended as a substitute for running mypy directly. +Allowlist +********* + If you wish to ignore some of stubtest's complaints, stubtest supports a -pretty handy allowlist system. +pretty handy :option:`--allowlist` system. + +Let's say that you have this python module called ``ex``: + +.. code-block:: python + + try: + import optional_expensive_dep + except ImportError: + optional_expensive_dep = None + + first = 1 + if optional_expensive_dep: + second = 2 + +Let's say that you can't install ``optional_expensive_dep`` in CI for some reason, +but you still want to include ``second: int`` in the stub file: + +.. code-block:: python + + first: int + second: int + +In this case stubtest will correctly complain: + +.. code-block:: shell + + error: ex.second is not present at runtime + Stub: in file /.../ex.pyi:2 + builtins.int + Runtime: + MISSING + + Found 1 error (checked 1 module) + +To fix this, you can add an ``allowlist`` entry: + +.. code-block:: ini + + # Allowlist entries in `allowlist.txt` file: + + # Does not exist if `optional_expensive_dep` is not installed: + ex.second + +And now when running stubtest with ``--allowlist=allowlist.txt``, +no errors will be generated anymore. + +Allowlists also support regular expressions, +which can be useful to ignore many similar errors at once. +They can also be useful for suppressing stubtest errors that occur sometimes, +but not on every CI run. For example, if some CI workers have +``optional_expensive_dep`` installed, stubtest might complain with this message +on those workers if you had the ``ex.second`` allowlist entry: + +.. code-block:: ini + + note: unused allowlist entry ex.second + Found 1 error (checked 1 module) + +Changing ``ex.second`` to be ``(ex\.second)?`` will make this error optional, +meaning that stubtest will pass whether or not a CI runner +has``optional_expensive_dep`` installed. + +CLI +*** The rest of this section documents the command line interface of stubtest. @@ -119,15 +186,15 @@ The rest of this section documents the command line interface of stubtest. .. option:: --allowlist FILE Use file as an allowlist. Can be passed multiple times to combine multiple - allowlists. Allowlists can be created with --generate-allowlist. Allowlists - support regular expressions. + allowlists. Allowlists can be created with :option:`--generate-allowlist`. + Allowlists support regular expressions. The presence of an entry in the allowlist means stubtest will not generate any errors for the corresponding definition. .. option:: --generate-allowlist - Print an allowlist (to stdout) to be used with --allowlist + Print an allowlist (to stdout) to be used with :option:`--allowlist`. When introducing stubtest to an existing project, this is an easy way to silence all existing errors. @@ -141,17 +208,17 @@ The rest of this section documents the command line interface of stubtest. Note if an allowlist entry is a regex that matches the empty string, stubtest will never consider it unused. For example, to get - `--ignore-unused-allowlist` behaviour for a single allowlist entry like + ``--ignore-unused-allowlist`` behaviour for a single allowlist entry like ``foo.bar`` you could add an allowlist entry ``(foo\.bar)?``. This can be useful when an error only occurs on a specific platform. .. option:: --mypy-config-file FILE - Use specified mypy config file to determine mypy plugins and mypy path + Use specified mypy config *file* to determine mypy plugins and mypy path .. option:: --custom-typeshed-dir DIR - Use the custom typeshed in DIR + Use the custom typeshed in *DIR* .. option:: --check-typeshed From 3807423e9d98e678bf16b13ec8b4f909fe181908 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Sun, 5 Oct 2025 21:14:59 -0700 Subject: [PATCH 302/424] Make untyped decorator its own code (#19911) Since this apparently comes up a lot, it shouldn't just be misc (it is, however, a subcode of misc, at least at the moment, for extra backwards compatibility). Fixes https://github.com/python/mypy/issues/19148 I didn't add any tests for this and it seems like our old tests don't have codes enabled because they didn't have to be changed. I did add documentation for this, as required by the relevant test. --------- Co-authored-by: A5rocks --- docs/source/error_code_list2.rst | 23 +++++++++++++++++++++++ mypy/errorcodes.py | 4 ++++ mypy/messages.py | 6 +++++- test-data/unit/check-errorcodes.test | 15 +++++++++++++++ 4 files changed, 47 insertions(+), 1 deletion(-) diff --git a/docs/source/error_code_list2.rst b/docs/source/error_code_list2.rst index 125671bc2bef4..bd24360619748 100644 --- a/docs/source/error_code_list2.rst +++ b/docs/source/error_code_list2.rst @@ -676,3 +676,26 @@ Example: print("red") case _: print("other") + +.. _code-untyped-decorator: + +Error if an untyped decorator makes a typed function effectively untyped [untyped-decorator] +-------------------------------------------------------------------------------------------- + +If enabled with :option:`--disallow-untyped-decorators ` +mypy generates an error if a typed function is wrapped by an untyped decorator +(as this would effectively remove the benefits of typing the function). + +Example: + +.. code-block:: python + + def printing_decorator(func): + def wrapper(*args, **kwds): + print("Calling", func) + return func(*args, **kwds) + return wrapper + # A decorated function. + @printing_decorator # E: Untyped decorator makes function "add_forty_two" untyped [untyped-decorator] + def add_forty_two(value: int) -> int: + return value + 42 diff --git a/mypy/errorcodes.py b/mypy/errorcodes.py index a96f5f723a7d2..fbfa572b94397 100644 --- a/mypy/errorcodes.py +++ b/mypy/errorcodes.py @@ -302,6 +302,10 @@ def __hash__(self) -> int: sub_code_of=MISC, ) +UNTYPED_DECORATOR: Final = ErrorCode( + "untyped-decorator", "Error if an untyped decorator makes a typed function untyped", "General" +) + NARROWED_TYPE_NOT_SUBTYPE: Final = ErrorCode( "narrowed-type-not-subtype", "Warn if a TypeIs function's narrowed type is not a subtype of the original type", diff --git a/mypy/messages.py b/mypy/messages.py index 6329cad687f6d..c6378c2647578 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -2008,7 +2008,11 @@ def untyped_decorated_function(self, typ: Type, context: Context) -> None: ) def typed_function_untyped_decorator(self, func_name: str, context: Context) -> None: - self.fail(f'Untyped decorator makes function "{func_name}" untyped', context) + self.fail( + f'Untyped decorator makes function "{func_name}" untyped', + context, + code=codes.UNTYPED_DECORATOR, + ) def bad_proto_variance( self, actual: int, tvar_name: str, expected: int, context: Context diff --git a/test-data/unit/check-errorcodes.test b/test-data/unit/check-errorcodes.test index bb5f658ebb50b..06c5753db5a74 100644 --- a/test-data/unit/check-errorcodes.test +++ b/test-data/unit/check-errorcodes.test @@ -394,6 +394,21 @@ def f() -> None: def g(): pass +[case testErrorCodeUntypedDecorator] +# flags: --disallow-untyped-decorators --warn-unused-ignores +def d(f): return f + +@d # E: Untyped decorator makes function "x" untyped [untyped-decorator] +def x() -> int: return 1 +@d # type: ignore +def y() -> int: return 2 +@d # type: ignore[untyped-decorator] +def best() -> int: return 3 +@d # type: ignore[misc] # E: Unused "type: ignore" comment [unused-ignore] \ + # E: Untyped decorator makes function "z" untyped [untyped-decorator] \ + # N: Error code "untyped-decorator" not covered by "type: ignore" comment +def z() -> int: return 4 + [case testErrorCodeIndexing] from typing import Dict x: Dict[int, int] From 374fefbcfc3e75b3b5ea2550ff2826dd3474f7ef Mon Sep 17 00:00:00 2001 From: Piotr Sawicki Date: Mon, 6 Oct 2025 17:25:51 +0200 Subject: [PATCH 303/424] [mypyc] Support deleting attributes in __setattr__ wrapper (#19997) The `__setattr__` wrapper that mypyc generates needs to handle deleting attributes as well because `del` statements go through the same `tp_setattro` pointer but with the value argument set to `NULL`. The wrapper calls `__delattr__` in this case if it's overridden in the native class (or its parent). Handling of dynamic attributes is different without `__dict__` which makes a custom `__delattr__` required if the dynamic attributes are stored in a custom dictionary. If `__delattr__` is not overridden it calls the implementation of `object.__delattr__` which results in `AttributeError` because there's no `__dict__`. If it's defined without `__setattr__`, mypyc reports an error. It's possible to support just `__delattr__` but since it shares a slot with `__setattr__`, the wrapper generation would be more complicated. It seems like an unlikely use case to only need `__delattr__` so I think it makes sense to leave it for later. --- mypyc/irbuild/function.py | 37 +++++- mypyc/test-data/fixtures/ir.py | 1 + mypyc/test-data/irbuild-classes.test | 161 ++++++++++++++++++++++++++- mypyc/test-data/run-classes.test | 142 +++++++++++++++++++++++ 4 files changed, 333 insertions(+), 8 deletions(-) diff --git a/mypyc/irbuild/function.py b/mypyc/irbuild/function.py index 51bdc76495f2f..c9f999597d307 100644 --- a/mypyc/irbuild/function.py +++ b/mypyc/irbuild/function.py @@ -83,7 +83,7 @@ dict_new_op, exact_dict_set_item_op, ) -from mypyc.primitives.generic_ops import generic_getattr, py_setattr_op +from mypyc.primitives.generic_ops import generic_getattr, generic_setattr, py_setattr_op from mypyc.primitives.misc_ops import register_function from mypyc.primitives.registry import builtin_names from mypyc.sametype import is_same_method_signature, is_same_type @@ -423,8 +423,10 @@ def generate_setattr_wrapper(builder: IRBuilder, cdef: ClassDef, setattr: FuncDe Returns 0 on success and -1 on failure. Restrictions are similar to the __getattr__ wrapper above. - This one is simpler because to match interpreted python semantics it's enough to always - call the user-provided function, including for names matching regular attributes. + The wrapper calls the user-defined __setattr__ when the value to set is not NULL. + When it's NULL, this means that the call to tp_setattro comes from a del statement, + so it calls __delattr__ instead. If __delattr__ is not overridden in the native class, + this will call the base implementation in object which doesn't work without __dict__. """ name = setattr.name + "__wrapper" ir = builder.mapper.type_to_ir[cdef.info] @@ -440,6 +442,27 @@ def generate_setattr_wrapper(builder: IRBuilder, cdef: ClassDef, setattr: FuncDe attr_arg = builder.add_argument("attr", object_rprimitive) value_arg = builder.add_argument("value", object_rprimitive) + call_delattr, call_setattr = BasicBlock(), BasicBlock() + null = Integer(0, object_rprimitive, line) + is_delattr = builder.add(ComparisonOp(value_arg, null, ComparisonOp.EQ, line)) + builder.add_bool_branch(is_delattr, call_delattr, call_setattr) + + builder.activate_block(call_delattr) + delattr_symbol = cdef.info.get("__delattr__") + delattr = delattr_symbol.node if delattr_symbol else None + delattr_override = delattr is not None and not delattr.fullname.startswith("builtins.") + if delattr_override: + builder.gen_method_call(builder.self(), "__delattr__", [attr_arg], None, line) + else: + # Call internal function that cpython normally calls when deleting an attribute. + # Cannot call object.__delattr__ here because it calls PyObject_SetAttr internally + # which in turn calls our wrapper and recurses infinitely. + # Note that since native classes don't have __dict__, this will raise AttributeError + # for dynamic attributes. + builder.call_c(generic_setattr, [builder.self(), attr_arg, null], line) + builder.add(Return(Integer(0, c_int_rprimitive), line)) + + builder.activate_block(call_setattr) builder.gen_method_call(builder.self(), setattr.name, [attr_arg, value_arg], None, line) builder.add(Return(Integer(0, c_int_rprimitive), line)) @@ -514,6 +537,14 @@ def handle_ext_method(builder: IRBuilder, cdef: ClassDef, fdef: FuncDef) -> None generate_getattr_wrapper(builder, cdef, fdef) elif fdef.name == "__setattr__": generate_setattr_wrapper(builder, cdef, fdef) + elif fdef.name == "__delattr__": + setattr = cdef.info.get("__setattr__") + if not setattr or not setattr.node or setattr.node.fullname.startswith("builtins."): + builder.error( + '"__delattr__" supported only in classes that also override "__setattr__", ' + + "or inherit from a native class that overrides it.", + fdef.line, + ) def handle_non_ext_method( diff --git a/mypyc/test-data/fixtures/ir.py b/mypyc/test-data/fixtures/ir.py index 22a6a5986cbd9..4d0aaba12cab4 100644 --- a/mypyc/test-data/fixtures/ir.py +++ b/mypyc/test-data/fixtures/ir.py @@ -46,6 +46,7 @@ def __eq__(self, x: object) -> bool: pass def __ne__(self, x: object) -> bool: pass def __str__(self) -> str: pass def __setattr__(self, k: str, v: object) -> None: pass + def __delattr__(self, k: str) -> None: pass class type: def __init__(self, o: object) -> None: ... diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index a98b3a7d3dcf9..a2d3b23ccfd9f 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -2222,18 +2222,41 @@ class AllowsInterpreted: def __setattr__(self, attr: str, val: object) -> None: # E: "__setattr__" not supported in class "AllowsInterpreted" because it allows interpreted subclasses pass + def __delattr__(self, attr: str) -> None: + pass + class InheritsInterpreted(dict): def __setattr__(self, attr: str, val: object) -> None: # E: "__setattr__" not supported in class "InheritsInterpreted" because it inherits from a non-native class pass + def __delattr__(self, attr: str) -> None: + pass + @mypyc_attr(native_class=False) class NonNative: - pass + def __setattr__(self, attr: str, val: object) -> None: + pass class InheritsNonNative(NonNative): def __setattr__(self, attr: str, val: object) -> None: # E: "__setattr__" not supported in class "InheritsNonNative" because it inherits from a non-native class pass + def __delattr__(self, attr: str) -> None: + pass + +[case testUnsupportedDelAttr] +class SetAttr: + def __setattr__(self, attr: str, val: object) -> None: + pass + +class NoSetAttr: + def __delattr__(self, attr: str) -> None: # E: "__delattr__" supported only in classes that also override "__setattr__", or inherit from a native class that overrides it. + pass + +class InheritedSetAttr(SetAttr): + def __delattr__(self, attr: str) -> None: + pass + [case testSetAttr] from typing import ClassVar class SetAttr: @@ -2329,11 +2352,21 @@ L6: def SetAttr.__setattr____wrapper(__mypyc_self__, attr, value): __mypyc_self__ :: __main__.SetAttr attr, value :: object - r0 :: str - r1 :: None + r0 :: bit + r1 :: i32 + r2 :: bit + r3 :: str + r4 :: None L0: - r0 = cast(str, attr) - r1 = __mypyc_self__.__setattr__(r0, value) + r0 = value == 0 + if r0 goto L1 else goto L2 :: bool +L1: + r1 = CPyObject_GenericSetAttr(__mypyc_self__, attr, 0) + r2 = r1 >= 0 :: signed + return 0 +L2: + r3 = cast(str, attr) + r4 = __mypyc_self__.__setattr__(r3, value) return 0 def test(attr, val): attr :: str @@ -2372,6 +2405,124 @@ L0: r14 = r13 >= 0 :: signed return 1 +[case testSetAttrAndDelAttr] +from typing import ClassVar +class SetAttr: + _attributes: dict[str, object] + regular_attr: int + class_var: ClassVar[str] = "x" + + def __init__(self, regular_attr: int, extra_attrs: dict[str, object], new_attr: str, new_val: object) -> None: + super().__setattr__("_attributes", extra_attrs) + object.__setattr__(self, "regular_attr", regular_attr) + + super().__setattr__(new_attr, new_val) + object.__setattr__(self, new_attr, new_val) + + def __setattr__(self, key: str, val: object) -> None: + if key == "regular_attr": + super().__setattr__("regular_attr", val) + elif key == "class_var": + raise AttributeError() + else: + self._attributes[key] = val + + def __delattr__(self, key: str) -> None: + del self._attributes[key] + +[typing fixtures/typing-full.pyi] +[out] +def SetAttr.__init__(self, regular_attr, extra_attrs, new_attr, new_val): + self :: __main__.SetAttr + regular_attr :: int + extra_attrs :: dict + new_attr :: str + new_val :: object + r0 :: i32 + r1 :: bit + r2 :: i32 + r3 :: bit +L0: + self._attributes = extra_attrs + self.regular_attr = regular_attr + r0 = CPyObject_GenericSetAttr(self, new_attr, new_val) + r1 = r0 >= 0 :: signed + r2 = CPyObject_GenericSetAttr(self, new_attr, new_val) + r3 = r2 >= 0 :: signed + return 1 +def SetAttr.__setattr__(self, key, val): + self :: __main__.SetAttr + key :: str + val :: object + r0 :: str + r1 :: bool + r2 :: int + r3 :: bool + r4 :: str + r5 :: bool + r6 :: object + r7 :: str + r8, r9 :: object + r10 :: dict + r11 :: i32 + r12 :: bit +L0: + r0 = 'regular_attr' + r1 = CPyStr_Equal(key, r0) + if r1 goto L1 else goto L2 :: bool +L1: + r2 = unbox(int, val) + self.regular_attr = r2; r3 = is_error + goto L6 +L2: + r4 = 'class_var' + r5 = CPyStr_Equal(key, r4) + if r5 goto L3 else goto L4 :: bool +L3: + r6 = builtins :: module + r7 = 'AttributeError' + r8 = CPyObject_GetAttr(r6, r7) + r9 = PyObject_Vectorcall(r8, 0, 0, 0) + CPy_Raise(r9) + unreachable +L4: + r10 = self._attributes + r11 = CPyDict_SetItem(r10, key, val) + r12 = r11 >= 0 :: signed +L5: +L6: + return 1 +def SetAttr.__setattr____wrapper(__mypyc_self__, attr, value): + __mypyc_self__ :: __main__.SetAttr + attr, value :: object + r0 :: bit + r1 :: str + r2 :: None + r3 :: str + r4 :: None +L0: + r0 = value == 0 + if r0 goto L1 else goto L2 :: bool +L1: + r1 = cast(str, attr) + r2 = __mypyc_self__.__delattr__(r1) + return 0 +L2: + r3 = cast(str, attr) + r4 = __mypyc_self__.__setattr__(r3, value) + return 0 +def SetAttr.__delattr__(self, key): + self :: __main__.SetAttr + key :: str + r0 :: dict + r1 :: i32 + r2 :: bit +L0: + r0 = self._attributes + r1 = PyObject_DelItem(r0, key) + r2 = r1 >= 0 :: signed + return 1 + [case testUntransformedSetAttr_64bit] from mypy_extensions import mypyc_attr diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index d10f7b19067cb..ab1dcb926c34f 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -4566,6 +4566,9 @@ class SetAttrOverridden(SetAttr): else: super().__setattr__(key, val) + def __delattr__(self, key: str) -> None: + del self._attributes[key] + @mypyc_attr(native_class=False) class SetAttrNonNative: _attributes: dict[str, object] @@ -4645,6 +4648,10 @@ def test_setattr() -> None: with assertRaises(AttributeError): i.const = 45 + # Doesn't work because there's no __delattr__. + with assertRaises(AttributeError): + del i.four + def test_setattr_inherited() -> None: i = SetAttrInherited(99, {"one": 1}) assert i.class_var == "x" @@ -4678,6 +4685,10 @@ def test_setattr_inherited() -> None: with assertRaises(AttributeError): i.const = 45 + # Doesn't work because there's no __delattr__. + with assertRaises(AttributeError): + del i.four + def test_setattr_overridden() -> None: i = SetAttrOverridden(99, 1, {"one": 1}) assert i.class_var == "x" @@ -4723,6 +4734,15 @@ def test_setattr_overridden() -> None: with assertRaises(AttributeError): i.const = 45 + del i.four + assert "four" not in i._attributes + + delattr(i, "three") + assert "three" not in i._attributes + + i.__delattr__("two") + assert "two" not in i._attributes + base_ref: SetAttr = i setattr(base_ref, "sub_attr", 5) assert base_ref.sub_attr == 5 @@ -4733,6 +4753,12 @@ def test_setattr_overridden() -> None: with assertRaises(AttributeError): setattr(base_ref, "subclass_var", "c") + base_ref.new_attr = "new_attr" + assert base_ref.new_attr == "new_attr" + + del base_ref.new_attr + assert "new_attr" not in base_ref._attributes + def test_setattr_nonnative() -> None: i = SetAttrNonNative(99, {"one": 1}) assert i.class_var == "x" @@ -4766,6 +4792,10 @@ def test_setattr_nonnative() -> None: with assertRaises(AttributeError): i.const = 45 + # Doesn't work because there's no __delattr__. + with assertRaises(AttributeError): + del i.four + def test_no_setattr() -> None: i = NoSetAttr(99) i.super_setattr("attr", 100) @@ -4806,6 +4836,15 @@ def test_no_setattr_nonnative() -> None: object.__setattr__(i, "three", 102) assert i.three == 102 + del i.three + assert i.three == None + + delattr(i, "two") + assert i.two == None + + object.__delattr__(i, "one") + assert i.one == None + [typing fixtures/typing-full.pyi] [case testDunderSetAttrInterpreted] @@ -4853,6 +4892,9 @@ class SetAttrOverridden(SetAttr): else: super().__setattr__(key, val) + def __delattr__(self, key: str) -> None: + del self._attributes[key] + @mypyc_attr(native_class=False) class SetAttrNonNative: _attributes: dict[str, object] @@ -4936,6 +4978,10 @@ def test_setattr() -> None: with assertRaises(AttributeError): i.const = 45 + # Doesn't work because there's no __delattr__. + with assertRaises(AttributeError): + del i.four + def test_setattr_inherited() -> None: i = SetAttrInherited(99, {"one": 1}) assert i.class_var == "x" @@ -4969,6 +5015,10 @@ def test_setattr_inherited() -> None: with assertRaises(AttributeError): i.const = 45 + # Doesn't work because there's no __delattr__. + with assertRaises(AttributeError): + del i.four + def test_setattr_overridden() -> None: i = SetAttrOverridden(99, 1, {"one": 1}) assert i.class_var == "x" @@ -5014,6 +5064,15 @@ def test_setattr_overridden() -> None: with assertRaises(AttributeError): i.const = 45 + del i.four + assert "four" not in i._attributes + + delattr(i, "three") + assert "three" not in i._attributes + + i.__delattr__("two") + assert "two" not in i._attributes + base_ref: SetAttr = i setattr(base_ref, "sub_attr", 5) assert base_ref.sub_attr == 5 @@ -5024,6 +5083,12 @@ def test_setattr_overridden() -> None: with assertRaises(AttributeError): setattr(base_ref, "subclass_var", "c") + base_ref.new_attr = "new_attr" + assert base_ref.new_attr == "new_attr" + + del base_ref.new_attr + assert "new_attr" not in base_ref._attributes + def test_setattr_nonnative() -> None: i = SetAttrNonNative(99, {"one": 1}) assert i.class_var == "x" @@ -5057,6 +5122,10 @@ def test_setattr_nonnative() -> None: with assertRaises(AttributeError): i.const = 45 + # Doesn't work because there's no __delattr__. + with assertRaises(AttributeError): + del i.four + def test_no_setattr() -> None: i = NoSetAttr(99) i.super_setattr("attr", 100) @@ -5097,6 +5166,15 @@ def test_no_setattr_nonnative() -> None: object.__setattr__(i, "three", 102) assert i.three == 102 + del i.three + assert i.three == None + + delattr(i, "two") + assert i.two == None + + object.__delattr__(i, "one") + assert i.one == None + test_setattr() test_setattr_inherited() test_setattr_overridden() @@ -5105,3 +5183,67 @@ test_no_setattr() test_no_setattr_nonnative() [typing fixtures/typing-full.pyi] + +[case testDelAttrWithDeletableAttr] +from testutil import assertRaises + +class DelAttr: + __deletable__ = ["del_counter"] + + _attributes: dict[str, object] + del_counter: int = 0 + + def __init__(self) -> None: + object.__setattr__(self, "_attributes", {}) + + def __setattr__(self, key: str, val: object) -> None: + if key == "del_counter": + object.__setattr__(self, "del_counter", val) + else: + self._attributes[key] = val + + def __delattr__(self, key: str) -> None: + if key == "del_counter": + self.del_counter += 1 + else: + del self._attributes[key] + +def test_deletable_attr() -> None: + i = DelAttr() + assert i.del_counter == 0 + del i.del_counter + assert i.del_counter == 1 + +[case testDelAttrWithDeletableAttrInterpreted] +class DelAttr: + __deletable__ = ["del_counter"] + + _attributes: dict[str, object] + del_counter: int = 0 + + def __init__(self) -> None: + object.__setattr__(self, "_attributes", {}) + + def __setattr__(self, key: str, val: object) -> None: + if key == "del_counter": + object.__setattr__(self, "del_counter", val) + else: + self._attributes[key] = val + + def __delattr__(self, key: str) -> None: + if key == "del_counter": + self.del_counter += 1 + else: + del self._attributes[key] + +[file driver.py] +from native import DelAttr +from testutil import assertRaises + +def test_deletable_attr() -> None: + i = DelAttr() + assert i.del_counter == 0 + del i.del_counter + assert i.del_counter == 1 + +test_deletable_attr() From 9dc611f57639f899b5f71d286c3efe6a385dadf9 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 7 Oct 2025 01:56:50 +0100 Subject: [PATCH 304/424] Pin librt version (#20010) This is to prepare for renaming `native_internal` -> `librt.internal` --- mypy-requirements.txt | 2 +- pyproject.toml | 4 ++-- test-requirements.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index 6927ddd25d81e..d356ca0b59d97 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -4,4 +4,4 @@ typing_extensions>=4.6.0 mypy_extensions>=1.0.0 pathspec>=0.9.0 tomli>=1.1.0; python_version<'3.11' -librt>=0.1.0 +librt==0.1.1 diff --git a/pyproject.toml b/pyproject.toml index 589679113c3b0..1575a15e9909f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ requires = [ "mypy_extensions>=1.0.0", "pathspec>=0.9.0", "tomli>=1.1.0; python_version<'3.11'", - "librt>=0.1.0", + "librt==0.1.1", # the following is from build-requirements.txt "types-psutil", "types-setuptools", @@ -54,7 +54,7 @@ dependencies = [ "mypy_extensions>=1.0.0", "pathspec>=0.9.0", "tomli>=1.1.0; python_version<'3.11'", - "librt>=0.1.0", + "librt==0.1.1", ] dynamic = ["version"] diff --git a/test-requirements.txt b/test-requirements.txt index 5402adcb682ca..0983dc362c8ae 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -22,7 +22,7 @@ identify==2.6.13 # via pre-commit iniconfig==2.1.0 # via pytest -librt==0.1.0 +librt==0.1.1 # via -r mypy-requirements.txt lxml==6.0.1 ; python_version < "3.15" # via -r test-requirements.in From d2a8800e314fb3aa8c88a5fe9d6000d839ca6254 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Tue, 7 Oct 2025 07:57:11 -0400 Subject: [PATCH 305/424] [mypyc] fix: reject invalid `mypyc_attr` args [1/1] (#19963) This PR emits a builder error when an invalid key is passed into `mypyc_attr`. This change could have saved me ~20 minutes or so, when the root of my problem was just a simple typo. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- mypyc/irbuild/prepare.py | 2 +- mypyc/irbuild/util.py | 57 ++++++++++++++++++++++------ mypyc/test-data/irbuild-classes.test | 15 ++++++++ 3 files changed, 62 insertions(+), 12 deletions(-) diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index 20f2aeef8e6ef..e4f43b38b0dcc 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -357,7 +357,7 @@ def prepare_class_def( ir = mapper.type_to_ir[cdef.info] info = cdef.info - attrs, attrs_lines = get_mypyc_attrs(cdef) + attrs, attrs_lines = get_mypyc_attrs(cdef, path, errors) if attrs.get("allow_interpreted_subclasses") is True: ir.allow_interpreted_subclasses = True if attrs.get("serializable") is True: diff --git a/mypyc/irbuild/util.py b/mypyc/irbuild/util.py index eca2cac7e9dba..3028e940f7f99 100644 --- a/mypyc/irbuild/util.py +++ b/mypyc/irbuild/util.py @@ -2,7 +2,8 @@ from __future__ import annotations -from typing import Any +from typing import Any, Final, Literal, TypedDict, cast +from typing_extensions import NotRequired from mypy.nodes import ( ARG_NAMED, @@ -31,7 +32,23 @@ from mypy.types import FINAL_DECORATOR_NAMES from mypyc.errors import Errors -DATACLASS_DECORATORS = {"dataclasses.dataclass", "attr.s", "attr.attrs"} +MYPYC_ATTRS: Final[frozenset[MypycAttr]] = frozenset( + ["native_class", "allow_interpreted_subclasses", "serializable", "free_list_len"] +) + +DATACLASS_DECORATORS: Final = frozenset(["dataclasses.dataclass", "attr.s", "attr.attrs"]) + + +MypycAttr = Literal[ + "native_class", "allow_interpreted_subclasses", "serializable", "free_list_len" +] + + +class MypycAttrs(TypedDict): + native_class: NotRequired[bool] + allow_interpreted_subclasses: NotRequired[bool] + serializable: NotRequired[bool] + free_list_len: NotRequired[int] def is_final_decorator(d: Expression) -> bool: @@ -112,21 +129,39 @@ def get_mypyc_attr_call(d: Expression) -> CallExpr | None: return None -def get_mypyc_attrs(stmt: ClassDef | Decorator) -> tuple[dict[str, Any], dict[str, int]]: +def get_mypyc_attrs( + stmt: ClassDef | Decorator, path: str, errors: Errors +) -> tuple[MypycAttrs, dict[MypycAttr, int]]: """Collect all the mypyc_attr attributes on a class definition or a function.""" - attrs: dict[str, Any] = {} - lines: dict[str, int] = {} + attrs: MypycAttrs = {} + lines: dict[MypycAttr, int] = {} + + def set_mypyc_attr(key: str, value: Any, line: int) -> None: + if key in MYPYC_ATTRS: + key = cast(MypycAttr, key) + attrs[key] = value + lines[key] = line + else: + errors.error(f'"{key}" is not a supported "mypyc_attr"', path, line) + supported_keys = '", "'.join(sorted(MYPYC_ATTRS)) + errors.note(f'supported keys: "{supported_keys}"', path, line) + for dec in stmt.decorators: - d = get_mypyc_attr_call(dec) - if d: + if d := get_mypyc_attr_call(dec): + line = d.line for name, arg in zip(d.arg_names, d.args): if name is None: if isinstance(arg, StrExpr): - attrs[arg.value] = True - lines[arg.value] = d.line + set_mypyc_attr(arg.value, True, line) + else: + errors.error( + 'All "mypyc_attr" positional arguments must be string literals.', + path, + line, + ) else: - attrs[name] = get_mypyc_attr_literal(arg) - lines[name] = d.line + arg_value = get_mypyc_attr_literal(arg) + set_mypyc_attr(name, arg_value, line) return attrs, lines diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index a2d3b23ccfd9f..94e89f276eeb0 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -2837,3 +2837,18 @@ L0: r6 = PyObject_VectorcallMethod(r3, r5, 9223372036854775812, 0) keep_alive r2, self, key, val return 1 + +[case testInvalidMypycAttr] +from mypy_extensions import mypyc_attr + +@mypyc_attr("allow_interpreted_subclasses", "invalid_arg") # E: "invalid_arg" is not a supported "mypyc_attr" \ + # N: supported keys: "allow_interpreted_subclasses", "free_list_len", "native_class", "serializable" +class InvalidArg: + pass +@mypyc_attr(invalid_kwarg=True) # E: "invalid_kwarg" is not a supported "mypyc_attr" \ + # N: supported keys: "allow_interpreted_subclasses", "free_list_len", "native_class", "serializable" +class InvalidKwarg: + pass +@mypyc_attr(str()) # E: All "mypyc_attr" positional arguments must be string literals. +class InvalidLiteral: + pass From 139071c2c3017f29cb2c9d2883a53cddd28c3493 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Tue, 7 Oct 2025 07:58:37 -0400 Subject: [PATCH 306/424] [mypyc] feat: support negative index in TupleGet op (#19990) This PR modifies `TupleGet.__init__` to automatically convert negative indexes to positive indexes instead of crashing at the assert This won't change functionality on its own, since none of the existing calling locations can pass a negative value, but will allow us to pass negative values in https://github.com/python/mypy/pull/19972 so I think we should consider this PR a prerequisite to that one --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Piotr Sawicki --- mypyc/ir/ops.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/mypyc/ir/ops.py b/mypyc/ir/ops.py index 76c1e07a79d5a..ffce529f0756c 100644 --- a/mypyc/ir/ops.py +++ b/mypyc/ir/ops.py @@ -1045,10 +1045,17 @@ class TupleGet(RegisterOp): def __init__(self, src: Value, index: int, line: int = -1, *, borrow: bool = False) -> None: super().__init__(line) + assert isinstance( + src.type, RTuple + ), f"TupleGet only operates on tuples, not {type(src.type).__name__}" + src_len = len(src.type.types) self.src = src self.index = index - assert isinstance(src.type, RTuple), "TupleGet only operates on tuples" - assert index >= 0 + if index < 0: + self.index += src_len + assert ( + self.index <= src_len - 1 + ), f"Index out of range.\nsource type: {src.type}\nindex: {index}" self.type = src.type.types[index] self.is_borrowed = borrow From f2ebd79f2484402967f5206b9fc6ce014b6760dd Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 7 Oct 2025 20:46:30 +0100 Subject: [PATCH 307/424] Rename native_internal to librt.internal (#20014) Also adjust build/test logic slightly to prepare for more modules in `librt`. This PR should fix https://github.com/python/mypy/issues/20006 --- mypy-requirements.txt | 2 +- mypy/cache.py | 4 +- mypy/typeshed/stubs/librt/METADATA.toml | 2 +- mypy/typeshed/stubs/librt/librt/__init__.pyi | 0 .../internal.pyi} | 0 mypyc/build.py | 46 +++++++++++-------- mypyc/codegen/emitmodule.py | 12 ++--- mypyc/ir/rtypes.py | 2 +- .../{native_internal.c => librt_internal.c} | 28 +++++------ .../{native_internal.h => librt_internal.h} | 22 +++++---- mypyc/lib-rt/setup.py | 4 +- mypyc/options.py | 8 ++-- mypyc/primitives/misc_ops.py | 26 +++++------ mypyc/test-data/irbuild-classes.test | 6 +-- mypyc/test-data/run-classes.test | 4 +- mypyc/test/test_run.py | 11 +++-- pyproject.toml | 4 +- setup.py | 2 +- test-requirements.txt | 2 +- 19 files changed, 99 insertions(+), 86 deletions(-) create mode 100644 mypy/typeshed/stubs/librt/librt/__init__.pyi rename mypy/typeshed/stubs/librt/{native_internal.pyi => librt/internal.pyi} (100%) rename mypyc/lib-rt/{native_internal.c => librt_internal.c} (96%) rename mypyc/lib-rt/{native_internal.h => librt_internal.h} (80%) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index d356ca0b59d97..229f5624e8863 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -4,4 +4,4 @@ typing_extensions>=4.6.0 mypy_extensions>=1.0.0 pathspec>=0.9.0 tomli>=1.1.0; python_version<'3.11' -librt==0.1.1 +librt>=0.2.1 diff --git a/mypy/cache.py b/mypy/cache.py index 51deac914efc8..f8d3e6a05ebac 100644 --- a/mypy/cache.py +++ b/mypy/cache.py @@ -3,8 +3,7 @@ from collections.abc import Sequence from typing import Final -from mypy_extensions import u8 -from native_internal import ( +from librt.internal import ( Buffer as Buffer, read_bool as read_bool, read_float as read_float, @@ -17,6 +16,7 @@ write_str as write_str, write_tag as write_tag, ) +from mypy_extensions import u8 # Always use this type alias to refer to type tags. Tag = u8 diff --git a/mypy/typeshed/stubs/librt/METADATA.toml b/mypy/typeshed/stubs/librt/METADATA.toml index 37dc09b102d40..a42da251bed52 100644 --- a/mypy/typeshed/stubs/librt/METADATA.toml +++ b/mypy/typeshed/stubs/librt/METADATA.toml @@ -1 +1 @@ -version = "0.1.*" +version = "0.2.*" diff --git a/mypy/typeshed/stubs/librt/librt/__init__.pyi b/mypy/typeshed/stubs/librt/librt/__init__.pyi new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/mypy/typeshed/stubs/librt/native_internal.pyi b/mypy/typeshed/stubs/librt/librt/internal.pyi similarity index 100% rename from mypy/typeshed/stubs/librt/native_internal.pyi rename to mypy/typeshed/stubs/librt/librt/internal.pyi diff --git a/mypyc/build.py b/mypyc/build.py index efbd0dce31db8..40638b31d0006 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -42,6 +42,8 @@ from mypyc.namegen import exported_name from mypyc.options import CompilerOptions +LIBRT_MODULES = [("librt.internal", "librt_internal.c")] + try: # Import setuptools so that it monkey-patch overrides distutils import setuptools @@ -492,8 +494,8 @@ def mypycify( strict_dunder_typing: bool = False, group_name: str | None = None, log_trace: bool = False, - depends_on_native_internal: bool = False, - install_native_libs: bool = False, + depends_on_librt_internal: bool = False, + install_librt: bool = False, ) -> list[Extension]: """Main entry point to building using mypyc. @@ -544,11 +546,11 @@ def mypycify( mypyc_trace.txt (derived from executed operations). This is useful for performance analysis, such as analyzing which primitive ops are used the most and on which lines. - depends_on_native_internal: This is True only for mypy itself. - install_native_libs: If True, also build the native extension modules. Normally, - those are build and published on PyPI separately, but during - tests, we want to use their development versions (i.e. from - current commit). + depends_on_librt_internal: This is True only for mypy itself. + install_librt: If True, also build the librt extension modules. Normally, + those are build and published on PyPI separately, but during + tests, we want to use their development versions (i.e. from + current commit). """ # Figure out our configuration @@ -562,7 +564,7 @@ def mypycify( strict_dunder_typing=strict_dunder_typing, group_name=group_name, log_trace=log_trace, - depends_on_native_internal=depends_on_native_internal, + depends_on_librt_internal=depends_on_librt_internal, ) # Generate all the actual important C code @@ -661,21 +663,25 @@ def mypycify( build_single_module(group_sources, cfilenames + shared_cfilenames, cflags) ) - if install_native_libs: - for name in ["native_internal.c"] + RUNTIME_C_FILES: + if install_librt: + os.makedirs("librt", exist_ok=True) + for name in RUNTIME_C_FILES: rt_file = os.path.join(build_dir, name) with open(os.path.join(include_dir(), name), encoding="utf-8") as f: write_file(rt_file, f.read()) - extensions.append( - get_extension()( - "native_internal", - sources=[ - os.path.join(build_dir, file) - for file in ["native_internal.c"] + RUNTIME_C_FILES - ], - include_dirs=[include_dir()], - extra_compile_args=cflags, + for mod, file_name in LIBRT_MODULES: + rt_file = os.path.join(build_dir, file_name) + with open(os.path.join(include_dir(), file_name), encoding="utf-8") as f: + write_file(rt_file, f.read()) + extensions.append( + get_extension()( + mod, + sources=[ + os.path.join(build_dir, file) for file in [file_name] + RUNTIME_C_FILES + ], + include_dirs=[include_dir()], + extra_compile_args=cflags, + ) ) - ) return extensions diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index ca5db52ab7da3..3602b3c26e03b 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -601,12 +601,12 @@ def generate_c_for_modules(self) -> list[tuple[str, str]]: ext_declarations.emit_line(f"#define MYPYC_NATIVE{self.group_suffix}_H") ext_declarations.emit_line("#include ") ext_declarations.emit_line("#include ") - if self.compiler_options.depends_on_native_internal: - ext_declarations.emit_line("#include ") + if self.compiler_options.depends_on_librt_internal: + ext_declarations.emit_line("#include ") declarations = Emitter(self.context) - declarations.emit_line(f"#ifndef MYPYC_NATIVE_INTERNAL{self.group_suffix}_H") - declarations.emit_line(f"#define MYPYC_NATIVE_INTERNAL{self.group_suffix}_H") + declarations.emit_line(f"#ifndef MYPYC_LIBRT_INTERNAL{self.group_suffix}_H") + declarations.emit_line(f"#define MYPYC_LIBRT_INTERNAL{self.group_suffix}_H") declarations.emit_line("#include ") declarations.emit_line("#include ") declarations.emit_line(f'#include "__native{self.short_group_suffix}.h"') @@ -1029,8 +1029,8 @@ def emit_module_exec_func( declaration = f"int CPyExec_{exported_name(module_name)}(PyObject *module)" module_static = self.module_internal_static_name(module_name, emitter) emitter.emit_lines(declaration, "{") - if self.compiler_options.depends_on_native_internal: - emitter.emit_line("if (import_native_internal() < 0) {") + if self.compiler_options.depends_on_librt_internal: + emitter.emit_line("if (import_librt_internal() < 0) {") emitter.emit_line("return -1;") emitter.emit_line("}") emitter.emit_line("PyObject* modname = NULL;") diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 34824a59cd5c3..941670ab230dd 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -514,7 +514,7 @@ def __hash__(self) -> int: KNOWN_NATIVE_TYPES: Final = { name: RPrimitive(name, is_unboxed=False, is_refcounted=True) - for name in ["native_internal.Buffer"] + for name in ["librt.internal.Buffer"] } diff --git a/mypyc/lib-rt/native_internal.c b/mypyc/lib-rt/librt_internal.c similarity index 96% rename from mypyc/lib-rt/native_internal.c rename to mypyc/lib-rt/librt_internal.c index a6511a1caf259..cb9aa10258211 100644 --- a/mypyc/lib-rt/native_internal.c +++ b/mypyc/lib-rt/librt_internal.c @@ -2,8 +2,8 @@ #include #include #include "CPy.h" -#define NATIVE_INTERNAL_MODULE -#include "native_internal.h" +#define LIBRT_INTERNAL_MODULE +#include "librt_internal.h" #define START_SIZE 512 #define MAX_SHORT_INT_TAGGED (255 << 1) @@ -558,7 +558,7 @@ write_tag(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames return Py_None; } -static PyMethodDef native_internal_module_methods[] = { +static PyMethodDef librt_internal_module_methods[] = { {"write_bool", (PyCFunction)write_bool, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("write a bool")}, {"read_bool", (PyCFunction)read_bool, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("read a bool")}, {"write_str", (PyCFunction)write_str, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("write a string")}, @@ -574,11 +574,11 @@ static PyMethodDef native_internal_module_methods[] = { static int NativeInternal_ABI_Version(void) { - return NATIVE_INTERNAL_ABI_VERSION; + return LIBRT_INTERNAL_ABI_VERSION; } static int -native_internal_module_exec(PyObject *m) +librt_internal_module_exec(PyObject *m) { if (PyType_Ready(&BufferType) < 0) { return -1; @@ -604,32 +604,32 @@ native_internal_module_exec(PyObject *m) (void *)read_tag_internal, (void *)NativeInternal_ABI_Version, }; - PyObject *c_api_object = PyCapsule_New((void *)NativeInternal_API, "native_internal._C_API", NULL); + PyObject *c_api_object = PyCapsule_New((void *)NativeInternal_API, "librt.internal._C_API", NULL); if (PyModule_Add(m, "_C_API", c_api_object) < 0) { return -1; } return 0; } -static PyModuleDef_Slot native_internal_module_slots[] = { - {Py_mod_exec, native_internal_module_exec}, +static PyModuleDef_Slot librt_internal_module_slots[] = { + {Py_mod_exec, librt_internal_module_exec}, #ifdef Py_MOD_GIL_NOT_USED {Py_mod_gil, Py_MOD_GIL_NOT_USED}, #endif {0, NULL} }; -static PyModuleDef native_internal_module = { +static PyModuleDef librt_internal_module = { .m_base = PyModuleDef_HEAD_INIT, - .m_name = "native_internal", + .m_name = "internal", .m_doc = "Mypy cache serialization utils", .m_size = 0, - .m_methods = native_internal_module_methods, - .m_slots = native_internal_module_slots, + .m_methods = librt_internal_module_methods, + .m_slots = librt_internal_module_slots, }; PyMODINIT_FUNC -PyInit_native_internal(void) +PyInit_internal(void) { - return PyModuleDef_Init(&native_internal_module); + return PyModuleDef_Init(&librt_internal_module); } diff --git a/mypyc/lib-rt/native_internal.h b/mypyc/lib-rt/librt_internal.h similarity index 80% rename from mypyc/lib-rt/native_internal.h rename to mypyc/lib-rt/librt_internal.h index 63e902a6e1bf2..fd8ec2422cc5b 100644 --- a/mypyc/lib-rt/native_internal.h +++ b/mypyc/lib-rt/librt_internal.h @@ -1,9 +1,9 @@ -#ifndef NATIVE_INTERNAL_H -#define NATIVE_INTERNAL_H +#ifndef LIBRT_INTERNAL_H +#define LIBRT_INTERNAL_H -#define NATIVE_INTERNAL_ABI_VERSION 0 +#define LIBRT_INTERNAL_ABI_VERSION 0 -#ifdef NATIVE_INTERNAL_MODULE +#ifdef LIBRT_INTERNAL_MODULE static PyObject *Buffer_internal(PyObject *source); static PyObject *Buffer_internal_empty(void); @@ -40,17 +40,21 @@ static void **NativeInternal_API; #define NativeInternal_ABI_Version (*(int (*)(void)) NativeInternal_API[13]) static int -import_native_internal(void) +import_librt_internal(void) { - NativeInternal_API = (void **)PyCapsule_Import("native_internal._C_API", 0); + PyObject *mod = PyImport_ImportModule("librt.internal"); + if (mod == NULL) + return -1; + Py_DECREF(mod); // we import just for the side effect of making the below work. + NativeInternal_API = (void **)PyCapsule_Import("librt.internal._C_API", 0); if (NativeInternal_API == NULL) return -1; - if (NativeInternal_ABI_Version() != NATIVE_INTERNAL_ABI_VERSION) { - PyErr_SetString(PyExc_ValueError, "ABI version conflict for native_internal"); + if (NativeInternal_ABI_Version() != LIBRT_INTERNAL_ABI_VERSION) { + PyErr_SetString(PyExc_ValueError, "ABI version conflict for librt.internal"); return -1; } return 0; } #endif -#endif // NATIVE_INTERNAL_H +#endif // LIBRT_INTERNAL_H diff --git a/mypyc/lib-rt/setup.py b/mypyc/lib-rt/setup.py index 36b55e44dcd1d..b78ad0dbc23e3 100644 --- a/mypyc/lib-rt/setup.py +++ b/mypyc/lib-rt/setup.py @@ -75,9 +75,9 @@ def run(self) -> None: setup( ext_modules=[ Extension( - "native_internal", + "librt.internal", [ - "native_internal.c", + "librt_internal.c", "init.c", "int_ops.c", "exc_ops.c", diff --git a/mypyc/options.py b/mypyc/options.py index c009d3c6a7a45..e004a0a52c958 100644 --- a/mypyc/options.py +++ b/mypyc/options.py @@ -17,7 +17,7 @@ def __init__( strict_dunder_typing: bool = False, group_name: str | None = None, log_trace: bool = False, - depends_on_native_internal: bool = False, + depends_on_librt_internal: bool = False, ) -> None: self.strip_asserts = strip_asserts self.multi_file = multi_file @@ -51,7 +51,7 @@ def __init__( # mypyc_trace.txt when compiled module is executed. This is useful for # performance analysis. self.log_trace = log_trace - # If enabled, add capsule imports of native_internal API. This should be used + # If enabled, add capsule imports of librt.internal API. This should be used # only for mypy itself, third-party code compiled with mypyc should not use - # native_internal. - self.depends_on_native_internal = depends_on_native_internal + # librt.internal. + self.depends_on_librt_internal = depends_on_librt_internal diff --git a/mypyc/primitives/misc_ops.py b/mypyc/primitives/misc_ops.py index 8e6e450c64dca..18d475fe89d41 100644 --- a/mypyc/primitives/misc_ops.py +++ b/mypyc/primitives/misc_ops.py @@ -333,11 +333,11 @@ error_kind=ERR_NEVER, ) -buffer_rprimitive = KNOWN_NATIVE_TYPES["native_internal.Buffer"] +buffer_rprimitive = KNOWN_NATIVE_TYPES["librt.internal.Buffer"] # Buffer(source) function_op( - name="native_internal.Buffer", + name="librt.internal.Buffer", arg_types=[bytes_rprimitive], return_type=buffer_rprimitive, c_function_name="Buffer_internal", @@ -346,7 +346,7 @@ # Buffer() function_op( - name="native_internal.Buffer", + name="librt.internal.Buffer", arg_types=[], return_type=buffer_rprimitive, c_function_name="Buffer_internal_empty", @@ -362,7 +362,7 @@ ) function_op( - name="native_internal.write_bool", + name="librt.internal.write_bool", arg_types=[object_rprimitive, bool_rprimitive], return_type=none_rprimitive, c_function_name="write_bool_internal", @@ -370,7 +370,7 @@ ) function_op( - name="native_internal.read_bool", + name="librt.internal.read_bool", arg_types=[object_rprimitive], return_type=bool_rprimitive, c_function_name="read_bool_internal", @@ -378,7 +378,7 @@ ) function_op( - name="native_internal.write_str", + name="librt.internal.write_str", arg_types=[object_rprimitive, str_rprimitive], return_type=none_rprimitive, c_function_name="write_str_internal", @@ -386,7 +386,7 @@ ) function_op( - name="native_internal.read_str", + name="librt.internal.read_str", arg_types=[object_rprimitive], return_type=str_rprimitive, c_function_name="read_str_internal", @@ -394,7 +394,7 @@ ) function_op( - name="native_internal.write_float", + name="librt.internal.write_float", arg_types=[object_rprimitive, float_rprimitive], return_type=none_rprimitive, c_function_name="write_float_internal", @@ -402,7 +402,7 @@ ) function_op( - name="native_internal.read_float", + name="librt.internal.read_float", arg_types=[object_rprimitive], return_type=float_rprimitive, c_function_name="read_float_internal", @@ -410,7 +410,7 @@ ) function_op( - name="native_internal.write_int", + name="librt.internal.write_int", arg_types=[object_rprimitive, int_rprimitive], return_type=none_rprimitive, c_function_name="write_int_internal", @@ -418,7 +418,7 @@ ) function_op( - name="native_internal.read_int", + name="librt.internal.read_int", arg_types=[object_rprimitive], return_type=int_rprimitive, c_function_name="read_int_internal", @@ -426,7 +426,7 @@ ) function_op( - name="native_internal.write_tag", + name="librt.internal.write_tag", arg_types=[object_rprimitive, uint8_rprimitive], return_type=none_rprimitive, c_function_name="write_tag_internal", @@ -434,7 +434,7 @@ ) function_op( - name="native_internal.read_tag", + name="librt.internal.read_tag", arg_types=[object_rprimitive], return_type=uint8_rprimitive, c_function_name="read_tag_internal", diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index 94e89f276eeb0..c4410b3b19d2b 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -1452,7 +1452,7 @@ class TestOverload: [case testNativeBufferFastPath] from typing import Final from mypy_extensions import u8 -from native_internal import ( +from librt.internal import ( Buffer, write_bool, read_bool, write_str, read_str, write_float, read_float, write_int, read_int, write_tag, read_tag ) @@ -1476,11 +1476,11 @@ def foo() -> None: u = read_tag(b) [out] def foo(): - r0, b :: native_internal.Buffer + r0, b :: librt.internal.Buffer r1 :: str r2, r3, r4, r5, r6 :: None r7 :: bytes - r8 :: native_internal.Buffer + r8 :: librt.internal.Buffer r9, x :: str r10, y :: bool r11, z :: float diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index ab1dcb926c34f..8755169fdb0b2 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -2710,10 +2710,10 @@ from native import Player [out] Player.MIN = -[case testBufferRoundTrip_native_libs] +[case testBufferRoundTrip_librt_internal] from typing import Final from mypy_extensions import u8 -from native_internal import ( +from librt.internal import ( Buffer, write_bool, read_bool, write_str, read_str, write_float, read_float, write_int, read_int, write_tag, read_tag ) diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py index 172a1016dd918..22ab18e97293c 100644 --- a/mypyc/test/test_run.py +++ b/mypyc/test/test_run.py @@ -86,7 +86,7 @@ setup(name='test_run_output', ext_modules=mypycify({}, separate={}, skip_cgen_input={!r}, strip_asserts=False, - multi_file={}, opt_level='{}', install_native_libs={}), + multi_file={}, opt_level='{}', install_librt={}), ) """ @@ -239,13 +239,16 @@ def run_case_step(self, testcase: DataDrivenTestCase, incremental_step: int) -> groups = construct_groups(sources, separate, len(module_names) > 1, None) - native_libs = "_native_libs" in testcase.name + # Use _librt_internal to test mypy-specific parts of librt (they have + # some special-casing in mypyc), for everything else use _librt suffix. + librt_internal = testcase.name.endswith("_librt_internal") + librt = librt_internal or testcase.name.endswith("_librt") try: compiler_options = CompilerOptions( multi_file=self.multi_file, separate=self.separate, strict_dunder_typing=self.strict_dunder_typing, - depends_on_native_internal=native_libs, + depends_on_librt_internal=librt_internal, ) result = emitmodule.parse_and_typecheck( sources=sources, @@ -278,7 +281,7 @@ def run_case_step(self, testcase: DataDrivenTestCase, incremental_step: int) -> with open(setup_file, "w", encoding="utf-8") as f: f.write( setup_format.format( - module_paths, separate, cfiles, self.multi_file, opt_level, native_libs + module_paths, separate, cfiles, self.multi_file, opt_level, librt ) ) diff --git a/pyproject.toml b/pyproject.toml index 1575a15e9909f..adcca65bf0156 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ requires = [ "mypy_extensions>=1.0.0", "pathspec>=0.9.0", "tomli>=1.1.0; python_version<'3.11'", - "librt==0.1.1", + "librt>=0.2.1", # the following is from build-requirements.txt "types-psutil", "types-setuptools", @@ -54,7 +54,7 @@ dependencies = [ "mypy_extensions>=1.0.0", "pathspec>=0.9.0", "tomli>=1.1.0; python_version<'3.11'", - "librt==0.1.1", + "librt>=0.2.1", ] dynamic = ["version"] diff --git a/setup.py b/setup.py index 3f53d96dbd85c..1d093ec3b9e2c 100644 --- a/setup.py +++ b/setup.py @@ -155,7 +155,7 @@ def run(self) -> None: multi_file=sys.platform == "win32" or force_multifile, log_trace=log_trace, # Mypy itself is allowed to use native_internal extension. - depends_on_native_internal=True, + depends_on_librt_internal=True, ) else: diff --git a/test-requirements.txt b/test-requirements.txt index 0983dc362c8ae..7e8e167300dc1 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -22,7 +22,7 @@ identify==2.6.13 # via pre-commit iniconfig==2.1.0 # via pytest -librt==0.1.1 +librt==0.2.1 # via -r mypy-requirements.txt lxml==6.0.1 ; python_version < "3.15" # via -r test-requirements.in From 4ff18305d710b3a5491615eddf056db14d337b63 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Wed, 8 Oct 2025 03:32:12 +0200 Subject: [PATCH 308/424] Filter SyntaxWarnings during AST parsing (#20023) Especially with [PEP 765](https://peps.python.org/pep-0765/) in 3.14, Python has been getting more liberal in emitting SyntaxWarnings during AST parsing and compilation. Generally, they aren't really helpful for mypy itself. There are open discussions to add a flag which would disable these. Until that's implemented, filter the warnings manually. _This also get's rid of the warnings emitted on test code. If at some point `return in finally` will be made an error, those tests could be adjust / removed. Until then, we can continue to test these as is without issues._ ```py def func() -> None: try: x = 1/0 finally: return None # return in finally "Hello \P world" # invalid escape sequence "" is 1 # "is" with 'int' literal ``` --- mypy/fastparse.py | 5 ++++- pyproject.toml | 3 --- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 6b2eb532003c9..aa5c89cd0f41f 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -232,9 +232,12 @@ def parse( assert options.python_version[0] >= 3 feature_version = options.python_version[1] try: - # Disable deprecation warnings about \u + # Disable + # - deprecation warnings about \u + # - syntax warnings for 'invalid escape sequence' (3.12+) and 'return in finally' (3.14+) with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=DeprecationWarning) + warnings.filterwarnings("ignore", category=SyntaxWarning) ast = ast3_parse(source, fnam, "exec", feature_version=feature_version) tree = ASTConverter( diff --git a/pyproject.toml b/pyproject.toml index adcca65bf0156..96b05ba459b19 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -227,9 +227,6 @@ xfail_strict = true # Force warnings as errors filterwarnings = [ "error", - # Some testcases may contain code that emits SyntaxWarnings, and they are not yet - # handled consistently in 3.14 (PEP 765) - "default::SyntaxWarning", ] [tool.coverage.run] From 59e9e7d0bb4e40e41c002f6fb399609991beccd9 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 8 Oct 2025 12:54:47 +0100 Subject: [PATCH 309/424] Fix infinite loop on empty Buffer (#20024) Currently `Buffer(b"")` (as opposite to `Buffer()`) can go into an infinite loop when resizing. Fix this by allocating one more byte, that is conveniently guaranteed by `bytes` ABI. I also move the `librt/__init__.py` hack to tests, since this is the only place where it is needed. I am still not sure why `*.so` from `site-packages` is preferred over a "local" namespace package with the same `*.so`. --- mypyc/build.py | 1 - mypyc/lib-rt/librt_internal.c | 8 +++++--- mypyc/test-data/run-classes.test | 9 +++++++++ mypyc/test/test_run.py | 6 ++++++ 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/mypyc/build.py b/mypyc/build.py index 40638b31d0006..13648911c0b5a 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -664,7 +664,6 @@ def mypycify( ) if install_librt: - os.makedirs("librt", exist_ok=True) for name in RUNTIME_C_FILES: rt_file = os.path.join(build_dir, name) with open(os.path.join(include_dir(), name), encoding="utf-8") as f: diff --git a/mypyc/lib-rt/librt_internal.c b/mypyc/lib-rt/librt_internal.c index cb9aa10258211..b97d6665b515a 100644 --- a/mypyc/lib-rt/librt_internal.c +++ b/mypyc/lib-rt/librt_internal.c @@ -70,12 +70,14 @@ Buffer_init_internal(BufferObject *self, PyObject *source) { PyErr_SetString(PyExc_TypeError, "source must be a bytes object"); return -1; } - self->size = PyBytes_GET_SIZE(source); - self->end = self->size; + self->end = PyBytes_GET_SIZE(source); + // Allocate at least one byte to simplify resizing logic. + // The original bytes buffer has last null byte, so this is safe. + self->size = self->end + 1; // This returns a pointer to internal bytes data, so make our own copy. char *buf = PyBytes_AsString(source); self->buf = PyMem_Malloc(self->size); - memcpy(self->buf, buf, self->size); + memcpy(self->buf, buf, self->end); } else { self->buf = PyMem_Malloc(START_SIZE); self->size = START_SIZE; diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 8755169fdb0b2..84704ce66c81d 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -2866,6 +2866,15 @@ test_buffer_roundtrip_interpreted() test_buffer_int_size_interpreted() test_buffer_str_size_interpreted() +[case testBufferEmpty_librt_internal] +from librt.internal import Buffer, write_int, read_int + +def test_empty() -> None: + b = Buffer(b"") + write_int(b, 42) + b1 = Buffer(b.getvalue()) + assert read_int(b1) == 42 + [case testEnumMethodCalls] from enum import Enum from typing import overload, Optional, Union diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py index 22ab18e97293c..953f613293953 100644 --- a/mypyc/test/test_run.py +++ b/mypyc/test/test_run.py @@ -285,6 +285,12 @@ def run_case_step(self, testcase: DataDrivenTestCase, incremental_step: int) -> ) ) + if librt: + # This hack forces Python to prefer the local "installation". + os.makedirs("librt", exist_ok=True) + with open(os.path.join("librt", "__init__.py"), "a"): + pass + if not run_setup(setup_file, ["build_ext", "--inplace"]): if testcase.config.getoption("--mypyc-showc"): show_c(cfiles) From eec826e940a9c5dcee54b418bc9aa6668922cf2e Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 8 Oct 2025 13:24:42 +0100 Subject: [PATCH 310/424] Make lib-rt/setup.py similar to mypyc extensions (#20022) Two small things here: * Use `setuptools` extension class. * Use same optimization levels we use when compiling mypy. --- mypyc/lib-rt/setup.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/mypyc/lib-rt/setup.py b/mypyc/lib-rt/setup.py index b78ad0dbc23e3..afbceba060f49 100644 --- a/mypyc/lib-rt/setup.py +++ b/mypyc/lib-rt/setup.py @@ -8,10 +8,12 @@ import os import subprocess import sys -from distutils.command.build_ext import build_ext -from distutils.core import Extension, setup +from distutils import ccompiler, sysconfig from typing import Any +from setuptools import Extension, setup +from setuptools.command.build_ext import build_ext + C_APIS_TO_TEST = [ "init.c", "int_ops.c", @@ -72,6 +74,14 @@ def run(self) -> None: else: # TODO: we need a way to share our preferred C flags and get_extension() logic with # mypyc/build.py without code duplication. + compiler = ccompiler.new_compiler() + sysconfig.customize_compiler(compiler) + cflags: list[str] = [] + if compiler.compiler_type == "unix": + cflags += ["-O3"] + elif compiler.compiler_type == "msvc": + cflags += ["/O2"] + setup( ext_modules=[ Extension( @@ -85,6 +95,7 @@ def run(self) -> None: "getargsfast.c", ], include_dirs=["."], + extra_compile_args=cflags, ) ] ) From 712afc5b204e9b07e46fb8d1b12d2393b03411a1 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Wed, 8 Oct 2025 20:59:53 +0200 Subject: [PATCH 311/424] Adjust stubtest test stubs for PEP 728 (Python 3.15) (#20009) Add stubs for [PEP 728](https://peps.python.org/pep-0728/). --- mypy/test/teststubtest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mypy/test/teststubtest.py b/mypy/test/teststubtest.py index 800f522d90a01..dfbde217e82f5 100644 --- a/mypy/test/teststubtest.py +++ b/mypy/test/teststubtest.py @@ -88,6 +88,8 @@ class _TypedDict(Mapping[str, object]): __total__: ClassVar[bool] __readonly_keys__: ClassVar[frozenset[str]] __mutable_keys__: ClassVar[frozenset[str]] + __closed__: ClassVar[bool | None] + __extra_items__: ClassVar[Any] def overload(func: _T) -> _T: ... def type_check_only(func: _T) -> _T: ... def final(func: _T) -> _T: ... From 77b4cfb167b2ef837eddccaf338b3b52e6cf4ba5 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 9 Oct 2025 01:23:38 +0300 Subject: [PATCH 312/424] Fix `[name-defined]` false-positive in `class A[X, Y=X]:` case (#20021) This a WIP to see the test result before adding my own tests :) Closes https://github.com/python/mypy/issues/20020 --- mypy/semanal.py | 7 +++---- test-data/unit/check-python313.test | 29 +++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 17dc9bfadc1f2..08f9eb03c9d74 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1791,11 +1791,10 @@ def push_type_args( return None tvs.append((p.name, tv)) - for name, tv in tvs: - if self.is_defined_type_param(name): - self.fail(f'"{name}" already defined as a type parameter', context) + if self.is_defined_type_param(p.name): + self.fail(f'"{p.name}" already defined as a type parameter', context) else: - self.add_symbol(name, tv, context, no_progress=True, type_param=True) + self.add_symbol(p.name, tv, context, no_progress=True, type_param=True) return tvs diff --git a/test-data/unit/check-python313.test b/test-data/unit/check-python313.test index b46ae0fecfc42..117d20ceaf0b0 100644 --- a/test-data/unit/check-python313.test +++ b/test-data/unit/check-python313.test @@ -290,3 +290,32 @@ reveal_type(A1().x) # N: Revealed type is "builtins.int" reveal_type(A2().x) # N: Revealed type is "builtins.int" reveal_type(A3().x) # N: Revealed type is "builtins.int" [builtins fixtures/tuple.pyi] + +[case testTypeVarDefaultToAnotherTypeVar] +class A[X, Y = X, Z = Y]: + x: X + y: Y + z: Z + +a1: A[int] +reveal_type(a1.x) # N: Revealed type is "builtins.int" +reveal_type(a1.y) # N: Revealed type is "builtins.int" +# TODO: this must reveal `int` as well: +reveal_type(a1.z) # N: Revealed type is "X`1" + +a2: A[int, str] +reveal_type(a2.x) # N: Revealed type is "builtins.int" +reveal_type(a2.y) # N: Revealed type is "builtins.str" +reveal_type(a2.z) # N: Revealed type is "builtins.str" + +a3: A[int, str, bool] +reveal_type(a3.x) # N: Revealed type is "builtins.int" +reveal_type(a3.y) # N: Revealed type is "builtins.str" +reveal_type(a3.z) # N: Revealed type is "builtins.bool" +[builtins fixtures/tuple.pyi] + +[case testTypeVarDefaultToAnotherTypeVarWrong] +class A[Y = X, X = int]: ... # E: Name "X" is not defined + +class B[Y = X]: ... # E: Name "X" is not defined +[builtins fixtures/tuple.pyi] From d51fa0098d2488ce135293f4310229c5b700f5f6 Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Wed, 8 Oct 2025 17:58:26 -0700 Subject: [PATCH 313/424] Update shellcheck version in actionlint pre-commit to 0.11.0 (#20030) I was getting bizarre wasm failures from runtests running pre-commit running actionlint running shellcheck (shellcheck is written in haskell but we depend on go-shellcheck which is a packaged up version of it using wasm?). (https://github.com/python/mypy/issues/19958#issuecomment-3383449423) If I update it, I don't get bizarre wasm failures and it still doesn't complain about anything. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3b323f03b99c9..a410585a52d4b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,7 +41,7 @@ repos: # actionlint has a shellcheck integration which extracts shell scripts in `run:` steps from GitHub Actions # and checks these with shellcheck. This is arguably its most useful feature, # but the integration only works if shellcheck is installed - - "github.com/wasilibs/go-shellcheck/cmd/shellcheck@v0.10.0" + - "github.com/wasilibs/go-shellcheck/cmd/shellcheck@v0.11.0" - repo: https://github.com/woodruffw/zizmor-pre-commit rev: v1.5.2 hooks: From c928847e8c990e0eb0306a04beda82a48eb18428 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Wed, 8 Oct 2025 17:59:11 -0700 Subject: [PATCH 314/424] [nit] clarify comment about deprecation warnings about \u (#20026) This is the same as 'invalid escape sequence', just a different type of warning on lower Python versions. Chases https://github.com/python/mypy/pull/20023. More background information is available in https://github.com/python/mypy/pull/19606, although that's probably not very helpful all told. --- mypy/fastparse.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index aa5c89cd0f41f..276e183a6bf0e 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -233,8 +233,8 @@ def parse( feature_version = options.python_version[1] try: # Disable - # - deprecation warnings about \u - # - syntax warnings for 'invalid escape sequence' (3.12+) and 'return in finally' (3.14+) + # - deprecation warnings for 'invalid escape sequence' (Python 3.11 and below) + # - syntax warnings for 'invalid escape sequence' (3.12+) and 'return in finally' (3.14+) with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=DeprecationWarning) warnings.filterwarnings("ignore", category=SyntaxWarning) From 6e9fb5948502ccaaca4e4869121c1fe8064f8a93 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 9 Oct 2025 08:24:46 +0100 Subject: [PATCH 315/424] Update test requirements snapshot (#20031) I actually wanted this for `librt`, but it looks like there is a bunch of other deps worth updating. --- test-requirements.in | 3 ++- test-requirements.txt | 32 +++++++++++++++++--------------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/test-requirements.in b/test-requirements.in index df074965a1e83..556edf5077d2b 100644 --- a/test-requirements.in +++ b/test-requirements.in @@ -4,7 +4,7 @@ -r mypy-requirements.txt -r build-requirements.txt attrs>=18.0 -filelock>=3.3.0 +filelock>=3.3.0,<3.20.0 # latest version is not available on 3.9 that we still support lxml>=5.3.0; python_version<'3.15' psutil>=4.0 pytest>=8.1.0 @@ -13,3 +13,4 @@ pytest-cov>=2.10.0 setuptools>=75.1.0 tomli>=1.1.0 # needed even on py311+ so the self check passes with --python-version 3.9 pre_commit>=3.5.0 +platformdirs<4.5.0 # latest version is not available on 3.9 that we still support diff --git a/test-requirements.txt b/test-requirements.txt index 7e8e167300dc1..c16708dfdbafa 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -4,11 +4,11 @@ # # pip-compile --allow-unsafe --output-file=test-requirements.txt --strip-extras test-requirements.in # -attrs==25.3.0 +attrs==25.4.0 # via -r test-requirements.in cfgv==3.4.0 # via pre-commit -coverage==7.10.5 +coverage==7.10.7 # via pytest-cov distlib==0.4.0 # via virtualenv @@ -18,13 +18,13 @@ filelock==3.19.1 # via # -r test-requirements.in # virtualenv -identify==2.6.13 +identify==2.6.15 # via pre-commit iniconfig==2.1.0 # via pytest -librt==0.2.1 +librt==0.2.2 # via -r mypy-requirements.txt -lxml==6.0.1 ; python_version < "3.15" +lxml==6.0.2 ; python_version < "3.15" # via -r test-requirements.in mypy-extensions==1.1.0 # via -r mypy-requirements.txt @@ -34,38 +34,40 @@ packaging==25.0 # via pytest pathspec==0.12.1 # via -r mypy-requirements.txt -platformdirs==4.3.8 - # via virtualenv +platformdirs==4.4.0 + # via + # -r test-requirements.in + # virtualenv pluggy==1.6.0 # via # pytest # pytest-cov pre-commit==4.3.0 # via -r test-requirements.in -psutil==7.0.0 +psutil==7.1.0 # via -r test-requirements.in pygments==2.19.2 # via pytest -pytest==8.4.1 +pytest==8.4.2 # via # -r test-requirements.in # pytest-cov # pytest-xdist -pytest-cov==6.2.1 +pytest-cov==7.0.0 # via -r test-requirements.in pytest-xdist==3.8.0 # via -r test-requirements.in -pyyaml==6.0.2 +pyyaml==6.0.3 # via pre-commit -tomli==2.2.1 +tomli==2.3.0 # via -r test-requirements.in -types-psutil==7.0.0.20250822 +types-psutil==7.0.0.20251001 # via -r build-requirements.txt types-setuptools==80.9.0.20250822 # via -r build-requirements.txt -typing-extensions==4.14.1 +typing-extensions==4.15.0 # via -r mypy-requirements.txt -virtualenv==20.34.0 +virtualenv==20.35.0 # via pre-commit # The following packages are considered to be unsafe in a requirements file: From 895d0cf9b60af0aac8c4c56709825cbca3feca1f Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 9 Oct 2025 11:27:54 +0100 Subject: [PATCH 316/424] Re-run pip-compile to get rid of yanked version of virtualenv (#20037) --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index c16708dfdbafa..bbaf1ce6010ff 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -67,7 +67,7 @@ types-setuptools==80.9.0.20250822 # via -r build-requirements.txt typing-extensions==4.15.0 # via -r mypy-requirements.txt -virtualenv==20.35.0 +virtualenv==20.34.0 # via pre-commit # The following packages are considered to be unsafe in a requirements file: From f8ecfe5658f33d5d254e7d161c25b2909691c5ce Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 9 Oct 2025 15:11:05 +0100 Subject: [PATCH 317/424] Bump librt again (#20040) Just in case to test the new MacOS wheels. Also fix `lib-rt/setup.py` docstring. --- mypyc/lib-rt/setup.py | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mypyc/lib-rt/setup.py b/mypyc/lib-rt/setup.py index afbceba060f49..299b0acd96e70 100644 --- a/mypyc/lib-rt/setup.py +++ b/mypyc/lib-rt/setup.py @@ -1,4 +1,4 @@ -"""Build script for mypyc C runtime library unit tests. +"""Build script for mypyc C runtime library and C API unit tests. The tests are written in C++ and use the Google Test framework. """ diff --git a/test-requirements.txt b/test-requirements.txt index bbaf1ce6010ff..0dc2a4cf8f189 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -22,7 +22,7 @@ identify==2.6.15 # via pre-commit iniconfig==2.1.0 # via pytest -librt==0.2.2 +librt==0.2.3 # via -r mypy-requirements.txt lxml==6.0.2 ; python_version < "3.15" # via -r test-requirements.in From 826e0adcb4b6b2855788927d220b965660df9294 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Fri, 10 Oct 2025 09:05:44 -0400 Subject: [PATCH 318/424] [mypyc] feat: support constant folding in `translate_index_expr` [1/1] (#19972) This PR attempts to constant fold the index value in `translate_index_expr` I'm not sure any test changes are warranted for a small PR of this nature. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- mypyc/irbuild/expression.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index 59ecc4ac2c5c2..f6636a0e7b624 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -590,8 +590,12 @@ def transform_index_expr(builder: IRBuilder, expr: IndexExpr) -> Value: base = builder.accept(expr.base, can_borrow=can_borrow_base) - if isinstance(base.type, RTuple) and isinstance(index, IntExpr): - return builder.add(TupleGet(base, index.value, expr.line)) + if isinstance(base.type, RTuple): + folded_index = constant_fold_expr(builder, index) + if isinstance(folded_index, int): + length = len(base.type.types) + if -length <= folded_index <= length - 1: + return builder.add(TupleGet(base, folded_index, expr.line)) if isinstance(index, SliceExpr): value = try_gen_slice_op(builder, base, index) From 320ea65a7043e62d88ac7dfaab54bb474b6bc7b0 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 10 Oct 2025 14:26:39 +0100 Subject: [PATCH 319/424] [mypyc] Refactor: adjust generator return types in a later pass (#20043) Move the inference of a more precise generator return type to a later pass. This way we have access to full class inheritance hierarchies. This is in preparation to fixing mypyc/mypyc#1141. --- mypyc/irbuild/main.py | 11 ++++--- mypyc/irbuild/mapper.py | 9 +----- mypyc/irbuild/prepare.py | 70 ++++++++++++++++++++++++---------------- 3 files changed, 51 insertions(+), 39 deletions(-) diff --git a/mypyc/irbuild/main.py b/mypyc/irbuild/main.py index d2c8924a7298e..f08911a1bc4c9 100644 --- a/mypyc/irbuild/main.py +++ b/mypyc/irbuild/main.py @@ -38,8 +38,9 @@ def f(x: int) -> int: from mypyc.irbuild.mapper import Mapper from mypyc.irbuild.prebuildvisitor import PreBuildVisitor from mypyc.irbuild.prepare import ( + adjust_generator_classes_of_methods, build_type_map, - create_generator_class_if_needed, + create_generator_class_for_func, find_singledispatch_register_impls, ) from mypyc.irbuild.visitor import IRBuilderVisitor @@ -68,6 +69,7 @@ def build_ir( """ build_type_map(mapper, modules, graph, types, options, errors) + adjust_generator_classes_of_methods(mapper) singledispatch_info = find_singledispatch_register_impls(modules, errors) result: ModuleIRs = {} @@ -87,9 +89,10 @@ def build_ir( if isinstance(fdef, FuncDef): # Make generator class name sufficiently unique. suffix = f"___{fdef.line}" - create_generator_class_if_needed( - module.fullname, None, fdef, mapper, name_suffix=suffix - ) + if fdef.is_coroutine or fdef.is_generator: + create_generator_class_for_func( + module.fullname, None, fdef, mapper, name_suffix=suffix + ) # Construct and configure builder objects (cyclic runtime dependency). visitor = IRBuilderVisitor() diff --git a/mypyc/irbuild/mapper.py b/mypyc/irbuild/mapper.py index 05aa0e45c569a..c986499b6f65e 100644 --- a/mypyc/irbuild/mapper.py +++ b/mypyc/irbuild/mapper.py @@ -180,14 +180,7 @@ def fdef_to_sig(self, fdef: FuncDef, strict_dunders_typing: bool) -> FuncSignatu for typ, kind in zip(fdef.type.arg_types, fdef.type.arg_kinds) ] arg_pos_onlys = [name is None for name in fdef.type.arg_names] - # TODO: We could probably support decorators sometimes (static and class method?) - if (fdef.is_coroutine or fdef.is_generator) and not fdef.is_decorated: - # Give a more precise type for generators, so that we can optimize - # code that uses them. They return a generator object, which has a - # specific class. Without this, the type would have to be 'object'. - ret: RType = RInstance(self.fdef_to_generator[fdef]) - else: - ret = self.type_to_rtype(fdef.type.ret_type) + ret = self.type_to_rtype(fdef.type.ret_type) else: # Handle unannotated functions arg_types = [object_rprimitive for _ in fdef.arguments] diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index e4f43b38b0dcc..2d0a1a8f03bfb 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -196,8 +196,6 @@ def prepare_func_def( mapper: Mapper, options: CompilerOptions, ) -> FuncDecl: - create_generator_class_if_needed(module_name, class_name, fdef, mapper) - kind = ( FUNC_CLASSMETHOD if fdef.is_class @@ -209,38 +207,37 @@ def prepare_func_def( return decl -def create_generator_class_if_needed( +def create_generator_class_for_func( module_name: str, class_name: str | None, fdef: FuncDef, mapper: Mapper, name_suffix: str = "" -) -> None: - """If function is a generator/async function, declare a generator class. +) -> ClassIR: + """For a generator/async function, declare a generator class. Each generator and async function gets a dedicated class that implements the generator protocol with generated methods. """ - if fdef.is_coroutine or fdef.is_generator: - name = "_".join(x for x in [fdef.name, class_name] if x) + "_gen" + name_suffix - cir = ClassIR(name, module_name, is_generated=True, is_final_class=True) - cir.reuse_freed_instance = True - mapper.fdef_to_generator[fdef] = cir + assert fdef.is_coroutine or fdef.is_generator + name = "_".join(x for x in [fdef.name, class_name] if x) + "_gen" + name_suffix + cir = ClassIR(name, module_name, is_generated=True, is_final_class=True) + cir.reuse_freed_instance = True + mapper.fdef_to_generator[fdef] = cir - helper_sig = FuncSignature( - ( - RuntimeArg(SELF_NAME, object_rprimitive), - RuntimeArg("type", object_rprimitive), - RuntimeArg("value", object_rprimitive), - RuntimeArg("traceback", object_rprimitive), - RuntimeArg("arg", object_rprimitive), - # If non-NULL, used to store return value instead of raising StopIteration(retv) - RuntimeArg("stop_iter_ptr", object_pointer_rprimitive), - ), - object_rprimitive, - ) + helper_sig = FuncSignature( + ( + RuntimeArg(SELF_NAME, object_rprimitive), + RuntimeArg("type", object_rprimitive), + RuntimeArg("value", object_rprimitive), + RuntimeArg("traceback", object_rprimitive), + RuntimeArg("arg", object_rprimitive), + # If non-NULL, used to store return value instead of raising StopIteration(retv) + RuntimeArg("stop_iter_ptr", object_pointer_rprimitive), + ), + object_rprimitive, + ) - # The implementation of most generator functionality is behind this magic method. - helper_fn_decl = FuncDecl( - GENERATOR_HELPER_NAME, name, module_name, helper_sig, internal=True - ) - cir.method_decls[helper_fn_decl.name] = helper_fn_decl + # The implementation of most generator functionality is behind this magic method. + helper_fn_decl = FuncDecl(GENERATOR_HELPER_NAME, name, module_name, helper_sig, internal=True) + cir.method_decls[helper_fn_decl.name] = helper_fn_decl + return cir def prepare_method_def( @@ -811,3 +808,22 @@ def registered_impl_from_possible_register_call( if isinstance(node, Decorator): return RegisteredImpl(node.func, dispatch_type) return None + + +def adjust_generator_classes_of_methods(mapper: Mapper) -> None: + """Make optimizations and adjustments to generated generator classes of methods. + + This is a separate pass after type map has been built, since we need all classes + to be processed to analyze class hierarchies. + """ + for fdef, ir in mapper.func_to_decl.items(): + if isinstance(fdef, FuncDef) and (fdef.is_coroutine or fdef.is_generator): + gen_ir = create_generator_class_for_func(ir.module_name, ir.class_name, fdef, mapper) + # TODO: We could probably support decorators sometimes (static and class method?) + if not fdef.is_decorated: + # Give a more precise type for generators, so that we can optimize + # code that uses them. They return a generator object, which has a + # specific class. Without this, the type would have to be 'object'. + ir.sig.ret_type = RInstance(gen_ir) + if ir.bound_sig: + ir.bound_sig.ret_type = RInstance(gen_ir) From 5b7279b7dc554e8ba21a159be584da0ddf7f0010 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Fri, 10 Oct 2025 06:37:12 -0700 Subject: [PATCH 320/424] Do not sort unused error codes in unused error codes warning (#20036) I intuit the previous author of this code sorted the codes for stability, but it actually should be in default order, to match what the user typed in. This will be more intuitive for the user. In my first commit, I add the failing testUnusedIgnoreCodeOrder test. In my second commit, I fix the code. --- mypy/errors.py | 6 +++--- test-data/unit/check-ignore.test | 10 ++++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/mypy/errors.py b/mypy/errors.py index f1b2faf67401c..1b092fb50e4af 100644 --- a/mypy/errors.py +++ b/mypy/errors.py @@ -806,8 +806,8 @@ def generate_unused_ignore_errors(self, file: str) -> None: continue if codes.UNUSED_IGNORE.code in ignored_codes: continue - used_ignored_codes = used_ignored_lines[line] - unused_ignored_codes = set(ignored_codes) - set(used_ignored_codes) + used_ignored_codes = set(used_ignored_lines[line]) + unused_ignored_codes = [c for c in ignored_codes if c not in used_ignored_codes] # `ignore` is used if not ignored_codes and used_ignored_codes: continue @@ -817,7 +817,7 @@ def generate_unused_ignore_errors(self, file: str) -> None: # Display detail only when `ignore[...]` specifies more than one error code unused_codes_message = "" if len(ignored_codes) > 1 and unused_ignored_codes: - unused_codes_message = f"[{', '.join(sorted(unused_ignored_codes))}]" + unused_codes_message = f"[{', '.join(unused_ignored_codes)}]" message = f'Unused "type: ignore{unused_codes_message}" comment' for unused in unused_ignored_codes: narrower = set(used_ignored_codes) & codes.sub_code_map[unused] diff --git a/test-data/unit/check-ignore.test b/test-data/unit/check-ignore.test index a4234e7a37a10..d0f6bb6aeb603 100644 --- a/test-data/unit/check-ignore.test +++ b/test-data/unit/check-ignore.test @@ -275,6 +275,16 @@ class CD(six.with_metaclass(M)): # E: Multiple metaclass definitions [builtins fixtures/tuple.pyi] +[case testUnusedIgnoreCodeOrder] +# flags: --warn-unused-ignores +5 # type: ignore[import, steven] # E: Unused "type: ignore[import, steven]" comment +-- User ordering of codes is preserved +5 # type: ignore[steven, import] # E: Unused "type: ignore[steven, import]" comment +-- Spacing is not preserved +5 # type: ignore[ steven, import ] # E: Unused "type: ignore[steven, import]" comment +-- Make sure it works as intended in more complex situations +1 + "ok" + "ok".foo # type: ignore[ operator,steven,attr-defined, import] # E: Unused "type: ignore[steven, import]" comment + [case testUnusedIgnoreTryExcept] # flags: --warn-unused-ignores try: From e03d3c1cffb2185cf3eac199db122e6364e459cc Mon Sep 17 00:00:00 2001 From: A5rocks Date: Sat, 11 Oct 2025 04:58:57 +0900 Subject: [PATCH 321/424] Check class references to catch non-existant classes in match cases (#20042) Fixes https://github.com/python/mypy/issues/20018. --- mypy/checkpattern.py | 36 ++++----- mypyc/test-data/irbuild-match.test | 121 ++++++++++++++-------------- test-data/unit/check-python310.test | 16 ++++ 3 files changed, 92 insertions(+), 81 deletions(-) diff --git a/mypy/checkpattern.py b/mypy/checkpattern.py index f81684d2f44ae..6f00c6c431771 100644 --- a/mypy/checkpattern.py +++ b/mypy/checkpattern.py @@ -14,7 +14,7 @@ from mypy.maptype import map_instance_to_supertype from mypy.meet import narrow_declared_type from mypy.messages import MessageBuilder -from mypy.nodes import ARG_POS, Context, Expression, NameExpr, TypeAlias, TypeInfo, Var +from mypy.nodes import ARG_POS, Context, Expression, NameExpr, TypeAlias, Var from mypy.options import Options from mypy.patterns import ( AsPattern, @@ -37,6 +37,7 @@ ) from mypy.types import ( AnyType, + FunctionLike, Instance, LiteralType, NoneType, @@ -538,27 +539,20 @@ def visit_class_pattern(self, o: ClassPattern) -> PatternType: # Check class type # type_info = o.class_ref.node - if type_info is None: - typ: Type = AnyType(TypeOfAny.from_error) - elif isinstance(type_info, TypeAlias) and not type_info.no_args: + typ = self.chk.expr_checker.accept(o.class_ref) + p_typ = get_proper_type(typ) + if isinstance(type_info, TypeAlias) and not type_info.no_args: self.msg.fail(message_registry.CLASS_PATTERN_GENERIC_TYPE_ALIAS, o) return self.early_non_match() - elif isinstance(type_info, TypeInfo): - typ = fill_typevars_with_any(type_info) - elif isinstance(type_info, TypeAlias): - typ = type_info.target - elif ( - isinstance(type_info, Var) - and type_info.type is not None - and isinstance(get_proper_type(type_info.type), AnyType) - ): - typ = type_info.type - else: - if isinstance(type_info, Var) and type_info.type is not None: - name = type_info.type.str_with_options(self.options) - else: - name = type_info.name - self.msg.fail(message_registry.CLASS_PATTERN_TYPE_REQUIRED.format(name), o) + elif isinstance(p_typ, FunctionLike) and p_typ.is_type_obj(): + typ = fill_typevars_with_any(p_typ.type_object()) + elif not isinstance(p_typ, AnyType): + self.msg.fail( + message_registry.CLASS_PATTERN_TYPE_REQUIRED.format( + typ.str_with_options(self.options) + ), + o, + ) return self.early_non_match() new_type, rest_type = self.chk.conditional_types_with_intersection( @@ -697,6 +691,8 @@ def should_self_match(self, typ: Type) -> bool: typ = get_proper_type(typ) if isinstance(typ, TupleType): typ = typ.partial_fallback + if isinstance(typ, AnyType): + return False if isinstance(typ, Instance) and typ.type.get("__match_args__") is not None: # Named tuples and other subtypes of builtins that define __match_args__ # should not self match. diff --git a/mypyc/test-data/irbuild-match.test b/mypyc/test-data/irbuild-match.test index 28aff3dcfc454..1e84c385100a7 100644 --- a/mypyc/test-data/irbuild-match.test +++ b/mypyc/test-data/irbuild-match.test @@ -563,10 +563,9 @@ def f(): def f(): r0, r1 :: object r2 :: bool - i :: int - r3 :: object - r4 :: str - r5, r6 :: object + r3, i, r4 :: object + r5 :: str + r6 :: object r7 :: object[1] r8 :: object_ptr r9, r10 :: object @@ -576,21 +575,22 @@ L0: r2 = CPy_TypeCheck(r1, r0) if r2 goto L1 else goto L3 :: bool L1: - i = 246 + r3 = object 123 + i = r3 L2: - r3 = builtins :: module - r4 = 'print' - r5 = CPyObject_GetAttr(r3, r4) - r6 = box(int, i) - r7 = [r6] + r4 = builtins :: module + r5 = 'print' + r6 = CPyObject_GetAttr(r4, r5) + r7 = [i] r8 = load_address r7 - r9 = PyObject_Vectorcall(r5, r8, 1, 0) - keep_alive r6 + r9 = PyObject_Vectorcall(r6, r8, 1, 0) + keep_alive i goto L4 L3: L4: r10 = box(None, 1) return r10 + [case testMatchClassPatternWithPositionalArgs_python3_10] class Position: __match_args__ = ("x", "y", "z") @@ -599,7 +599,7 @@ class Position: y: int z: int -def f(x): +def f(x) -> None: match x: case Position(1, 2, 3): print("matched") @@ -641,7 +641,7 @@ def f(x): r28 :: object r29 :: object[1] r30 :: object_ptr - r31, r32 :: object + r31 :: object L0: r0 = __main__.Position :: type r1 = PyObject_IsInstance(x, r0) @@ -687,8 +687,8 @@ L4: goto L6 L5: L6: - r32 = box(None, 1) - return r32 + return 1 + [case testMatchClassPatternWithKeywordPatterns_python3_10] class Position: x: int @@ -848,7 +848,7 @@ class C: a: int b: int -def f(x): +def f(x) -> None: match x: case C(1, 2) as y: print("matched") @@ -885,7 +885,7 @@ def f(x): r22 :: object r23 :: object[1] r24 :: object_ptr - r25, r26 :: object + r25 :: object L0: r0 = __main__.C :: type r1 = PyObject_IsInstance(x, r0) @@ -925,8 +925,8 @@ L4: goto L6 L5: L6: - r26 = box(None, 1) - return r26 + return 1 + [case testMatchClassPatternPositionalCapture_python3_10] class C: __match_args__ = ("x",) @@ -953,15 +953,14 @@ def f(x): r2 :: bit r3 :: bool r4 :: str - r5 :: object - r6, num :: int - r7 :: str - r8 :: object - r9 :: str - r10 :: object - r11 :: object[1] - r12 :: object_ptr - r13, r14 :: object + r5, num :: object + r6 :: str + r7 :: object + r8 :: str + r9 :: object + r10 :: object[1] + r11 :: object_ptr + r12, r13 :: object L0: r0 = __main__.C :: type r1 = PyObject_IsInstance(x, r0) @@ -971,22 +970,22 @@ L0: L1: r4 = 'x' r5 = CPyObject_GetAttr(x, r4) - r6 = unbox(int, r5) - num = r6 + num = r5 L2: - r7 = 'matched' - r8 = builtins :: module - r9 = 'print' - r10 = CPyObject_GetAttr(r8, r9) - r11 = [r7] - r12 = load_address r11 - r13 = PyObject_Vectorcall(r10, r12, 1, 0) - keep_alive r7 + r6 = 'matched' + r7 = builtins :: module + r8 = 'print' + r9 = CPyObject_GetAttr(r7, r8) + r10 = [r6] + r11 = load_address r10 + r12 = PyObject_Vectorcall(r9, r11, 1, 0) + keep_alive r6 goto L4 L3: L4: - r14 = box(None, 1) - return r14 + r13 = box(None, 1) + return r13 + [case testMatchMappingEmpty_python3_10] def f(x): match x: @@ -1601,35 +1600,35 @@ def f(x): def f(x): x, r0 :: object r1 :: bool - r2, y :: int - r3 :: str - r4 :: object - r5 :: str - r6 :: object - r7 :: object[1] - r8 :: object_ptr - r9, r10 :: object + y :: object + r2 :: str + r3 :: object + r4 :: str + r5 :: object + r6 :: object[1] + r7 :: object_ptr + r8, r9 :: object L0: r0 = load_address PyLong_Type r1 = CPy_TypeCheck(x, r0) if r1 goto L1 else goto L3 :: bool L1: - r2 = unbox(int, x) - y = r2 + y = x L2: - r3 = 'matched' - r4 = builtins :: module - r5 = 'print' - r6 = CPyObject_GetAttr(r4, r5) - r7 = [r3] - r8 = load_address r7 - r9 = PyObject_Vectorcall(r6, r8, 1, 0) - keep_alive r3 + r2 = 'matched' + r3 = builtins :: module + r4 = 'print' + r5 = CPyObject_GetAttr(r3, r4) + r6 = [r2] + r7 = load_address r6 + r8 = PyObject_Vectorcall(r5, r7, 1, 0) + keep_alive r2 goto L4 L3: L4: - r10 = box(None, 1) - return r10 + r9 = box(None, 1) + return r9 + [case testMatchSequenceCaptureAll_python3_10] def f(x): match x: diff --git a/test-data/unit/check-python310.test b/test-data/unit/check-python310.test index 7d76c09b61514..2c4597e212ea7 100644 --- a/test-data/unit/check-python310.test +++ b/test-data/unit/check-python310.test @@ -2990,3 +2990,19 @@ def foo(e: Literal[0, 1]) -> None: ... defer = unknown_module.foo + +[case testMatchErrorsIncorrectName] +class A: + pass + +match 5: + case A.blah(): # E: "type[A]" has no attribute "blah" + pass + +[case testMatchAllowsAnyClassArgsForAny] +match 5: + case BlahBlah(a, b): # E: Name "BlahBlah" is not defined + reveal_type(a) # N: Revealed type is "Any" + reveal_type(b) # N: Revealed type is "Any" + case BlahBlah(c=c): # E: Name "BlahBlah" is not defined + reveal_type(c) # N: Revealed type is "Any" From 04a586c6dbe9a29c8b2f38f037828c437682fc45 Mon Sep 17 00:00:00 2001 From: iap Date: Fri, 10 Oct 2025 21:45:17 -0400 Subject: [PATCH 322/424] stubgenc: small fix in get_default_function_sig (#19822) This small change fixes a crash in the case when a function arg has both a default value and a non-string type annotation. Here is an example: ``` def f(i: int = 0): pass ``` --- mypy/stubgenc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/stubgenc.py b/mypy/stubgenc.py index e64dbcdd9d408..e0e063927aadb 100755 --- a/mypy/stubgenc.py +++ b/mypy/stubgenc.py @@ -322,7 +322,7 @@ def add_args( default_value = get_default_value(i, arg) if default_value is not _Missing.VALUE: if arg in annotations: - argtype = annotations[arg] + argtype = get_annotation(arg) else: argtype = self.get_type_annotation(default_value) if argtype == "None": From 18bfc016eb2d413d1fb3389fa8f454cb40fcebd0 Mon Sep 17 00:00:00 2001 From: Christoph Tyralla Date: Sun, 12 Oct 2025 02:15:15 +0200 Subject: [PATCH 323/424] prevent false unreachable warnings for @final instances that occur when strict optional checking is disabled (#20045) Fixes #19849 See #11717 for some background information on `--no-strict-optional`. --- mypy/typeops.py | 4 ++-- test-data/unit/check-inference.test | 25 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/mypy/typeops.py b/mypy/typeops.py index d058bb8201d38..341c96c089315 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -789,9 +789,9 @@ def false_only(t: Type) -> ProperType: if not ret_type.can_be_false: return UninhabitedType(line=t.line) elif isinstance(t, Instance): - if t.type.is_final or t.type.is_enum: + if (t.type.is_final or t.type.is_enum) and state.strict_optional: return UninhabitedType(line=t.line) - elif isinstance(t, LiteralType) and t.is_enum_literal(): + elif isinstance(t, LiteralType) and t.is_enum_literal() and state.strict_optional: return UninhabitedType(line=t.line) new_t = copy_type(t) diff --git a/test-data/unit/check-inference.test b/test-data/unit/check-inference.test index 63278d6c4547a..24ea61f2c7156 100644 --- a/test-data/unit/check-inference.test +++ b/test-data/unit/check-inference.test @@ -1964,6 +1964,31 @@ if 'x' in d: # E: "None" has no attribute "__iter__" (not iterable) reveal_type(d) # N: Revealed type is "None" [builtins fixtures/dict.pyi] +[case testNoWrongUnreachableWarningWithNoStrictOptionalAndFinalInstance] +# flags: --no-strict-optional --warn-unreachable +from typing import final, Optional + +@final +class C: ... + +x: Optional[C] +if not x: + x = C() +[builtins fixtures/dict.pyi] + +[case testNoWrongUnreachableWarningWithNoStrictOptionalAndEnumLiteral] +# flags: --no-strict-optional --warn-unreachable +from enum import Enum +from typing import Literal, Optional + +class E(Enum): + a = 1 + +x: Optional[Literal[E.a]] +if not x: + x = E.a +[builtins fixtures/dict.pyi] + [case testInferFromEmptyListWhenUsingInWithStrictEquality] # flags: --strict-equality def f() -> None: From 6aa44da630a9a277b6e7b9c77f9083bb6e00c26b Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Sun, 12 Oct 2025 02:22:01 +0200 Subject: [PATCH 324/424] Prevent TypeGuardedType leak from `narrow_declared_type` as part of typevar bound (#20046) Fixes #20015, refs #18895 as a previous example of the same issue. I don't see any similar problems in other branches. --- mypy/meet.py | 4 +++- test-data/unit/check-typeguard.test | 20 ++++++++++++++++++++ test-data/unit/fixtures/typing-full.pyi | 1 + 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/mypy/meet.py b/mypy/meet.py index 353af59367ad1..63305c2bb236e 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -170,7 +170,9 @@ def narrow_declared_type(declared: Type, narrowed: Type) -> Type: ): # We put this branch early to get T(bound=Union[A, B]) instead of # Union[T(bound=A), T(bound=B)] that will be confusing for users. - return declared.copy_modified(upper_bound=original_narrowed) + return declared.copy_modified( + upper_bound=narrow_declared_type(declared.upper_bound, original_narrowed) + ) elif not is_overlapping_types(declared, narrowed, prohibit_none_typevar_overlap=True): if state.strict_optional: return UninhabitedType() diff --git a/test-data/unit/check-typeguard.test b/test-data/unit/check-typeguard.test index 93e665e4548c3..b15458d5819a4 100644 --- a/test-data/unit/check-typeguard.test +++ b/test-data/unit/check-typeguard.test @@ -825,6 +825,26 @@ def handle(model: Model) -> int: return 0 [builtins fixtures/tuple.pyi] +[case testTypeGuardedTypeDoesNotLeakTypeVar] +# flags: --debug-serialize +# https://github.com/python/mypy/issues/20015 +from typing import Generic, TypeVar, TypeGuard + +class A: ... +class B: ... + +def is_a(_: object) -> TypeGuard[A]: return True +def is_b(_: object) -> TypeGuard[B]: return True + +_T = TypeVar("_T") + +class Foo(Generic[_T]): + def __init__(self, v: _T) -> None: + if is_a(v) or is_b(v): + self.v = v +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] + [case testTypeGuardRestrictTypeVarUnion] from typing import Union, TypeVar from typing_extensions import TypeGuard diff --git a/test-data/unit/fixtures/typing-full.pyi b/test-data/unit/fixtures/typing-full.pyi index 3757e868552e1..1a63deaa727d0 100644 --- a/test-data/unit/fixtures/typing-full.pyi +++ b/test-data/unit/fixtures/typing-full.pyi @@ -34,6 +34,7 @@ no_type_check = 0 ClassVar = 0 Final = 0 TypedDict = 0 +TypeGuard = 0 NoReturn = 0 NewType = 0 Self = 0 From b3e26e7d68a792eeb207aeb8dd1e903593bfc097 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 14 Oct 2025 10:45:17 +0100 Subject: [PATCH 325/424] [mypyc] Fix inheritance of async defs (#20044) When inferring a precise generator return type for an async def (or generator), make the generate returned by an override in a subclass inherit from the base class generator. This means that the environment has to be moved to a separate class in the base class generator. Don't infer a precise generator return type when an override might have a less precise return type, since it would break LSP. Fixes mypyc/mypyc#1141. --- mypyc/codegen/emitclass.py | 8 +-- mypyc/codegen/emitmodule.py | 2 + mypyc/ir/func_ir.py | 17 ++++-- mypyc/irbuild/context.py | 6 ++- mypyc/irbuild/function.py | 12 +++-- mypyc/irbuild/generator.py | 4 +- mypyc/irbuild/prepare.py | 84 +++++++++++++++++++++++++---- mypyc/test-data/run-async.test | 74 +++++++++++++++++++++++++ mypyc/test-data/run-generators.test | 29 ++++++++++ 9 files changed, 212 insertions(+), 24 deletions(-) diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 122f62a0d5826..d64940084f12e 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -410,7 +410,7 @@ def setter_name(cl: ClassIR, attribute: str, names: NameGenerator) -> str: def generate_object_struct(cl: ClassIR, emitter: Emitter) -> None: - seen_attrs: set[tuple[str, RType]] = set() + seen_attrs: set[str] = set() lines: list[str] = [] lines += ["typedef struct {", "PyObject_HEAD", "CPyVTableItem *vtable;"] if cl.has_method("__call__"): @@ -427,9 +427,11 @@ def generate_object_struct(cl: ClassIR, emitter: Emitter) -> None: lines.append(f"{BITMAP_TYPE} {attr};") bitmap_attrs.append(attr) for attr, rtype in base.attributes.items(): - if (attr, rtype) not in seen_attrs: + # Generated class may redefine certain attributes with different + # types in subclasses (this would be unsafe for user-defined classes). + if attr not in seen_attrs: lines.append(f"{emitter.ctype_spaced(rtype)}{emitter.attr(attr)};") - seen_attrs.add((attr, rtype)) + seen_attrs.add(attr) if isinstance(rtype, RTuple): emitter.declare_tuple_struct(rtype) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 3602b3c26e03b..7ec315a6bd349 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -1064,6 +1064,8 @@ def emit_module_exec_func( "(PyObject *){t}_template, NULL, modname);".format(t=type_struct) ) emitter.emit_lines(f"if (unlikely(!{type_struct}))", " goto fail;") + name_prefix = cl.name_prefix(emitter.names) + emitter.emit_line(f"CPyDef_{name_prefix}_trait_vtable_setup();") emitter.emit_lines("if (CPyGlobalsInit() < 0)", " goto fail;") diff --git a/mypyc/ir/func_ir.py b/mypyc/ir/func_ir.py index 881ac5939c275..d11fef42feb54 100644 --- a/mypyc/ir/func_ir.py +++ b/mypyc/ir/func_ir.py @@ -149,8 +149,11 @@ def __init__( module_name: str, sig: FuncSignature, kind: int = FUNC_NORMAL, + *, is_prop_setter: bool = False, is_prop_getter: bool = False, + is_generator: bool = False, + is_coroutine: bool = False, implicit: bool = False, internal: bool = False, ) -> None: @@ -161,6 +164,8 @@ def __init__( self.kind = kind self.is_prop_setter = is_prop_setter self.is_prop_getter = is_prop_getter + self.is_generator = is_generator + self.is_coroutine = is_coroutine if class_name is None: self.bound_sig: FuncSignature | None = None else: @@ -219,6 +224,8 @@ def serialize(self) -> JsonDict: "kind": self.kind, "is_prop_setter": self.is_prop_setter, "is_prop_getter": self.is_prop_getter, + "is_generator": self.is_generator, + "is_coroutine": self.is_coroutine, "implicit": self.implicit, "internal": self.internal, } @@ -240,10 +247,12 @@ def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> FuncDecl: data["module_name"], FuncSignature.deserialize(data["sig"], ctx), data["kind"], - data["is_prop_setter"], - data["is_prop_getter"], - data["implicit"], - data["internal"], + is_prop_setter=data["is_prop_setter"], + is_prop_getter=data["is_prop_getter"], + is_generator=data["is_generator"], + is_coroutine=data["is_coroutine"], + implicit=data["implicit"], + internal=data["internal"], ) diff --git a/mypyc/irbuild/context.py b/mypyc/irbuild/context.py index 8d2e55ed96fb9..d5a48bf838c83 100644 --- a/mypyc/irbuild/context.py +++ b/mypyc/irbuild/context.py @@ -98,7 +98,11 @@ def curr_env_reg(self) -> Value: def can_merge_generator_and_env_classes(self) -> bool: # In simple cases we can place the environment into the generator class, # instead of having two separate classes. - return self.is_generator and not self.is_nested and not self.contains_nested + if self._generator_class and not self._generator_class.ir.is_final_class: + result = False + else: + result = self.is_generator and not self.is_nested and not self.contains_nested + return result class ImplicitClass: diff --git a/mypyc/irbuild/function.py b/mypyc/irbuild/function.py index c9f999597d307..738d19ea67485 100644 --- a/mypyc/irbuild/function.py +++ b/mypyc/irbuild/function.py @@ -69,7 +69,7 @@ instantiate_callable_class, setup_callable_class, ) -from mypyc.irbuild.context import FuncInfo +from mypyc.irbuild.context import FuncInfo, GeneratorClass from mypyc.irbuild.env_class import ( add_vars_to_env, finalize_env_class, @@ -246,6 +246,12 @@ def c() -> None: is_generator = fn_info.is_generator builder.enter(fn_info, ret_type=sig.ret_type) + if is_generator: + fitem = builder.fn_info.fitem + assert isinstance(fitem, FuncDef), fitem + generator_class_ir = builder.mapper.fdef_to_generator[fitem] + builder.fn_info.generator_class = GeneratorClass(generator_class_ir) + # Functions that contain nested functions need an environment class to store variables that # are free in their nested functions. Generator functions need an environment class to # store a variable denoting the next instruction to be executed when the __next__ function @@ -357,8 +363,8 @@ def gen_func_ir( builder.module_name, sig, func_decl.kind, - func_decl.is_prop_getter, - func_decl.is_prop_setter, + is_prop_getter=func_decl.is_prop_getter, + is_prop_setter=func_decl.is_prop_setter, ) func_ir = FuncIR(func_decl, args, blocks, fitem.line, traceback_name=fitem.name) else: diff --git a/mypyc/irbuild/generator.py b/mypyc/irbuild/generator.py index b3a417ed6a3ef..4dcd748f6effc 100644 --- a/mypyc/irbuild/generator.py +++ b/mypyc/irbuild/generator.py @@ -39,7 +39,7 @@ object_rprimitive, ) from mypyc.irbuild.builder import IRBuilder, calculate_arg_defaults, gen_arg_defaults -from mypyc.irbuild.context import FuncInfo, GeneratorClass +from mypyc.irbuild.context import FuncInfo from mypyc.irbuild.env_class import ( add_args_to_env, add_vars_to_env, @@ -166,10 +166,8 @@ def setup_generator_class(builder: IRBuilder) -> ClassIR: builder.fn_info.env_class = generator_class_ir else: generator_class_ir.attributes[ENV_ATTR_NAME] = RInstance(builder.fn_info.env_class) - generator_class_ir.mro = [generator_class_ir] builder.classes.append(generator_class_ir) - builder.fn_info.generator_class = GeneratorClass(generator_class_ir) return generator_class_ir diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index 2d0a1a8f03bfb..0f7cc7e3b3c55 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -202,7 +202,15 @@ def prepare_func_def( else (FUNC_STATICMETHOD if fdef.is_static else FUNC_NORMAL) ) sig = mapper.fdef_to_sig(fdef, options.strict_dunders_typing) - decl = FuncDecl(fdef.name, class_name, module_name, sig, kind) + decl = FuncDecl( + fdef.name, + class_name, + module_name, + sig, + kind, + is_generator=fdef.is_generator, + is_coroutine=fdef.is_coroutine, + ) mapper.func_to_decl[fdef] = decl return decl @@ -217,7 +225,7 @@ def create_generator_class_for_func( """ assert fdef.is_coroutine or fdef.is_generator name = "_".join(x for x in [fdef.name, class_name] if x) + "_gen" + name_suffix - cir = ClassIR(name, module_name, is_generated=True, is_final_class=True) + cir = ClassIR(name, module_name, is_generated=True, is_final_class=class_name is None) cir.reuse_freed_instance = True mapper.fdef_to_generator[fdef] = cir @@ -816,14 +824,70 @@ def adjust_generator_classes_of_methods(mapper: Mapper) -> None: This is a separate pass after type map has been built, since we need all classes to be processed to analyze class hierarchies. """ - for fdef, ir in mapper.func_to_decl.items(): + + generator_methods = [] + + for fdef, fn_ir in mapper.func_to_decl.items(): if isinstance(fdef, FuncDef) and (fdef.is_coroutine or fdef.is_generator): - gen_ir = create_generator_class_for_func(ir.module_name, ir.class_name, fdef, mapper) + gen_ir = create_generator_class_for_func( + fn_ir.module_name, fn_ir.class_name, fdef, mapper + ) # TODO: We could probably support decorators sometimes (static and class method?) if not fdef.is_decorated: - # Give a more precise type for generators, so that we can optimize - # code that uses them. They return a generator object, which has a - # specific class. Without this, the type would have to be 'object'. - ir.sig.ret_type = RInstance(gen_ir) - if ir.bound_sig: - ir.bound_sig.ret_type = RInstance(gen_ir) + name = fn_ir.name + precise_ret_type = True + if fn_ir.class_name is not None: + class_ir = mapper.type_to_ir[fdef.info] + subcls = class_ir.subclasses() + if subcls is None: + # Override could be of a different type, so we can't make assumptions. + precise_ret_type = False + else: + for s in subcls: + if name in s.method_decls: + m = s.method_decls[name] + if ( + m.is_generator != fn_ir.is_generator + or m.is_coroutine != fn_ir.is_coroutine + ): + # Override is of a different kind, and the optimization + # to use a precise generator return type doesn't work. + precise_ret_type = False + else: + class_ir = None + + if precise_ret_type: + # Give a more precise type for generators, so that we can optimize + # code that uses them. They return a generator object, which has a + # specific class. Without this, the type would have to be 'object'. + fn_ir.sig.ret_type = RInstance(gen_ir) + if fn_ir.bound_sig: + fn_ir.bound_sig.ret_type = RInstance(gen_ir) + if class_ir is not None: + if class_ir.is_method_final(name): + gen_ir.is_final_class = True + generator_methods.append((name, class_ir, gen_ir)) + + new_bases = {} + + for name, class_ir, gen in generator_methods: + # For generator methods, we need to have subclass generator classes inherit from + # baseclass generator classes when there are overrides to maintain LSP. + base = class_ir.real_base() + if base is not None: + if base.has_method(name): + base_sig = base.method_sig(name) + if isinstance(base_sig.ret_type, RInstance): + base_gen = base_sig.ret_type.class_ir + new_bases[gen] = base_gen + + # Add generator inheritance relationships by adjusting MROs. + for deriv, base in new_bases.items(): + if base.children is not None: + base.children.append(deriv) + while True: + deriv.mro.append(base) + deriv.base_mro.append(base) + if base not in new_bases: + break + base = new_bases[base] diff --git a/mypyc/test-data/run-async.test b/mypyc/test-data/run-async.test index 94a1cd2e97c5f..cf063310fd895 100644 --- a/mypyc/test-data/run-async.test +++ b/mypyc/test-data/run-async.test @@ -1291,3 +1291,77 @@ class CancelledError(Exception): ... def run(x: object) -> object: ... def get_running_loop() -> Any: ... def create_task(x: object) -> Any: ... + +[case testAsyncInheritance1] +from typing import final, Coroutine, Any, TypeVar + +import asyncio + +class Base1: + async def foo(self) -> int: + return 1 + +class Derived1(Base1): + async def foo(self) -> int: + return await super().foo() + 1 + +async def base1_foo(b: Base1) -> int: + return await b.foo() + +async def derived1_foo(b: Derived1) -> int: + return await b.foo() + +def test_async_inheritance() -> None: + assert asyncio.run(base1_foo(Base1())) == 1 + assert asyncio.run(base1_foo(Derived1())) == 2 + assert asyncio.run(derived1_foo(Derived1())) == 2 + +@final +class FinalClass: + async def foo(self) -> int: + return 3 + +async def final_class_foo(b: FinalClass) -> int: + return await b.foo() + +def test_final_class() -> None: + assert asyncio.run(final_class_foo(FinalClass())) == 3 + +class Base2: + async def foo(self) -> int: + return 4 + + async def bar(self) -> int: + return 5 + +class Derived2(Base2): + # Does not override "foo" + async def bar(self) -> int: + return 6 + +async def base2_foo(b: Base2) -> int: + return await b.foo() + +def test_no_override() -> None: + assert asyncio.run(base2_foo(Base2())) == 4 + assert asyncio.run(base2_foo(Derived2())) == 4 + +class Base3: + async def foo(self) -> int: + return 7 + +class Derived3(Base3): + def foo(self) -> Coroutine[Any, Any, int]: + async def inner() -> int: + return 8 + return inner() + +async def base3_foo(b: Base3) -> int: + return await b.foo() + +def test_override_non_async() -> None: + assert asyncio.run(base3_foo(Base3())) == 7 + assert asyncio.run(base3_foo(Derived3())) == 8 + +[file asyncio/__init__.pyi] +def run(x: object) -> object: ... diff --git a/mypyc/test-data/run-generators.test b/mypyc/test-data/run-generators.test index bfbd5b83696b1..c8e83173474de 100644 --- a/mypyc/test-data/run-generators.test +++ b/mypyc/test-data/run-generators.test @@ -907,3 +907,32 @@ def test_same_names() -> None: # matches the variable name in the input code, since internally it's generated # with a prefix. list(undefined()) + +[case testGeneratorInheritance] +from typing import Iterator + +class Base1: + def foo(self) -> Iterator[int]: + yield 1 + +class Derived1(Base1): + def foo(self) -> Iterator[int]: + yield 2 + yield 3 + +def base1_foo(b: Base1) -> list[int]: + a = [] + for x in b.foo(): + a.append(x) + return a + +def derived1_foo(b: Derived1) -> list[int]: + a = [] + for x in b.foo(): + a.append(x) + return a + +def test_generator_override() -> None: + assert base1_foo(Base1()) == [1] + assert base1_foo(Derived1()) == [2, 3] + assert derived1_foo(Derived1()) == [2, 3] From 8e57622c45b153c84f773bccae1db5f337f5a68c Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Tue, 14 Oct 2025 08:12:06 -0400 Subject: [PATCH 326/424] [mypyc] feat: new primitive for `int.bit_length` (#19673) This PR adds a new primitive for `int.bit_length`. --- mypyc/lib-rt/CPy.h | 1 + mypyc/lib-rt/int_ops.c | 64 +++++++++++++++++++++++++++++++ mypyc/primitives/int_ops.py | 18 ++++++++- mypyc/test-data/fixtures/ir.py | 1 + mypyc/test-data/irbuild-int.test | 10 +++++ mypyc/test-data/run-integers.test | 14 +++++++ 6 files changed, 107 insertions(+), 1 deletion(-) diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index e9dfd8de36834..e2fe129786d39 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -148,6 +148,7 @@ CPyTagged CPyTagged_Remainder_(CPyTagged left, CPyTagged right); CPyTagged CPyTagged_BitwiseLongOp_(CPyTagged a, CPyTagged b, char op); CPyTagged CPyTagged_Rshift_(CPyTagged left, CPyTagged right); CPyTagged CPyTagged_Lshift_(CPyTagged left, CPyTagged right); +CPyTagged CPyTagged_BitLength(CPyTagged self); PyObject *CPyTagged_Str(CPyTagged n); CPyTagged CPyTagged_FromFloat(double f); diff --git a/mypyc/lib-rt/int_ops.c b/mypyc/lib-rt/int_ops.c index e2c302eea5760..333783ae619d4 100644 --- a/mypyc/lib-rt/int_ops.c +++ b/mypyc/lib-rt/int_ops.c @@ -5,6 +5,10 @@ #include #include "CPy.h" +#ifdef _MSC_VER +#include +#endif + #ifndef _WIN32 // On 64-bit Linux and macOS, ssize_t and long are both 64 bits, and // PyLong_FromLong is faster than PyLong_FromSsize_t, so use the faster one @@ -15,6 +19,17 @@ #define CPyLong_FromSsize_t PyLong_FromSsize_t #endif +#if defined(__GNUC__) || defined(__clang__) +# if defined(__x86_64__) || defined(_M_X64) || defined(__aarch64__) || (defined(__SIZEOF_POINTER__) && __SIZEOF_POINTER__ == 8) +# define CPY_CLZ(x) __builtin_clzll((unsigned long long)(x)) +# define CPY_BITS 64 +# else +# define CPY_CLZ(x) __builtin_clz((unsigned int)(x)) +# define CPY_BITS 32 +# endif +#endif + + CPyTagged CPyTagged_FromSsize_t(Py_ssize_t value) { // We use a Python object if the value shifted left by 1 is too // large for Py_ssize_t @@ -581,3 +596,52 @@ double CPyTagged_TrueDivide(CPyTagged x, CPyTagged y) { } return 1.0; } + +// int.bit_length() +CPyTagged CPyTagged_BitLength(CPyTagged self) { + // Handle zero + if (self == 0) { + return 0; + } + + // Fast path for small (tagged) ints + if (CPyTagged_CheckShort(self)) { + Py_ssize_t val = CPyTagged_ShortAsSsize_t(self); + Py_ssize_t absval = val < 0 ? -val : val; + int bits = 0; + if (absval) { +#if defined(_MSC_VER) + #if defined(_WIN64) + unsigned long idx; + if (_BitScanReverse64(&idx, (unsigned __int64)absval)) { + bits = (int)(idx + 1); + } + #else + unsigned long idx; + if (_BitScanReverse(&idx, (unsigned long)absval)) { + bits = (int)(idx + 1); + } + #endif +#elif defined(__GNUC__) || defined(__clang__) + bits = (int)(CPY_BITS - CPY_CLZ(absval)); +#else + // Fallback to loop if no builtin + while (absval) { + absval >>= 1; + bits++; + } +#endif + } + return bits << 1; + } + + // Slow path for big ints + PyObject *pyint = CPyTagged_AsObject(self); + int bits = _PyLong_NumBits(pyint); + Py_DECREF(pyint); + if (bits < 0) { + // _PyLong_NumBits sets an error on failure + return CPY_INT_TAG; + } + return bits << 1; +} diff --git a/mypyc/primitives/int_ops.py b/mypyc/primitives/int_ops.py index d723c9b63a86c..8f43140dd2559 100644 --- a/mypyc/primitives/int_ops.py +++ b/mypyc/primitives/int_ops.py @@ -31,7 +31,14 @@ str_rprimitive, void_rtype, ) -from mypyc.primitives.registry import binary_op, custom_op, function_op, load_address_op, unary_op +from mypyc.primitives.registry import ( + binary_op, + custom_op, + function_op, + load_address_op, + method_op, + unary_op, +) # Constructors for builtins.int and native int types have the same behavior. In # interpreted mode, native int types are just aliases to 'int'. @@ -305,3 +312,12 @@ def int_unary_op(name: str, c_function_name: str) -> PrimitiveDescription: c_function_name="PyLong_Check", error_kind=ERR_NEVER, ) + +# int.bit_length() +method_op( + name="bit_length", + arg_types=[int_rprimitive], + return_type=int_rprimitive, + c_function_name="CPyTagged_BitLength", + error_kind=ERR_MAGIC, +) diff --git a/mypyc/test-data/fixtures/ir.py b/mypyc/test-data/fixtures/ir.py index 4d0aaba12cab4..5033100223a3d 100644 --- a/mypyc/test-data/fixtures/ir.py +++ b/mypyc/test-data/fixtures/ir.py @@ -86,6 +86,7 @@ def __lt__(self, n: int) -> bool: pass def __gt__(self, n: int) -> bool: pass def __le__(self, n: int) -> bool: pass def __ge__(self, n: int) -> bool: pass + def bit_length(self) -> int: pass class str: @overload diff --git a/mypyc/test-data/irbuild-int.test b/mypyc/test-data/irbuild-int.test index bdf9127b722a4..184c66fafb7c3 100644 --- a/mypyc/test-data/irbuild-int.test +++ b/mypyc/test-data/irbuild-int.test @@ -210,3 +210,13 @@ L0: r0 = CPyTagged_Invert(n) x = r0 return x + +[case testIntBitLength] +def f(x: int) -> int: + return x.bit_length() +[out] +def f(x): + x, r0 :: int +L0: + r0 = CPyTagged_BitLength(x) + return r0 diff --git a/mypyc/test-data/run-integers.test b/mypyc/test-data/run-integers.test index 1163c9d942f78..c02f7d808883c 100644 --- a/mypyc/test-data/run-integers.test +++ b/mypyc/test-data/run-integers.test @@ -572,3 +572,17 @@ class subc(int): [file userdefinedint.py] class int: pass + +[case testBitLength] +def bit_length(n: int) -> int: + return n.bit_length() +def bit_length_python(n: int) -> int: + return getattr(n, "bit_length")() +def test_bit_length() -> None: + for n in range(256): + i = 1 << n + assert bit_length(i) == bit_length_python(i) + assert bit_length(-(i)) == bit_length_python(-(i)) + i -= 1 + assert bit_length(i) == bit_length_python(i) + assert bit_length(-(i)) == bit_length_python(-(i)) From d69419cc452fa9a628e8ab0f6d5d7abd2b5aa015 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Tue, 14 Oct 2025 08:44:14 -0400 Subject: [PATCH 327/424] [mypyc] feat: extend `get_expr_length` to work with RTuple [2/4] (#19929) This PR extends `get_expr_length` to work with type information from RTuple types. --- mypyc/irbuild/for_helpers.py | 13 ++- mypyc/test-data/irbuild-tuple.test | 130 +++++++++++++---------------- 2 files changed, 69 insertions(+), 74 deletions(-) diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index 20440d4a26f49..715f5432cd133 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -1203,18 +1203,18 @@ def gen_cleanup(self) -> None: gen.gen_cleanup() -def get_expr_length(expr: Expression) -> int | None: +def get_expr_length(builder: IRBuilder, expr: Expression) -> int | None: if isinstance(expr, (StrExpr, BytesExpr)): return len(expr.value) elif isinstance(expr, (ListExpr, TupleExpr)): # if there are no star expressions, or we know the length of them, # we know the length of the expression - stars = [get_expr_length(i) for i in expr.items if isinstance(i, StarExpr)] + stars = [get_expr_length(builder, i) for i in expr.items if isinstance(i, StarExpr)] if None not in stars: other = sum(not isinstance(i, StarExpr) for i in expr.items) return other + sum(stars) # type: ignore [arg-type] elif isinstance(expr, StarExpr): - return get_expr_length(expr.expr) + return get_expr_length(builder, expr.expr) elif ( isinstance(expr, RefExpr) and isinstance(expr.node, Var) @@ -1227,6 +1227,11 @@ def get_expr_length(expr: Expression) -> int | None: # performance boost and can be (sometimes) figured out pretty easily. set and dict # comps *can* be done as well but will need special logic to consider the possibility # of key conflicts. Range, enumerate, zip are all simple logic. + + # we might still be able to get the length directly from the type + rtype = builder.node_type(expr) + if isinstance(rtype, RTuple): + return len(rtype.types) return None @@ -1235,7 +1240,7 @@ def get_expr_length_value( ) -> Value: rtype = builder.node_type(expr) assert is_sequence_rprimitive(rtype) or isinstance(rtype, RTuple), rtype - length = get_expr_length(expr) + length = get_expr_length(builder, expr) if length is None: # We cannot compute the length at compile time, so we will fetch it. return builder.builder.builtin_len(expr_reg, line, use_pyssize_t=use_pyssize_t) diff --git a/mypyc/test-data/irbuild-tuple.test b/mypyc/test-data/irbuild-tuple.test index 3613c5f0101d7..0fdd8e87a1542 100644 --- a/mypyc/test-data/irbuild-tuple.test +++ b/mypyc/test-data/irbuild-tuple.test @@ -694,51 +694,46 @@ L0: return r1 def test(): r0, source :: tuple[int, int, int] - r1 :: object - r2 :: native_int - r3 :: bit - r4, r5, r6 :: int - r7, r8, r9 :: object - r10, r11 :: tuple - r12 :: native_int - r13 :: bit + r1, r2, r3 :: int + r4, r5, r6 :: object + r7, r8 :: tuple + r9 :: native_int + r10 :: bit + r11 :: object + r12, x :: int + r13 :: bool r14 :: object - r15, x :: int - r16 :: bool - r17 :: object - r18 :: native_int + r15 :: native_int a :: tuple L0: r0 = (2, 4, 6) source = r0 - r1 = box(tuple[int, int, int], source) - r2 = PyObject_Size(r1) - r3 = r2 >= 0 :: signed - r4 = source[0] - r5 = source[1] - r6 = source[2] - r7 = box(int, r4) - r8 = box(int, r5) - r9 = box(int, r6) - r10 = PyTuple_Pack(3, r7, r8, r9) - r11 = PyTuple_New(r2) - r12 = 0 + r1 = source[0] + r2 = source[1] + r3 = source[2] + r4 = box(int, r1) + r5 = box(int, r2) + r6 = box(int, r3) + r7 = PyTuple_Pack(3, r4, r5, r6) + r8 = PyTuple_New(3) + r9 = 0 + goto L2 L1: - r13 = r12 < r2 :: signed - if r13 goto L2 else goto L4 :: bool + r10 = r9 < 3 :: signed + if r10 goto L2 else goto L4 :: bool L2: - r14 = CPySequenceTuple_GetItemUnsafe(r10, r12) - r15 = unbox(int, r14) - x = r15 - r16 = f(x) - r17 = box(bool, r16) - CPySequenceTuple_SetItemUnsafe(r11, r12, r17) + r11 = CPySequenceTuple_GetItemUnsafe(r7, r9) + r12 = unbox(int, r11) + x = r12 + r13 = f(x) + r14 = box(bool, r13) + CPySequenceTuple_SetItemUnsafe(r8, r9, r14) L3: - r18 = r12 + 1 - r12 = r18 + r15 = r9 + 1 + r9 = r15 goto L1 L4: - a = r11 + a = r8 return 1 [case testTupleBuiltFromFinalFixedLengthTuple] @@ -762,19 +757,16 @@ L0: def test(): r0 :: tuple[int, int, int] r1 :: bool - r2 :: object - r3 :: native_int - r4 :: bit - r5, r6, r7 :: int - r8, r9, r10 :: object - r11, r12 :: tuple - r13 :: native_int - r14 :: bit + r2, r3, r4 :: int + r5, r6, r7 :: object + r8, r9 :: tuple + r10 :: native_int + r11 :: bit + r12 :: object + r13, x :: int + r14 :: bool r15 :: object - r16, x :: int - r17 :: bool - r18 :: object - r19 :: native_int + r16 :: native_int a :: tuple L0: r0 = __main__.source :: static @@ -783,34 +775,32 @@ L1: r1 = raise NameError('value for final name "source" was not set') unreachable L2: - r2 = box(tuple[int, int, int], r0) - r3 = PyObject_Size(r2) - r4 = r3 >= 0 :: signed - r5 = r0[0] - r6 = r0[1] - r7 = r0[2] - r8 = box(int, r5) - r9 = box(int, r6) - r10 = box(int, r7) - r11 = PyTuple_Pack(3, r8, r9, r10) - r12 = PyTuple_New(r3) - r13 = 0 + r2 = r0[0] + r3 = r0[1] + r4 = r0[2] + r5 = box(int, r2) + r6 = box(int, r3) + r7 = box(int, r4) + r8 = PyTuple_Pack(3, r5, r6, r7) + r9 = PyTuple_New(3) + r10 = 0 + goto L4 L3: - r14 = r13 < r3 :: signed - if r14 goto L4 else goto L6 :: bool + r11 = r10 < 3 :: signed + if r11 goto L4 else goto L6 :: bool L4: - r15 = CPySequenceTuple_GetItemUnsafe(r11, r13) - r16 = unbox(int, r15) - x = r16 - r17 = f(x) - r18 = box(bool, r17) - CPySequenceTuple_SetItemUnsafe(r12, r13, r18) + r12 = CPySequenceTuple_GetItemUnsafe(r8, r10) + r13 = unbox(int, r12) + x = r13 + r14 = f(x) + r15 = box(bool, r14) + CPySequenceTuple_SetItemUnsafe(r9, r10, r15) L5: - r19 = r13 + 1 - r13 = r19 + r16 = r10 + 1 + r10 = r16 goto L3 L6: - a = r12 + a = r9 return 1 [case testTupleBuiltFromVariableLengthTuple] From b8f57fda22b6fc5f57664e212db5578eaf4d3c84 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Tue, 14 Oct 2025 09:20:47 -0400 Subject: [PATCH 328/424] [mypyc] feat: further optimize equality check with string literals [1/1] (#19883) This PR further optimizes string equality checks against literals by getting rid of the PyUnicode_GET_LENGTH call against the literal value, which is not necessary since the value is known at compile-time I think this optimization will be helpful in cases where the non-literal string DOES match but is actually a subtype of string (actual strings instances that match would be caught by the identity check), or in cases where an exact string does NOT match. --- mypyc/irbuild/ll_builder.py | 30 +++++++++++++++++++++-- mypyc/lib-rt/CPy.h | 1 + mypyc/lib-rt/str_ops.c | 29 ++++++++++++++++------ mypyc/primitives/str_ops.py | 8 ++++++ mypyc/test-data/irbuild-classes.test | 8 +++--- mypyc/test-data/irbuild-dict.test | 2 +- mypyc/test-data/irbuild-str.test | 31 ++++++++++++++++++++++++ mypyc/test-data/irbuild-unreachable.test | 4 +-- 8 files changed, 96 insertions(+), 17 deletions(-) diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index 37f2add4abbd9..d33497d4987bd 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -8,7 +8,8 @@ import sys from collections.abc import Sequence -from typing import Callable, Final, Optional +from typing import Callable, Final, Optional, cast +from typing_extensions import TypeGuard from mypy.argmap import map_actuals_to_formals from mypy.nodes import ARG_POS, ARG_STAR, ARG_STAR2, ArgKind @@ -185,6 +186,7 @@ from mypyc.primitives.str_ops import ( str_check_if_true, str_eq, + str_eq_literal, str_ssize_t_size_op, unicode_compare, ) @@ -1551,9 +1553,33 @@ def check_tagged_short_int(self, val: Value, line: int, negated: bool = False) - def compare_strings(self, lhs: Value, rhs: Value, op: str, line: int) -> Value: """Compare two strings""" if op == "==": + # We can specialize this case if one or both values are string literals + literal_fastpath = False + + def is_string_literal(value: Value) -> TypeGuard[LoadLiteral]: + return isinstance(value, LoadLiteral) and is_str_rprimitive(value.type) + + if is_string_literal(lhs): + if is_string_literal(rhs): + # we can optimize out the check entirely in some constant-folded cases + return self.true() if lhs.value == rhs.value else self.false() + + # if lhs argument is string literal, switch sides to match specializer C api + lhs, rhs = rhs, lhs + literal_fastpath = True + elif is_string_literal(rhs): + literal_fastpath = True + + if literal_fastpath: + literal_string = cast(str, cast(LoadLiteral, rhs).value) + literal_length = Integer(len(literal_string), c_pyssize_t_rprimitive, line) + return self.primitive_op(str_eq_literal, [lhs, rhs, literal_length], line) + return self.primitive_op(str_eq, [lhs, rhs], line) + elif op == "!=": - eq = self.primitive_op(str_eq, [lhs, rhs], line) + # perform a standard equality check, then negate + eq = self.compare_strings(lhs, rhs, "==", line) return self.add(ComparisonOp(eq, self.false(), ComparisonOp.EQ, line)) # TODO: modify 'str' to use same interface as 'compare_bytes' as it would avoid diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index e2fe129786d39..c79923f69e691 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -735,6 +735,7 @@ static inline char CPyDict_CheckSize(PyObject *dict, Py_ssize_t size) { #define BOTHSTRIP 2 char CPyStr_Equal(PyObject *str1, PyObject *str2); +char CPyStr_EqualLiteral(PyObject *str, PyObject *literal_str, Py_ssize_t literal_length); PyObject *CPyStr_Build(Py_ssize_t len, ...); PyObject *CPyStr_GetItem(PyObject *str, CPyTagged index); PyObject *CPyStr_GetItemUnsafe(PyObject *str, Py_ssize_t index); diff --git a/mypyc/lib-rt/str_ops.c b/mypyc/lib-rt/str_ops.c index 337ef14fc955f..f16e99bb4159b 100644 --- a/mypyc/lib-rt/str_ops.c +++ b/mypyc/lib-rt/str_ops.c @@ -64,20 +64,33 @@ make_bloom_mask(int kind, const void* ptr, Py_ssize_t len) #undef BLOOM_UPDATE } -// Adapted from CPython 3.13.1 (_PyUnicode_Equal) -char CPyStr_Equal(PyObject *str1, PyObject *str2) { - if (str1 == str2) { - return 1; - } - Py_ssize_t len = PyUnicode_GET_LENGTH(str1); - if (PyUnicode_GET_LENGTH(str2) != len) +static inline char _CPyStr_Equal_NoIdentCheck(PyObject *str1, PyObject *str2, Py_ssize_t str2_length) { + // This helper function only exists to deduplicate code in CPyStr_Equal and CPyStr_EqualLiteral + Py_ssize_t str1_length = PyUnicode_GET_LENGTH(str1); + if (str1_length != str2_length) return 0; int kind = PyUnicode_KIND(str1); if (PyUnicode_KIND(str2) != kind) return 0; const void *data1 = PyUnicode_DATA(str1); const void *data2 = PyUnicode_DATA(str2); - return memcmp(data1, data2, len * kind) == 0; + return memcmp(data1, data2, str1_length * kind) == 0; +} + +// Adapted from CPython 3.13.1 (_PyUnicode_Equal) +char CPyStr_Equal(PyObject *str1, PyObject *str2) { + if (str1 == str2) { + return 1; + } + Py_ssize_t str2_length = PyUnicode_GET_LENGTH(str2); + return _CPyStr_Equal_NoIdentCheck(str1, str2, str2_length); +} + +char CPyStr_EqualLiteral(PyObject *str, PyObject *literal_str, Py_ssize_t literal_length) { + if (str == literal_str) { + return 1; + } + return _CPyStr_Equal_NoIdentCheck(str, literal_str, literal_length); } PyObject *CPyStr_GetItem(PyObject *str, CPyTagged index) { diff --git a/mypyc/primitives/str_ops.py b/mypyc/primitives/str_ops.py index a8f4e4df74c2d..d39f1f872763e 100644 --- a/mypyc/primitives/str_ops.py +++ b/mypyc/primitives/str_ops.py @@ -88,6 +88,14 @@ error_kind=ERR_NEVER, ) +str_eq_literal = custom_primitive_op( + name="str_eq_literal", + c_function_name="CPyStr_EqualLiteral", + arg_types=[str_rprimitive, str_rprimitive, c_pyssize_t_rprimitive], + return_type=bool_rprimitive, + error_kind=ERR_NEVER, +) + unicode_compare = custom_op( arg_types=[str_rprimitive, str_rprimitive], return_type=c_int_rprimitive, diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index c4410b3b19d2b..3280b21cf7e66 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -2325,7 +2325,7 @@ def SetAttr.__setattr__(self, key, val): r12 :: bit L0: r0 = 'regular_attr' - r1 = CPyStr_Equal(key, r0) + r1 = CPyStr_EqualLiteral(key, r0, 12) if r1 goto L1 else goto L2 :: bool L1: r2 = unbox(int, val) @@ -2333,7 +2333,7 @@ L1: goto L6 L2: r4 = 'class_var' - r5 = CPyStr_Equal(key, r4) + r5 = CPyStr_EqualLiteral(key, r4, 9) if r5 goto L3 else goto L4 :: bool L3: r6 = builtins :: module @@ -2468,7 +2468,7 @@ def SetAttr.__setattr__(self, key, val): r12 :: bit L0: r0 = 'regular_attr' - r1 = CPyStr_Equal(key, r0) + r1 = CPyStr_EqualLiteral(key, r0, 12) if r1 goto L1 else goto L2 :: bool L1: r2 = unbox(int, val) @@ -2476,7 +2476,7 @@ L1: goto L6 L2: r4 = 'class_var' - r5 = CPyStr_Equal(key, r4) + r5 = CPyStr_EqualLiteral(key, r4, 9) if r5 goto L3 else goto L4 :: bool L3: r6 = builtins :: module diff --git a/mypyc/test-data/irbuild-dict.test b/mypyc/test-data/irbuild-dict.test index e0c014f078138..e7a330951ab0d 100644 --- a/mypyc/test-data/irbuild-dict.test +++ b/mypyc/test-data/irbuild-dict.test @@ -410,7 +410,7 @@ L2: k = r8 v = r7 r9 = 'name' - r10 = CPyStr_Equal(k, r9) + r10 = CPyStr_EqualLiteral(k, r9, 4) if r10 goto L3 else goto L4 :: bool L3: name = v diff --git a/mypyc/test-data/irbuild-str.test b/mypyc/test-data/irbuild-str.test index 3fa39819498de..056f120c7bac0 100644 --- a/mypyc/test-data/irbuild-str.test +++ b/mypyc/test-data/irbuild-str.test @@ -740,3 +740,34 @@ L2: L3: keep_alive x return r2 + +[case testStrEqLiteral] +from typing import Final +literal: Final = "literal" +def literal_rhs(x: str) -> bool: + return x == literal +def literal_lhs(x: str) -> bool: + return literal == x +def literal_both() -> bool: + return literal == "literal" +[out] +def literal_rhs(x): + x, r0 :: str + r1 :: bool +L0: + r0 = 'literal' + r1 = CPyStr_EqualLiteral(x, r0, 7) + return r1 +def literal_lhs(x): + x, r0 :: str + r1 :: bool +L0: + r0 = 'literal' + r1 = CPyStr_EqualLiteral(x, r0, 7) + return r1 +def literal_both(): + r0, r1 :: str +L0: + r0 = 'literal' + r1 = 'literal' + return 1 diff --git a/mypyc/test-data/irbuild-unreachable.test b/mypyc/test-data/irbuild-unreachable.test index a4f1ef8c7dba4..8eafede66b56e 100644 --- a/mypyc/test-data/irbuild-unreachable.test +++ b/mypyc/test-data/irbuild-unreachable.test @@ -20,7 +20,7 @@ L0: r2 = CPyObject_GetAttr(r0, r1) r3 = cast(str, r2) r4 = 'x' - r5 = CPyStr_Equal(r3, r4) + r5 = CPyStr_EqualLiteral(r3, r4, 1) if r5 goto L2 else goto L1 :: bool L1: r6 = r5 @@ -54,7 +54,7 @@ L0: r2 = CPyObject_GetAttr(r0, r1) r3 = cast(str, r2) r4 = 'x' - r5 = CPyStr_Equal(r3, r4) + r5 = CPyStr_EqualLiteral(r3, r4, 1) if r5 goto L2 else goto L1 :: bool L1: r6 = r5 From 843d133c12e34c781fb469fe50e02a7cacaae9ae Mon Sep 17 00:00:00 2001 From: Randolf Scholz Date: Tue, 14 Oct 2025 16:50:21 +0200 Subject: [PATCH 329/424] More precise return types for `TypedDict.get` (#19897) Fixes #19896, #19902 - `TypedDict.get` now ignores the type of the default when the key is required. - `reveal_type(d.get)` now gives an appropriate list of overloads - I kept the special casing for `get(subdict, {})`, but this is not visible in the overloads. Implementing this via overloads is blocked by #19895 Some additional changes: - I added some code that ensures that the default type always appears last in the union (relevant when a union of multiple keys is given) - I ensure that the original value-type is use instead of its `proper_type`. This simplifies the return in `testRecursiveTypedDictMethods`. --------- Co-authored-by: Ivan Levkivskyi --- mypy/checkexpr.py | 2 +- mypy/plugins/default.py | 34 ++- test-data/unit/check-incremental.test | 201 ++++++++++++++ test-data/unit/check-literal.test | 19 +- test-data/unit/check-recursive-types.test | 3 +- test-data/unit/check-typeddict.test | 272 +++++++++++++++++-- test-data/unit/fixtures/typing-typeddict.pyi | 4 +- test-data/unit/pythoneval.test | 55 ++-- 8 files changed, 533 insertions(+), 57 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index b8f9bf0874678..3eb54579a0500 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1502,7 +1502,7 @@ def check_call_expr_with_callee_type( def check_union_call_expr(self, e: CallExpr, object_type: UnionType, member: str) -> Type: """Type check calling a member expression where the base type is a union.""" res: list[Type] = [] - for typ in object_type.relevant_items(): + for typ in flatten_nested_unions(object_type.relevant_items()): # Member access errors are already reported when visiting the member expression. with self.msg.filter_errors(): item = analyze_member_access( diff --git a/mypy/plugins/default.py b/mypy/plugins/default.py index e492b8dd73351..7a58307fc5597 100644 --- a/mypy/plugins/default.py +++ b/mypy/plugins/default.py @@ -5,7 +5,7 @@ import mypy.errorcodes as codes from mypy import message_registry -from mypy.nodes import DictExpr, IntExpr, StrExpr, UnaryExpr +from mypy.nodes import DictExpr, Expression, IntExpr, StrExpr, UnaryExpr from mypy.plugin import ( AttributeContext, ClassDefContext, @@ -263,30 +263,40 @@ def typed_dict_get_callback(ctx: MethodContext) -> Type: if keys is None: return ctx.default_return_type + default_type: Type + default_arg: Expression | None + if len(ctx.arg_types) <= 1 or not ctx.arg_types[1]: + default_arg = None + default_type = NoneType() + elif len(ctx.arg_types[1]) == 1 and len(ctx.args[1]) == 1: + default_arg = ctx.args[1][0] + default_type = ctx.arg_types[1][0] + else: + return ctx.default_return_type + output_types: list[Type] = [] for key in keys: - value_type = get_proper_type(ctx.type.items.get(key)) + value_type: Type | None = ctx.type.items.get(key) if value_type is None: return ctx.default_return_type - if len(ctx.arg_types) == 1: + if key in ctx.type.required_keys: output_types.append(value_type) - elif len(ctx.arg_types) == 2 and len(ctx.arg_types[1]) == 1 and len(ctx.args[1]) == 1: - default_arg = ctx.args[1][0] + else: + # HACK to deal with get(key, {}) if ( isinstance(default_arg, DictExpr) and len(default_arg.items) == 0 - and isinstance(value_type, TypedDictType) + and isinstance(vt := get_proper_type(value_type), TypedDictType) ): - # Special case '{}' as the default for a typed dict type. - output_types.append(value_type.copy_modified(required_keys=set())) + output_types.append(vt.copy_modified(required_keys=set())) else: output_types.append(value_type) - output_types.append(ctx.arg_types[1][0]) - - if len(ctx.arg_types) == 1: - output_types.append(NoneType()) + output_types.append(default_type) + # for nicer reveal_type, put default at the end, if it is present + if default_type in output_types: + output_types = [t for t in output_types if t != default_type] + [default_type] return make_simplified_union(output_types) return ctx.default_return_type diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index e91b8778e9868..94f65a950062c 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -7311,3 +7311,204 @@ x = 2 [out] [rechecked bar] [stale] + + +[case testIncrementalTypedDictGetMethodTotalFalse] +import impl +[file lib.py] +from typing import TypedDict +class Unrelated: pass +D = TypedDict('D', {'x': int, 'y': str}, total=False) +[file impl.py] +pass +[file impl.py.2] +from typing import Literal +from lib import D, Unrelated +d: D +u: Unrelated +x: Literal['x'] +y: Literal['y'] +z: Literal['z'] +x_or_y: Literal['x', 'y'] +x_or_z: Literal['x', 'z'] +x_or_y_or_z: Literal['x', 'y', 'z'] + +# test with literal expression +reveal_type(d.get('x')) +reveal_type(d.get('y')) +reveal_type(d.get('z')) +reveal_type(d.get('x', u)) +reveal_type(d.get('x', 1)) +reveal_type(d.get('y', None)) + +# test with literal type / union of literal types with implicit default +reveal_type(d.get(x)) +reveal_type(d.get(y)) +reveal_type(d.get(z)) +reveal_type(d.get(x_or_y)) +reveal_type(d.get(x_or_z)) +reveal_type(d.get(x_or_y_or_z)) + +# test with literal type / union of literal types with explicit default +reveal_type(d.get(x, u)) +reveal_type(d.get(y, u)) +reveal_type(d.get(z, u)) +reveal_type(d.get(x_or_y, u)) +reveal_type(d.get(x_or_z, u)) +reveal_type(d.get(x_or_y_or_z, u)) +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] +[out] +[out2] +tmp/impl.py:13: note: Revealed type is "Union[builtins.int, None]" +tmp/impl.py:14: note: Revealed type is "Union[builtins.str, None]" +tmp/impl.py:15: note: Revealed type is "builtins.object" +tmp/impl.py:16: note: Revealed type is "Union[builtins.int, lib.Unrelated]" +tmp/impl.py:17: note: Revealed type is "builtins.int" +tmp/impl.py:18: note: Revealed type is "Union[builtins.str, None]" +tmp/impl.py:21: note: Revealed type is "Union[builtins.int, None]" +tmp/impl.py:22: note: Revealed type is "Union[builtins.str, None]" +tmp/impl.py:23: note: Revealed type is "builtins.object" +tmp/impl.py:24: note: Revealed type is "Union[builtins.int, builtins.str, None]" +tmp/impl.py:25: note: Revealed type is "builtins.object" +tmp/impl.py:26: note: Revealed type is "builtins.object" +tmp/impl.py:29: note: Revealed type is "Union[builtins.int, lib.Unrelated]" +tmp/impl.py:30: note: Revealed type is "Union[builtins.str, lib.Unrelated]" +tmp/impl.py:31: note: Revealed type is "builtins.object" +tmp/impl.py:32: note: Revealed type is "Union[builtins.int, builtins.str, lib.Unrelated]" +tmp/impl.py:33: note: Revealed type is "builtins.object" +tmp/impl.py:34: note: Revealed type is "builtins.object" + +[case testIncrementalTypedDictGetMethodTotalTrue] +import impl +[file lib.py] +from typing import TypedDict +class Unrelated: pass +D = TypedDict('D', {'x': int, 'y': str}, total=True) +[file impl.py] +pass +[file impl.py.2] +from typing import Literal +from lib import D, Unrelated +d: D +u: Unrelated +x: Literal['x'] +y: Literal['y'] +z: Literal['z'] +x_or_y: Literal['x', 'y'] +x_or_z: Literal['x', 'z'] +x_or_y_or_z: Literal['x', 'y', 'z'] + +# test with literal expression +reveal_type(d.get('x')) +reveal_type(d.get('y')) +reveal_type(d.get('z')) +reveal_type(d.get('x', u)) +reveal_type(d.get('x', 1)) +reveal_type(d.get('y', None)) + +# test with literal type / union of literal types with implicit default +reveal_type(d.get(x)) +reveal_type(d.get(y)) +reveal_type(d.get(z)) +reveal_type(d.get(x_or_y)) +reveal_type(d.get(x_or_z)) +reveal_type(d.get(x_or_y_or_z)) + +# test with literal type / union of literal types with explicit default +reveal_type(d.get(x, u)) +reveal_type(d.get(y, u)) +reveal_type(d.get(z, u)) +reveal_type(d.get(x_or_y, u)) +reveal_type(d.get(x_or_z, u)) +reveal_type(d.get(x_or_y_or_z, u)) +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] +[out] +[out2] +tmp/impl.py:13: note: Revealed type is "builtins.int" +tmp/impl.py:14: note: Revealed type is "builtins.str" +tmp/impl.py:15: note: Revealed type is "builtins.object" +tmp/impl.py:16: note: Revealed type is "builtins.int" +tmp/impl.py:17: note: Revealed type is "builtins.int" +tmp/impl.py:18: note: Revealed type is "builtins.str" +tmp/impl.py:21: note: Revealed type is "builtins.int" +tmp/impl.py:22: note: Revealed type is "builtins.str" +tmp/impl.py:23: note: Revealed type is "builtins.object" +tmp/impl.py:24: note: Revealed type is "Union[builtins.int, builtins.str]" +tmp/impl.py:25: note: Revealed type is "builtins.object" +tmp/impl.py:26: note: Revealed type is "builtins.object" +tmp/impl.py:29: note: Revealed type is "builtins.int" +tmp/impl.py:30: note: Revealed type is "builtins.str" +tmp/impl.py:31: note: Revealed type is "builtins.object" +tmp/impl.py:32: note: Revealed type is "Union[builtins.int, builtins.str]" +tmp/impl.py:33: note: Revealed type is "builtins.object" +tmp/impl.py:34: note: Revealed type is "builtins.object" + + +[case testIncrementalTypedDictGetMethodTotalMixed] +import impl +[file lib.py] +from typing import TypedDict +from typing_extensions import Required, NotRequired +class Unrelated: pass +D = TypedDict('D', {'x': Required[int], 'y': NotRequired[str]}) +[file impl.py] +pass +[file impl.py.2] +from typing import Literal +from lib import D, Unrelated +d: D +u: Unrelated +x: Literal['x'] +y: Literal['y'] +z: Literal['z'] +x_or_y: Literal['x', 'y'] +x_or_z: Literal['x', 'z'] +x_or_y_or_z: Literal['x', 'y', 'z'] + +# test with literal expression +reveal_type(d.get('x')) +reveal_type(d.get('y')) +reveal_type(d.get('z')) +reveal_type(d.get('x', u)) +reveal_type(d.get('x', 1)) +reveal_type(d.get('y', None)) + +# test with literal type / union of literal types with implicit default +reveal_type(d.get(x)) +reveal_type(d.get(y)) +reveal_type(d.get(z)) +reveal_type(d.get(x_or_y)) +reveal_type(d.get(x_or_z)) +reveal_type(d.get(x_or_y_or_z)) + +# test with literal type / union of literal types with explicit default +reveal_type(d.get(x, u)) +reveal_type(d.get(y, u)) +reveal_type(d.get(z, u)) +reveal_type(d.get(x_or_y, u)) +reveal_type(d.get(x_or_z, u)) +reveal_type(d.get(x_or_y_or_z, u)) +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] +[out] +[out2] +tmp/impl.py:13: note: Revealed type is "builtins.int" +tmp/impl.py:14: note: Revealed type is "Union[builtins.str, None]" +tmp/impl.py:15: note: Revealed type is "builtins.object" +tmp/impl.py:16: note: Revealed type is "builtins.int" +tmp/impl.py:17: note: Revealed type is "builtins.int" +tmp/impl.py:18: note: Revealed type is "Union[builtins.str, None]" +tmp/impl.py:21: note: Revealed type is "builtins.int" +tmp/impl.py:22: note: Revealed type is "Union[builtins.str, None]" +tmp/impl.py:23: note: Revealed type is "builtins.object" +tmp/impl.py:24: note: Revealed type is "Union[builtins.int, builtins.str, None]" +tmp/impl.py:25: note: Revealed type is "builtins.object" +tmp/impl.py:26: note: Revealed type is "builtins.object" +tmp/impl.py:29: note: Revealed type is "builtins.int" +tmp/impl.py:30: note: Revealed type is "Union[builtins.str, lib.Unrelated]" +tmp/impl.py:31: note: Revealed type is "builtins.object" +tmp/impl.py:32: note: Revealed type is "Union[builtins.int, builtins.str, lib.Unrelated]" +tmp/impl.py:33: note: Revealed type is "builtins.object" +tmp/impl.py:34: note: Revealed type is "builtins.object" diff --git a/test-data/unit/check-literal.test b/test-data/unit/check-literal.test index 3c9290b8dbbba..ce0ae2844bae1 100644 --- a/test-data/unit/check-literal.test +++ b/test-data/unit/check-literal.test @@ -1884,7 +1884,7 @@ reveal_type(d[a_key]) # N: Revealed type is "builtins.int" reveal_type(d[b_key]) # N: Revealed type is "builtins.str" d[c_key] # E: TypedDict "Outer" has no key "c" -reveal_type(d.get(a_key, u)) # N: Revealed type is "Union[builtins.int, __main__.Unrelated]" +reveal_type(d.get(a_key, u)) # N: Revealed type is "builtins.int" reveal_type(d.get(b_key, u)) # N: Revealed type is "Union[builtins.str, __main__.Unrelated]" reveal_type(d.get(c_key, u)) # N: Revealed type is "builtins.object" @@ -1928,7 +1928,7 @@ u: Unrelated reveal_type(a[int_key_good]) # N: Revealed type is "builtins.int" reveal_type(b[int_key_good]) # N: Revealed type is "builtins.int" reveal_type(c[str_key_good]) # N: Revealed type is "builtins.int" -reveal_type(c.get(str_key_good, u)) # N: Revealed type is "Union[builtins.int, __main__.Unrelated]" +reveal_type(c.get(str_key_good, u)) # N: Revealed type is "builtins.int" reveal_type(c.get(str_key_bad, u)) # N: Revealed type is "builtins.object" a[int_key_bad] # E: Tuple index out of range @@ -1993,8 +1993,8 @@ optional_keys: Literal["d", "e"] bad_keys: Literal["a", "bad"] reveal_type(test[good_keys]) # N: Revealed type is "Union[__main__.A, __main__.B]" -reveal_type(test.get(good_keys)) # N: Revealed type is "Union[__main__.A, __main__.B, None]" -reveal_type(test.get(good_keys, 3)) # N: Revealed type is "Union[__main__.A, Literal[3]?, __main__.B]" +reveal_type(test.get(good_keys)) # N: Revealed type is "Union[__main__.A, __main__.B]" +reveal_type(test.get(good_keys, 3)) # N: Revealed type is "Union[__main__.A, __main__.B]" reveal_type(test.pop(optional_keys)) # N: Revealed type is "Union[__main__.D, __main__.E]" reveal_type(test.pop(optional_keys, 3)) # N: Revealed type is "Union[__main__.D, __main__.E, Literal[3]?]" reveal_type(test.setdefault(good_keys, AAndB())) # N: Revealed type is "Union[__main__.A, __main__.B]" @@ -2037,15 +2037,18 @@ class D2(TypedDict): d: D x: Union[D1, D2] -bad_keys: Literal['a', 'b', 'c', 'd'] good_keys: Literal['b', 'c'] +mixed_keys: Literal['a', 'b', 'c', 'd'] +bad_keys: Literal['e', 'f'] -x[bad_keys] # E: TypedDict "D1" has no key "d" \ +x[mixed_keys] # E: TypedDict "D1" has no key "d" \ # E: TypedDict "D2" has no key "a" reveal_type(x[good_keys]) # N: Revealed type is "Union[__main__.B, __main__.C]" -reveal_type(x.get(good_keys)) # N: Revealed type is "Union[__main__.B, __main__.C, None]" -reveal_type(x.get(good_keys, 3)) # N: Revealed type is "Union[__main__.B, Literal[3]?, __main__.C]" +reveal_type(x.get(good_keys)) # N: Revealed type is "Union[__main__.B, __main__.C]" +reveal_type(x.get(good_keys, 3)) # N: Revealed type is "Union[__main__.B, __main__.C]" +reveal_type(x.get(mixed_keys)) # N: Revealed type is "builtins.object" +reveal_type(x.get(mixed_keys, 3)) # N: Revealed type is "builtins.object" reveal_type(x.get(bad_keys)) # N: Revealed type is "builtins.object" reveal_type(x.get(bad_keys, 3)) # N: Revealed type is "builtins.object" diff --git a/test-data/unit/check-recursive-types.test b/test-data/unit/check-recursive-types.test index 4f451aa062d69..c82111322fe17 100644 --- a/test-data/unit/check-recursive-types.test +++ b/test-data/unit/check-recursive-types.test @@ -690,10 +690,11 @@ class TD(TypedDict, total=False): y: TD td: TD +reveal_type(td.get("y")) # N: Revealed type is "Union[TypedDict('__main__.TD', {'x'?: builtins.int, 'y'?: ...}), None]" td["y"] = {"x": 0, "y": {}} td["y"] = {"x": 0, "y": {"x": 0, "y": 42}} # E: Incompatible types (expression has type "int", TypedDict item "y" has type "TD") -reveal_type(td.get("y")) # N: Revealed type is "Union[TypedDict('__main__.TD', {'x'?: builtins.int, 'y'?: TypedDict('__main__.TD', {'x'?: builtins.int, 'y'?: ...})}), None]" +reveal_type(td.get("y")) # N: Revealed type is "Union[TypedDict('__main__.TD', {'x'?: builtins.int, 'y'?: ...}), None]" s: str = td.get("y") # E: Incompatible types in assignment (expression has type "Optional[TD]", variable has type "str") td.update({"x": 0, "y": {"x": 1, "y": {}}}) diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index 34cae74d795b0..e1a70efe9316d 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -997,26 +997,149 @@ if int(): -- Other TypedDict methods -[case testTypedDictGetMethod] + +[case testTypedDictGetMethodOverloads] from typing import TypedDict -class A: pass -D = TypedDict('D', {'x': int, 'y': str}) +from typing_extensions import Required, NotRequired + +class D(TypedDict): + a: int + b: NotRequired[str] + +def test(d: D) -> None: + reveal_type(d.get) # N: Revealed type is "Overload(def (k: builtins.str) -> builtins.object, def (builtins.str, builtins.object) -> builtins.object, def [V] (builtins.str, V`4) -> builtins.object)" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictGetMethodTotalFalse] +from typing import TypedDict, Literal +class Unrelated: pass +D = TypedDict('D', {'x': int, 'y': str}, total=False) d: D +u: Unrelated +x: Literal['x'] +y: Literal['y'] +z: Literal['z'] +x_or_y: Literal['x', 'y'] +x_or_z: Literal['x', 'z'] +x_or_y_or_z: Literal['x', 'y', 'z'] + +# test with literal expression reveal_type(d.get('x')) # N: Revealed type is "Union[builtins.int, None]" reveal_type(d.get('y')) # N: Revealed type is "Union[builtins.str, None]" -reveal_type(d.get('x', A())) # N: Revealed type is "Union[builtins.int, __main__.A]" +reveal_type(d.get('z')) # N: Revealed type is "builtins.object" +reveal_type(d.get('x', u)) # N: Revealed type is "Union[builtins.int, __main__.Unrelated]" reveal_type(d.get('x', 1)) # N: Revealed type is "builtins.int" reveal_type(d.get('y', None)) # N: Revealed type is "Union[builtins.str, None]" + +# test with literal type / union of literal types with implicit default +reveal_type(d.get(x)) # N: Revealed type is "Union[builtins.int, None]" +reveal_type(d.get(y)) # N: Revealed type is "Union[builtins.str, None]" +reveal_type(d.get(z)) # N: Revealed type is "builtins.object" +reveal_type(d.get(x_or_y)) # N: Revealed type is "Union[builtins.int, builtins.str, None]" +reveal_type(d.get(x_or_z)) # N: Revealed type is "builtins.object" +reveal_type(d.get(x_or_y_or_z)) # N: Revealed type is "builtins.object" + +# test with literal type / union of literal types with explicit default +reveal_type(d.get(x, u)) # N: Revealed type is "Union[builtins.int, __main__.Unrelated]" +reveal_type(d.get(y, u)) # N: Revealed type is "Union[builtins.str, __main__.Unrelated]" +reveal_type(d.get(z, u)) # N: Revealed type is "builtins.object" +reveal_type(d.get(x_or_y, u)) # N: Revealed type is "Union[builtins.int, builtins.str, __main__.Unrelated]" +reveal_type(d.get(x_or_z, u)) # N: Revealed type is "builtins.object" +reveal_type(d.get(x_or_y_or_z, u)) # N: Revealed type is "builtins.object" [builtins fixtures/dict.pyi] [typing fixtures/typing-typeddict.pyi] +[case testTypedDictGetMethodTotalTrue] +from typing import TypedDict, Literal +class Unrelated: pass +D = TypedDict('D', {'x': int, 'y': str}, total=True) +d: D +u: Unrelated +x: Literal['x'] +y: Literal['y'] +z: Literal['z'] +x_or_y: Literal['x', 'y'] +x_or_z: Literal['x', 'z'] +x_or_y_or_z: Literal['x', 'y', 'z'] + +# test with literal expression +reveal_type(d.get('x')) # N: Revealed type is "builtins.int" +reveal_type(d.get('y')) # N: Revealed type is "builtins.str" +reveal_type(d.get('z')) # N: Revealed type is "builtins.object" +reveal_type(d.get('x', u)) # N: Revealed type is "builtins.int" +reveal_type(d.get('x', 1)) # N: Revealed type is "builtins.int" +reveal_type(d.get('y', None)) # N: Revealed type is "builtins.str" + +# test with literal type / union of literal types with implicit default +reveal_type(d.get(x)) # N: Revealed type is "builtins.int" +reveal_type(d.get(y)) # N: Revealed type is "builtins.str" +reveal_type(d.get(z)) # N: Revealed type is "builtins.object" +reveal_type(d.get(x_or_y)) # N: Revealed type is "Union[builtins.int, builtins.str]" +reveal_type(d.get(x_or_z)) # N: Revealed type is "builtins.object" +reveal_type(d.get(x_or_y_or_z)) # N: Revealed type is "builtins.object" + +# test with literal type / union of literal types with explicit default +reveal_type(d.get(x, u)) # N: Revealed type is "builtins.int" +reveal_type(d.get(y, u)) # N: Revealed type is "builtins.str" +reveal_type(d.get(z, u)) # N: Revealed type is "builtins.object" +reveal_type(d.get(x_or_y, u)) # N: Revealed type is "Union[builtins.int, builtins.str]" +reveal_type(d.get(x_or_z, u)) # N: Revealed type is "builtins.object" +reveal_type(d.get(x_or_y_or_z, u)) # N: Revealed type is "builtins.object" + +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictGetMethodTotalMixed] +from typing import TypedDict, Literal +from typing_extensions import Required, NotRequired +class Unrelated: pass +D = TypedDict('D', {'x': Required[int], 'y': NotRequired[str]}) +d: D +u: Unrelated +x: Literal['x'] +y: Literal['y'] +z: Literal['z'] +x_or_y: Literal['x', 'y'] +x_or_z: Literal['x', 'z'] +x_or_y_or_z: Literal['x', 'y', 'z'] + +# test with literal expression +reveal_type(d.get('x')) # N: Revealed type is "builtins.int" +reveal_type(d.get('y')) # N: Revealed type is "Union[builtins.str, None]" +reveal_type(d.get('z')) # N: Revealed type is "builtins.object" +reveal_type(d.get('x', u)) # N: Revealed type is "builtins.int" +reveal_type(d.get('x', 1)) # N: Revealed type is "builtins.int" +reveal_type(d.get('y', None)) # N: Revealed type is "Union[builtins.str, None]" + +# test with literal type / union of literal types with implicit default +reveal_type(d.get(x)) # N: Revealed type is "builtins.int" +reveal_type(d.get(y)) # N: Revealed type is "Union[builtins.str, None]" +reveal_type(d.get(z)) # N: Revealed type is "builtins.object" +reveal_type(d.get(x_or_y)) # N: Revealed type is "Union[builtins.int, builtins.str, None]" +reveal_type(d.get(x_or_z)) # N: Revealed type is "builtins.object" +reveal_type(d.get(x_or_y_or_z)) # N: Revealed type is "builtins.object" + +# test with literal type / union of literal types with explicit default +reveal_type(d.get(x, u)) # N: Revealed type is "builtins.int" +reveal_type(d.get(y, u)) # N: Revealed type is "Union[builtins.str, __main__.Unrelated]" +reveal_type(d.get(z, u)) # N: Revealed type is "builtins.object" +reveal_type(d.get(x_or_y, u)) # N: Revealed type is "Union[builtins.int, builtins.str, __main__.Unrelated]" +reveal_type(d.get(x_or_z, u)) # N: Revealed type is "builtins.object" +reveal_type(d.get(x_or_y_or_z, u)) # N: Revealed type is "builtins.object" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + + [case testTypedDictGetMethodTypeContext] from typing import List, TypedDict class A: pass -D = TypedDict('D', {'x': List[int], 'y': int}) +D = TypedDict('D', {'x': List[int], 'y': int}, total=False) d: D reveal_type(d.get('x', [])) # N: Revealed type is "builtins.list[builtins.int]" -d.get('x', ['x']) # E: List item 0 has incompatible type "str"; expected "int" +reveal_type(d.get('x', ['x'])) # N: Revealed type is "Union[builtins.list[builtins.int], builtins.list[builtins.str]]" a = [''] reveal_type(d.get('x', a)) # N: Revealed type is "Union[builtins.list[builtins.int], builtins.list[builtins.str]]" [builtins fixtures/dict.pyi] @@ -1029,11 +1152,13 @@ d: D d.get() # E: All overload variants of "get" of "Mapping" require at least one argument \ # N: Possible overload variants: \ # N: def get(self, k: str) -> object \ - # N: def [V] get(self, k: str, default: object) -> object + # N: def get(self, str, object, /) -> object \ + # N: def [V] get(self, str, V, /) -> object d.get('x', 1, 2) # E: No overload variant of "get" of "Mapping" matches argument types "str", "int", "int" \ # N: Possible overload variants: \ # N: def get(self, k: str) -> object \ - # N: def [V] get(self, k: str, default: Union[int, V]) -> object + # N: def get(self, str, object, /) -> object \ + # N: def [V] get(self, str, Union[int, V], /) -> object x = d.get('z') reveal_type(x) # N: Revealed type is "builtins.object" s = '' @@ -1069,19 +1194,134 @@ p.get('x', 1 + 'y') # E: Unsupported operand types for + ("int" and "str") [case testTypedDictChainedGetWithEmptyDictDefault] from typing import TypedDict -C = TypedDict('C', {'a': int}) -D = TypedDict('D', {'x': C, 'y': str}) +C = TypedDict('C', {'a': int}, total=True) +D = TypedDict('D', {'x': C, 'y': str}, total=False) d: D -reveal_type(d.get('x', {})) \ - # N: Revealed type is "TypedDict('__main__.C', {'a'?: builtins.int})" -reveal_type(d.get('x', None)) \ - # N: Revealed type is "Union[TypedDict('__main__.C', {'a': builtins.int}), None]" +reveal_type(d.get('x', {})) # N: Revealed type is "TypedDict('__main__.C', {'a'?: builtins.int})" +reveal_type(d.get('x', None)) # N: Revealed type is "Union[TypedDict('__main__.C', {'a': builtins.int}), None]" reveal_type(d.get('x', {}).get('a')) # N: Revealed type is "Union[builtins.int, None]" reveal_type(d.get('x', {})['a']) # N: Revealed type is "builtins.int" [builtins fixtures/dict.pyi] [typing fixtures/typing-typeddict.pyi] +[case testTypedDictChainedGetWithEmptyDictDefault2] +from typing import TypedDict +C = TypedDict('C', {'a': int}, total=False) +D = TypedDict('D', {'x': C, 'y': str}, total=True) +d: D +reveal_type(d.get('x', {})) # N: Revealed type is "TypedDict('__main__.C', {'a'?: builtins.int})" +reveal_type(d.get('x', None)) # N: Revealed type is "TypedDict('__main__.C', {'a'?: builtins.int})" +reveal_type(d.get('x', {}).get('a')) # N: Revealed type is "Union[builtins.int, None]" +reveal_type(d.get('x', {})['a']) # N: Revealed type is "builtins.int" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictChainedGetWithEmptyDictDefault3] +from typing import TypedDict +C = TypedDict('C', {'a': int}, total=True) +D = TypedDict('D', {'x': C, 'y': str}, total=True) +d: D +reveal_type(d.get('x', {})) # N: Revealed type is "TypedDict('__main__.C', {'a': builtins.int})" +reveal_type(d.get('x', None)) # N: Revealed type is "TypedDict('__main__.C', {'a': builtins.int})" +reveal_type(d.get('x', {}).get('a')) # N: Revealed type is "builtins.int" +reveal_type(d.get('x', {})['a']) # N: Revealed type is "builtins.int" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictChainedGetWithEmptyDictDefault4] +from typing import TypedDict +C = TypedDict('C', {'a': int}, total=False) +D = TypedDict('D', {'x': C, 'y': str}, total=False) +d: D +reveal_type(d.get('x', {})) # N: Revealed type is "TypedDict('__main__.C', {'a'?: builtins.int})" +reveal_type(d.get('x', None)) # N: Revealed type is "Union[TypedDict('__main__.C', {'a'?: builtins.int}), None]" +reveal_type(d.get('x', {}).get('a')) # N: Revealed type is "Union[builtins.int, None]" +reveal_type(d.get('x', {})['a']) # N: Revealed type is "builtins.int" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictGetMethodChained] +# check that chaining with get like ``.get(key, {}).get(subkey, {})`` works. +from typing import TypedDict, Mapping +from typing_extensions import Required, NotRequired, Never + +class Total(TypedDict, total=True): # no keys optional + key_one: int + key_two: str + +class Maybe(TypedDict, total=False): # all keys are optional + key_one: int + key_two: str + +class Mixed(TypedDict): # some keys optional + key_one: Required[int] + key_two: NotRequired[str] + +class Config(TypedDict): + required_total: Required[Total] + optional_total: NotRequired[Total] + required_mixed: Required[Mixed] + optional_mixed: NotRequired[Mixed] + required_maybe: Required[Maybe] + optional_maybe: NotRequired[Maybe] + +def test_chaining(d: Config) -> None: + reveal_type( d.get("required_total", {}) ) # N: Revealed type is "TypedDict('__main__.Total', {'key_one': builtins.int, 'key_two': builtins.str})" + reveal_type( d.get("optional_total", {}) ) # N: Revealed type is "TypedDict('__main__.Total', {'key_one'?: builtins.int, 'key_two'?: builtins.str})" + reveal_type( d.get("required_maybe", {}) ) # N: Revealed type is "TypedDict('__main__.Maybe', {'key_one'?: builtins.int, 'key_two'?: builtins.str})" + reveal_type( d.get("optional_maybe", {}) ) # N: Revealed type is "TypedDict('__main__.Maybe', {'key_one'?: builtins.int, 'key_two'?: builtins.str})" + reveal_type( d.get("required_mixed", {}) ) # N: Revealed type is "TypedDict('__main__.Mixed', {'key_one': builtins.int, 'key_two'?: builtins.str})" + reveal_type( d.get("optional_mixed", {}) ) # N: Revealed type is "TypedDict('__main__.Mixed', {'key_one'?: builtins.int, 'key_two'?: builtins.str})" + + reveal_type( d.get("required_total", {}).get("key_one") ) # N: Revealed type is "builtins.int" + reveal_type( d.get("required_total", {}).get("key_two") ) # N: Revealed type is "builtins.str" + reveal_type( d.get("required_total", {}).get("bad_key") ) # N: Revealed type is "builtins.object" + reveal_type( d.get("optional_total", {}).get("key_one") ) # N: Revealed type is "Union[builtins.int, None]" + reveal_type( d.get("optional_total", {}).get("key_two") ) # N: Revealed type is "Union[builtins.str, None]" + reveal_type( d.get("optional_total", {}).get("bad_key") ) # N: Revealed type is "builtins.object" + + reveal_type( d.get("required_maybe", {}).get("key_one") ) # N: Revealed type is "Union[builtins.int, None]" + reveal_type( d.get("required_maybe", {}).get("key_two") ) # N: Revealed type is "Union[builtins.str, None]" + reveal_type( d.get("required_maybe", {}).get("bad_key") ) # N: Revealed type is "builtins.object" + reveal_type( d.get("optional_maybe", {}).get("key_one") ) # N: Revealed type is "Union[builtins.int, None]" + reveal_type( d.get("optional_maybe", {}).get("key_two") ) # N: Revealed type is "Union[builtins.str, None]" + reveal_type( d.get("optional_maybe", {}).get("bad_key") ) # N: Revealed type is "builtins.object" + + reveal_type( d.get("required_mixed", {}).get("key_one") ) # N: Revealed type is "builtins.int" + reveal_type( d.get("required_mixed", {}).get("key_two") ) # N: Revealed type is "Union[builtins.str, None]" + reveal_type( d.get("required_mixed", {}).get("bad_key") ) # N: Revealed type is "builtins.object" + reveal_type( d.get("optional_mixed", {}).get("key_one") ) # N: Revealed type is "Union[builtins.int, None]" + reveal_type( d.get("optional_mixed", {}).get("key_two") ) # N: Revealed type is "Union[builtins.str, None]" + reveal_type( d.get("optional_mixed", {}).get("bad_key") ) # N: Revealed type is "builtins.object" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictGetWithNestedUnionOfTypedDicts] +# https://github.com/python/mypy/issues/19902 +from typing import TypedDict, Union +from typing_extensions import TypeAlias, NotRequired +class A(TypedDict): + key: NotRequired[int] + +class B(TypedDict): + key: NotRequired[int] + +class C(TypedDict): + key: NotRequired[int] + +A_or_B: TypeAlias = Union[A, B] +A_or_B_or_C: TypeAlias = Union[A_or_B, C] + +def test(d: A_or_B_or_C) -> None: + reveal_type(d.get("key")) # N: Revealed type is "Union[builtins.int, None]" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + -- Totality (the "total" keyword argument) [case testTypedDictWithTotalTrue] @@ -1769,8 +2009,8 @@ class TDB(TypedDict): td: Union[TDA, TDB] -reveal_type(td.get('a')) # N: Revealed type is "Union[builtins.int, None]" -reveal_type(td.get('b')) # N: Revealed type is "Union[builtins.str, None, builtins.int]" +reveal_type(td.get('a')) # N: Revealed type is "builtins.int" +reveal_type(td.get('b')) # N: Revealed type is "Union[builtins.str, builtins.int]" reveal_type(td.get('c')) # N: Revealed type is "builtins.object" reveal_type(td['a']) # N: Revealed type is "builtins.int" diff --git a/test-data/unit/fixtures/typing-typeddict.pyi b/test-data/unit/fixtures/typing-typeddict.pyi index 16658c82528b8..29635b6518706 100644 --- a/test-data/unit/fixtures/typing-typeddict.pyi +++ b/test-data/unit/fixtures/typing-typeddict.pyi @@ -56,7 +56,9 @@ class Mapping(Iterable[T], Generic[T, T_co], metaclass=ABCMeta): @overload def get(self, k: T) -> Optional[T_co]: pass @overload - def get(self, k: T, default: Union[T_co, V]) -> Union[T_co, V]: pass + def get(self, k: T, default: T_co, /) -> Optional[T_co]: pass # type: ignore[misc] + @overload + def get(self, k: T, default: V, /) -> Union[T_co, V]: pass def values(self) -> Iterable[T_co]: pass # Approximate return type def __len__(self) -> int: ... def __contains__(self, arg: object) -> int: pass diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index 93b67bfa813a8..2069d082df178 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -1034,24 +1034,43 @@ _program.py:17: note: Revealed type is "builtins.str" # Test that TypedDict get plugin works with typeshed stubs from typing import TypedDict class A: pass -D = TypedDict('D', {'x': int, 'y': str}) -d: D -reveal_type(d.get('x')) -reveal_type(d.get('y')) -reveal_type(d.get('z')) -d.get() -s = '' -reveal_type(d.get(s)) -[out] -_testTypedDictGet.py:6: note: Revealed type is "Union[builtins.int, None]" -_testTypedDictGet.py:7: note: Revealed type is "Union[builtins.str, None]" -_testTypedDictGet.py:8: note: Revealed type is "builtins.object" -_testTypedDictGet.py:9: error: All overload variants of "get" of "Mapping" require at least one argument -_testTypedDictGet.py:9: note: Possible overload variants: -_testTypedDictGet.py:9: note: def get(self, str, /) -> object -_testTypedDictGet.py:9: note: def get(self, str, /, default: object) -> object -_testTypedDictGet.py:9: note: def [_T] get(self, str, /, default: _T) -> object -_testTypedDictGet.py:11: note: Revealed type is "builtins.object" +D_total = TypedDict('D_total', {'x': int, 'y': str}, total=True) +D_not_total = TypedDict('D_not_total', {'x': int, 'y': str}, total=False) + +def test_total(d: D_total) -> None: + reveal_type(d.get('x')) + reveal_type(d.get('y')) + reveal_type(d.get('z')) + d.get() + s = '' + reveal_type(d.get(s)) + +def test_not_total(d: D_not_total) -> None: + reveal_type(d.get('x')) + reveal_type(d.get('y')) + reveal_type(d.get('z')) + d.get() + s = '' + reveal_type(d.get(s)) +[out] +_testTypedDictGet.py:8: note: Revealed type is "builtins.int" +_testTypedDictGet.py:9: note: Revealed type is "builtins.str" +_testTypedDictGet.py:10: note: Revealed type is "builtins.object" +_testTypedDictGet.py:11: error: All overload variants of "get" of "Mapping" require at least one argument +_testTypedDictGet.py:11: note: Possible overload variants: +_testTypedDictGet.py:11: note: def get(self, str, /) -> object +_testTypedDictGet.py:11: note: def get(self, str, /, default: object) -> object +_testTypedDictGet.py:11: note: def [_T] get(self, str, /, default: _T) -> object +_testTypedDictGet.py:13: note: Revealed type is "builtins.object" +_testTypedDictGet.py:16: note: Revealed type is "Union[builtins.int, None]" +_testTypedDictGet.py:17: note: Revealed type is "Union[builtins.str, None]" +_testTypedDictGet.py:18: note: Revealed type is "builtins.object" +_testTypedDictGet.py:19: error: All overload variants of "get" of "Mapping" require at least one argument +_testTypedDictGet.py:19: note: Possible overload variants: +_testTypedDictGet.py:19: note: def get(self, str, /) -> object +_testTypedDictGet.py:19: note: def get(self, str, /, default: object) -> object +_testTypedDictGet.py:19: note: def [_T] get(self, str, /, default: _T) -> object +_testTypedDictGet.py:21: note: Revealed type is "builtins.object" [case testTypedDictMappingMethods] from typing import TypedDict From 958e45097b5fc99a18c7f6ebfc7120f9c810ec19 Mon Sep 17 00:00:00 2001 From: Guo Ci Date: Tue, 14 Oct 2025 15:29:51 -0400 Subject: [PATCH 330/424] Fix typo in generics documentation (#20065) --- docs/source/generics.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/generics.rst b/docs/source/generics.rst index 4755c4f17ec82..bdd6e333f8959 100644 --- a/docs/source/generics.rst +++ b/docs/source/generics.rst @@ -1165,7 +1165,7 @@ This example correctly uses a covariant type variable: See :ref:`variance-of-generics` for more about variance. -Generic protocols can also be recursive. Example (Python 3.12 synta): +Generic protocols can also be recursive. Example (Python 3.12 syntax): .. code-block:: python From 94d0d7e3e57e7ecd408c2c7324d8550e76de2f55 Mon Sep 17 00:00:00 2001 From: Joren Hammudoglu Date: Tue, 14 Oct 2025 23:55:30 +0200 Subject: [PATCH 331/424] stubtest: include function name in overload assertion messages (#20063) I just managed to cause this assertion to fail (I'll open an issue about this soon), but the traceback did not tell me *where* in the stubs this occurred. Knowing the full name of the relevant function would've made the debugging process a lot easier. --- mypy/stubtest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index 4126f3959ee15..99404dbe52abd 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -968,7 +968,7 @@ def from_overloadedfuncdef(stub: nodes.OverloadedFuncDef) -> Signature[nodes.Arg is_arg_pos_only: defaultdict[str, set[bool]] = defaultdict(set) for func in map(_resolve_funcitem_from_decorator, stub.items): - assert func is not None, "Failed to resolve decorated overload" + assert func is not None, f"Failed to resolve decorated overload of {stub.fullname!r}" args = maybe_strip_cls(stub.name, func.arguments) for index, arg in enumerate(args): if ( @@ -984,7 +984,7 @@ def from_overloadedfuncdef(stub: nodes.OverloadedFuncDef) -> Signature[nodes.Arg all_args: dict[str, list[tuple[nodes.Argument, int]]] = {} for func in map(_resolve_funcitem_from_decorator, stub.items): - assert func is not None, "Failed to resolve decorated overload" + assert func is not None, f"Failed to resolve decorated overload of {stub.fullname!r}" args = maybe_strip_cls(stub.name, func.arguments) for index, arg in enumerate(args): # For positional-only args, we allow overloads to have different names for the same From 2c6c3959356674262d9b2c2dc43a33486e807a9c Mon Sep 17 00:00:00 2001 From: A5rocks Date: Wed, 15 Oct 2025 08:32:21 +0900 Subject: [PATCH 332/424] Make --pretty work better on multi-line issues (#20056) We can just print the first line for things where there's multiple lines being the issue. Ideally, we would print the first line, last line, and explicitly elide the lines in between. That's for a future change! (maybe) Fixes https://github.com/python/mypy/issues/18522. NOTE: this is an aesthetic thing, there's no right formatting. So I would be fine if this is closed because others think it looks worse. --- mypy/errors.py | 3 +++ test-data/unit/check-unreachable-code.test | 11 +++++++++++ test-data/unit/daemon.test | 2 +- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/mypy/errors.py b/mypy/errors.py index 1b092fb50e4af..69e4fb4cf0654 100644 --- a/mypy/errors.py +++ b/mypy/errors.py @@ -1003,6 +1003,9 @@ def format_messages( marker = "^" if end_line == line and end_column > column: marker = f'^{"~" * (end_column - column - 1)}' + elif end_line != line: + # just highlight the first line instead + marker = f'^{"~" * (len(source_line_expanded) - column - 1)}' a.append(" " * (DEFAULT_SOURCE_OFFSET + column) + marker) return a diff --git a/test-data/unit/check-unreachable-code.test b/test-data/unit/check-unreachable-code.test index 645f81e89ca13..7e00671dfd114 100644 --- a/test-data/unit/check-unreachable-code.test +++ b/test-data/unit/check-unreachable-code.test @@ -1619,3 +1619,14 @@ reveal_type(bar().attr) # N: Revealed type is "Never" 1 # not unreachable reveal_type(foo().attr) # N: Revealed type is "Never" 1 # E: Statement is unreachable + +[case testUnreachableStatementPrettyHighlighting] +# flags: --warn-unreachable --pretty +def x() -> None: + assert False + if 5: + pass +[out] +main:4: error: Statement is unreachable + if 5: + ^~~~~ diff --git a/test-data/unit/daemon.test b/test-data/unit/daemon.test index 295eb4000d812..c02f78be18340 100644 --- a/test-data/unit/daemon.test +++ b/test-data/unit/daemon.test @@ -122,7 +122,7 @@ Daemon stopped Daemon started foo.py:1: error: Function is missing a return type annotation def f(): - ^ + ^~~~~~~~ foo.py:1: note: Use "-> None" if function does not return a value Found 1 error in 1 file (checked 1 source file) == Return code: 1 From 2eaafe905fc00c2b6c1ea4ec74c7f84cdf09c50e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 16 Oct 2025 10:09:46 +0100 Subject: [PATCH 333/424] Add tool to convert binary cache files to JSON (#20071) Work to enable #19697 (see the issue for motivation). Copy the old `serialize` methods with some modifications and use them in the export tool. This will let us remove the old `serialize` and `deserialize` methods once we drop support for the old JSON serialization format at some point. This should be enough to support existing use cases that inspect the JSON files. Instead of directly inspecting them, users will have to run the tool first if they use the binary cache format. Example: ``` $ python -m mypy.exportjson .mypy_cache/3.13/foobar.data.ff .mypy_cache/3.13/foobar.data.ff -> .mypy_cache/3.13/foobar.data.ff.json ``` The run generates the `.mypy_cache/3.13/foobar.data.ff.json` file, which is similar to existing to cache json files. I added some tests and manually checked that the JSON file for `builtins` module is identical to the one generated by mypy. However, we won't guarantee that all new symbol table or cache features will be added to the exporter, to simplify maintenance. Also I didn't test all features of the exporter in the tests -- I just ensure that the basics work. The tool is primarily there to support existing use cases and debugging workflows, and it would be better to use `MypyFile.read(...)` in new use cases that require cache inspection. --- mypy/exportjson.py | 578 +++++++++++++++++++++++++++++++++ mypy/test/testexportjson.py | 70 ++++ setup.py | 1 + test-data/unit/exportjson.test | 280 ++++++++++++++++ 4 files changed, 929 insertions(+) create mode 100644 mypy/exportjson.py create mode 100644 mypy/test/testexportjson.py create mode 100644 test-data/unit/exportjson.test diff --git a/mypy/exportjson.py b/mypy/exportjson.py new file mode 100644 index 0000000000000..09945f0ef28f0 --- /dev/null +++ b/mypy/exportjson.py @@ -0,0 +1,578 @@ +"""Tool to convert mypy cache file to a JSON format (print to stdout). + +Usage: + python -m mypy.exportjson .mypy_cache/.../my_module.data.ff + +The idea is to make caches introspectable once we've switched to a binary +cache format and removed support for the older JSON cache format. + +This is primarily to support existing use cases that need to inspect +cache files, and to support debugging mypy caching issues. This means that +this doesn't necessarily need to be kept 1:1 up to date with changes in the +binary cache format (to simplify maintenance -- we don't want this to slow +down mypy development). +""" + +import argparse +import json +import sys +from typing import Any, Union +from typing_extensions import TypeAlias as _TypeAlias + +from librt.internal import Buffer + +from mypy.nodes import ( + FUNCBASE_FLAGS, + FUNCDEF_FLAGS, + VAR_FLAGS, + ClassDef, + DataclassTransformSpec, + Decorator, + FuncDef, + MypyFile, + OverloadedFuncDef, + OverloadPart, + ParamSpecExpr, + SymbolNode, + SymbolTable, + SymbolTableNode, + TypeAlias, + TypeInfo, + TypeVarExpr, + TypeVarTupleExpr, + Var, + get_flags, + node_kinds, +) +from mypy.types import ( + NOT_READY, + AnyType, + CallableType, + ExtraAttrs, + Instance, + LiteralType, + NoneType, + Overloaded, + Parameters, + ParamSpecType, + TupleType, + Type, + TypeAliasType, + TypedDictType, + TypeType, + TypeVarTupleType, + TypeVarType, + UnboundType, + UninhabitedType, + UnionType, + UnpackType, + get_proper_type, +) + +Json: _TypeAlias = Union[dict[str, Any], str] + + +class Config: + def __init__(self, *, implicit_names: bool = True) -> None: + self.implicit_names = implicit_names + + +def convert_binary_cache_to_json(data: bytes, *, implicit_names: bool = True) -> Json: + tree = MypyFile.read(Buffer(data)) + return convert_mypy_file_to_json(tree, Config(implicit_names=implicit_names)) + + +def convert_mypy_file_to_json(self: MypyFile, cfg: Config) -> Json: + return { + ".class": "MypyFile", + "_fullname": self._fullname, + "names": convert_symbol_table(self.names, cfg), + "is_stub": self.is_stub, + "path": self.path, + "is_partial_stub_package": self.is_partial_stub_package, + "future_import_flags": sorted(self.future_import_flags), + } + + +def convert_symbol_table(self: SymbolTable, cfg: Config) -> Json: + data: dict[str, Any] = {".class": "SymbolTable"} + for key, value in self.items(): + # Skip __builtins__: it's a reference to the builtins + # module that gets added to every module by + # SemanticAnalyzerPass2.visit_file(), but it shouldn't be + # accessed by users of the module. + if key == "__builtins__" or value.no_serialize: + continue + if not cfg.implicit_names and key in { + "__spec__", + "__package__", + "__file__", + "__doc__", + "__annotations__", + "__name__", + }: + continue + data[key] = convert_symbol_table_node(value, cfg) + return data + + +def convert_symbol_table_node(self: SymbolTableNode, cfg: Config) -> Json: + data: dict[str, Any] = {".class": "SymbolTableNode", "kind": node_kinds[self.kind]} + if self.module_hidden: + data["module_hidden"] = True + if not self.module_public: + data["module_public"] = False + if self.implicit: + data["implicit"] = True + if self.plugin_generated: + data["plugin_generated"] = True + if self.cross_ref: + data["cross_ref"] = self.cross_ref + elif self.node is not None: + data["node"] = convert_symbol_node(self.node, cfg) + return data + + +def convert_symbol_node(self: SymbolNode, cfg: Config) -> Json: + if isinstance(self, FuncDef): + return convert_func_def(self) + elif isinstance(self, OverloadedFuncDef): + return convert_overloaded_func_def(self) + elif isinstance(self, Decorator): + return convert_decorator(self) + elif isinstance(self, Var): + return convert_var(self) + elif isinstance(self, TypeInfo): + return convert_type_info(self, cfg) + elif isinstance(self, TypeAlias): + return convert_type_alias(self) + elif isinstance(self, TypeVarExpr): + return convert_type_var_expr(self) + elif isinstance(self, ParamSpecExpr): + return convert_param_spec_expr(self) + elif isinstance(self, TypeVarTupleExpr): + return convert_type_var_tuple_expr(self) + return {"ERROR": f"{type(self)!r} unrecognized"} + + +def convert_func_def(self: FuncDef) -> Json: + return { + ".class": "FuncDef", + "name": self._name, + "fullname": self._fullname, + "arg_names": self.arg_names, + "arg_kinds": [int(x.value) for x in self.arg_kinds], + "type": None if self.type is None else convert_type(self.type), + "flags": get_flags(self, FUNCDEF_FLAGS), + "abstract_status": self.abstract_status, + # TODO: Do we need expanded, original_def? + "dataclass_transform_spec": ( + None + if self.dataclass_transform_spec is None + else convert_dataclass_transform_spec(self.dataclass_transform_spec) + ), + "deprecated": self.deprecated, + "original_first_arg": self.original_first_arg, + } + + +def convert_dataclass_transform_spec(self: DataclassTransformSpec) -> Json: + return { + "eq_default": self.eq_default, + "order_default": self.order_default, + "kw_only_default": self.kw_only_default, + "frozen_default": self.frozen_default, + "field_specifiers": list(self.field_specifiers), + } + + +def convert_overloaded_func_def(self: OverloadedFuncDef) -> Json: + return { + ".class": "OverloadedFuncDef", + "items": [convert_overload_part(i) for i in self.items], + "type": None if self.type is None else convert_type(self.type), + "fullname": self._fullname, + "impl": None if self.impl is None else convert_overload_part(self.impl), + "flags": get_flags(self, FUNCBASE_FLAGS), + "deprecated": self.deprecated, + "setter_index": self.setter_index, + } + + +def convert_overload_part(self: OverloadPart) -> Json: + if isinstance(self, FuncDef): + return convert_func_def(self) + else: + return convert_decorator(self) + + +def convert_decorator(self: Decorator) -> Json: + return { + ".class": "Decorator", + "func": convert_func_def(self.func), + "var": convert_var(self.var), + "is_overload": self.is_overload, + } + + +def convert_var(self: Var) -> Json: + data: dict[str, Any] = { + ".class": "Var", + "name": self._name, + "fullname": self._fullname, + "type": None if self.type is None else convert_type(self.type), + "setter_type": None if self.setter_type is None else convert_type(self.setter_type), + "flags": get_flags(self, VAR_FLAGS), + } + if self.final_value is not None: + data["final_value"] = self.final_value + return data + + +def convert_type_info(self: TypeInfo, cfg: Config) -> Json: + data = { + ".class": "TypeInfo", + "module_name": self.module_name, + "fullname": self.fullname, + "names": convert_symbol_table(self.names, cfg), + "defn": convert_class_def(self.defn), + "abstract_attributes": self.abstract_attributes, + "type_vars": self.type_vars, + "has_param_spec_type": self.has_param_spec_type, + "bases": [convert_type(b) for b in self.bases], + "mro": self._mro_refs, + "_promote": [convert_type(p) for p in self._promote], + "alt_promote": None if self.alt_promote is None else convert_type(self.alt_promote), + "declared_metaclass": ( + None if self.declared_metaclass is None else convert_type(self.declared_metaclass) + ), + "metaclass_type": ( + None if self.metaclass_type is None else convert_type(self.metaclass_type) + ), + "tuple_type": None if self.tuple_type is None else convert_type(self.tuple_type), + "typeddict_type": ( + None if self.typeddict_type is None else convert_typeddict_type(self.typeddict_type) + ), + "flags": get_flags(self, TypeInfo.FLAGS), + "metadata": self.metadata, + "slots": sorted(self.slots) if self.slots is not None else None, + "deletable_attributes": self.deletable_attributes, + "self_type": convert_type(self.self_type) if self.self_type is not None else None, + "dataclass_transform_spec": ( + convert_dataclass_transform_spec(self.dataclass_transform_spec) + if self.dataclass_transform_spec is not None + else None + ), + "deprecated": self.deprecated, + } + return data + + +def convert_class_def(self: ClassDef) -> Json: + return { + ".class": "ClassDef", + "name": self.name, + "fullname": self.fullname, + "type_vars": [convert_type(v) for v in self.type_vars], + } + + +def convert_type_alias(self: TypeAlias) -> Json: + data: Json = { + ".class": "TypeAlias", + "fullname": self._fullname, + "module": self.module, + "target": convert_type(self.target), + "alias_tvars": [convert_type(v) for v in self.alias_tvars], + "no_args": self.no_args, + "normalized": self.normalized, + "python_3_12_type_alias": self.python_3_12_type_alias, + } + return data + + +def convert_type_var_expr(self: TypeVarExpr) -> Json: + return { + ".class": "TypeVarExpr", + "name": self._name, + "fullname": self._fullname, + "values": [convert_type(t) for t in self.values], + "upper_bound": convert_type(self.upper_bound), + "default": convert_type(self.default), + "variance": self.variance, + } + + +def convert_param_spec_expr(self: ParamSpecExpr) -> Json: + return { + ".class": "ParamSpecExpr", + "name": self._name, + "fullname": self._fullname, + "upper_bound": convert_type(self.upper_bound), + "default": convert_type(self.default), + "variance": self.variance, + } + + +def convert_type_var_tuple_expr(self: TypeVarTupleExpr) -> Json: + return { + ".class": "TypeVarTupleExpr", + "name": self._name, + "fullname": self._fullname, + "upper_bound": convert_type(self.upper_bound), + "tuple_fallback": convert_type(self.tuple_fallback), + "default": convert_type(self.default), + "variance": self.variance, + } + + +def convert_type(typ: Type) -> Json: + if type(typ) is TypeAliasType: + return convert_type_alias_type(typ) + typ = get_proper_type(typ) + if isinstance(typ, Instance): + return convert_instance(typ) + elif isinstance(typ, AnyType): + return convert_any_type(typ) + elif isinstance(typ, NoneType): + return convert_none_type(typ) + elif isinstance(typ, UnionType): + return convert_union_type(typ) + elif isinstance(typ, TupleType): + return convert_tuple_type(typ) + elif isinstance(typ, CallableType): + return convert_callable_type(typ) + elif isinstance(typ, Overloaded): + return convert_overloaded(typ) + elif isinstance(typ, LiteralType): + return convert_literal_type(typ) + elif isinstance(typ, TypeVarType): + return convert_type_var_type(typ) + elif isinstance(typ, TypeType): + return convert_type_type(typ) + elif isinstance(typ, UninhabitedType): + return convert_uninhabited_type(typ) + elif isinstance(typ, UnpackType): + return convert_unpack_type(typ) + elif isinstance(typ, ParamSpecType): + return convert_param_spec_type(typ) + elif isinstance(typ, TypeVarTupleType): + return convert_type_var_tuple_type(typ) + elif isinstance(typ, Parameters): + return convert_parameters(typ) + elif isinstance(typ, TypedDictType): + return convert_typeddict_type(typ) + elif isinstance(typ, UnboundType): + return convert_unbound_type(typ) + return {"ERROR": f"{type(typ)!r} unrecognized"} + + +def convert_instance(self: Instance) -> Json: + ready = self.type is not NOT_READY + if not self.args and not self.last_known_value and not self.extra_attrs: + if ready: + return self.type.fullname + elif self.type_ref: + return self.type_ref + + data: dict[str, Any] = { + ".class": "Instance", + "type_ref": self.type.fullname if ready else self.type_ref, + "args": [convert_type(arg) for arg in self.args], + } + if self.last_known_value is not None: + data["last_known_value"] = convert_type(self.last_known_value) + data["extra_attrs"] = convert_extra_attrs(self.extra_attrs) if self.extra_attrs else None + return data + + +def convert_extra_attrs(self: ExtraAttrs) -> Json: + return { + ".class": "ExtraAttrs", + "attrs": {k: convert_type(v) for k, v in self.attrs.items()}, + "immutable": sorted(self.immutable), + "mod_name": self.mod_name, + } + + +def convert_type_alias_type(self: TypeAliasType) -> Json: + data: Json = { + ".class": "TypeAliasType", + "type_ref": self.type_ref, + "args": [convert_type(arg) for arg in self.args], + } + return data + + +def convert_any_type(self: AnyType) -> Json: + return { + ".class": "AnyType", + "type_of_any": self.type_of_any, + "source_any": convert_type(self.source_any) if self.source_any is not None else None, + "missing_import_name": self.missing_import_name, + } + + +def convert_none_type(self: NoneType) -> Json: + return {".class": "NoneType"} + + +def convert_union_type(self: UnionType) -> Json: + return { + ".class": "UnionType", + "items": [convert_type(t) for t in self.items], + "uses_pep604_syntax": self.uses_pep604_syntax, + } + + +def convert_tuple_type(self: TupleType) -> Json: + return { + ".class": "TupleType", + "items": [convert_type(t) for t in self.items], + "partial_fallback": convert_type(self.partial_fallback), + "implicit": self.implicit, + } + + +def convert_literal_type(self: LiteralType) -> Json: + return {".class": "LiteralType", "value": self.value, "fallback": convert_type(self.fallback)} + + +def convert_type_var_type(self: TypeVarType) -> Json: + assert not self.id.is_meta_var() + return { + ".class": "TypeVarType", + "name": self.name, + "fullname": self.fullname, + "id": self.id.raw_id, + "namespace": self.id.namespace, + "values": [convert_type(v) for v in self.values], + "upper_bound": convert_type(self.upper_bound), + "default": convert_type(self.default), + "variance": self.variance, + } + + +def convert_callable_type(self: CallableType) -> Json: + return { + ".class": "CallableType", + "arg_types": [convert_type(t) for t in self.arg_types], + "arg_kinds": [int(x.value) for x in self.arg_kinds], + "arg_names": self.arg_names, + "ret_type": convert_type(self.ret_type), + "fallback": convert_type(self.fallback), + "name": self.name, + # We don't serialize the definition (only used for error messages). + "variables": [convert_type(v) for v in self.variables], + "is_ellipsis_args": self.is_ellipsis_args, + "implicit": self.implicit, + "is_bound": self.is_bound, + "type_guard": convert_type(self.type_guard) if self.type_guard is not None else None, + "type_is": convert_type(self.type_is) if self.type_is is not None else None, + "from_concatenate": self.from_concatenate, + "imprecise_arg_kinds": self.imprecise_arg_kinds, + "unpack_kwargs": self.unpack_kwargs, + } + + +def convert_overloaded(self: Overloaded) -> Json: + return {".class": "Overloaded", "items": [convert_type(t) for t in self.items]} + + +def convert_type_type(self: TypeType) -> Json: + return {".class": "TypeType", "item": convert_type(self.item)} + + +def convert_uninhabited_type(self: UninhabitedType) -> Json: + return {".class": "UninhabitedType"} + + +def convert_unpack_type(self: UnpackType) -> Json: + return {".class": "UnpackType", "type": convert_type(self.type)} + + +def convert_param_spec_type(self: ParamSpecType) -> Json: + assert not self.id.is_meta_var() + return { + ".class": "ParamSpecType", + "name": self.name, + "fullname": self.fullname, + "id": self.id.raw_id, + "namespace": self.id.namespace, + "flavor": self.flavor, + "upper_bound": convert_type(self.upper_bound), + "default": convert_type(self.default), + "prefix": convert_type(self.prefix), + } + + +def convert_type_var_tuple_type(self: TypeVarTupleType) -> Json: + assert not self.id.is_meta_var() + return { + ".class": "TypeVarTupleType", + "name": self.name, + "fullname": self.fullname, + "id": self.id.raw_id, + "namespace": self.id.namespace, + "upper_bound": convert_type(self.upper_bound), + "tuple_fallback": convert_type(self.tuple_fallback), + "default": convert_type(self.default), + "min_len": self.min_len, + } + + +def convert_parameters(self: Parameters) -> Json: + return { + ".class": "Parameters", + "arg_types": [convert_type(t) for t in self.arg_types], + "arg_kinds": [int(x.value) for x in self.arg_kinds], + "arg_names": self.arg_names, + "variables": [convert_type(tv) for tv in self.variables], + "imprecise_arg_kinds": self.imprecise_arg_kinds, + } + + +def convert_typeddict_type(self: TypedDictType) -> Json: + return { + ".class": "TypedDictType", + "items": [[n, convert_type(t)] for (n, t) in self.items.items()], + "required_keys": sorted(self.required_keys), + "readonly_keys": sorted(self.readonly_keys), + "fallback": convert_type(self.fallback), + } + + +def convert_unbound_type(self: UnboundType) -> Json: + return { + ".class": "UnboundType", + "name": self.name, + "args": [convert_type(a) for a in self.args], + "expr": self.original_str_expr, + "expr_fallback": self.original_str_fallback, + } + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Convert binary cache files to JSON. " + "Create files in the same directory with extra .json extension." + ) + parser.add_argument( + "path", nargs="+", help="mypy cache data file to convert (.data.ff extension)" + ) + args = parser.parse_args() + fnams: list[str] = args.path + for fnam in fnams: + if not fnam.endswith(".data.ff"): + sys.exit(f"error: Expected .data.ff extension, but got {fnam}") + with open(fnam, "rb") as f: + data = f.read() + json_data = convert_binary_cache_to_json(data) + new_fnam = fnam + ".json" + with open(new_fnam, "w") as f: + json.dump(json_data, f) + print(f"{fnam} -> {new_fnam}") + + +if __name__ == "__main__": + main() diff --git a/mypy/test/testexportjson.py b/mypy/test/testexportjson.py new file mode 100644 index 0000000000000..13bd96d066427 --- /dev/null +++ b/mypy/test/testexportjson.py @@ -0,0 +1,70 @@ +"""Test cases for the mypy cache JSON export tool.""" + +from __future__ import annotations + +import json +import os +import re +import sys + +from mypy import build +from mypy.errors import CompileError +from mypy.exportjson import convert_binary_cache_to_json +from mypy.modulefinder import BuildSource +from mypy.options import Options +from mypy.test.config import test_temp_dir +from mypy.test.data import DataDrivenTestCase, DataSuite +from mypy.test.helpers import assert_string_arrays_equal + + +class TypeExportSuite(DataSuite): + required_out_section = True + files = ["exportjson.test"] + + def run_case(self, testcase: DataDrivenTestCase) -> None: + error = False + src = "\n".join(testcase.input) + try: + options = Options() + options.use_builtins_fixtures = True + options.show_traceback = True + options.allow_empty_bodies = True + options.fixed_format_cache = True + fnam = os.path.join(self.base_path, "main.py") + with open(fnam, "w") as f: + f.write(src) + result = build.build( + sources=[BuildSource(fnam, "main")], options=options, alt_lib_path=test_temp_dir + ) + a = result.errors + error = bool(a) + + major, minor = sys.version_info[:2] + cache_dir = os.path.join(".mypy_cache", f"{major}.{minor}") + + for module in result.files: + if module in ( + "builtins", + "typing", + "_typeshed", + "__future__", + "typing_extensions", + "sys", + ): + continue + fnam = os.path.join(cache_dir, f"{module}.data.ff") + with open(fnam, "rb") as f: + json_data = convert_binary_cache_to_json(f.read(), implicit_names=False) + for line in json.dumps(json_data, indent=4).splitlines(): + if '"path": ' in line: + # We source file path is unpredictable, so filter it out + line = re.sub(r'"[^"]+\.pyi?"', "...", line) + assert "ERROR" not in line, line + a.append(line) + except CompileError as e: + a = e.messages + error = True + if error or "\n".join(testcase.output).strip() != "": + assert_string_arrays_equal( + testcase.output, a, f"Invalid output ({testcase.file}, line {testcase.line})" + ) diff --git a/setup.py b/setup.py index 1d093ec3b9e2c..0037624f9bbc2 100644 --- a/setup.py +++ b/setup.py @@ -81,6 +81,7 @@ def run(self) -> None: "__main__.py", "pyinfo.py", os.path.join("dmypy", "__main__.py"), + "exportjson.py", # Uses __getattr__/__setattr__ "split_namespace.py", # Lies to mypy about code reachability diff --git a/test-data/unit/exportjson.test b/test-data/unit/exportjson.test new file mode 100644 index 0000000000000..14295281a48f8 --- /dev/null +++ b/test-data/unit/exportjson.test @@ -0,0 +1,280 @@ +-- Test cases for exporting mypy cache files to JSON (mypy.exportjson). +-- +-- The tool is maintained on a best effort basis so we don't attempt to have +-- full test coverage. +-- +-- Some tests only ensure that *some* JSON is generated successfully. These +-- have as [out]. + +[case testExportVar] +x = 0 +[out] +{ + ".class": "MypyFile", + "_fullname": "main", + "names": { + ".class": "SymbolTable", + "x": { + ".class": "SymbolTableNode", + "kind": "Gdef", + "node": { + ".class": "Var", + "name": "x", + "fullname": "main.x", + "type": "builtins.int", + "setter_type": null, + "flags": [ + "is_ready", + "is_inferred", + "has_explicit_value" + ] + } + } + }, + "is_stub": false, + "path": ..., + "is_partial_stub_package": false, + "future_import_flags": [] +} + +[case testExportClass] +class C: + x: int +[out] +{ + ".class": "MypyFile", + "_fullname": "main", + "names": { + ".class": "SymbolTable", + "C": { + ".class": "SymbolTableNode", + "kind": "Gdef", + "node": { + ".class": "TypeInfo", + "module_name": "main", + "fullname": "main.C", + "names": { + ".class": "SymbolTable", + "x": { + ".class": "SymbolTableNode", + "kind": "Mdef", + "node": { + ".class": "Var", + "name": "x", + "fullname": "main.C.x", + "type": "builtins.int", + "setter_type": null, + "flags": [ + "is_initialized_in_class", + "is_ready" + ] + } + } + }, + "defn": { + ".class": "ClassDef", + "name": "C", + "fullname": "main.C", + "type_vars": [] + }, + "abstract_attributes": [], + "type_vars": [], + "has_param_spec_type": false, + "bases": [ + "builtins.object" + ], + "mro": [ + "main.C", + "builtins.object" + ], + "_promote": [], + "alt_promote": null, + "declared_metaclass": null, + "metaclass_type": null, + "tuple_type": null, + "typeddict_type": null, + "flags": [], + "metadata": {}, + "slots": null, + "deletable_attributes": [], + "self_type": null, + "dataclass_transform_spec": null, + "deprecated": null + } + } + }, + "is_stub": false, + "path": ..., + "is_partial_stub_package": false, + "future_import_flags": [] +} + +[case testExportCrossRef] +from typing import Any +[out] +{ + ".class": "MypyFile", + "_fullname": "main", + "names": { + ".class": "SymbolTable", + "Any": { + ".class": "SymbolTableNode", + "kind": "Gdef", + "cross_ref": "typing.Any" + } + }, + "is_stub": false, + "path": ..., + "is_partial_stub_package": false, + "future_import_flags": [] +} + +[case testExportFuncDef] +def foo(a: int) -> None: ... +[out] +{ + ".class": "MypyFile", + "_fullname": "main", + "names": { + ".class": "SymbolTable", + "foo": { + ".class": "SymbolTableNode", + "kind": "Gdef", + "node": { + ".class": "FuncDef", + "name": "foo", + "fullname": "main.foo", + "arg_names": [ + "a" + ], + "arg_kinds": [ + 0 + ], + "type": { + ".class": "CallableType", + "arg_types": [ + "builtins.int" + ], + "arg_kinds": [ + 0 + ], + "arg_names": [ + "a" + ], + "ret_type": { + ".class": "NoneType" + }, + "fallback": "builtins.function", + "name": "foo", + "variables": [], + "is_ellipsis_args": false, + "implicit": false, + "is_bound": false, + "type_guard": null, + "type_is": null, + "from_concatenate": false, + "imprecise_arg_kinds": false, + "unpack_kwargs": false + }, + "flags": [], + "abstract_status": 0, + "dataclass_transform_spec": null, + "deprecated": null, + "original_first_arg": "a" + } + } + }, + "is_stub": false, + "path": ..., + "is_partial_stub_package": false, + "future_import_flags": [] +} + +[case testExportDifferentTypes] +from __future__ import annotations + +from typing import Callable, Any, Literal, NoReturn, TypedDict, NamedTuple + +list_ann: list[int] +any_ann: Any +tuple_ann: tuple[int, str] +union_ann: int | None +callable_ann: Callable[[int], str] +type_type_ann: type[int] +literal_ann: Literal['x', 5, False] + +def f() -> NoReturn: + assert False + +BadType = 1 +x: BadType # type: ignore + +class TD(TypedDict): + x: int + +td = TD(x=1) + +NT = NamedTuple("NT", [("x", int)]) + +nt = NT(x=1) + +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-medium.pyi] +[out] + + +[case testExportGenericTypes] +from __future__ import annotations + +from typing import TypeVar, Callable +from typing_extensions import TypeVarTuple, ParamSpec, Unpack, Concatenate + +T = TypeVar("T") + +def ident(x: T) -> T: + return x + +Ts = TypeVarTuple("Ts") + +def ts(t: tuple[Unpack[Ts]]) -> tuple[Unpack[Ts]]: + return t + +P = ParamSpec("P") + +def pspec(f: Callable[P, None], *args: P.args, **kwargs: P.kwargs) -> None: + f(*args, **kwargs) + +def concat(f: Callable[Concatenate[int, P], None], *args: P.args, **kwargs: P.kwargs) -> None: + f(1, *args, **kwargs) + +[builtins fixtures/tuple.pyi] +[out] + + +[case testExportDifferentNodes] +from __future__ import annotations + +import typing + +from typing import overload, TypeVar + +@overload +def f(x: int) -> int: ... +@overload +def f(x: str) -> str: ... +def f(x: int | str) -> int | str: ... + +T = TypeVar("T") + +def deco(f: T) -> T: + return f + +@deco +def foo(x: int) -> int: ... + +X = int +x: X = 2 + +[builtins fixtures/tuple.pyi] +[out] + From fd20f348813e471d4630253d3023a5b579c47017 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 16 Oct 2025 12:54:50 +0100 Subject: [PATCH 334/424] Support writing/reading bytes in librt.internal (#20069) This is for storing actual `bytes` objects (not arbitrary sub-buffers). This will be useful for storing various hashes in cache metas, since storing them as hex strings takes twice more space and currently these hashes take a macroscopic part of the cache metas' sizes. @JukkaL for training purposes you can sync/release this one yourself (if you want to, of course). --- mypy/typeshed/stubs/librt/librt/internal.pyi | 2 + mypyc/lib-rt/librt_internal.c | 100 ++++++++++++++++++- mypyc/lib-rt/librt_internal.h | 4 + mypyc/primitives/misc_ops.py | 16 +++ mypyc/test-data/irbuild-classes.test | 61 ++++++----- mypyc/test-data/run-classes.test | 22 +++- 6 files changed, 177 insertions(+), 28 deletions(-) diff --git a/mypy/typeshed/stubs/librt/librt/internal.pyi b/mypy/typeshed/stubs/librt/librt/internal.pyi index a47a4849fe204..8a5fc262931e7 100644 --- a/mypy/typeshed/stubs/librt/librt/internal.pyi +++ b/mypy/typeshed/stubs/librt/librt/internal.pyi @@ -8,6 +8,8 @@ def write_bool(data: Buffer, value: bool) -> None: ... def read_bool(data: Buffer) -> bool: ... def write_str(data: Buffer, value: str) -> None: ... def read_str(data: Buffer) -> str: ... +def write_bytes(data: Buffer, value: bytes) -> None: ... +def read_bytes(data: Buffer) -> bytes: ... def write_float(data: Buffer, value: float) -> None: ... def read_float(data: Buffer) -> float: ... def write_int(data: Buffer, value: int) -> None: ... diff --git a/mypyc/lib-rt/librt_internal.c b/mypyc/lib-rt/librt_internal.c index b97d6665b515a..6f6a110446ade 100644 --- a/mypyc/lib-rt/librt_internal.c +++ b/mypyc/lib-rt/librt_internal.c @@ -346,6 +346,100 @@ write_str(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames return Py_None; } +/* +bytes format: size followed by bytes + short bytes (len <= 127): single byte for size as `(uint8_t)size << 1` + long bytes: \x01 followed by size as Py_ssize_t +*/ + +static PyObject* +read_bytes_internal(PyObject *data) { + _CHECK_BUFFER(data, NULL) + + // Read length. + Py_ssize_t size; + _CHECK_READ(data, 1, NULL) + uint8_t first = _READ(data, uint8_t) + if (likely(first != LONG_STR_TAG)) { + // Common case: short bytes (len <= 127). + size = (Py_ssize_t)(first >> 1); + } else { + _CHECK_READ(data, sizeof(CPyTagged), NULL) + size = _READ(data, Py_ssize_t) + } + // Read bytes content. + char *buf = ((BufferObject *)data)->buf; + _CHECK_READ(data, size, NULL) + PyObject *res = PyBytes_FromStringAndSize( + buf + ((BufferObject *)data)->pos, (Py_ssize_t)size + ); + if (unlikely(res == NULL)) + return NULL; + ((BufferObject *)data)->pos += size; + return res; +} + +static PyObject* +read_bytes(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames) { + static const char * const kwlist[] = {"data", 0}; + static CPyArg_Parser parser = {"O:read_bytes", kwlist, 0}; + PyObject *data; + if (unlikely(!CPyArg_ParseStackAndKeywordsOneArg(args, nargs, kwnames, &parser, &data))) { + return NULL; + } + return read_bytes_internal(data); +} + +static char +write_bytes_internal(PyObject *data, PyObject *value) { + _CHECK_BUFFER(data, CPY_NONE_ERROR) + + const char *chunk = PyBytes_AsString(value); + if (unlikely(chunk == NULL)) + return CPY_NONE_ERROR; + Py_ssize_t size = PyBytes_GET_SIZE(value); + + Py_ssize_t need; + // Write length. + if (likely(size <= MAX_SHORT_LEN)) { + // Common case: short bytes (len <= 127) store as single byte. + need = size + 1; + _CHECK_SIZE(data, need) + _WRITE(data, uint8_t, (uint8_t)size << 1) + } else { + need = size + sizeof(Py_ssize_t) + 1; + _CHECK_SIZE(data, need) + _WRITE(data, uint8_t, LONG_STR_TAG) + _WRITE(data, Py_ssize_t, size) + } + // Write bytes content. + char *buf = ((BufferObject *)data)->buf; + memcpy(buf + ((BufferObject *)data)->pos, chunk, size); + ((BufferObject *)data)->pos += size; + ((BufferObject *)data)->end += need; + return CPY_NONE; +} + +static PyObject* +write_bytes(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames) { + static const char * const kwlist[] = {"data", "value", 0}; + static CPyArg_Parser parser = {"OO:write_bytes", kwlist, 0}; + PyObject *data; + PyObject *value; + if (unlikely(!CPyArg_ParseStackAndKeywordsSimple(args, nargs, kwnames, &parser, &data, &value))) { + return NULL; + } + if (unlikely(!PyBytes_Check(value))) { + PyErr_SetString(PyExc_TypeError, "value must be a bytes object"); + return NULL; + } + if (unlikely(write_bytes_internal(data, value) == CPY_NONE_ERROR)) { + return NULL; + } + Py_INCREF(Py_None); + return Py_None; +} + /* float format: stored as a C double @@ -565,6 +659,8 @@ static PyMethodDef librt_internal_module_methods[] = { {"read_bool", (PyCFunction)read_bool, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("read a bool")}, {"write_str", (PyCFunction)write_str, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("write a string")}, {"read_str", (PyCFunction)read_str, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("read a string")}, + {"write_bytes", (PyCFunction)write_bytes, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("write bytes")}, + {"read_bytes", (PyCFunction)read_bytes, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("read bytes")}, {"write_float", (PyCFunction)write_float, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("write a float")}, {"read_float", (PyCFunction)read_float, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("read a float")}, {"write_int", (PyCFunction)write_int, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("write an int")}, @@ -590,7 +686,7 @@ librt_internal_module_exec(PyObject *m) } // Export mypy internal C API, be careful with the order! - static void *NativeInternal_API[14] = { + static void *NativeInternal_API[16] = { (void *)Buffer_internal, (void *)Buffer_internal_empty, (void *)Buffer_getvalue_internal, @@ -605,6 +701,8 @@ librt_internal_module_exec(PyObject *m) (void *)write_tag_internal, (void *)read_tag_internal, (void *)NativeInternal_ABI_Version, + (void *)write_bytes_internal, + (void *)read_bytes_internal, }; PyObject *c_api_object = PyCapsule_New((void *)NativeInternal_API, "librt.internal._C_API", NULL); if (PyModule_Add(m, "_C_API", c_api_object) < 0) { diff --git a/mypyc/lib-rt/librt_internal.h b/mypyc/lib-rt/librt_internal.h index fd8ec2422cc5b..d996b8fd95c1e 100644 --- a/mypyc/lib-rt/librt_internal.h +++ b/mypyc/lib-rt/librt_internal.h @@ -19,6 +19,8 @@ static CPyTagged read_int_internal(PyObject *data); static char write_tag_internal(PyObject *data, uint8_t value); static uint8_t read_tag_internal(PyObject *data); static int NativeInternal_ABI_Version(void); +static char write_bytes_internal(PyObject *data, PyObject *value); +static PyObject *read_bytes_internal(PyObject *data); #else @@ -38,6 +40,8 @@ static void **NativeInternal_API; #define write_tag_internal (*(char (*)(PyObject *source, uint8_t value)) NativeInternal_API[11]) #define read_tag_internal (*(uint8_t (*)(PyObject *source)) NativeInternal_API[12]) #define NativeInternal_ABI_Version (*(int (*)(void)) NativeInternal_API[13]) +#define write_bytes_internal (*(char (*)(PyObject *source, PyObject *value)) NativeInternal_API[14]) +#define read_bytes_internal (*(PyObject* (*)(PyObject *source)) NativeInternal_API[15]) static int import_librt_internal(void) diff --git a/mypyc/primitives/misc_ops.py b/mypyc/primitives/misc_ops.py index 18d475fe89d41..c12172875e8ba 100644 --- a/mypyc/primitives/misc_ops.py +++ b/mypyc/primitives/misc_ops.py @@ -393,6 +393,22 @@ error_kind=ERR_MAGIC, ) +function_op( + name="librt.internal.write_bytes", + arg_types=[object_rprimitive, bytes_rprimitive], + return_type=none_rprimitive, + c_function_name="write_bytes_internal", + error_kind=ERR_MAGIC, +) + +function_op( + name="librt.internal.read_bytes", + arg_types=[object_rprimitive], + return_type=bytes_rprimitive, + c_function_name="read_bytes_internal", + error_kind=ERR_MAGIC, +) + function_op( name="librt.internal.write_float", arg_types=[object_rprimitive, float_rprimitive], diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index 3280b21cf7e66..27ffba45ba396 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -1454,7 +1454,7 @@ from typing import Final from mypy_extensions import u8 from librt.internal import ( Buffer, write_bool, read_bool, write_str, read_str, write_float, read_float, - write_int, read_int, write_tag, read_tag + write_int, read_int, write_tag, read_tag, write_bytes, read_bytes, ) Tag = u8 @@ -1463,6 +1463,7 @@ TAG: Final[Tag] = 1 def foo() -> None: b = Buffer() write_str(b, "foo") + write_bytes(b, b"bar") write_bool(b, True) write_float(b, 0.1) write_int(b, 1) @@ -1470,6 +1471,7 @@ def foo() -> None: b = Buffer(b.getvalue()) x = read_str(b) + xb = read_bytes(b) y = read_bool(b) z = read_float(b) t = read_int(b) @@ -1478,36 +1480,43 @@ def foo() -> None: def foo(): r0, b :: librt.internal.Buffer r1 :: str - r2, r3, r4, r5, r6 :: None - r7 :: bytes - r8 :: librt.internal.Buffer - r9, x :: str - r10, y :: bool - r11, z :: float - r12, t :: int - r13, u :: u8 + r2 :: None + r3 :: bytes + r4, r5, r6, r7, r8 :: None + r9 :: bytes + r10 :: librt.internal.Buffer + r11, x :: str + r12, xb :: bytes + r13, y :: bool + r14, z :: float + r15, t :: int + r16, u :: u8 L0: r0 = Buffer_internal_empty() b = r0 r1 = 'foo' r2 = write_str_internal(b, r1) - r3 = write_bool_internal(b, 1) - r4 = write_float_internal(b, 0.1) - r5 = write_int_internal(b, 2) - r6 = write_tag_internal(b, 1) - r7 = Buffer_getvalue_internal(b) - r8 = Buffer_internal(r7) - b = r8 - r9 = read_str_internal(b) - x = r9 - r10 = read_bool_internal(b) - y = r10 - r11 = read_float_internal(b) - z = r11 - r12 = read_int_internal(b) - t = r12 - r13 = read_tag_internal(b) - u = r13 + r3 = b'bar' + r4 = write_bytes_internal(b, r3) + r5 = write_bool_internal(b, 1) + r6 = write_float_internal(b, 0.1) + r7 = write_int_internal(b, 2) + r8 = write_tag_internal(b, 1) + r9 = Buffer_getvalue_internal(b) + r10 = Buffer_internal(r9) + b = r10 + r11 = read_str_internal(b) + x = r11 + r12 = read_bytes_internal(b) + xb = r12 + r13 = read_bool_internal(b) + y = r13 + r14 = read_float_internal(b) + z = r14 + r15 = read_int_internal(b) + t = r15 + r16 = read_tag_internal(b) + u = r16 return 1 [case testEnumFastPath] diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 84704ce66c81d..efa6c225ecabd 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -2715,7 +2715,7 @@ from typing import Final from mypy_extensions import u8 from librt.internal import ( Buffer, write_bool, read_bool, write_str, read_str, write_float, read_float, - write_int, read_int, write_tag, read_tag + write_int, read_int, write_tag, read_tag, write_bytes, read_bytes ) Tag = u8 @@ -2733,6 +2733,11 @@ def test_buffer_roundtrip() -> None: write_bool(b, True) write_str(b, "bar" * 1000) write_bool(b, False) + write_bytes(b, b"bar") + write_bytes(b, b"bar" * 100) + write_bytes(b, b"") + write_bytes(b, b"a" * 127) + write_bytes(b, b"a" * 128) write_float(b, 0.1) write_int(b, 0) write_int(b, 1) @@ -2752,6 +2757,11 @@ def test_buffer_roundtrip() -> None: assert read_bool(b) is True assert read_str(b) == "bar" * 1000 assert read_bool(b) is False + assert read_bytes(b) == b"bar" + assert read_bytes(b) == b"bar" * 100 + assert read_bytes(b) == b"" + assert read_bytes(b) == b"a" * 127 + assert read_bytes(b) == b"a" * 128 assert read_float(b) == 0.1 assert read_int(b) == 0 assert read_int(b) == 1 @@ -2806,6 +2816,11 @@ def test_buffer_roundtrip_interpreted() -> None: write_bool(b, True) write_str(b, "bar" * 1000) write_bool(b, False) + write_bytes(b, b"bar") + write_bytes(b, b"bar" * 100) + write_bytes(b, b"") + write_bytes(b, b"a" * 127) + write_bytes(b, b"a" * 128) write_float(b, 0.1) write_int(b, 0) write_int(b, 1) @@ -2825,6 +2840,11 @@ def test_buffer_roundtrip_interpreted() -> None: assert read_bool(b) is True assert read_str(b) == "bar" * 1000 assert read_bool(b) is False + assert read_bytes(b) == b"bar" + assert read_bytes(b) == b"bar" * 100 + assert read_bytes(b) == b"" + assert read_bytes(b) == b"a" * 127 + assert read_bytes(b) == b"a" * 128 assert read_float(b) == 0.1 assert read_int(b) == 0 assert read_int(b) == 1 From 72b0fcacb720121ebe241c12f25ef6c66435e5c9 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Thu, 16 Oct 2025 05:36:16 -0700 Subject: [PATCH 335/424] Bump version of wasilibs/go-shellcheck to fix lint on Windows (#20050) Currently, if you try to run our lint step on Windows, shellcheck crashes with a complaint about an empty env var key. However, that has just been fixed in https://github.com/wasilibs/go-shellcheck/issues/10. Closes https://github.com/python/mypy/issues/19958, which I have already preemptively closed. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a410585a52d4b..1f4282e4f65b5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,7 +41,7 @@ repos: # actionlint has a shellcheck integration which extracts shell scripts in `run:` steps from GitHub Actions # and checks these with shellcheck. This is arguably its most useful feature, # but the integration only works if shellcheck is installed - - "github.com/wasilibs/go-shellcheck/cmd/shellcheck@v0.11.0" + - "github.com/wasilibs/go-shellcheck/cmd/shellcheck@v0.11.1" - repo: https://github.com/woodruffw/zizmor-pre-commit rev: v1.5.2 hooks: From 9c26271945ee7c9095d769f2138f4f2e969445ce Mon Sep 17 00:00:00 2001 From: Randolf Scholz Date: Thu, 16 Oct 2025 16:39:52 +0200 Subject: [PATCH 336/424] [match-case] fix matching against `typing.Callable` and `Protocol` types. (#19471) - Fixes #14014 - Partially addresses #19470 Added extra logic in `checker.py:conditional_types` function to deal with structural types such as `typing.Callable` or protocols. ## new tests - `testMatchClassPatternCallable`: tests `case Callable() as fn` usage - `testMatchClassPatternProtocol`: tests `case Proto()` usage, where `Proto` is a Protocol - `testMatchClassPatternCallbackProtocol`: tests `case Proto()` usage, where `Proto` is a Callback-Protocol - `testGenericAliasIsinstanceUnreachable`: derived from a mypy-primer failure in mesonbuild. Tests that `isinstance(x, Proto)` can produce unreachable error. - `testGenericAliasRedundantExprCompoundIfExpr`: derived from a CI failure of `python runtest.py self` of an earlier version of this PR. ## modified tests - `testOverloadOnProtocol` added annotations to overload implementation, which wasn't getting checked. Added missing return. Fixed return type in second branch. --------- Co-authored-by: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Ivan Levkivskyi --- mypy/checker.py | 33 +++-- mypy/checkpattern.py | 10 ++ test-data/unit/check-generic-alias.test | 32 +++++ test-data/unit/check-protocols.test | 5 +- test-data/unit/check-python310.test | 156 ++++++++++++++++++++++++ 5 files changed, 225 insertions(+), 11 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 3bee7b633339d..754bd59a49625 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -8174,11 +8174,15 @@ def conditional_types( ) -> tuple[Type | None, Type | None]: """Takes in the current type and a proposed type of an expression. - Returns a 2-tuple: The first element is the proposed type, if the expression - can be the proposed type. The second element is the type it would hold - if it was not the proposed type, if any. UninhabitedType means unreachable. - None means no new information can be inferred. If default is set it is returned - instead.""" + Returns a 2-tuple: + The first element is the proposed type, if the expression can be the proposed type. + (or default, if default is set and the expression is a subtype of the proposed type). + The second element is the type it would hold if it was not the proposed type, if any. + (or default, if default is set and the expression is not a subtype of the proposed type). + + UninhabitedType means unreachable. + None means no new information can be inferred. + """ if proposed_type_ranges: if len(proposed_type_ranges) == 1: target = proposed_type_ranges[0].item @@ -8190,14 +8194,25 @@ def conditional_types( current_type = try_expanding_sum_type_to_union(current_type, enum_name) proposed_items = [type_range.item for type_range in proposed_type_ranges] proposed_type = make_simplified_union(proposed_items) - if isinstance(proposed_type, AnyType): + if isinstance(get_proper_type(current_type), AnyType): + return proposed_type, current_type + elif isinstance(proposed_type, AnyType): # We don't really know much about the proposed type, so we shouldn't # attempt to narrow anything. Instead, we broaden the expr to Any to # avoid false positives return proposed_type, default - elif not any( - type_range.is_upper_bound for type_range in proposed_type_ranges - ) and is_proper_subtype(current_type, proposed_type, ignore_promotions=True): + elif not any(type_range.is_upper_bound for type_range in proposed_type_ranges) and ( + # concrete subtypes + is_proper_subtype(current_type, proposed_type, ignore_promotions=True) + # structural subtypes + or ( + ( + isinstance(proposed_type, CallableType) + or (isinstance(proposed_type, Instance) and proposed_type.type.is_protocol) + ) + and is_subtype(current_type, proposed_type, ignore_promotions=True) + ) + ): # Expression is always of one of the types in proposed_type_ranges return default, UninhabitedType() elif not is_overlapping_types(current_type, proposed_type, ignore_promotions=True): diff --git a/mypy/checkpattern.py b/mypy/checkpattern.py index 6f00c6c431771..3c51c41069097 100644 --- a/mypy/checkpattern.py +++ b/mypy/checkpattern.py @@ -51,6 +51,7 @@ UninhabitedType, UnionType, UnpackType, + callable_with_ellipsis, find_unpack_in_list, get_proper_type, split_with_prefix_and_suffix, @@ -546,6 +547,15 @@ def visit_class_pattern(self, o: ClassPattern) -> PatternType: return self.early_non_match() elif isinstance(p_typ, FunctionLike) and p_typ.is_type_obj(): typ = fill_typevars_with_any(p_typ.type_object()) + elif ( + isinstance(type_info, Var) + and type_info.type is not None + and type_info.fullname == "typing.Callable" + ): + # Create a `Callable[..., Any]` + fallback = self.chk.named_type("builtins.function") + any_type = AnyType(TypeOfAny.unannotated) + typ = callable_with_ellipsis(any_type, ret_type=any_type, fallback=fallback) elif not isinstance(p_typ, AnyType): self.msg.fail( message_registry.CLASS_PATTERN_TYPE_REQUIRED.format( diff --git a/test-data/unit/check-generic-alias.test b/test-data/unit/check-generic-alias.test index 678950a1e18bd..3f088308da64e 100644 --- a/test-data/unit/check-generic-alias.test +++ b/test-data/unit/check-generic-alias.test @@ -149,6 +149,38 @@ t23: collections.abc.ValuesView[str] # reveal_type(t23) # Nx Revealed type is "collections.abc.ValuesView[builtins.str]" [builtins fixtures/tuple.pyi] +[case testGenericAliasIsinstanceUnreachable] +# flags: --warn-unreachable --python-version 3.10 +from collections.abc import Iterable + +class A: ... + +def test(dependencies: list[A] | None) -> None: + if dependencies is None: + dependencies = [] + elif not isinstance(dependencies, Iterable): + dependencies = [dependencies] # E: Statement is unreachable + +[builtins fixtures/isinstancelist.pyi] +[typing fixtures/typing-full.pyi] + +[case testGenericAliasRedundantExprCompoundIfExpr] +# flags: --warn-unreachable --enable-error-code=redundant-expr --python-version 3.10 + +from typing import Any, reveal_type +from collections.abc import Iterable + +def test_example(x: Iterable[Any]) -> None: + if isinstance(x, Iterable) and not isinstance(x, str): # E: Left operand of "and" is always true + reveal_type(x) # N: Revealed type is "typing.Iterable[Any]" + +def test_counterexample(x: Any) -> None: + if isinstance(x, Iterable) and not isinstance(x, str): + reveal_type(x) # N: Revealed type is "typing.Iterable[Any]" + +[builtins fixtures/isinstancelist.pyi] +[typing fixtures/typing-full.pyi] + [case testGenericBuiltinTupleTyping] from typing import Tuple diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index 0f19b404082ef..ae6f603555127 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -1506,11 +1506,12 @@ class C: pass def f(x: P1) -> int: ... @overload def f(x: P2) -> str: ... -def f(x): +def f(x: object) -> object: if isinstance(x, P1): return P1.attr1 if isinstance(x, P2): # E: Only @runtime_checkable protocols can be used with instance and class checks - return P1.attr2 + return P2.attr2 + return None reveal_type(f(C1())) # N: Revealed type is "builtins.int" reveal_type(f(C2())) # N: Revealed type is "builtins.str" diff --git a/test-data/unit/check-python310.test b/test-data/unit/check-python310.test index 2c4597e212ea7..3a9f12e8a5500 100644 --- a/test-data/unit/check-python310.test +++ b/test-data/unit/check-python310.test @@ -11,11 +11,30 @@ match m: -- Literal Pattern -- [case testMatchLiteralPatternNarrows] +# flags: --warn-unreachable m: object match m: case 1: reveal_type(m) # N: Revealed type is "Literal[1]" + case 2: + reveal_type(m) # N: Revealed type is "Literal[2]" + case other: + reveal_type(other) # N: Revealed type is "builtins.object" + +[case testMatchLiteralPatternNarrows2] +# flags: --warn-unreachable +from typing import Any + +m: Any + +match m: + case 1: + reveal_type(m) # N: Revealed type is "Literal[1]" + case 2: + reveal_type(m) # N: Revealed type is "Literal[2]" + case other: + reveal_type(other) # N: Revealed type is "Any" [case testMatchLiteralPatternAlreadyNarrower-skip] m: bool @@ -1079,6 +1098,143 @@ match m: case Foo(): pass +[case testMatchClassPatternCallable] +# flags: --warn-unreachable +from typing import Callable, Any + +class FnImpl: + def __call__(self, x: object, /) -> int: ... + +def test_any(x: Any) -> None: + match x: + case Callable() as fn: + reveal_type(fn) # N: Revealed type is "def (*Any, **Any) -> Any" + case other: + reveal_type(other) # N: Revealed type is "Any" + +def test_object(x: object) -> None: + match x: + case Callable() as fn: + reveal_type(fn) # N: Revealed type is "def (*Any, **Any) -> Any" + case other: + reveal_type(other) # N: Revealed type is "builtins.object" + +def test_impl(x: FnImpl) -> None: + match x: + case Callable() as fn: + reveal_type(fn) # N: Revealed type is "__main__.FnImpl" + case other: + reveal_type(other) # E: Statement is unreachable + +def test_callable(x: Callable[[object], int]) -> None: + match x: + case Callable() as fn: + reveal_type(fn) # N: Revealed type is "def (builtins.object) -> builtins.int" + case other: + reveal_type(other) # E: Statement is unreachable + +[case testMatchClassPatternCallbackProtocol] +# flags: --warn-unreachable +from typing import Any, Callable +from typing_extensions import Protocol, runtime_checkable + +@runtime_checkable +class FnProto(Protocol): + def __call__(self, x: int, /) -> object: ... + +class FnImpl: + def __call__(self, x: object, /) -> int: ... + +def test_any(x: Any) -> None: + match x: + case FnProto() as fn: + reveal_type(fn) # N: Revealed type is "__main__.FnProto" + case other: + reveal_type(other) # N: Revealed type is "Any" + +def test_object(x: object) -> None: + match x: + case FnProto() as fn: + reveal_type(fn) # N: Revealed type is "__main__.FnProto" + case other: + reveal_type(other) # N: Revealed type is "builtins.object" + +def test_impl(x: FnImpl) -> None: + match x: + case FnProto() as fn: + reveal_type(fn) # N: Revealed type is "__main__.FnImpl" + case other: + reveal_type(other) # E: Statement is unreachable + +def test_callable(x: Callable[[object], int]) -> None: + match x: + case FnProto() as fn: + reveal_type(fn) # N: Revealed type is "def (builtins.object) -> builtins.int" + case other: + reveal_type(other) # E: Statement is unreachable + +[builtins fixtures/dict.pyi] + +[case testMatchClassPatternAnyCallableProtocol] +# flags: --warn-unreachable +from typing import Any, Callable +from typing_extensions import Protocol, runtime_checkable + +@runtime_checkable +class AnyCallable(Protocol): + def __call__(self, *args: Any, **kwargs: Any) -> Any: ... + +class FnImpl: + def __call__(self, x: object, /) -> int: ... + +def test_object(x: object) -> None: + match x: + case AnyCallable() as fn: + reveal_type(fn) # N: Revealed type is "__main__.AnyCallable" + case other: + reveal_type(other) # N: Revealed type is "builtins.object" + +def test_impl(x: FnImpl) -> None: + match x: + case AnyCallable() as fn: + reveal_type(fn) # N: Revealed type is "__main__.FnImpl" + case other: + reveal_type(other) # E: Statement is unreachable + +def test_callable(x: Callable[[object], int]) -> None: + match x: + case AnyCallable() as fn: + reveal_type(fn) # N: Revealed type is "def (builtins.object) -> builtins.int" + case other: + reveal_type(other) # E: Statement is unreachable + +[builtins fixtures/dict.pyi] + + +[case testMatchClassPatternProtocol] +from typing import Any +from typing_extensions import Protocol, runtime_checkable + +@runtime_checkable +class Proto(Protocol): + def foo(self, x: int, /) -> object: ... + +class Impl: + def foo(self, x: object, /) -> int: ... + +def test_object(x: object) -> None: + match x: + case Proto() as y: + reveal_type(y) # N: Revealed type is "__main__.Proto" + +def test_impl(x: Impl) -> None: + match x: + case Proto() as y: + reveal_type(y) # N: Revealed type is "__main__.Impl" + +[builtins fixtures/dict.pyi] + + [case testMatchClassPatternNestedGenerics] # From cpython test_patma.py x = [[{0: 0}]] From e1643aec5229e37cf62700dabae294b9a5344c97 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 16 Oct 2025 16:31:35 +0100 Subject: [PATCH 337/424] Switch to a more dynamic SCC processing logic (#20053) Ref https://github.com/python/mypy/issues/933 Instead of processing SCCs layer by layer, we will now process an SCC as soon as it is ready. This logic is easier to adapt for parallel processing, and should get us more benefit from parallelization (as more SCCs can be processed in parallel). I tried to make order with single worker stable and very similar (or maybe even identical) to the current order. Note I already add some methods to the build manager to emulate parallel processing, but they are not parallel _yet_. --- mypy/build.py | 378 ++++++++++++++++++++++++------------ mypy/test/testcheck.py | 10 +- mypy/test/testgraph.py | 6 +- mypyc/codegen/emitmodule.py | 4 +- 4 files changed, 265 insertions(+), 133 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 9f840499fcc23..f9137d8b1a32f 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -122,6 +122,28 @@ Graph: _TypeAlias = dict[str, "State"] +class SCC: + """A simple class that represents a strongly connected component (import cycle).""" + + id_counter: ClassVar[int] = 0 + + def __init__(self, ids: set[str]) -> None: + self.id = SCC.id_counter + SCC.id_counter += 1 + # Ids of modules in this cycle. + self.mod_ids = ids + # Direct dependencies, should be populated by the caller. + self.deps: set[int] = set() + # Direct dependencies that have not been processed yet. + # Should be populated by the caller. This set may change during graph + # processing, while the above stays constant. + self.not_ready_deps: set[int] = set() + # SCCs that (directly) depend on this SCC. Note this is a list to + # make processing order more predictable. Dependents will be notified + # that they may be ready in the order in this list. + self.direct_dependents: list[int] = [] + + # TODO: Get rid of BuildResult. We might as well return a BuildManager. class BuildResult: """The result of a successful build. @@ -725,6 +747,18 @@ def __init__( self.ast_cache: dict[str, tuple[MypyFile, list[ErrorInfo]]] = {} # Number of times we used GC optimization hack for fresh SCCs. self.gc_freeze_cycles = 0 + # Mapping from SCC id to corresponding SCC instance. This is populated + # in process_graph(). + self.scc_by_id: dict[int, SCC] = {} + # Global topological order for SCCs. This exists to make order of processing + # SCCs more predictable. + self.top_order: list[int] = [] + # Stale SCCs that are queued for processing. Note that as of now we have just + # one worker, that is the same process. In the future, we will support multiple + # parallel worker processes. + self.scc_queue: list[SCC] = [] + # SCCs that have been fully processed. + self.done_sccs: set[int] = set() def dump_stats(self) -> None: if self.options.dump_build_stats: @@ -925,6 +959,23 @@ def add_stats(self, **kwds: Any) -> None: def stats_summary(self) -> Mapping[str, object]: return self.stats + def submit(self, sccs: list[SCC]) -> None: + """Submit a stale SCC for processing in current process.""" + self.scc_queue.extend(sccs) + + def wait_for_done(self, graph: Graph) -> tuple[list[SCC], bool]: + """Wait for a stale SCC processing (in process) to finish. + + Return next processed SCC and whether we have more in the queue. + This emulates the API we will have for parallel processing + in multiple worker processes. + """ + if not self.scc_queue: + return [], False + next_scc = self.scc_queue.pop(0) + process_stale_scc(graph, next_scc, self) + return [next_scc], bool(self.scc_queue) + def deps_to_json(x: dict[str, set[str]]) -> bytes: return json_dumps({k: list(v) for k, v in x.items()}) @@ -3012,7 +3063,7 @@ def dump_graph(graph: Graph, stdout: TextIO | None = None) -> None: nodes = [] sccs = sorted_components(graph) for i, ascc in enumerate(sccs): - scc = order_ascc(graph, ascc) + scc = order_ascc(graph, ascc.mod_ids) node = NodeInfo(i, scc) nodes.append(node) inv_nodes = {} # module -> node_id @@ -3203,58 +3254,51 @@ def load_graph( return graph -def process_graph(graph: Graph, manager: BuildManager) -> None: - """Process everything in dependency order.""" - sccs = sorted_components(graph) - manager.log("Found %d SCCs; largest has %d nodes" % (len(sccs), max(len(scc) for scc in sccs))) - - fresh_scc_queue: list[list[str]] = [] +def order_ascc_ex(graph: Graph, ascc: SCC) -> list[str]: + """Apply extra heuristics on top of order_ascc(). - # We're processing SCCs from leaves (those without further - # dependencies) to roots (those from which everything else can be - # reached). + This should be used only for actual SCCs, not for "inner" SCCs + we create recursively during ordering of the SCC. Currently, this + has only some special handling for builtin SCC. + """ + scc = order_ascc(graph, ascc.mod_ids) + # Make the order of the SCC that includes 'builtins' and 'typing', + # among other things, predictable. Various things may break if + # the order changes. + if "builtins" in ascc.mod_ids: + scc = sorted(scc, reverse=True) + # If builtins is in the list, move it last. (This is a bit of + # a hack, but it's necessary because the builtins module is + # part of a small cycle involving at least {builtins, abc, + # typing}. Of these, builtins must be processed last or else + # some builtin objects will be incompletely processed.) + scc.remove("builtins") + scc.append("builtins") + return scc + + +def find_stale_sccs( + sccs: list[SCC], graph: Graph, manager: BuildManager +) -> tuple[list[SCC], list[SCC]]: + """Split a list of ready SCCs into stale and fresh. + + Fresh SCCs are those where: + * We have valid cache files for all modules in the SCC. + * The interface hashes of direct dependents matches those recorded in the cache. + * There are no new (un)suppressed dependencies (files removed/added to the build). + """ + stale_sccs = [] + fresh_sccs = [] for ascc in sccs: - # Order the SCC's nodes using a heuristic. - # Note that ascc is a set, and scc is a list. - scc = order_ascc(graph, ascc) - # Make the order of the SCC that includes 'builtins' and 'typing', - # among other things, predictable. Various things may break if - # the order changes. - if "builtins" in ascc: - scc = sorted(scc, reverse=True) - # If builtins is in the list, move it last. (This is a bit of - # a hack, but it's necessary because the builtins module is - # part of a small cycle involving at least {builtins, abc, - # typing}. Of these, builtins must be processed last or else - # some builtin objects will be incompletely processed.) - scc.remove("builtins") - scc.append("builtins") - if manager.options.verbosity >= 2: - for id in scc: - manager.trace( - f"Priorities for {id}:", - " ".join( - "%s:%d" % (x, graph[id].priorities[x]) - for x in graph[id].dependencies - if x in ascc and x in graph[id].priorities - ), - ) - # Because the SCCs are presented in topological sort order, we - # don't need to look at dependencies recursively for staleness - # -- the immediate dependencies are sufficient. - stale_scc = {id for id in scc if not graph[id].is_fresh()} + stale_scc = {id for id in ascc.mod_ids if not graph[id].is_fresh()} fresh = not stale_scc - deps = set() - for id in scc: - deps.update(graph[id].dependencies) - deps -= ascc # Verify that interfaces of dependencies still present in graph are up-to-date (fresh). # Note: if a dependency is not in graph anymore, it should be considered interface-stale. # This is important to trigger any relevant updates from indirect dependencies that were # removed in load_graph(). stale_deps = set() - for id in ascc: + for id in ascc.mod_ids: for dep in graph[id].dep_hashes: if dep not in graph: stale_deps.add(dep) @@ -3262,98 +3306,101 @@ def process_graph(graph: Graph, manager: BuildManager) -> None: if graph[dep].interface_hash != graph[id].dep_hashes[dep]: stale_deps.add(dep) fresh = fresh and not stale_deps + undeps = set() if fresh: # Check if any dependencies that were suppressed according # to the cache have been added back in this run. # NOTE: Newly suppressed dependencies are handled by is_fresh(). - for id in scc: + for id in ascc.mod_ids: undeps.update(graph[id].suppressed) undeps &= graph.keys() if undeps: fresh = False + if fresh: fresh_msg = "fresh" elif undeps: fresh_msg = f"stale due to changed suppression ({' '.join(sorted(undeps))})" elif stale_scc: fresh_msg = "inherently stale" - if stale_scc != ascc: + if stale_scc != ascc.mod_ids: fresh_msg += f" ({' '.join(sorted(stale_scc))})" if stale_deps: fresh_msg += f" with stale deps ({' '.join(sorted(stale_deps))})" else: fresh_msg = f"stale due to deps ({' '.join(sorted(stale_deps))})" - scc_str = " ".join(scc) + scc_str = " ".join(ascc.mod_ids) if fresh: - manager.trace(f"Queuing {fresh_msg} SCC ({scc_str})") + manager.trace(f"Found {fresh_msg} SCC ({scc_str})") + # If there is at most one file with errors we can skip the ordering to save time. + mods_with_errors = [id for id in ascc.mod_ids if graph[id].error_lines] + if len(mods_with_errors) <= 1: + scc = mods_with_errors + else: + # Use exactly the same order as for stale SCCs for stability. + scc = order_ascc_ex(graph, ascc) for id in scc: if graph[id].error_lines: manager.flush_errors( manager.errors.simplify_path(graph[id].xpath), graph[id].error_lines, False ) - fresh_scc_queue.append(scc) + fresh_sccs.append(ascc) else: - if fresh_scc_queue: - manager.log(f"Processing {len(fresh_scc_queue)} queued fresh SCCs") - # Defer processing fresh SCCs until we actually run into a stale SCC - # and need the earlier modules to be loaded. - # - # Note that `process_graph` may end with us not having processed every - # single fresh SCC. This is intentional -- we don't need those modules - # loaded if there are no more stale SCCs to be rechecked. - # - # TODO: see if it's possible to determine if we need to process only a - # _subset_ of the past SCCs instead of having to process them all. - if ( - not manager.options.test_env - and platform.python_implementation() == "CPython" - and manager.gc_freeze_cycles < MAX_GC_FREEZE_CYCLES - ): - # When deserializing cache we create huge amount of new objects, so even - # with our generous GC thresholds, GC is still doing a lot of pointless - # work searching for garbage. So, we temporarily disable it when - # processing fresh SCCs, and then move all the new objects to the oldest - # generation with the freeze()/unfreeze() trick below. This is arguably - # a hack, but it gives huge performance wins for large third-party - # libraries, like torch. - gc.collect() - gc.disable() - for prev_scc in fresh_scc_queue: - process_fresh_modules(graph, prev_scc, manager) - if ( - not manager.options.test_env - and platform.python_implementation() == "CPython" - and manager.gc_freeze_cycles < MAX_GC_FREEZE_CYCLES - ): - manager.gc_freeze_cycles += 1 - gc.freeze() - gc.unfreeze() - gc.enable() - fresh_scc_queue = [] - size = len(scc) + size = len(ascc.mod_ids) if size == 1: - manager.log(f"Processing SCC singleton ({scc_str}) as {fresh_msg}") + manager.log(f"Scheduling SCC singleton ({scc_str}) as {fresh_msg}") else: - manager.log("Processing SCC of size %d (%s) as %s" % (size, scc_str, fresh_msg)) - process_stale_scc(graph, scc, manager) + manager.log("Scheduling SCC of size %d (%s) as %s" % (size, scc_str, fresh_msg)) + stale_sccs.append(ascc) + return stale_sccs, fresh_sccs - sccs_left = len(fresh_scc_queue) - nodes_left = sum(len(scc) for scc in fresh_scc_queue) - manager.add_stats(sccs_left=sccs_left, nodes_left=nodes_left) - if sccs_left: - manager.log( - "{} fresh SCCs ({} nodes) left in queue (and will remain unprocessed)".format( - sccs_left, nodes_left - ) - ) - manager.trace(str(fresh_scc_queue)) - else: - manager.log("No fresh SCCs left in queue") +def process_graph(graph: Graph, manager: BuildManager) -> None: + """Process everything in dependency order.""" + sccs = sorted_components(graph) + manager.log( + "Found %d SCCs; largest has %d nodes" % (len(sccs), max(len(scc.mod_ids) for scc in sccs)) + ) + + scc_by_id = {scc.id: scc for scc in sccs} + manager.scc_by_id = scc_by_id + manager.top_order = [scc.id for scc in sccs] -def order_ascc(graph: Graph, ascc: AbstractSet[str], pri_max: int = PRI_ALL) -> list[str]: + # Prime the ready list with leaf SCCs (that have no dependencies). + ready = [] + not_ready = [] + for scc in sccs: + if not scc.deps: + ready.append(scc) + else: + not_ready.append(scc) + + still_working = False + while ready or not_ready or still_working: + stale, fresh = find_stale_sccs(ready, graph, manager) + if stale: + manager.submit(stale) + still_working = True + # We eagerly walk over fresh SCCs to reach as many stale SCCs as soon + # as possible. Only when there are no fresh SCCs, we wait on scheduled stale ones. + # This strategy, similar to a naive strategy in minesweeper game, will allow us + # to leverage parallelism as much as possible. + if fresh: + done = fresh + else: + done, still_working = manager.wait_for_done(graph) + ready = [] + for done_scc in done: + for dependent in done_scc.direct_dependents: + scc_by_id[dependent].not_ready_deps.discard(done_scc.id) + if not scc_by_id[dependent].not_ready_deps: + not_ready.remove(scc_by_id[dependent]) + ready.append(scc_by_id[dependent]) + + +def order_ascc(graph: Graph, ascc: AbstractSet[str], pri_max: int = PRI_INDIRECT) -> list[str]: """Come up with the ideal processing order within an SCC. Using the priorities assigned by all_imported_modules_in_file(), @@ -3377,7 +3424,7 @@ def order_ascc(graph: Graph, ascc: AbstractSet[str], pri_max: int = PRI_ALL) -> In practice there are only a few priority levels (less than a dozen) and in the worst case we just carry out the same algorithm - for finding SCCs N times. Thus the complexity is no worse than + for finding SCCs N times. Thus, the complexity is no worse than the complexity of the original SCC-finding algorithm -- see strongly_connected_components() below for a reference. """ @@ -3395,7 +3442,7 @@ def order_ascc(graph: Graph, ascc: AbstractSet[str], pri_max: int = PRI_ALL) -> # Filtered dependencies are uniform -- order by global order. return sorted(ascc, key=lambda id: -graph[id].order) pri_max = max(pri_spread) - sccs = sorted_components(graph, ascc, pri_max) + sccs = sorted_components_inner(graph, ascc, pri_max) # The recursion is bounded by the len(pri_spread) check above. return [s for ss in sccs for s in order_ascc(graph, ss, pri_max)] @@ -3403,8 +3450,8 @@ def order_ascc(graph: Graph, ascc: AbstractSet[str], pri_max: int = PRI_ALL) -> def process_fresh_modules(graph: Graph, modules: list[str], manager: BuildManager) -> None: """Process the modules in one group of modules from their cached data. - This can be used to process an SCC of modules - This involves loading the tree from JSON and then doing various cleanups. + This can be used to process an SCC of modules. This involves loading the tree (i.e. + module symbol tables) from cache file and then fixing cross-references in the symbols. """ t0 = time.time() for id in modules: @@ -3416,11 +3463,54 @@ def process_fresh_modules(graph: Graph, modules: list[str], manager: BuildManage manager.add_stats(process_fresh_time=t2 - t0, load_tree_time=t1 - t0) -def process_stale_scc(graph: Graph, scc: list[str], manager: BuildManager) -> None: - """Process the modules in one SCC from source code. +def process_stale_scc(graph: Graph, ascc: SCC, manager: BuildManager) -> None: + """Process the modules in one SCC from source code.""" + # First verify if all transitive dependencies are loaded in the current process. + missing_sccs = set() + sccs_to_find = ascc.deps.copy() + while sccs_to_find: + dep_scc = sccs_to_find.pop() + if dep_scc in manager.done_sccs or dep_scc in missing_sccs: + continue + missing_sccs.add(dep_scc) + sccs_to_find.update(manager.scc_by_id[dep_scc].deps) + + if missing_sccs: + # Load missing SCCs from cache. + # TODO: speed-up ordering if this causes problems for large builds. + fresh_sccs_to_load = [ + manager.scc_by_id[sid] for sid in manager.top_order if sid in missing_sccs + ] + manager.log(f"Processing {len(fresh_sccs_to_load)} fresh SCCs") + if ( + not manager.options.test_env + and platform.python_implementation() == "CPython" + and manager.gc_freeze_cycles < MAX_GC_FREEZE_CYCLES + ): + # When deserializing cache we create huge amount of new objects, so even + # with our generous GC thresholds, GC is still doing a lot of pointless + # work searching for garbage. So, we temporarily disable it when + # processing fresh SCCs, and then move all the new objects to the oldest + # generation with the freeze()/unfreeze() trick below. This is arguably + # a hack, but it gives huge performance wins for large third-party + # libraries, like torch. + gc.collect() + gc.disable() + for prev_scc in fresh_sccs_to_load: + manager.done_sccs.add(prev_scc.id) + process_fresh_modules(graph, sorted(prev_scc.mod_ids), manager) + if ( + not manager.options.test_env + and platform.python_implementation() == "CPython" + and manager.gc_freeze_cycles < MAX_GC_FREEZE_CYCLES + ): + manager.gc_freeze_cycles += 1 + gc.freeze() + gc.unfreeze() + gc.enable() - Exception: If quick_and_dirty is set, use the cache for fresh modules. - """ + # Process the SCC in stable order. + scc = order_ascc_ex(graph, ascc) stale = scc for id in stale: # We may already have parsed the module, or not. @@ -3434,7 +3524,7 @@ def process_stale_scc(graph: Graph, scc: list[str], manager: BuildManager) -> No assert typing_mod, "The typing module was not parsed" mypy.semanal_main.semantic_analysis_for_scc(graph, scc, manager.errors) - # Track what modules aren't yet done so we can finish them as soon + # Track what modules aren't yet done, so we can finish them as soon # as possible, saving memory. unfinished_modules = set(stale) for id in stale: @@ -3478,27 +3568,44 @@ def process_stale_scc(graph: Graph, scc: list[str], manager: BuildManager) -> No } meta["error_lines"] = errors_by_id.get(id, []) write_cache_meta(meta, manager, meta_json) - - -def sorted_components( - graph: Graph, vertices: AbstractSet[str] | None = None, pri_max: int = PRI_INDIRECT -) -> list[AbstractSet[str]]: + manager.done_sccs.add(ascc.id) + + +def prepare_sccs_full( + raw_sccs: Iterator[set[str]], edges: dict[str, list[str]] +) -> dict[SCC, set[SCC]]: + """Turn raw SCC sets into SCC objects and build dependency graph for SCCs.""" + sccs = [SCC(raw_scc) for raw_scc in raw_sccs] + scc_map = {} + for scc in sccs: + for id in scc.mod_ids: + scc_map[id] = scc + scc_deps_map: dict[SCC, set[SCC]] = {} + for scc in sccs: + for id in scc.mod_ids: + scc_deps_map.setdefault(scc, set()).update(scc_map[dep] for dep in edges[id]) + for scc in sccs: + # Remove trivial dependency on itself. + scc_deps_map[scc].discard(scc) + for dep_scc in scc_deps_map[scc]: + scc.deps.add(dep_scc.id) + scc.not_ready_deps.add(dep_scc.id) + return scc_deps_map + + +def sorted_components(graph: Graph) -> list[SCC]: """Return the graph's SCCs, topologically sorted by dependencies. The sort order is from leaves (nodes without dependencies) to roots (nodes on which no other nodes depend). - - This works for a subset of the full dependency graph too; - dependencies that aren't present in graph.keys() are ignored. """ # Compute SCCs. - if vertices is None: - vertices = set(graph) - edges = {id: deps_filtered(graph, vertices, id, pri_max) for id in vertices} - sccs = list(strongly_connected_components(vertices, edges)) + vertices = set(graph) + edges = {id: deps_filtered(graph, vertices, id, PRI_INDIRECT) for id in vertices} + scc_dep_map = prepare_sccs_full(strongly_connected_components(vertices, edges), edges) # Topsort. res = [] - for ready in topsort(prepare_sccs(sccs, edges)): + for ready in topsort(scc_dep_map): # Sort the sets in ready by reversed smallest State.order. Examples: # # - If ready is [{x}, {y}], x.order == 1, y.order == 2, we get @@ -3507,6 +3614,27 @@ def sorted_components( # - If ready is [{a, b}, {c, d}], a.order == 1, b.order == 3, # c.order == 2, d.order == 4, the sort keys become [1, 2] # and the result is [{c, d}, {a, b}]. + sorted_ready = sorted(ready, key=lambda scc: -min(graph[id].order for id in scc.mod_ids)) + for scc in sorted_ready: + for dep in scc_dep_map[scc]: + dep.direct_dependents.append(scc.id) + res.extend(sorted_ready) + return res + + +def sorted_components_inner( + graph: Graph, vertices: AbstractSet[str], pri_max: int +) -> list[AbstractSet[str]]: + """Simplified version of sorted_components() to work with sub-graphs. + + This doesn't create SCC objects, and operates with raw sets. This function + also allows filtering dependencies to take into account when building SCCs. + This is used for heuristic ordering of modules within actual SCCs. + """ + edges = {id: deps_filtered(graph, vertices, id, pri_max) for id in vertices} + sccs = list(strongly_connected_components(vertices, edges)) + res = [] + for ready in topsort(prepare_sccs(sccs, edges)): res.extend(sorted(ready, key=lambda scc: -min(graph[id].order for id in scc))) return res diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index 73f33c0323af9..f59cce701ea6c 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -200,7 +200,7 @@ def run_case_once( if res: if options.cache_dir != os.devnull: - self.verify_cache(module_data, res.manager, blocker) + self.verify_cache(module_data, res.manager, blocker, incremental_step) name = "targets" if incremental_step: @@ -230,7 +230,11 @@ def run_case_once( check_test_output_files(testcase, incremental_step, strip_prefix="tmp/") def verify_cache( - self, module_data: list[tuple[str, str, str]], manager: build.BuildManager, blocker: bool + self, + module_data: list[tuple[str, str, str]], + manager: build.BuildManager, + blocker: bool, + step: int, ) -> None: if not blocker: # There should be valid cache metadata for each module except @@ -240,7 +244,7 @@ def verify_cache( modules.update({module_name: path for module_name, path, text in module_data}) missing_paths = self.find_missing_cache_files(modules, manager) if missing_paths: - raise AssertionError(f"cache data missing for {missing_paths}") + raise AssertionError(f"cache data missing for {missing_paths} on run {step}") assert os.path.isfile(os.path.join(manager.options.cache_dir, ".gitignore")) cachedir_tag = os.path.join(manager.options.cache_dir, "CACHEDIR.TAG") assert os.path.isfile(cachedir_tag) diff --git a/mypy/test/testgraph.py b/mypy/test/testgraph.py index 238869f36fdff..c87eb66c13046 100644 --- a/mypy/test/testgraph.py +++ b/mypy/test/testgraph.py @@ -65,8 +65,8 @@ def test_sorted_components(self) -> None: "b": State("b", None, "import c", manager), "c": State("c", None, "import b, d", manager), } - res = sorted_components(graph) - assert_equal(res, [frozenset({"d"}), frozenset({"c", "b"}), frozenset({"a"})]) + res = [scc.mod_ids for scc in sorted_components(graph)] + assert_equal(res, [{"d"}, {"c", "b"}, {"a"}]) def test_order_ascc(self) -> None: manager = self._make_manager() @@ -76,7 +76,7 @@ def test_order_ascc(self) -> None: "b": State("b", None, "import c", manager), "c": State("c", None, "import b, d", manager), } - res = sorted_components(graph) + res = [scc.mod_ids for scc in sorted_components(graph)] assert_equal(res, [frozenset({"a", "d", "c", "b"})]) ascc = res[0] scc = order_ascc(graph, ascc) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 7ec315a6bd349..996ec4c52b08f 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -280,13 +280,13 @@ def compile_modules_to_ir( # Process the graph by SCC in topological order, like we do in mypy.build for scc in sorted_components(result.graph): - scc_states = [result.graph[id] for id in scc] + scc_states = [result.graph[id] for id in scc.mod_ids] trees = [st.tree for st in scc_states if st.id in mapper.group_map and st.tree] if not trees: continue - fresh = all(id not in result.manager.rechecked_modules for id in scc) + fresh = all(id not in result.manager.rechecked_modules for id in scc.mod_ids) if fresh: load_scc_from_cache(trees, result, mapper, deser_ctx) else: From b38413d8959758167e96c9f5e140cfb4be1a94dd Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Fri, 17 Oct 2025 02:12:20 +0200 Subject: [PATCH 338/424] Sync typeshed (#20080) Source commit: https://github.com/python/typeshed/commit/11c7821a79a8ab7e1982f3ab506db16f1c4a22a9 --------- Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Co-authored-by: AlexWaygood --- ...e-of-LiteralString-in-builtins-13743.patch | 30 +-- ...redundant-inheritances-from-Iterator.patch | 18 +- mypy/typeshed/stdlib/_csv.pyi | 7 +- .../stdlib/_frozen_importlib_external.pyi | 10 +- mypy/typeshed/stdlib/_typeshed/__init__.pyi | 3 + mypy/typeshed/stdlib/ast.pyi | 39 ++- mypy/typeshed/stdlib/builtins.pyi | 131 +++++----- mypy/typeshed/stdlib/ctypes/__init__.pyi | 4 +- mypy/typeshed/stdlib/html/parser.pyi | 13 +- mypy/typeshed/stdlib/importlib/abc.pyi | 6 +- .../stdlib/importlib/resources/__init__.pyi | 8 +- .../importlib/resources/_functional.pyi | 3 +- mypy/typeshed/stdlib/operator.pyi | 6 +- mypy/typeshed/stdlib/pdb.pyi | 5 + mypy/typeshed/stdlib/tkinter/__init__.pyi | 227 ++++++++++++------ mypy/typeshed/stdlib/turtle.pyi | 24 +- mypy/typeshed/stdlib/types.pyi | 28 +-- mypy/typeshed/stdlib/typing.pyi | 13 +- mypy/typeshed/stdlib/typing_extensions.pyi | 2 +- .../typeshed/stdlib/xml/etree/ElementTree.pyi | 30 +-- 20 files changed, 375 insertions(+), 232 deletions(-) diff --git a/misc/typeshed_patches/0001-Remove-use-of-LiteralString-in-builtins-13743.patch b/misc/typeshed_patches/0001-Remove-use-of-LiteralString-in-builtins-13743.patch index a47d5db3cd222..f9334251c2bdb 100644 --- a/misc/typeshed_patches/0001-Remove-use-of-LiteralString-in-builtins-13743.patch +++ b/misc/typeshed_patches/0001-Remove-use-of-LiteralString-in-builtins-13743.patch @@ -1,4 +1,4 @@ -From 805d7fc06a8bee350959512e0908a18a87b7f8c2 Mon Sep 17 00:00:00 2001 +From 3229a6066cff3d80d6cb923322c2d42a300d0be3 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 26 Sep 2022 12:55:07 -0700 Subject: [PATCH] Remove use of LiteralString in builtins (#13743) @@ -8,7 +8,7 @@ Subject: [PATCH] Remove use of LiteralString in builtins (#13743) 1 file changed, 1 insertion(+), 99 deletions(-) diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi -index c7ab95482..3e93da36e 100644 +index 969d16876..044e264d2 100644 --- a/mypy/typeshed/stdlib/builtins.pyi +++ b/mypy/typeshed/stdlib/builtins.pyi @@ -63,7 +63,6 @@ from typing import ( # noqa: Y022,UP035 @@ -19,10 +19,10 @@ index c7ab95482..3e93da36e 100644 ParamSpec, Self, TypeAlias, -@@ -468,31 +467,16 @@ class str(Sequence[str]): - def __new__(cls, object: object = ...) -> Self: ... +@@ -480,31 +479,16 @@ class str(Sequence[str]): + def __new__(cls, object: object = "") -> Self: ... @overload - def __new__(cls, object: ReadableBuffer, encoding: str = ..., errors: str = ...) -> Self: ... + def __new__(cls, object: ReadableBuffer, encoding: str = "utf-8", errors: str = "strict") -> Self: ... - @overload - def capitalize(self: LiteralString) -> LiteralString: ... - @overload @@ -35,23 +35,23 @@ index c7ab95482..3e93da36e 100644 - def center(self: LiteralString, width: SupportsIndex, fillchar: LiteralString = " ", /) -> LiteralString: ... - @overload def center(self, width: SupportsIndex, fillchar: str = " ", /) -> str: ... # type: ignore[misc] - def count(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... + def count(self, sub: str, start: SupportsIndex | None = None, end: SupportsIndex | None = None, /) -> int: ... def encode(self, encoding: str = "utf-8", errors: str = "strict") -> bytes: ... def endswith( - self, suffix: str | tuple[str, ...], start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + self, suffix: str | tuple[str, ...], start: SupportsIndex | None = None, end: SupportsIndex | None = None, / ) -> bool: ... - @overload - def expandtabs(self: LiteralString, tabsize: SupportsIndex = 8) -> LiteralString: ... - @overload def expandtabs(self, tabsize: SupportsIndex = 8) -> str: ... # type: ignore[misc] - def find(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... + def find(self, sub: str, start: SupportsIndex | None = None, end: SupportsIndex | None = None, /) -> int: ... - @overload - def format(self: LiteralString, *args: LiteralString, **kwargs: LiteralString) -> LiteralString: ... - @overload def format(self, *args: object, **kwargs: object) -> str: ... def format_map(self, mapping: _FormatMapMapping, /) -> str: ... - def index(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... -@@ -508,98 +492,34 @@ class str(Sequence[str]): + def index(self, sub: str, start: SupportsIndex | None = None, end: SupportsIndex | None = None, /) -> int: ... +@@ -520,98 +504,34 @@ class str(Sequence[str]): def isspace(self) -> bool: ... def istitle(self) -> bool: ... def isupper(self) -> bool: ... @@ -98,8 +98,8 @@ index c7ab95482..3e93da36e 100644 - def removesuffix(self: LiteralString, suffix: LiteralString, /) -> LiteralString: ... - @overload def removesuffix(self, suffix: str, /) -> str: ... # type: ignore[misc] - def rfind(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... - def rindex(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... + def rfind(self, sub: str, start: SupportsIndex | None = None, end: SupportsIndex | None = None, /) -> int: ... + def rindex(self, sub: str, start: SupportsIndex | None = None, end: SupportsIndex | None = None, /) -> int: ... - @overload - def rjust(self: LiteralString, width: SupportsIndex, fillchar: LiteralString = " ", /) -> LiteralString: ... - @overload @@ -125,7 +125,7 @@ index c7ab95482..3e93da36e 100644 - @overload def splitlines(self, keepends: bool = False) -> list[str]: ... # type: ignore[misc] def startswith( - self, prefix: str | tuple[str, ...], start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + self, prefix: str | tuple[str, ...], start: SupportsIndex | None = None, end: SupportsIndex | None = None, / ) -> bool: ... - @overload - def strip(self: LiteralString, chars: LiteralString | None = None, /) -> LiteralString: ... @@ -150,7 +150,7 @@ index c7ab95482..3e93da36e 100644 def zfill(self, width: SupportsIndex, /) -> str: ... # type: ignore[misc] @staticmethod @overload -@@ -610,39 +530,21 @@ class str(Sequence[str]): +@@ -622,39 +542,21 @@ class str(Sequence[str]): @staticmethod @overload def maketrans(x: str, y: str, z: str, /) -> dict[int, int | None]: ... @@ -192,5 +192,5 @@ index c7ab95482..3e93da36e 100644 def __getnewargs__(self) -> tuple[str]: ... def __format__(self, format_spec: str, /) -> str: ... -- -2.50.1 +2.51.1 diff --git a/misc/typeshed_patches/0001-Revert-Remove-redundant-inheritances-from-Iterator.patch b/misc/typeshed_patches/0001-Revert-Remove-redundant-inheritances-from-Iterator.patch index fdcc14cec3c6a..7110eff5f148c 100644 --- a/misc/typeshed_patches/0001-Revert-Remove-redundant-inheritances-from-Iterator.patch +++ b/misc/typeshed_patches/0001-Revert-Remove-redundant-inheritances-from-Iterator.patch @@ -1,4 +1,4 @@ -From 438dbb1300b77331940d7db8f010e97305745116 Mon Sep 17 00:00:00 2001 +From 7678bc3f80e4d3f04a0ff0ee3a7d51f49ae4c465 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Sat, 21 Dec 2024 22:36:38 +0100 Subject: [PATCH] Revert Remove redundant inheritances from Iterator in @@ -36,10 +36,10 @@ index d663f5d93..f43178e4d 100644 @property def _exception(self) -> BaseException | None: ... diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi -index f2dd00079..784ee7eac 100644 +index 044e264d2..6d813f172 100644 --- a/mypy/typeshed/stdlib/builtins.pyi +++ b/mypy/typeshed/stdlib/builtins.pyi -@@ -1209,7 +1209,7 @@ class frozenset(AbstractSet[_T_co]): +@@ -1210,7 +1210,7 @@ class frozenset(AbstractSet[_T_co]): def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... @disjoint_base @@ -48,7 +48,7 @@ index f2dd00079..784ee7eac 100644 def __new__(cls, iterable: Iterable[_T], start: int = 0) -> Self: ... def __iter__(self) -> Self: ... def __next__(self) -> tuple[int, _T]: ... -@@ -1405,7 +1405,7 @@ else: +@@ -1404,7 +1404,7 @@ else: exit: _sitebuiltins.Quitter @disjoint_base @@ -57,7 +57,7 @@ index f2dd00079..784ee7eac 100644 @overload def __new__(cls, function: None, iterable: Iterable[_T | None], /) -> Self: ... @overload -@@ -1469,7 +1469,7 @@ license: _sitebuiltins._Printer +@@ -1468,7 +1468,7 @@ license: _sitebuiltins._Printer def locals() -> dict[str, Any]: ... @disjoint_base @@ -66,7 +66,7 @@ index f2dd00079..784ee7eac 100644 # 3.14 adds `strict` argument. if sys.version_info >= (3, 14): @overload -@@ -1776,7 +1776,7 @@ def pow(base: _SupportsSomeKindOfPow, exp: complex, mod: None = None) -> complex +@@ -1775,7 +1775,7 @@ def pow(base: _SupportsSomeKindOfPow, exp: complex, mod: None = None) -> complex quit: _sitebuiltins.Quitter @disjoint_base @@ -75,7 +75,7 @@ index f2dd00079..784ee7eac 100644 @overload def __new__(cls, sequence: Reversible[_T], /) -> Iterator[_T]: ... # type: ignore[misc] @overload -@@ -1840,7 +1840,7 @@ def vars(object: type, /) -> types.MappingProxyType[str, Any]: ... +@@ -1839,7 +1839,7 @@ def vars(object: type, /) -> types.MappingProxyType[str, Any]: ... @overload def vars(object: Any = ..., /) -> dict[str, Any]: ... @disjoint_base @@ -83,7 +83,7 @@ index f2dd00079..784ee7eac 100644 +class zip(Iterator[_T_co]): if sys.version_info >= (3, 10): @overload - def __new__(cls, *, strict: bool = ...) -> zip[Any]: ... + def __new__(cls, *, strict: bool = False) -> zip[Any]: ... diff --git a/mypy/typeshed/stdlib/csv.pyi b/mypy/typeshed/stdlib/csv.pyi index 2c8e7109c..4ed0ab1d8 100644 --- a/mypy/typeshed/stdlib/csv.pyi @@ -326,5 +326,5 @@ index 6b0f1ba94..882cd143c 100644 @property def connection(self) -> Connection: ... -- -2.51.0 +2.51.1 diff --git a/mypy/typeshed/stdlib/_csv.pyi b/mypy/typeshed/stdlib/_csv.pyi index 4128178c18b34..ea90766afee66 100644 --- a/mypy/typeshed/stdlib/_csv.pyi +++ b/mypy/typeshed/stdlib/_csv.pyi @@ -92,7 +92,7 @@ else: def writerows(self, rows: Iterable[Iterable[Any]]) -> None: ... def writer( - csvfile: SupportsWrite[str], + fileobj: SupportsWrite[str], /, dialect: _DialectLike = "excel", *, @@ -106,7 +106,7 @@ def writer( strict: bool = False, ) -> _writer: ... def reader( - csvfile: Iterable[str], + iterable: Iterable[str], /, dialect: _DialectLike = "excel", *, @@ -121,7 +121,8 @@ def reader( ) -> _reader: ... def register_dialect( name: str, - dialect: type[Dialect | csv.Dialect] = ..., + /, + dialect: type[Dialect | csv.Dialect] | str = "excel", *, delimiter: str = ",", quotechar: str | None = '"', diff --git a/mypy/typeshed/stdlib/_frozen_importlib_external.pyi b/mypy/typeshed/stdlib/_frozen_importlib_external.pyi index 71642c65dc07d..4778be3af1f39 100644 --- a/mypy/typeshed/stdlib/_frozen_importlib_external.pyi +++ b/mypy/typeshed/stdlib/_frozen_importlib_external.pyi @@ -100,7 +100,7 @@ class SourceLoader(_LoaderBasics): def get_source(self, fullname: str) -> str | None: ... def path_stats(self, path: str) -> Mapping[str, Any]: ... def source_to_code( - self, data: ReadableBuffer | str | _ast.Module | _ast.Expression | _ast.Interactive, path: ReadableBuffer | StrPath + self, data: ReadableBuffer | str | _ast.Module | _ast.Expression | _ast.Interactive, path: bytes | StrPath ) -> types.CodeType: ... def get_code(self, fullname: str) -> types.CodeType | None: ... @@ -109,8 +109,8 @@ class FileLoader: path: str def __init__(self, fullname: str, path: str) -> None: ... def get_data(self, path: str) -> bytes: ... - def get_filename(self, name: str | None = None) -> str: ... - def load_module(self, name: str | None = None) -> types.ModuleType: ... + def get_filename(self, fullname: str | None = None) -> str: ... + def load_module(self, fullname: str | None = None) -> types.ModuleType: ... if sys.version_info >= (3, 10): def get_resource_reader(self, name: str | None = None) -> importlib.readers.FileReader: ... else: @@ -126,7 +126,7 @@ class SourceFileLoader(importlib.abc.FileLoader, FileLoader, importlib.abc.Sourc def source_to_code( # type: ignore[override] # incompatible with InspectLoader.source_to_code self, data: ReadableBuffer | str | _ast.Module | _ast.Expression | _ast.Interactive, - path: ReadableBuffer | StrPath, + path: bytes | StrPath, *, _optimize: int = -1, ) -> types.CodeType: ... @@ -137,7 +137,7 @@ class SourcelessFileLoader(importlib.abc.FileLoader, FileLoader, _LoaderBasics): class ExtensionFileLoader(FileLoader, _LoaderBasics, importlib.abc.ExecutionLoader): def __init__(self, name: str, path: str) -> None: ... - def get_filename(self, name: str | None = None) -> str: ... + def get_filename(self, fullname: str | None = None) -> str: ... def get_source(self, fullname: str) -> None: ... def create_module(self, spec: ModuleSpec) -> types.ModuleType: ... def exec_module(self, module: types.ModuleType) -> None: ... diff --git a/mypy/typeshed/stdlib/_typeshed/__init__.pyi b/mypy/typeshed/stdlib/_typeshed/__init__.pyi index 25054b601a4f6..b786923880e13 100644 --- a/mypy/typeshed/stdlib/_typeshed/__init__.pyi +++ b/mypy/typeshed/stdlib/_typeshed/__init__.pyi @@ -142,6 +142,9 @@ class SupportsIter(Protocol[_T_co]): class SupportsAiter(Protocol[_T_co]): def __aiter__(self) -> _T_co: ... +class SupportsLen(Protocol): + def __len__(self) -> int: ... + class SupportsLenAndGetItem(Protocol[_T_co]): def __len__(self) -> int: ... def __getitem__(self, k: int, /) -> _T_co: ... diff --git a/mypy/typeshed/stdlib/ast.pyi b/mypy/typeshed/stdlib/ast.pyi index d360c2ed60e5c..e66e609ee6645 100644 --- a/mypy/typeshed/stdlib/ast.pyi +++ b/mypy/typeshed/stdlib/ast.pyi @@ -1744,10 +1744,20 @@ if sys.version_info < (3, 14): _T = _TypeVar("_T", bound=AST) if sys.version_info >= (3, 13): + @overload + def parse( + source: _T, + filename: str | bytes | os.PathLike[Any] = "", + mode: Literal["exec", "eval", "func_type", "single"] = "exec", + *, + type_comments: bool = False, + feature_version: None | int | tuple[int, int] = None, + optimize: Literal[-1, 0, 1, 2] = -1, + ) -> _T: ... @overload def parse( source: str | ReadableBuffer, - filename: str | ReadableBuffer | os.PathLike[Any] = "", + filename: str | bytes | os.PathLike[Any] = "", mode: Literal["exec"] = "exec", *, type_comments: bool = False, @@ -1757,7 +1767,7 @@ if sys.version_info >= (3, 13): @overload def parse( source: str | ReadableBuffer, - filename: str | ReadableBuffer | os.PathLike[Any], + filename: str | bytes | os.PathLike[Any], mode: Literal["eval"], *, type_comments: bool = False, @@ -1767,7 +1777,7 @@ if sys.version_info >= (3, 13): @overload def parse( source: str | ReadableBuffer, - filename: str | ReadableBuffer | os.PathLike[Any], + filename: str | bytes | os.PathLike[Any], mode: Literal["func_type"], *, type_comments: bool = False, @@ -1777,7 +1787,7 @@ if sys.version_info >= (3, 13): @overload def parse( source: str | ReadableBuffer, - filename: str | ReadableBuffer | os.PathLike[Any], + filename: str | bytes | os.PathLike[Any], mode: Literal["single"], *, type_comments: bool = False, @@ -1814,7 +1824,7 @@ if sys.version_info >= (3, 13): @overload def parse( source: str | ReadableBuffer, - filename: str | ReadableBuffer | os.PathLike[Any] = "", + filename: str | bytes | os.PathLike[Any] = "", mode: str = "exec", *, type_comments: bool = False, @@ -1823,10 +1833,19 @@ if sys.version_info >= (3, 13): ) -> mod: ... else: + @overload + def parse( + source: _T, + filename: str | bytes | os.PathLike[Any] = "", + mode: Literal["exec", "eval", "func_type", "single"] = "exec", + *, + type_comments: bool = False, + feature_version: None | int | tuple[int, int] = None, + ) -> _T: ... @overload def parse( source: str | ReadableBuffer, - filename: str | ReadableBuffer | os.PathLike[Any] = "", + filename: str | bytes | os.PathLike[Any] = "", mode: Literal["exec"] = "exec", *, type_comments: bool = False, @@ -1835,7 +1854,7 @@ else: @overload def parse( source: str | ReadableBuffer, - filename: str | ReadableBuffer | os.PathLike[Any], + filename: str | bytes | os.PathLike[Any], mode: Literal["eval"], *, type_comments: bool = False, @@ -1844,7 +1863,7 @@ else: @overload def parse( source: str | ReadableBuffer, - filename: str | ReadableBuffer | os.PathLike[Any], + filename: str | bytes | os.PathLike[Any], mode: Literal["func_type"], *, type_comments: bool = False, @@ -1853,7 +1872,7 @@ else: @overload def parse( source: str | ReadableBuffer, - filename: str | ReadableBuffer | os.PathLike[Any], + filename: str | bytes | os.PathLike[Any], mode: Literal["single"], *, type_comments: bool = False, @@ -1886,7 +1905,7 @@ else: @overload def parse( source: str | ReadableBuffer, - filename: str | ReadableBuffer | os.PathLike[Any] = "", + filename: str | bytes | os.PathLike[Any] = "", mode: str = "exec", *, type_comments: bool = False, diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi index e276441523c8c..ddf81db181bfa 100644 --- a/mypy/typeshed/stdlib/builtins.pyi +++ b/mypy/typeshed/stdlib/builtins.pyi @@ -225,8 +225,10 @@ class type: @classmethod def __prepare__(metacls, name: str, bases: tuple[type, ...], /, **kwds: Any) -> MutableMapping[str, object]: ... if sys.version_info >= (3, 10): - def __or__(self, value: Any, /) -> types.UnionType: ... - def __ror__(self, value: Any, /) -> types.UnionType: ... + # `int | str` produces an instance of `UnionType`, but `int | int` produces an instance of `type`, + # and `abc.ABC | abc.ABC` produces an instance of `abc.ABCMeta`. + def __or__(self: _typeshed.Self, value: Any, /) -> types.UnionType | _typeshed.Self: ... + def __ror__(self: _typeshed.Self, value: Any, /) -> types.UnionType | _typeshed.Self: ... if sys.version_info >= (3, 12): __type_params__: tuple[TypeVar | ParamSpec | TypeVarTuple, ...] __annotations__: dict[str, AnnotationForm] @@ -249,7 +251,7 @@ _LiteralInteger = _PositiveInteger | _NegativeInteger | Literal[0] # noqa: Y026 @disjoint_base class int: @overload - def __new__(cls, x: ConvertibleToInt = ..., /) -> Self: ... + def __new__(cls, x: ConvertibleToInt = 0, /) -> Self: ... @overload def __new__(cls, x: str | bytes | bytearray, /, base: SupportsIndex) -> Self: ... def as_integer_ratio(self) -> tuple[int, Literal[1]]: ... @@ -359,7 +361,7 @@ class int: @disjoint_base class float: - def __new__(cls, x: ConvertibleToFloat = ..., /) -> Self: ... + def __new__(cls, x: ConvertibleToFloat = 0, /) -> Self: ... def as_integer_ratio(self) -> tuple[int, int]: ... def hex(self) -> str: ... def is_integer(self) -> bool: ... @@ -429,8 +431,8 @@ class complex: @overload def __new__( cls, - real: complex | SupportsComplex | SupportsFloat | SupportsIndex = ..., - imag: complex | SupportsFloat | SupportsIndex = ..., + real: complex | SupportsComplex | SupportsFloat | SupportsIndex = 0, + imag: complex | SupportsFloat | SupportsIndex = 0, ) -> Self: ... @overload def __new__(cls, real: str | SupportsComplex | SupportsFloat | SupportsIndex | complex) -> Self: ... @@ -474,22 +476,22 @@ class _TranslateTable(Protocol): @disjoint_base class str(Sequence[str]): @overload - def __new__(cls, object: object = ...) -> Self: ... + def __new__(cls, object: object = "") -> Self: ... @overload - def __new__(cls, object: ReadableBuffer, encoding: str = ..., errors: str = ...) -> Self: ... + def __new__(cls, object: ReadableBuffer, encoding: str = "utf-8", errors: str = "strict") -> Self: ... def capitalize(self) -> str: ... # type: ignore[misc] def casefold(self) -> str: ... # type: ignore[misc] def center(self, width: SupportsIndex, fillchar: str = " ", /) -> str: ... # type: ignore[misc] - def count(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... + def count(self, sub: str, start: SupportsIndex | None = None, end: SupportsIndex | None = None, /) -> int: ... def encode(self, encoding: str = "utf-8", errors: str = "strict") -> bytes: ... def endswith( - self, suffix: str | tuple[str, ...], start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + self, suffix: str | tuple[str, ...], start: SupportsIndex | None = None, end: SupportsIndex | None = None, / ) -> bool: ... def expandtabs(self, tabsize: SupportsIndex = 8) -> str: ... # type: ignore[misc] - def find(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... + def find(self, sub: str, start: SupportsIndex | None = None, end: SupportsIndex | None = None, /) -> int: ... def format(self, *args: object, **kwargs: object) -> str: ... def format_map(self, mapping: _FormatMapMapping, /) -> str: ... - def index(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... + def index(self, sub: str, start: SupportsIndex | None = None, end: SupportsIndex | None = None, /) -> int: ... def isalnum(self) -> bool: ... def isalpha(self) -> bool: ... def isascii(self) -> bool: ... @@ -514,8 +516,8 @@ class str(Sequence[str]): def removeprefix(self, prefix: str, /) -> str: ... # type: ignore[misc] def removesuffix(self, suffix: str, /) -> str: ... # type: ignore[misc] - def rfind(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... - def rindex(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... + def rfind(self, sub: str, start: SupportsIndex | None = None, end: SupportsIndex | None = None, /) -> int: ... + def rindex(self, sub: str, start: SupportsIndex | None = None, end: SupportsIndex | None = None, /) -> int: ... def rjust(self, width: SupportsIndex, fillchar: str = " ", /) -> str: ... # type: ignore[misc] def rpartition(self, sep: str, /) -> tuple[str, str, str]: ... # type: ignore[misc] def rsplit(self, sep: str | None = None, maxsplit: SupportsIndex = -1) -> list[str]: ... # type: ignore[misc] @@ -523,7 +525,7 @@ class str(Sequence[str]): def split(self, sep: str | None = None, maxsplit: SupportsIndex = -1) -> list[str]: ... # type: ignore[misc] def splitlines(self, keepends: bool = False) -> list[str]: ... # type: ignore[misc] def startswith( - self, prefix: str | tuple[str, ...], start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + self, prefix: str | tuple[str, ...], start: SupportsIndex | None = None, end: SupportsIndex | None = None, / ) -> bool: ... def strip(self, chars: str | None = None, /) -> str: ... # type: ignore[misc] def swapcase(self) -> str: ... # type: ignore[misc] @@ -564,29 +566,29 @@ class bytes(Sequence[int]): @overload def __new__(cls, o: Iterable[SupportsIndex] | SupportsIndex | SupportsBytes | ReadableBuffer, /) -> Self: ... @overload - def __new__(cls, string: str, /, encoding: str, errors: str = ...) -> Self: ... + def __new__(cls, string: str, /, encoding: str, errors: str = "strict") -> Self: ... @overload def __new__(cls) -> Self: ... def capitalize(self) -> bytes: ... def center(self, width: SupportsIndex, fillchar: bytes = b" ", /) -> bytes: ... def count( - self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = None, end: SupportsIndex | None = None, / ) -> int: ... def decode(self, encoding: str = "utf-8", errors: str = "strict") -> str: ... def endswith( self, suffix: ReadableBuffer | tuple[ReadableBuffer, ...], - start: SupportsIndex | None = ..., - end: SupportsIndex | None = ..., + start: SupportsIndex | None = None, + end: SupportsIndex | None = None, /, ) -> bool: ... def expandtabs(self, tabsize: SupportsIndex = 8) -> bytes: ... def find( - self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = None, end: SupportsIndex | None = None, / ) -> int: ... - def hex(self, sep: str | bytes = ..., bytes_per_sep: SupportsIndex = ...) -> str: ... + def hex(self, sep: str | bytes = ..., bytes_per_sep: SupportsIndex = 1) -> str: ... def index( - self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = None, end: SupportsIndex | None = None, / ) -> int: ... def isalnum(self) -> bool: ... def isalpha(self) -> bool: ... @@ -605,10 +607,10 @@ class bytes(Sequence[int]): def removeprefix(self, prefix: ReadableBuffer, /) -> bytes: ... def removesuffix(self, suffix: ReadableBuffer, /) -> bytes: ... def rfind( - self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = None, end: SupportsIndex | None = None, / ) -> int: ... def rindex( - self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = None, end: SupportsIndex | None = None, / ) -> int: ... def rjust(self, width: SupportsIndex, fillchar: bytes | bytearray = b" ", /) -> bytes: ... def rpartition(self, sep: ReadableBuffer, /) -> tuple[bytes, bytes, bytes]: ... @@ -619,8 +621,8 @@ class bytes(Sequence[int]): def startswith( self, prefix: ReadableBuffer | tuple[ReadableBuffer, ...], - start: SupportsIndex | None = ..., - end: SupportsIndex | None = ..., + start: SupportsIndex | None = None, + end: SupportsIndex | None = None, /, ) -> bool: ... def strip(self, bytes: ReadableBuffer | None = None, /) -> bytes: ... @@ -665,30 +667,30 @@ class bytearray(MutableSequence[int]): @overload def __init__(self, ints: Iterable[SupportsIndex] | SupportsIndex | ReadableBuffer, /) -> None: ... @overload - def __init__(self, string: str, /, encoding: str, errors: str = ...) -> None: ... + def __init__(self, string: str, /, encoding: str, errors: str = "strict") -> None: ... def append(self, item: SupportsIndex, /) -> None: ... def capitalize(self) -> bytearray: ... def center(self, width: SupportsIndex, fillchar: bytes = b" ", /) -> bytearray: ... def count( - self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = None, end: SupportsIndex | None = None, / ) -> int: ... def copy(self) -> bytearray: ... def decode(self, encoding: str = "utf-8", errors: str = "strict") -> str: ... def endswith( self, suffix: ReadableBuffer | tuple[ReadableBuffer, ...], - start: SupportsIndex | None = ..., - end: SupportsIndex | None = ..., + start: SupportsIndex | None = None, + end: SupportsIndex | None = None, /, ) -> bool: ... def expandtabs(self, tabsize: SupportsIndex = 8) -> bytearray: ... def extend(self, iterable_of_ints: Iterable[SupportsIndex], /) -> None: ... def find( - self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = None, end: SupportsIndex | None = None, / ) -> int: ... - def hex(self, sep: str | bytes = ..., bytes_per_sep: SupportsIndex = ...) -> str: ... + def hex(self, sep: str | bytes = ..., bytes_per_sep: SupportsIndex = 1) -> str: ... def index( - self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = None, end: SupportsIndex | None = None, / ) -> int: ... def insert(self, index: SupportsIndex, item: SupportsIndex, /) -> None: ... def isalnum(self) -> bool: ... @@ -710,10 +712,10 @@ class bytearray(MutableSequence[int]): def removesuffix(self, suffix: ReadableBuffer, /) -> bytearray: ... def replace(self, old: ReadableBuffer, new: ReadableBuffer, count: SupportsIndex = -1, /) -> bytearray: ... def rfind( - self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = None, end: SupportsIndex | None = None, / ) -> int: ... def rindex( - self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = None, end: SupportsIndex | None = None, / ) -> int: ... def rjust(self, width: SupportsIndex, fillchar: bytes | bytearray = b" ", /) -> bytearray: ... def rpartition(self, sep: ReadableBuffer, /) -> tuple[bytearray, bytearray, bytearray]: ... @@ -724,8 +726,8 @@ class bytearray(MutableSequence[int]): def startswith( self, prefix: ReadableBuffer | tuple[ReadableBuffer, ...], - start: SupportsIndex | None = ..., - end: SupportsIndex | None = ..., + start: SupportsIndex | None = None, + end: SupportsIndex | None = None, /, ) -> bool: ... def strip(self, bytes: ReadableBuffer | None = None, /) -> bytearray: ... @@ -839,7 +841,7 @@ class memoryview(Sequence[_I]): def tolist(self) -> list[int]: ... def toreadonly(self) -> memoryview: ... def release(self) -> None: ... - def hex(self, sep: str | bytes = ..., bytes_per_sep: SupportsIndex = ...) -> str: ... + def hex(self, sep: str | bytes = ..., bytes_per_sep: SupportsIndex = 1) -> str: ... def __buffer__(self, flags: int, /) -> memoryview: ... def __release_buffer__(self, buffer: memoryview, /) -> None: ... @@ -852,7 +854,7 @@ class memoryview(Sequence[_I]): @final class bool(int): - def __new__(cls, o: object = ..., /) -> Self: ... + def __new__(cls, o: object = False, /) -> Self: ... # The following overloads could be represented more elegantly with a TypeVar("_B", bool, int), # however mypy has a bug regarding TypeVar constraints (https://github.com/python/mypy/issues/11880). @overload @@ -925,7 +927,7 @@ class slice(Generic[_StartT_co, _StopT_co, _StepT_co]): @disjoint_base class tuple(Sequence[_T_co]): - def __new__(cls, iterable: Iterable[_T_co] = ..., /) -> Self: ... + def __new__(cls, iterable: Iterable[_T_co] = (), /) -> Self: ... def __len__(self) -> int: ... def __contains__(self, key: object, /) -> bool: ... @overload @@ -1225,7 +1227,7 @@ class range(Sequence[int]): @overload def __new__(cls, stop: SupportsIndex, /) -> Self: ... @overload - def __new__(cls, start: SupportsIndex, stop: SupportsIndex, step: SupportsIndex = ..., /) -> Self: ... + def __new__(cls, start: SupportsIndex, stop: SupportsIndex, step: SupportsIndex = 1, /) -> Self: ... def count(self, value: int, /) -> int: ... def index(self, value: int, /) -> int: ... # type: ignore[override] def __len__(self) -> int: ... @@ -1250,10 +1252,10 @@ class property: def __init__( self, - fget: Callable[[Any], Any] | None = ..., - fset: Callable[[Any, Any], None] | None = ..., - fdel: Callable[[Any], None] | None = ..., - doc: str | None = ..., + fget: Callable[[Any], Any] | None = None, + fset: Callable[[Any, Any], None] | None = None, + fdel: Callable[[Any], None] | None = None, + doc: str | None = None, ) -> None: ... def getter(self, fget: Callable[[Any], Any], /) -> property: ... def setter(self, fset: Callable[[Any, Any], None], /) -> property: ... @@ -1301,7 +1303,7 @@ if sys.version_info >= (3, 10): @overload def compile( source: str | ReadableBuffer | _ast.Module | _ast.Expression | _ast.Interactive, - filename: str | ReadableBuffer | PathLike[Any], + filename: str | bytes | PathLike[Any], mode: str, flags: Literal[0], dont_inherit: bool = False, @@ -1312,7 +1314,7 @@ def compile( @overload def compile( source: str | ReadableBuffer | _ast.Module | _ast.Expression | _ast.Interactive, - filename: str | ReadableBuffer | PathLike[Any], + filename: str | bytes | PathLike[Any], mode: str, *, dont_inherit: bool = False, @@ -1322,7 +1324,7 @@ def compile( @overload def compile( source: str | ReadableBuffer | _ast.Module | _ast.Expression | _ast.Interactive, - filename: str | ReadableBuffer | PathLike[Any], + filename: str | bytes | PathLike[Any], mode: str, flags: Literal[1024], dont_inherit: bool = False, @@ -1333,7 +1335,7 @@ def compile( @overload def compile( source: str | ReadableBuffer | _ast.Module | _ast.Expression | _ast.Interactive, - filename: str | ReadableBuffer | PathLike[Any], + filename: str | bytes | PathLike[Any], mode: str, flags: int, dont_inherit: bool = False, @@ -1840,18 +1842,25 @@ def vars(object: Any = ..., /) -> dict[str, Any]: ... class zip(Iterator[_T_co]): if sys.version_info >= (3, 10): @overload - def __new__(cls, *, strict: bool = ...) -> zip[Any]: ... + def __new__(cls, *, strict: bool = False) -> zip[Any]: ... @overload - def __new__(cls, iter1: Iterable[_T1], /, *, strict: bool = ...) -> zip[tuple[_T1]]: ... + def __new__(cls, iter1: Iterable[_T1], /, *, strict: bool = False) -> zip[tuple[_T1]]: ... @overload - def __new__(cls, iter1: Iterable[_T1], iter2: Iterable[_T2], /, *, strict: bool = ...) -> zip[tuple[_T1, _T2]]: ... + def __new__(cls, iter1: Iterable[_T1], iter2: Iterable[_T2], /, *, strict: bool = False) -> zip[tuple[_T1, _T2]]: ... @overload def __new__( - cls, iter1: Iterable[_T1], iter2: Iterable[_T2], iter3: Iterable[_T3], /, *, strict: bool = ... + cls, iter1: Iterable[_T1], iter2: Iterable[_T2], iter3: Iterable[_T3], /, *, strict: bool = False ) -> zip[tuple[_T1, _T2, _T3]]: ... @overload def __new__( - cls, iter1: Iterable[_T1], iter2: Iterable[_T2], iter3: Iterable[_T3], iter4: Iterable[_T4], /, *, strict: bool = ... + cls, + iter1: Iterable[_T1], + iter2: Iterable[_T2], + iter3: Iterable[_T3], + iter4: Iterable[_T4], + /, + *, + strict: bool = False, ) -> zip[tuple[_T1, _T2, _T3, _T4]]: ... @overload def __new__( @@ -1863,7 +1872,7 @@ class zip(Iterator[_T_co]): iter5: Iterable[_T5], /, *, - strict: bool = ..., + strict: bool = False, ) -> zip[tuple[_T1, _T2, _T3, _T4, _T5]]: ... @overload def __new__( @@ -1876,7 +1885,7 @@ class zip(Iterator[_T_co]): iter6: Iterable[Any], /, *iterables: Iterable[Any], - strict: bool = ..., + strict: bool = False, ) -> zip[tuple[Any, ...]]: ... else: @overload @@ -1990,8 +1999,8 @@ class AssertionError(Exception): ... if sys.version_info >= (3, 10): @disjoint_base class AttributeError(Exception): - def __init__(self, *args: object, name: str | None = ..., obj: object = ...) -> None: ... - name: str + def __init__(self, *args: object, name: str | None = None, obj: object = None) -> None: ... + name: str | None obj: object else: @@ -2002,7 +2011,7 @@ class EOFError(Exception): ... @disjoint_base class ImportError(Exception): - def __init__(self, *args: object, name: str | None = ..., path: str | None = ...) -> None: ... + def __init__(self, *args: object, name: str | None = None, path: str | None = None) -> None: ... name: str | None path: str | None msg: str # undocumented @@ -2015,8 +2024,8 @@ class MemoryError(Exception): ... if sys.version_info >= (3, 10): @disjoint_base class NameError(Exception): - def __init__(self, *args: object, name: str | None = ...) -> None: ... - name: str + def __init__(self, *args: object, name: str | None = None) -> None: ... + name: str | None else: class NameError(Exception): ... diff --git a/mypy/typeshed/stdlib/ctypes/__init__.pyi b/mypy/typeshed/stdlib/ctypes/__init__.pyi index 9da972240abb7..19bd261c67e06 100644 --- a/mypy/typeshed/stdlib/ctypes/__init__.pyi +++ b/mypy/typeshed/stdlib/ctypes/__init__.pyi @@ -23,7 +23,7 @@ from _ctypes import ( set_errno as set_errno, sizeof as sizeof, ) -from _typeshed import StrPath +from _typeshed import StrPath, SupportsBool, SupportsLen from ctypes._endian import BigEndianStructure as BigEndianStructure, LittleEndianStructure as LittleEndianStructure from types import GenericAlias from typing import Any, ClassVar, Final, Generic, Literal, TypeVar, overload, type_check_only @@ -217,7 +217,7 @@ class py_object(_CanCastTo, _SimpleCData[_T]): class c_bool(_SimpleCData[bool]): _type_: ClassVar[Literal["?"]] - def __init__(self, value: bool = ...) -> None: ... + def __init__(self, value: SupportsBool | SupportsLen | None = ...) -> None: ... class c_byte(_SimpleCData[int]): _type_: ClassVar[Literal["b"]] diff --git a/mypy/typeshed/stdlib/html/parser.pyi b/mypy/typeshed/stdlib/html/parser.pyi index 8b3fce0010b78..7edd39e8c7037 100644 --- a/mypy/typeshed/stdlib/html/parser.pyi +++ b/mypy/typeshed/stdlib/html/parser.pyi @@ -1,4 +1,3 @@ -import sys from _markupbase import ParserBase from re import Pattern from typing import Final @@ -7,9 +6,8 @@ __all__ = ["HTMLParser"] class HTMLParser(ParserBase): CDATA_CONTENT_ELEMENTS: Final[tuple[str, ...]] - if sys.version_info >= (3, 13): - # Added in 3.13.6 - RCDATA_CONTENT_ELEMENTS: Final[tuple[str, ...]] + # Added in Python 3.9.23, 3.10.18, 3.11.13, 3.12.11, 3.13.6 + RCDATA_CONTENT_ELEMENTS: Final[tuple[str, ...]] def __init__(self, *, convert_charrefs: bool = True) -> None: ... def feed(self, data: str) -> None: ... @@ -32,11 +30,8 @@ class HTMLParser(ParserBase): def parse_html_declaration(self, i: int) -> int: ... # undocumented def parse_pi(self, i: int) -> int: ... # undocumented def parse_starttag(self, i: int) -> int: ... # undocumented - if sys.version_info >= (3, 13): - # `escapable` parameter added in 3.13.6 - def set_cdata_mode(self, elem: str, *, escapable: bool = False) -> None: ... # undocumented - else: - def set_cdata_mode(self, elem: str) -> None: ... # undocumented + # `escapable` parameter added in Python 3.9.23, 3.10.18, 3.11.13, 3.12.11, 3.13.6 + def set_cdata_mode(self, elem: str, *, escapable: bool = False) -> None: ... # undocumented rawdata: str # undocumented cdata_elem: str | None # undocumented convert_charrefs: bool # undocumented diff --git a/mypy/typeshed/stdlib/importlib/abc.pyi b/mypy/typeshed/stdlib/importlib/abc.pyi index 72031e0e3bd2e..ef7761f7119b9 100644 --- a/mypy/typeshed/stdlib/importlib/abc.pyi +++ b/mypy/typeshed/stdlib/importlib/abc.pyi @@ -53,7 +53,7 @@ class InspectLoader(Loader): def exec_module(self, module: types.ModuleType) -> None: ... @staticmethod def source_to_code( - data: ReadableBuffer | str | _ast.Module | _ast.Expression | _ast.Interactive, path: ReadableBuffer | StrPath = "" + data: ReadableBuffer | str | _ast.Module | _ast.Expression | _ast.Interactive, path: bytes | StrPath = "" ) -> types.CodeType: ... class ExecutionLoader(InspectLoader): @@ -114,8 +114,8 @@ class FileLoader(_bootstrap_external.FileLoader, ResourceLoader, ExecutionLoader path: str def __init__(self, fullname: str, path: str) -> None: ... def get_data(self, path: str) -> bytes: ... - def get_filename(self, name: str | None = None) -> str: ... - def load_module(self, name: str | None = None) -> types.ModuleType: ... + def get_filename(self, fullname: str | None = None) -> str: ... + def load_module(self, fullname: str | None = None) -> types.ModuleType: ... if sys.version_info < (3, 11): class ResourceReader(metaclass=ABCMeta): diff --git a/mypy/typeshed/stdlib/importlib/resources/__init__.pyi b/mypy/typeshed/stdlib/importlib/resources/__init__.pyi index e672a619bd17a..28adc37da4a42 100644 --- a/mypy/typeshed/stdlib/importlib/resources/__init__.pyi +++ b/mypy/typeshed/stdlib/importlib/resources/__init__.pyi @@ -5,7 +5,7 @@ from contextlib import AbstractContextManager from pathlib import Path from types import ModuleType from typing import Any, BinaryIO, Literal, TextIO -from typing_extensions import TypeAlias +from typing_extensions import TypeAlias, deprecated if sys.version_info >= (3, 11): from importlib.resources.abc import Traversable @@ -64,7 +64,11 @@ else: def read_text(package: Package, resource: Resource, encoding: str = "utf-8", errors: str = "strict") -> str: ... def path(package: Package, resource: Resource) -> AbstractContextManager[Path, Literal[False]]: ... def is_resource(package: Package, name: str) -> bool: ... - def contents(package: Package) -> Iterator[str]: ... + if sys.version_info >= (3, 11): + @deprecated("Deprecated since Python 3.11. Use `files(anchor).iterdir()`.") + def contents(package: Package) -> Iterator[str]: ... + else: + def contents(package: Package) -> Iterator[str]: ... if sys.version_info >= (3, 11): from importlib.resources._common import as_file as as_file diff --git a/mypy/typeshed/stdlib/importlib/resources/_functional.pyi b/mypy/typeshed/stdlib/importlib/resources/_functional.pyi index 50f3405f9a00b..71e01bcd3d5ec 100644 --- a/mypy/typeshed/stdlib/importlib/resources/_functional.pyi +++ b/mypy/typeshed/stdlib/importlib/resources/_functional.pyi @@ -9,7 +9,7 @@ if sys.version_info >= (3, 13): from io import TextIOWrapper from pathlib import Path from typing import BinaryIO, Literal, overload - from typing_extensions import Unpack + from typing_extensions import Unpack, deprecated def open_binary(anchor: Anchor, *path_names: StrPath) -> BinaryIO: ... @overload @@ -27,4 +27,5 @@ if sys.version_info >= (3, 13): def read_text(anchor: Anchor, *path_names: StrPath, encoding: str | None, errors: str | None = "strict") -> str: ... def path(anchor: Anchor, *path_names: StrPath) -> AbstractContextManager[Path, Literal[False]]: ... def is_resource(anchor: Anchor, *path_names: StrPath) -> bool: ... + @deprecated("Deprecated since Python 3.11. Use `files(anchor).iterdir()`.") def contents(anchor: Anchor, *path_names: StrPath) -> Iterator[str]: ... diff --git a/mypy/typeshed/stdlib/operator.pyi b/mypy/typeshed/stdlib/operator.pyi index bc2b5e0266171..2f919514b0b8b 100644 --- a/mypy/typeshed/stdlib/operator.pyi +++ b/mypy/typeshed/stdlib/operator.pyi @@ -205,8 +205,10 @@ class itemgetter(Generic[_T_co]): # "tuple[int, int]" is incompatible with protocol "SupportsIndex" # preventing [_T_co, ...] instead of [Any, ...] # - # A suspected mypy issue prevents using [..., _T] instead of [..., Any] here. - # https://github.com/python/mypy/issues/14032 + # If we can't infer a literal key from __new__ (ie: `itemgetter[Literal[0]]` for `itemgetter(0)`), + # then we can't annotate __call__'s return type or it'll break on tuples + # + # These issues are best demonstrated by the `itertools.check_itertools_recipes.unique_justseen` test. def __call__(self, obj: SupportsGetItem[Any, Any]) -> Any: ... @final diff --git a/mypy/typeshed/stdlib/pdb.pyi b/mypy/typeshed/stdlib/pdb.pyi index 0c16f48e2e220..2f114b20572df 100644 --- a/mypy/typeshed/stdlib/pdb.pyi +++ b/mypy/typeshed/stdlib/pdb.pyi @@ -5,6 +5,7 @@ from cmd import Cmd from collections.abc import Callable, Iterable, Mapping, Sequence from inspect import _SourceObjectType from linecache import _ModuleGlobals +from rlcompleter import Completer from types import CodeType, FrameType, TracebackType from typing import IO, Any, ClassVar, Final, Literal, TypeVar from typing_extensions import ParamSpec, Self, TypeAlias @@ -200,6 +201,10 @@ class Pdb(Bdb, Cmd): def completenames(self, text: str, line: str, begidx: int, endidx: int) -> list[str]: ... # type: ignore[override] if sys.version_info >= (3, 12): def set_convenience_variable(self, frame: FrameType, name: str, value: Any) -> None: ... + if sys.version_info >= (3, 13) and sys.version_info < (3, 14): + # Added in 3.13.8. + @property + def rlcompleter(self) -> type[Completer]: ... def _select_frame(self, number: int) -> None: ... def _getval_except(self, arg: str, frame: FrameType | None = None) -> object: ... diff --git a/mypy/typeshed/stdlib/tkinter/__init__.pyi b/mypy/typeshed/stdlib/tkinter/__init__.pyi index b653545f1d9ca..ef57faa2b0097 100644 --- a/mypy/typeshed/stdlib/tkinter/__init__.pyi +++ b/mypy/typeshed/stdlib/tkinter/__init__.pyi @@ -3145,7 +3145,6 @@ class Scrollbar(Widget): def get(self) -> tuple[float, float, float, float] | tuple[float, float]: ... def set(self, first: float | str, last: float | str) -> None: ... -_TextIndex: TypeAlias = _tkinter.Tcl_Obj | str | float | Misc _WhatToCount: TypeAlias = Literal[ "chars", "displaychars", "displayindices", "displaylines", "indices", "lines", "xpixels", "ypixels" ] @@ -3261,20 +3260,37 @@ class Text(Widget, XView, YView): @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... config = configure - def bbox(self, index: _TextIndex) -> tuple[int, int, int, int] | None: ... # type: ignore[override] - def compare(self, index1: _TextIndex, op: Literal["<", "<=", "==", ">=", ">", "!="], index2: _TextIndex) -> bool: ... + def bbox(self, index: str | float | _tkinter.Tcl_Obj | Widget) -> tuple[int, int, int, int] | None: ... # type: ignore[override] + def compare( + self, + index1: str | float | _tkinter.Tcl_Obj | Widget, + op: Literal["<", "<=", "==", ">=", ">", "!="], + index2: str | float | _tkinter.Tcl_Obj | Widget, + ) -> bool: ... if sys.version_info >= (3, 13): @overload - def count(self, index1: _TextIndex, index2: _TextIndex, *, return_ints: Literal[True]) -> int: ... + def count( + self, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget, + *, + return_ints: Literal[True], + ) -> int: ... @overload def count( - self, index1: _TextIndex, index2: _TextIndex, arg: _WhatToCount | Literal["update"], /, *, return_ints: Literal[True] + self, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget, + arg: _WhatToCount | Literal["update"], + /, + *, + return_ints: Literal[True], ) -> int: ... @overload def count( self, - index1: _TextIndex, - index2: _TextIndex, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget, arg1: Literal["update"], arg2: _WhatToCount, /, @@ -3284,8 +3300,8 @@ class Text(Widget, XView, YView): @overload def count( self, - index1: _TextIndex, - index2: _TextIndex, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget, arg1: _WhatToCount, arg2: Literal["update"], /, @@ -3294,13 +3310,20 @@ class Text(Widget, XView, YView): ) -> int: ... @overload def count( - self, index1: _TextIndex, index2: _TextIndex, arg1: _WhatToCount, arg2: _WhatToCount, /, *, return_ints: Literal[True] + self, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget, + arg1: _WhatToCount, + arg2: _WhatToCount, + /, + *, + return_ints: Literal[True], ) -> tuple[int, int]: ... @overload def count( self, - index1: _TextIndex, - index2: _TextIndex, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget, arg1: _WhatToCount | Literal["update"], arg2: _WhatToCount | Literal["update"], arg3: _WhatToCount | Literal["update"], @@ -3309,12 +3332,18 @@ class Text(Widget, XView, YView): return_ints: Literal[True], ) -> tuple[int, ...]: ... @overload - def count(self, index1: _TextIndex, index2: _TextIndex, *, return_ints: Literal[False] = False) -> tuple[int] | None: ... + def count( + self, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget, + *, + return_ints: Literal[False] = False, + ) -> tuple[int] | None: ... @overload def count( self, - index1: _TextIndex, - index2: _TextIndex, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget, arg: _WhatToCount | Literal["update"], /, *, @@ -3323,8 +3352,8 @@ class Text(Widget, XView, YView): @overload def count( self, - index1: _TextIndex, - index2: _TextIndex, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget, arg1: Literal["update"], arg2: _WhatToCount, /, @@ -3334,8 +3363,8 @@ class Text(Widget, XView, YView): @overload def count( self, - index1: _TextIndex, - index2: _TextIndex, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget, arg1: _WhatToCount, arg2: Literal["update"], /, @@ -3345,8 +3374,8 @@ class Text(Widget, XView, YView): @overload def count( self, - index1: _TextIndex, - index2: _TextIndex, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget, arg1: _WhatToCount, arg2: _WhatToCount, /, @@ -3356,8 +3385,8 @@ class Text(Widget, XView, YView): @overload def count( self, - index1: _TextIndex, - index2: _TextIndex, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget, arg1: _WhatToCount | Literal["update"], arg2: _WhatToCount | Literal["update"], arg3: _WhatToCount | Literal["update"], @@ -3367,22 +3396,49 @@ class Text(Widget, XView, YView): ) -> tuple[int, ...]: ... else: @overload - def count(self, index1: _TextIndex, index2: _TextIndex) -> tuple[int] | None: ... + def count( + self, index1: str | float | _tkinter.Tcl_Obj | Widget, index2: str | float | _tkinter.Tcl_Obj | Widget + ) -> tuple[int] | None: ... @overload def count( - self, index1: _TextIndex, index2: _TextIndex, arg: _WhatToCount | Literal["update"], / + self, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget, + arg: _WhatToCount | Literal["update"], + /, ) -> tuple[int] | None: ... @overload - def count(self, index1: _TextIndex, index2: _TextIndex, arg1: Literal["update"], arg2: _WhatToCount, /) -> int | None: ... + def count( + self, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget, + arg1: Literal["update"], + arg2: _WhatToCount, + /, + ) -> int | None: ... @overload - def count(self, index1: _TextIndex, index2: _TextIndex, arg1: _WhatToCount, arg2: Literal["update"], /) -> int | None: ... + def count( + self, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget, + arg1: _WhatToCount, + arg2: Literal["update"], + /, + ) -> int | None: ... @overload - def count(self, index1: _TextIndex, index2: _TextIndex, arg1: _WhatToCount, arg2: _WhatToCount, /) -> tuple[int, int]: ... + def count( + self, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget, + arg1: _WhatToCount, + arg2: _WhatToCount, + /, + ) -> tuple[int, int]: ... @overload def count( self, - index1: _TextIndex, - index2: _TextIndex, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget, arg1: _WhatToCount | Literal["update"], arg2: _WhatToCount | Literal["update"], arg3: _WhatToCount | Literal["update"], @@ -3394,13 +3450,15 @@ class Text(Widget, XView, YView): def debug(self, boolean: None = None) -> bool: ... @overload def debug(self, boolean: bool) -> None: ... - def delete(self, index1: _TextIndex, index2: _TextIndex | None = None) -> None: ... - def dlineinfo(self, index: _TextIndex) -> tuple[int, int, int, int, int] | None: ... + def delete( + self, index1: str | float | _tkinter.Tcl_Obj | Widget, index2: str | float | _tkinter.Tcl_Obj | Widget | None = None + ) -> None: ... + def dlineinfo(self, index: str | float | _tkinter.Tcl_Obj | Widget) -> tuple[int, int, int, int, int] | None: ... @overload def dump( self, - index1: _TextIndex, - index2: _TextIndex | None = None, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget | None = None, command: None = None, *, all: bool = ..., @@ -3413,8 +3471,8 @@ class Text(Widget, XView, YView): @overload def dump( self, - index1: _TextIndex, - index2: _TextIndex | None, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget | None, command: Callable[[str, str, str], object] | str, *, all: bool = ..., @@ -3427,8 +3485,8 @@ class Text(Widget, XView, YView): @overload def dump( self, - index1: _TextIndex, - index2: _TextIndex | None = None, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget | None = None, *, command: Callable[[str, str, str], object] | str, all: bool = ..., @@ -3447,21 +3505,27 @@ class Text(Widget, XView, YView): def edit_reset(self) -> None: ... # actually returns empty string def edit_separator(self) -> None: ... # actually returns empty string def edit_undo(self) -> None: ... # actually returns empty string - def get(self, index1: _TextIndex, index2: _TextIndex | None = None) -> str: ... + def get( + self, index1: str | float | _tkinter.Tcl_Obj | Widget, index2: str | float | _tkinter.Tcl_Obj | Widget | None = None + ) -> str: ... @overload - def image_cget(self, index: _TextIndex, option: Literal["image", "name"]) -> str: ... + def image_cget(self, index: str | float | _tkinter.Tcl_Obj | Widget, option: Literal["image", "name"]) -> str: ... @overload - def image_cget(self, index: _TextIndex, option: Literal["padx", "pady"]) -> int: ... + def image_cget(self, index: str | float | _tkinter.Tcl_Obj | Widget, option: Literal["padx", "pady"]) -> int: ... @overload - def image_cget(self, index: _TextIndex, option: Literal["align"]) -> Literal["baseline", "bottom", "center", "top"]: ... + def image_cget( + self, index: str | float | _tkinter.Tcl_Obj | Widget, option: Literal["align"] + ) -> Literal["baseline", "bottom", "center", "top"]: ... @overload - def image_cget(self, index: _TextIndex, option: str) -> Any: ... + def image_cget(self, index: str | float | _tkinter.Tcl_Obj | Widget, option: str) -> Any: ... @overload - def image_configure(self, index: _TextIndex, cnf: str) -> tuple[str, str, str, str, str | int]: ... + def image_configure( + self, index: str | float | _tkinter.Tcl_Obj | Widget, cnf: str + ) -> tuple[str, str, str, str, str | int]: ... @overload def image_configure( self, - index: _TextIndex, + index: str | float | _tkinter.Tcl_Obj | Widget, cnf: dict[str, Any] | None = None, *, align: Literal["baseline", "bottom", "center", "top"] = ..., @@ -3472,7 +3536,7 @@ class Text(Widget, XView, YView): ) -> dict[str, tuple[str, str, str, str, str | int]] | None: ... def image_create( self, - index: _TextIndex, + index: str | float | _tkinter.Tcl_Obj | Widget, cnf: dict[str, Any] | None = {}, *, align: Literal["baseline", "bottom", "center", "top"] = ..., @@ -3482,28 +3546,36 @@ class Text(Widget, XView, YView): pady: float | str = ..., ) -> str: ... def image_names(self) -> tuple[str, ...]: ... - def index(self, index: _TextIndex) -> str: ... - def insert(self, index: _TextIndex, chars: str, *args: str | list[str] | tuple[str, ...]) -> None: ... + def index(self, index: str | float | _tkinter.Tcl_Obj | Widget) -> str: ... + def insert( + self, index: str | float | _tkinter.Tcl_Obj | Widget, chars: str, *args: str | list[str] | tuple[str, ...] + ) -> None: ... @overload def mark_gravity(self, markName: str, direction: None = None) -> Literal["left", "right"]: ... @overload def mark_gravity(self, markName: str, direction: Literal["left", "right"]) -> None: ... # actually returns empty string def mark_names(self) -> tuple[str, ...]: ... - def mark_set(self, markName: str, index: _TextIndex) -> None: ... + def mark_set(self, markName: str, index: str | float | _tkinter.Tcl_Obj | Widget) -> None: ... def mark_unset(self, *markNames: str) -> None: ... - def mark_next(self, index: _TextIndex) -> str | None: ... - def mark_previous(self, index: _TextIndex) -> str | None: ... + def mark_next(self, index: str | float | _tkinter.Tcl_Obj | Widget) -> str | None: ... + def mark_previous(self, index: str | float | _tkinter.Tcl_Obj | Widget) -> str | None: ... # **kw of peer_create is same as the kwargs of Text.__init__ def peer_create(self, newPathName: str | Text, cnf: dict[str, Any] = {}, **kw) -> None: ... def peer_names(self) -> tuple[_tkinter.Tcl_Obj, ...]: ... - def replace(self, index1: _TextIndex, index2: _TextIndex, chars: str, *args: str | list[str] | tuple[str, ...]) -> None: ... + def replace( + self, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget, + chars: str, + *args: str | list[str] | tuple[str, ...], + ) -> None: ... def scan_mark(self, x: int, y: int) -> None: ... def scan_dragto(self, x: int, y: int) -> None: ... def search( self, pattern: str, - index: _TextIndex, - stopindex: _TextIndex | None = None, + index: str | float | _tkinter.Tcl_Obj | Widget, + stopindex: str | float | _tkinter.Tcl_Obj | Widget | None = None, forwards: bool | None = None, backwards: bool | None = None, exact: bool | None = None, @@ -3512,8 +3584,10 @@ class Text(Widget, XView, YView): count: Variable | None = None, elide: bool | None = None, ) -> str: ... # returns empty string for not found - def see(self, index: _TextIndex) -> None: ... - def tag_add(self, tagName: str, index1: _TextIndex, *args: _TextIndex) -> None: ... + def see(self, index: str | float | _tkinter.Tcl_Obj | Widget) -> None: ... + def tag_add( + self, tagName: str, index1: str | float | _tkinter.Tcl_Obj | Widget, *args: str | float | _tkinter.Tcl_Obj | Widget + ) -> None: ... # tag_bind stuff is very similar to Canvas @overload def tag_bind( @@ -3568,33 +3642,50 @@ class Text(Widget, XView, YView): tag_config = tag_configure def tag_delete(self, first_tag_name: str, /, *tagNames: str) -> None: ... # error if no tag names given def tag_lower(self, tagName: str, belowThis: str | None = None) -> None: ... - def tag_names(self, index: _TextIndex | None = None) -> tuple[str, ...]: ... + def tag_names(self, index: str | float | _tkinter.Tcl_Obj | Widget | None = None) -> tuple[str, ...]: ... def tag_nextrange( - self, tagName: str, index1: _TextIndex, index2: _TextIndex | None = None + self, + tagName: str, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget | None = None, ) -> tuple[str, str] | tuple[()]: ... def tag_prevrange( - self, tagName: str, index1: _TextIndex, index2: _TextIndex | None = None + self, + tagName: str, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget | None = None, ) -> tuple[str, str] | tuple[()]: ... def tag_raise(self, tagName: str, aboveThis: str | None = None) -> None: ... def tag_ranges(self, tagName: str) -> tuple[_tkinter.Tcl_Obj, ...]: ... # tag_remove and tag_delete are different - def tag_remove(self, tagName: str, index1: _TextIndex, index2: _TextIndex | None = None) -> None: ... + def tag_remove( + self, + tagName: str, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget | None = None, + ) -> None: ... @overload - def window_cget(self, index: _TextIndex, option: Literal["padx", "pady"]) -> int: ... + def window_cget(self, index: str | float | _tkinter.Tcl_Obj | Widget, option: Literal["padx", "pady"]) -> int: ... @overload - def window_cget(self, index: _TextIndex, option: Literal["stretch"]) -> bool: ... # actually returns Literal[0, 1] + def window_cget( + self, index: str | float | _tkinter.Tcl_Obj | Widget, option: Literal["stretch"] + ) -> bool: ... # actually returns Literal[0, 1] @overload - def window_cget(self, index: _TextIndex, option: Literal["align"]) -> Literal["baseline", "bottom", "center", "top"]: ... + def window_cget( + self, index: str | float | _tkinter.Tcl_Obj | Widget, option: Literal["align"] + ) -> Literal["baseline", "bottom", "center", "top"]: ... @overload # window is set to a widget, but read as the string name. - def window_cget(self, index: _TextIndex, option: Literal["create", "window"]) -> str: ... + def window_cget(self, index: str | float | _tkinter.Tcl_Obj | Widget, option: Literal["create", "window"]) -> str: ... @overload - def window_cget(self, index: _TextIndex, option: str) -> Any: ... + def window_cget(self, index: str | float | _tkinter.Tcl_Obj | Widget, option: str) -> Any: ... @overload - def window_configure(self, index: _TextIndex, cnf: str) -> tuple[str, str, str, str, str | int]: ... + def window_configure( + self, index: str | float | _tkinter.Tcl_Obj | Widget, cnf: str + ) -> tuple[str, str, str, str, str | int]: ... @overload def window_configure( self, - index: _TextIndex, + index: str | float | _tkinter.Tcl_Obj | Widget, cnf: dict[str, Any] | None = None, *, align: Literal["baseline", "bottom", "center", "top"] = ..., @@ -3607,7 +3698,7 @@ class Text(Widget, XView, YView): window_config = window_configure def window_create( self, - index: _TextIndex, + index: str | float | _tkinter.Tcl_Obj | Widget, cnf: dict[str, Any] | None = {}, *, align: Literal["baseline", "bottom", "center", "top"] = ..., diff --git a/mypy/typeshed/stdlib/turtle.pyi b/mypy/typeshed/stdlib/turtle.pyi index 39a995de26124..9b9b329bd74bc 100644 --- a/mypy/typeshed/stdlib/turtle.pyi +++ b/mypy/typeshed/stdlib/turtle.pyi @@ -221,16 +221,20 @@ class Terminator(Exception): ... class TurtleGraphicsError(Exception): ... class Shape: - def __init__(self, type_: str, data: _PolygonCoords | PhotoImage | None = None) -> None: ... + def __init__( + self, type_: Literal["polygon", "image", "compound"], data: _PolygonCoords | PhotoImage | None = None + ) -> None: ... def addcomponent(self, poly: _PolygonCoords, fill: _Color, outline: _Color | None = None) -> None: ... class TurtleScreen(TurtleScreenBase): - def __init__(self, cv: Canvas, mode: str = "standard", colormode: float = 1.0, delay: int = 10) -> None: ... + def __init__( + self, cv: Canvas, mode: Literal["standard", "logo", "world"] = "standard", colormode: float = 1.0, delay: int = 10 + ) -> None: ... def clear(self) -> None: ... @overload def mode(self, mode: None = None) -> str: ... @overload - def mode(self, mode: str) -> None: ... + def mode(self, mode: Literal["standard", "logo", "world"]) -> None: ... def setworldcoordinates(self, llx: float, lly: float, urx: float, ury: float) -> None: ... def register_shape(self, name: str, shape: _PolygonCoords | Shape | None = None) -> None: ... @overload @@ -289,7 +293,7 @@ class TNavigator: DEFAULT_MODE: str DEFAULT_ANGLEOFFSET: int DEFAULT_ANGLEORIENT: int - def __init__(self, mode: str = "standard") -> None: ... + def __init__(self, mode: Literal["standard", "logo", "world"] = "standard") -> None: ... def reset(self) -> None: ... def degrees(self, fullcircle: float = 360.0) -> None: ... def radians(self) -> None: ... @@ -333,11 +337,11 @@ class TNavigator: seth = setheading class TPen: - def __init__(self, resizemode: str = "noresize") -> None: ... + def __init__(self, resizemode: Literal["auto", "user", "noresize"] = "noresize") -> None: ... @overload def resizemode(self, rmode: None = None) -> str: ... @overload - def resizemode(self, rmode: str) -> None: ... + def resizemode(self, rmode: Literal["auto", "user", "noresize"]) -> None: ... @overload def pensize(self, width: None = None) -> int: ... @overload @@ -389,7 +393,7 @@ class TPen: fillcolor: _Color = ..., pensize: int = ..., speed: int = ..., - resizemode: str = ..., + resizemode: Literal["auto", "user", "noresize"] = ..., stretchfactor: tuple[float, float] = ..., outline: int = ..., tilt: float = ..., @@ -524,7 +528,7 @@ def clear() -> None: ... @overload def mode(mode: None = None) -> str: ... @overload -def mode(mode: str) -> None: ... +def mode(mode: Literal["standard", "logo", "world"]) -> None: ... def setworldcoordinates(llx: float, lly: float, urx: float, ury: float) -> None: ... def register_shape(name: str, shape: _PolygonCoords | Shape | None = None) -> None: ... @overload @@ -634,7 +638,7 @@ seth = setheading @overload def resizemode(rmode: None = None) -> str: ... @overload -def resizemode(rmode: str) -> None: ... +def resizemode(rmode: Literal["auto", "user", "noresize"]) -> None: ... @overload def pensize(width: None = None) -> int: ... @overload @@ -683,7 +687,7 @@ def pen( fillcolor: _Color = ..., pensize: int = ..., speed: int = ..., - resizemode: str = ..., + resizemode: Literal["auto", "user", "noresize"] = ..., stretchfactor: tuple[float, float] = ..., outline: int = ..., tilt: float = ..., diff --git a/mypy/typeshed/stdlib/types.pyi b/mypy/typeshed/stdlib/types.pyi index ba343ce9effc0..649e463ff71f8 100644 --- a/mypy/typeshed/stdlib/types.pyi +++ b/mypy/typeshed/stdlib/types.pyi @@ -65,7 +65,7 @@ if sys.version_info >= (3, 13): _T1 = TypeVar("_T1") _T2 = TypeVar("_T2") -_KT = TypeVar("_KT") +_KT_co = TypeVar("_KT_co", covariant=True) _VT_co = TypeVar("_VT_co", covariant=True) # Make sure this class definition stays roughly in line with `builtins.function` @@ -309,27 +309,27 @@ class CodeType: __replace__ = replace @final -class MappingProxyType(Mapping[_KT, _VT_co]): +class MappingProxyType(Mapping[_KT_co, _VT_co]): # type: ignore[type-var] # pyright: ignore[reportInvalidTypeArguments] __hash__: ClassVar[None] # type: ignore[assignment] - def __new__(cls, mapping: SupportsKeysAndGetItem[_KT, _VT_co]) -> Self: ... - def __getitem__(self, key: _KT, /) -> _VT_co: ... - def __iter__(self) -> Iterator[_KT]: ... + def __new__(cls, mapping: SupportsKeysAndGetItem[_KT_co, _VT_co]) -> Self: ... + def __getitem__(self, key: _KT_co, /) -> _VT_co: ... # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] + def __iter__(self) -> Iterator[_KT_co]: ... def __len__(self) -> int: ... def __eq__(self, value: object, /) -> bool: ... - def copy(self) -> dict[_KT, _VT_co]: ... - def keys(self) -> KeysView[_KT]: ... + def copy(self) -> dict[_KT_co, _VT_co]: ... + def keys(self) -> KeysView[_KT_co]: ... def values(self) -> ValuesView[_VT_co]: ... - def items(self) -> ItemsView[_KT, _VT_co]: ... + def items(self) -> ItemsView[_KT_co, _VT_co]: ... @overload - def get(self, key: _KT, /) -> _VT_co | None: ... + def get(self, key: _KT_co, /) -> _VT_co | None: ... # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] # Covariant type as parameter @overload - def get(self, key: _KT, default: _VT_co, /) -> _VT_co: ... # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] # Covariant type as parameter + def get(self, key: _KT_co, default: _VT_co, /) -> _VT_co: ... # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] # Covariant type as parameter @overload - def get(self, key: _KT, default: _T2, /) -> _VT_co | _T2: ... + def get(self, key: _KT_co, default: _T2, /) -> _VT_co | _T2: ... # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] # Covariant type as parameter def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... - def __reversed__(self) -> Iterator[_KT]: ... - def __or__(self, value: Mapping[_T1, _T2], /) -> dict[_KT | _T1, _VT_co | _T2]: ... - def __ror__(self, value: Mapping[_T1, _T2], /) -> dict[_KT | _T1, _VT_co | _T2]: ... + def __reversed__(self) -> Iterator[_KT_co]: ... + def __or__(self, value: Mapping[_T1, _T2], /) -> dict[_KT_co | _T1, _VT_co | _T2]: ... + def __ror__(self, value: Mapping[_T1, _T2], /) -> dict[_KT_co | _T1, _VT_co | _T2]: ... if sys.version_info >= (3, 12): @disjoint_base diff --git a/mypy/typeshed/stdlib/typing.pyi b/mypy/typeshed/stdlib/typing.pyi index ca25c92d5c34a..2ca65dad4562f 100644 --- a/mypy/typeshed/stdlib/typing.pyi +++ b/mypy/typeshed/stdlib/typing.pyi @@ -1133,14 +1133,23 @@ if sys.version_info >= (3, 10): def _type_repr(obj: object) -> str: ... if sys.version_info >= (3, 12): + _TypeParameter: typing_extensions.TypeAlias = ( + TypeVar + | typing_extensions.TypeVar + | ParamSpec + | typing_extensions.ParamSpec + | TypeVarTuple + | typing_extensions.TypeVarTuple + ) + def override(method: _F, /) -> _F: ... @final class TypeAliasType: - def __new__(cls, name: str, value: Any, *, type_params: tuple[TypeVar | ParamSpec | TypeVarTuple, ...] = ()) -> Self: ... + def __new__(cls, name: str, value: Any, *, type_params: tuple[_TypeParameter, ...] = ()) -> Self: ... @property def __value__(self) -> Any: ... # AnnotationForm @property - def __type_params__(self) -> tuple[TypeVar | ParamSpec | TypeVarTuple, ...]: ... + def __type_params__(self) -> tuple[_TypeParameter, ...]: ... @property def __parameters__(self) -> tuple[Any, ...]: ... # AnnotationForm @property diff --git a/mypy/typeshed/stdlib/typing_extensions.pyi b/mypy/typeshed/stdlib/typing_extensions.pyi index f5ea13f67733f..5fd3f4578a8bd 100644 --- a/mypy/typeshed/stdlib/typing_extensions.pyi +++ b/mypy/typeshed/stdlib/typing_extensions.pyi @@ -239,7 +239,7 @@ class _TypedDict(Mapping[str, object], metaclass=abc.ABCMeta): __readonly_keys__: ClassVar[frozenset[str]] __mutable_keys__: ClassVar[frozenset[str]] # PEP 728 - __closed__: ClassVar[bool] + __closed__: ClassVar[bool | None] __extra_items__: ClassVar[AnnotationForm] def copy(self) -> Self: ... # Using Never so that only calls using mypy plugin hook that specialize the signature diff --git a/mypy/typeshed/stdlib/xml/etree/ElementTree.pyi b/mypy/typeshed/stdlib/xml/etree/ElementTree.pyi index e8f737778040c..d42db1bc0c571 100644 --- a/mypy/typeshed/stdlib/xml/etree/ElementTree.pyi +++ b/mypy/typeshed/stdlib/xml/etree/ElementTree.pyi @@ -93,7 +93,7 @@ class Element(Generic[_Tag]): def __init__(self, tag: _Tag, attrib: dict[str, str] = {}, **extra: str) -> None: ... def append(self, subelement: Element[Any], /) -> None: ... def clear(self) -> None: ... - def extend(self, elements: Iterable[Element], /) -> None: ... + def extend(self, elements: Iterable[Element[Any]], /) -> None: ... def find(self, path: str, namespaces: dict[str, str] | None = None) -> Element | None: ... def findall(self, path: str, namespaces: dict[str, str] | None = None) -> list[Element]: ... @overload @@ -104,7 +104,7 @@ class Element(Generic[_Tag]): def get(self, key: str, default: None = None) -> str | None: ... @overload def get(self, key: str, default: _T) -> str | _T: ... - def insert(self, index: int, subelement: Element, /) -> None: ... + def insert(self, index: int, subelement: Element[Any], /) -> None: ... def items(self) -> ItemsView[str, str]: ... def iter(self, tag: str | None = None) -> Generator[Element, None, None]: ... @overload @@ -115,7 +115,7 @@ class Element(Generic[_Tag]): def keys(self) -> dict_keys[str, str]: ... # makeelement returns the type of self in Python impl, but not in C impl def makeelement(self, tag: _OtherTag, attrib: dict[str, str], /) -> Element[_OtherTag]: ... - def remove(self, subelement: Element, /) -> None: ... + def remove(self, subelement: Element[Any], /) -> None: ... def set(self, key: str, value: str, /) -> None: ... def __copy__(self) -> Element[_Tag]: ... # returns the type of self in Python impl, but not in C impl def __deepcopy__(self, memo: Any, /) -> Element: ... # Only exists in C impl @@ -128,15 +128,15 @@ class Element(Generic[_Tag]): # Doesn't actually exist at runtime, but instance of the class are indeed iterable due to __getitem__. def __iter__(self) -> Iterator[Element]: ... @overload - def __setitem__(self, key: SupportsIndex, value: Element, /) -> None: ... + def __setitem__(self, key: SupportsIndex, value: Element[Any], /) -> None: ... @overload - def __setitem__(self, key: slice, value: Iterable[Element], /) -> None: ... + def __setitem__(self, key: slice, value: Iterable[Element[Any]], /) -> None: ... # Doesn't really exist in earlier versions, where __len__ is called implicitly instead @deprecated("Testing an element's truth value is deprecated.") def __bool__(self) -> bool: ... -def SubElement(parent: Element, tag: str, attrib: dict[str, str] = ..., **extra: str) -> Element: ... +def SubElement(parent: Element[Any], tag: str, attrib: dict[str, str] = ..., **extra: str) -> Element: ... def Comment(text: str | None = None) -> Element[_ElementCallable]: ... def ProcessingInstruction(target: str, text: str | None = None) -> Element[_ElementCallable]: ... @@ -155,7 +155,7 @@ class QName: _Root = TypeVar("_Root", Element, Element | None, default=Element | None) class ElementTree(Generic[_Root]): - def __init__(self, element: Element | None = None, file: _FileRead | None = None) -> None: ... + def __init__(self, element: Element[Any] | None = None, file: _FileRead | None = None) -> None: ... def getroot(self) -> _Root: ... def parse(self, source: _FileRead, parser: XMLParser | None = None) -> Element: ... def iter(self, tag: str | None = None) -> Generator[Element, None, None]: ... @@ -186,7 +186,7 @@ HTML_EMPTY: Final[set[str]] def register_namespace(prefix: str, uri: str) -> None: ... @overload def tostring( - element: Element, + element: Element[Any], encoding: None = None, method: Literal["xml", "html", "text", "c14n"] | None = None, *, @@ -196,7 +196,7 @@ def tostring( ) -> bytes: ... @overload def tostring( - element: Element, + element: Element[Any], encoding: Literal["unicode"], method: Literal["xml", "html", "text", "c14n"] | None = None, *, @@ -206,7 +206,7 @@ def tostring( ) -> str: ... @overload def tostring( - element: Element, + element: Element[Any], encoding: str, method: Literal["xml", "html", "text", "c14n"] | None = None, *, @@ -216,7 +216,7 @@ def tostring( ) -> Any: ... @overload def tostringlist( - element: Element, + element: Element[Any], encoding: None = None, method: Literal["xml", "html", "text", "c14n"] | None = None, *, @@ -226,7 +226,7 @@ def tostringlist( ) -> list[bytes]: ... @overload def tostringlist( - element: Element, + element: Element[Any], encoding: Literal["unicode"], method: Literal["xml", "html", "text", "c14n"] | None = None, *, @@ -236,7 +236,7 @@ def tostringlist( ) -> list[str]: ... @overload def tostringlist( - element: Element, + element: Element[Any], encoding: str, method: Literal["xml", "html", "text", "c14n"] | None = None, *, @@ -244,8 +244,8 @@ def tostringlist( default_namespace: str | None = None, short_empty_elements: bool = True, ) -> list[Any]: ... -def dump(elem: Element | ElementTree[Any]) -> None: ... -def indent(tree: Element | ElementTree[Any], space: str = " ", level: int = 0) -> None: ... +def dump(elem: Element[Any] | ElementTree[Any]) -> None: ... +def indent(tree: Element[Any] | ElementTree[Any], space: str = " ", level: int = 0) -> None: ... def parse(source: _FileRead, parser: XMLParser[Any] | None = None) -> ElementTree[Element]: ... # This class is defined inside the body of iterparse From 37333c7741fecc2980e7ec6c983d3f7ce9186cb2 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 17 Oct 2025 19:16:05 +0100 Subject: [PATCH 339/424] Make metas more compact; fix indirect suppression (#20075) This makes cache metas ~twice smaller with two things: * Don't store individual options that don't require any special handling, just take a hash of them all. This behavior is disabled if `--debug-cache` is set. * Store `dep_hashes` as a list instead of a dictionary. Note that while implementing the second part, the assert I added failed, so I started digging and fond that suppression was handled inconsistency for indirect dependencies. I am fixing this here, now indirect dependencies are (un)suppressed just like any other dependency. Note this allowed to simplify/delete some parts of the code. I the next PR I am going to use fixed format serialization for `CacheMeta` (when enabled). --------- Co-authored-by: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> --- mypy/build.py | 112 ++++++++++++++++++++++-------------------------- mypy/options.py | 3 +- 2 files changed, 52 insertions(+), 63 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index f9137d8b1a32f..489fcf69c22c5 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -363,7 +363,7 @@ class CacheMeta(NamedTuple): # dep_prios and dep_lines are in parallel with dependencies + suppressed dep_prios: list[int] dep_lines: list[int] - dep_hashes: dict[str, str] + dep_hashes: list[str] interface_hash: str # hash representing the public interface error_lines: list[str] version_id: str # mypy version for cache invalidation @@ -403,7 +403,7 @@ def cache_meta_from_dict(meta: dict[str, Any], data_json: str) -> CacheMeta: meta.get("options"), meta.get("dep_prios", []), meta.get("dep_lines", []), - meta.get("dep_hashes", {}), + meta.get("dep_hashes", []), meta.get("interface_hash", ""), meta.get("error_lines", []), meta.get("version_id", sentinel), @@ -1310,8 +1310,7 @@ def get_cache_names(id: str, path: str, options: Options) -> tuple[str, str, str Args: id: module ID path: module path - cache_dir: cache directory - pyversion: Python version (major, minor) + options: build options Returns: A tuple with the file names to be used for the meta JSON, the @@ -1328,7 +1327,7 @@ def get_cache_names(id: str, path: str, options: Options) -> tuple[str, str, str # Solve this by rewriting the paths as relative to the root dir. # This only makes sense when using the filesystem backed cache. root = _cache_dir_prefix(options) - return (os.path.relpath(pair[0], root), os.path.relpath(pair[1], root), None) + return os.path.relpath(pair[0], root), os.path.relpath(pair[1], root), None prefix = os.path.join(*id.split(".")) is_package = os.path.basename(path).startswith("__init__.py") if is_package: @@ -1341,7 +1340,20 @@ def get_cache_names(id: str, path: str, options: Options) -> tuple[str, str, str data_suffix = ".data.ff" else: data_suffix = ".data.json" - return (prefix + ".meta.json", prefix + data_suffix, deps_json) + return prefix + ".meta.json", prefix + data_suffix, deps_json + + +def options_snapshot(id: str, manager: BuildManager) -> dict[str, object]: + """Make compact snapshot of options for a module. + + Separately store only the options we may compare individually, and take a hash + of everything else. If --debug-cache is specified, fall back to full snapshot. + """ + snapshot = manager.options.clone_for_module(id).select_options_affecting_cache() + if manager.options.debug_cache: + return snapshot + platform_opt = snapshot.pop("platform") + return {"platform": platform_opt, "other_options": hash_digest(json_dumps(snapshot))} def find_cache_meta(id: str, path: str, manager: BuildManager) -> CacheMeta | None: @@ -1403,7 +1415,7 @@ def find_cache_meta(id: str, path: str, manager: BuildManager) -> CacheMeta | No # Ignore cache if (relevant) options aren't the same. # Note that it's fine to mutilate cached_options since it's only used here. cached_options = m.options - current_options = manager.options.clone_for_module(id).select_options_affecting_cache() + current_options = options_snapshot(id, manager) if manager.options.skip_version_check: # When we're lax about version we're also lax about platform. cached_options["platform"] = current_options["platform"] @@ -1556,7 +1568,7 @@ def validate_meta( "data_mtime": meta.data_mtime, "dependencies": meta.dependencies, "suppressed": meta.suppressed, - "options": (manager.options.clone_for_module(id).select_options_affecting_cache()), + "options": options_snapshot(id, manager), "dep_prios": meta.dep_prios, "dep_lines": meta.dep_lines, "dep_hashes": meta.dep_hashes, @@ -1701,7 +1713,6 @@ def write_cache( # updates made by inline config directives in the file. This is # important, or otherwise the options would never match when # verifying the cache. - options = manager.options.clone_for_module(id) assert source_hash is not None meta = { "id": id, @@ -1712,7 +1723,7 @@ def write_cache( "data_mtime": data_mtime, "dependencies": dependencies, "suppressed": suppressed, - "options": options.select_options_affecting_cache(), + "options": options_snapshot(id, manager), "dep_prios": dep_prios, "dep_lines": dep_lines, "interface_hash": interface_hash, @@ -2029,7 +2040,10 @@ def __init__( self.priorities = {id: pri for id, pri in zip(all_deps, self.meta.dep_prios)} assert len(all_deps) == len(self.meta.dep_lines) self.dep_line_map = {id: line for id, line in zip(all_deps, self.meta.dep_lines)} - self.dep_hashes = self.meta.dep_hashes + assert len(self.meta.dep_hashes) == len(self.meta.dependencies) + self.dep_hashes = { + k: v for (k, v) in zip(self.meta.dependencies, self.meta.dep_hashes) + } self.error_lines = self.meta.error_lines if temporary: self.load_tree(temporary=True) @@ -2346,6 +2360,7 @@ def compute_dependencies(self) -> None: self.suppressed_set = set() self.priorities = {} # id -> priority self.dep_line_map = {} # id -> line + self.dep_hashes = {} dep_entries = manager.all_imported_modules_in_file( self.tree ) + self.manager.plugin.get_additional_deps(self.tree) @@ -2433,7 +2448,7 @@ def finish_passes(self) -> None: # We should always patch indirect dependencies, even in full (non-incremental) builds, # because the cache still may be written, and it must be correct. - self._patch_indirect_dependencies( + self.patch_indirect_dependencies( # Two possible sources of indirect dependencies: # * Symbols not directly imported in this module but accessed via an attribute # or via a re-export (vast majority of these recorded in semantic analysis). @@ -2470,21 +2485,17 @@ def free_state(self) -> None: self._type_checker.reset() self._type_checker = None - def _patch_indirect_dependencies(self, module_refs: set[str], types: set[Type]) -> None: - assert None not in types - valid = self.valid_references() + def patch_indirect_dependencies(self, module_refs: set[str], types: set[Type]) -> None: + assert self.ancestors is not None + existing_deps = set(self.dependencies + self.suppressed + self.ancestors) + existing_deps.add(self.id) encountered = self.manager.indirection_detector.find_modules(types) | module_refs - extra = encountered - valid - - for dep in sorted(extra): + for dep in sorted(encountered - existing_deps): if dep not in self.manager.modules: continue - if dep not in self.suppressed_set and dep not in self.manager.missing_modules: - self.add_dependency(dep) - self.priorities[dep] = PRI_INDIRECT - elif dep not in self.suppressed_set and dep in self.manager.missing_modules: - self.suppress_dependency(dep) + self.add_dependency(dep) + self.priorities[dep] = PRI_INDIRECT def compute_fine_grained_deps(self) -> dict[str, set[str]]: assert self.tree is not None @@ -2514,16 +2525,6 @@ def update_fine_grained_deps(self, deps: dict[str, set[str]]) -> None: merge_dependencies(self.compute_fine_grained_deps(), deps) type_state.update_protocol_deps(deps) - def valid_references(self) -> set[str]: - assert self.ancestors is not None - valid_refs = set(self.dependencies + self.suppressed + self.ancestors) - valid_refs.add(self.id) - - if "os" in valid_refs: - valid_refs.add("os.path") - - return valid_refs - def write_cache(self) -> tuple[dict[str, Any], str] | None: assert self.tree is not None, "Internal error: method must be called on parsed file only" # We don't support writing cache files in fine-grained incremental mode. @@ -2577,14 +2578,16 @@ def verify_dependencies(self, suppressed_only: bool = False) -> None: """ manager = self.manager assert self.ancestors is not None + # Strip out indirect dependencies. See comment in build.load_graph(). if suppressed_only: - all_deps = self.suppressed + all_deps = [dep for dep in self.suppressed if self.priorities.get(dep) != PRI_INDIRECT] else: - # Strip out indirect dependencies. See comment in build.load_graph(). dependencies = [ - dep for dep in self.dependencies if self.priorities.get(dep) != PRI_INDIRECT + dep + for dep in self.dependencies + self.suppressed + if self.priorities.get(dep) != PRI_INDIRECT ] - all_deps = dependencies + self.suppressed + self.ancestors + all_deps = dependencies + self.ancestors for dep in all_deps: if dep in manager.modules: continue @@ -3250,6 +3253,13 @@ def load_graph( if dep in graph and dep in st.suppressed_set: # Previously suppressed file is now visible st.add_dependency(dep) + # In the loop above we skip indirect dependencies, so to make indirect dependencies behave + # more consistently with regular ones, we suppress them manually here (when needed). + for st in graph.values(): + indirect = [dep for dep in st.dependencies if st.priorities.get(dep) == PRI_INDIRECT] + for dep in indirect: + if dep not in graph: + st.suppress_dependency(dep) manager.plugin.set_modules(manager.modules) return graph @@ -3284,8 +3294,9 @@ def find_stale_sccs( Fresh SCCs are those where: * We have valid cache files for all modules in the SCC. + * There are no changes in dependencies (files removed from/added to the build). * The interface hashes of direct dependents matches those recorded in the cache. - * There are no new (un)suppressed dependencies (files removed/added to the build). + The first and second conditions are verified by is_fresh(). """ stale_sccs = [] fresh_sccs = [] @@ -3294,34 +3305,15 @@ def find_stale_sccs( fresh = not stale_scc # Verify that interfaces of dependencies still present in graph are up-to-date (fresh). - # Note: if a dependency is not in graph anymore, it should be considered interface-stale. - # This is important to trigger any relevant updates from indirect dependencies that were - # removed in load_graph(). stale_deps = set() for id in ascc.mod_ids: for dep in graph[id].dep_hashes: - if dep not in graph: - stale_deps.add(dep) - continue - if graph[dep].interface_hash != graph[id].dep_hashes[dep]: + if dep in graph and graph[dep].interface_hash != graph[id].dep_hashes[dep]: stale_deps.add(dep) fresh = fresh and not stale_deps - undeps = set() - if fresh: - # Check if any dependencies that were suppressed according - # to the cache have been added back in this run. - # NOTE: Newly suppressed dependencies are handled by is_fresh(). - for id in ascc.mod_ids: - undeps.update(graph[id].suppressed) - undeps &= graph.keys() - if undeps: - fresh = False - if fresh: fresh_msg = "fresh" - elif undeps: - fresh_msg = f"stale due to changed suppression ({' '.join(sorted(undeps))})" elif stale_scc: fresh_msg = "inherently stale" if stale_scc != ascc.mod_ids: @@ -3563,9 +3555,7 @@ def process_stale_scc(graph: Graph, ascc: SCC, manager: BuildManager) -> None: if meta_tuple is None: continue meta, meta_json = meta_tuple - meta["dep_hashes"] = { - dep: graph[dep].interface_hash for dep in graph[id].dependencies if dep in graph - } + meta["dep_hashes"] = [graph[dep].interface_hash for dep in graph[id].dependencies] meta["error_lines"] = errors_by_id.get(id, []) write_cache_meta(meta, manager, meta_json) manager.done_sccs.add(ascc.id) diff --git a/mypy/options.py b/mypy/options.py index b1456934c6c9e..209759763a5ac 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -5,7 +5,6 @@ import sys import sysconfig import warnings -from collections.abc import Mapping from re import Pattern from typing import Any, Callable, Final @@ -621,7 +620,7 @@ def compile_glob(self, s: str) -> Pattern[str]: expr += re.escape("." + part) if part != "*" else r"(\..*)?" return re.compile(expr + "\\Z") - def select_options_affecting_cache(self) -> Mapping[str, object]: + def select_options_affecting_cache(self) -> dict[str, object]: result: dict[str, object] = {} for opt in OPTIONS_AFFECTING_CACHE: val = getattr(self, opt) From 80d00663650cec5e957f0923093473ed72f83536 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Sat, 18 Oct 2025 15:35:33 +0200 Subject: [PATCH 340/424] Run CI with Python 3.14.0 final (#20086) --- .github/workflows/test.yml | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 47f725170bd8b..de3f8877ee676 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -59,19 +59,25 @@ jobs: toxenv: py tox_extra_args: "-n 4" test_mypyc: true - - name: Test suite with py313-windows-64 - python: '3.13' - os: windows-latest - toxenv: py - tox_extra_args: "-n 4" - - - name: Test suite with py314-dev-ubuntu - python: '3.14-dev' + - name: Test suite with py314-ubuntu, mypyc-compiled + python: '3.14' os: ubuntu-24.04-arm toxenv: py tox_extra_args: "-n 4" - # allow_failure: true test_mypyc: true + - name: Test suite with py314-windows-64 + python: '3.14' + os: windows-latest + toxenv: py + tox_extra_args: "-n 4" + + # - name: Test suite with py315-dev-ubuntu + # python: '3.15-dev' + # os: ubuntu-24.04-arm + # toxenv: py + # tox_extra_args: "-n 4" + # # allow_failure: true + # test_mypyc: true - name: mypyc runtime tests with py39-macos python: '3.9.21' From c2a82b95bd68f7f41ebbb823123a2120dd0c770c Mon Sep 17 00:00:00 2001 From: Christoph Tyralla Date: Sun, 19 Oct 2025 23:29:43 +0200 Subject: [PATCH 341/424] Do not emit unreachable warnings for lines that return `NotImplemented`. (#20083) I think no one has complained so far. I just encountered this (in my understanding) lack in `TypeChecker.is_noop_for_reachability` working on #20068. --- mypy/checker.py | 4 +++- test-data/unit/check-unreachable-code.test | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/mypy/checker.py b/mypy/checker.py index 754bd59a49625..b6a9bb3b22cd5 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -3147,6 +3147,8 @@ def is_noop_for_reachability(self, s: Statement) -> bool: """ if isinstance(s, AssertStmt) and is_false_literal(s.expr): return True + elif isinstance(s, ReturnStmt) and is_literal_not_implemented(s.expr): + return True elif isinstance(s, (RaiseStmt, PassStmt)): return True elif isinstance(s, ExpressionStmt): @@ -8281,7 +8283,7 @@ def is_literal_none(n: Expression) -> bool: return isinstance(n, NameExpr) and n.fullname == "builtins.None" -def is_literal_not_implemented(n: Expression) -> bool: +def is_literal_not_implemented(n: Expression | None) -> bool: return isinstance(n, NameExpr) and n.fullname == "builtins.NotImplemented" diff --git a/test-data/unit/check-unreachable-code.test b/test-data/unit/check-unreachable-code.test index 7e00671dfd114..34d800404903d 100644 --- a/test-data/unit/check-unreachable-code.test +++ b/test-data/unit/check-unreachable-code.test @@ -1620,6 +1620,26 @@ reveal_type(bar().attr) # N: Revealed type is "Never" reveal_type(foo().attr) # N: Revealed type is "Never" 1 # E: Statement is unreachable +[case testIgnoreReturningNotImplemented] +# flags: --warn-unreachable + +class C: + def __add__(self, o: C) -> C: + if not isinstance(o, C): + return NotImplemented + return C() + def __sub__(self, o: C) -> C: + if isinstance(o, C): + return C() + return NotImplemented + def __mul__(self, o: C) -> C: + if isinstance(o, C): + return C() + else: + return NotImplemented + +[builtins fixtures/isinstance.pyi] + [case testUnreachableStatementPrettyHighlighting] # flags: --warn-unreachable --pretty def x() -> None: From 583c5f7efe5b851c140dbd2458619182137261bc Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Mon, 20 Oct 2025 20:16:52 +0200 Subject: [PATCH 342/424] Allow type parameters reusing the name missing from current module (#20081) Fixes #18507. Fixes #19526. Fixes #19946 (already closed without reproducer, but star imports may cause such issues, `testPEP695TypeVarNameClashStarImport` fails on current master). Even if a type parameter is reusing the name from outer scope, it still cannot be a redefinition, so we can safely store it even in presence of unresolved star imports or variables that do not refer to a type. --- mypy/semanal.py | 9 +++++-- test-data/unit/check-python312.test | 37 +++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 08f9eb03c9d74..d7b50bd09496e 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1794,7 +1794,9 @@ def push_type_args( if self.is_defined_type_param(p.name): self.fail(f'"{p.name}" already defined as a type parameter', context) else: - self.add_symbol(p.name, tv, context, no_progress=True, type_param=True) + assert self.add_symbol( + p.name, tv, context, no_progress=True, type_param=True + ), "Type parameter should not be discarded" return tvs @@ -6830,6 +6832,7 @@ def add_symbol_table_node( else: # see note in docstring describing None contexts self.defer() + if ( existing is not None and context is not None @@ -6849,7 +6852,9 @@ def add_symbol_table_node( self.add_redefinition(names, name, symbol) if not (isinstance(new, (FuncDef, Decorator)) and self.set_original_def(old, new)): self.name_already_defined(name, context, existing) - elif name not in self.missing_names[-1] and "*" not in self.missing_names[-1]: + elif type_param or ( + name not in self.missing_names[-1] and "*" not in self.missing_names[-1] + ): names[name] = symbol if not no_progress: self.progress = True diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 6a72a7e4d5b51..be46ff6ee5c0b 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -1228,6 +1228,43 @@ class C[T]: def f[S, S](x: S) -> S: # E: "S" already defined as a type parameter return x +[case testPEP695TypeVarNameClashNoCrashForwardReference] +# https://github.com/python/mypy/issues/18507 +from typing import TypeVar +T = TypeVar("T", bound=Foo) # E: Name "Foo" is used before definition + +class Foo: ... +class Bar[T]: ... + +[case testPEP695TypeVarNameClashNoCrashDeferredSymbol] +# https://github.com/python/mypy/issues/19526 +T = Unknown # E: Name "Unknown" is not defined + +class Foo[T]: ... +class Bar[*T]: ... +class Baz[**T]: ... +[builtins fixtures/tuple.pyi] + +[case testPEP695TypeVarNameClashTypeAlias] +type Tb = object +type Ta[Tb] = 'B[Tb]' +class A[Ta]: ... +class B[Tb](A[Ta]): ... + +[case testPEP695TypeVarNameClashStarImport] +# Similar to +# https://github.com/python/mypy/issues/19946 +import a + +[file a.py] +from b import * +class Foo[T]: ... + +[file b.py] +from a import * +class Bar[T]: ... +[builtins fixtures/tuple.pyi] + [case testPEP695ClassDecorator] from typing import Any From b266dd1a238e867e6b135c54664a76dae5a00fcb Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 20 Oct 2025 23:27:56 +0100 Subject: [PATCH 343/424] Use fixed format for cache metas (#20088) This makes cache meta files ~1.5x smaller. (I hoped together with previous diff it will give us 4x, but it is more like 3x). Implementation is mostly straightforward, here are some comments: * I make `CacheMeta` a regular class, it doesn't need to be immutable IMO (since we actually mutate it in few places). * I remove all uses of untyped dicts in favour of `CacheMeta` (note this might make JSON format slightly slower actually, but difference is below noise level) * Instead of manually checking some individual keys, I use blanket `try/except (KeyError, ValueError)` when deserializing metas. * In one place (where we update meta file after _read_), I update the loaded view to match the updated file 1:1. This should be more robust. * I still use JSON dumps for options and plugins snapshots. Serializing these using FF is tricky (and will be simpler with type tags). * I rename `data_json`/`meta_json` paths to `data_file`/`meta_file` everywhere. --- mypy-requirements.txt | 2 +- mypy/build.py | 277 ++++++++++++++---------------------- mypy/cache.py | 157 +++++++++++++++++++- mypy/util.py | 8 ++ mypyc/codegen/emitmodule.py | 3 +- pyproject.toml | 4 +- test-requirements.txt | 2 +- 7 files changed, 277 insertions(+), 176 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index 229f5624e8863..622a8c3f36135 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -4,4 +4,4 @@ typing_extensions>=4.6.0 mypy_extensions>=1.0.0 pathspec>=0.9.0 tomli>=1.1.0; python_version<'3.11' -librt>=0.2.1 +librt>=0.3.0 diff --git a/mypy/build.py b/mypy/build.py index 489fcf69c22c5..0058fb7eaaa06 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -25,21 +25,11 @@ import time import types from collections.abc import Iterator, Mapping, Sequence, Set as AbstractSet -from typing import ( - TYPE_CHECKING, - Any, - Callable, - ClassVar, - Final, - NamedTuple, - NoReturn, - TextIO, - TypedDict, -) +from typing import TYPE_CHECKING, Any, Callable, ClassVar, Final, NoReturn, TextIO, TypedDict from typing_extensions import TypeAlias as _TypeAlias import mypy.semanal_main -from mypy.cache import Buffer +from mypy.cache import Buffer, CacheMeta from mypy.checker import TypeChecker from mypy.error_formatter import OUTPUT_CHOICES, ErrorFormatter from mypy.errors import CompileError, ErrorInfo, Errors, report_internal_error @@ -55,6 +45,7 @@ decode_python_encoding, get_mypy_comments, hash_digest, + hash_digest_bytes, is_stub_package_file, is_sub_path_normabs, is_typeshed_file, @@ -349,28 +340,6 @@ def normpath(path: str, options: Options) -> str: return os.path.abspath(path) -class CacheMeta(NamedTuple): - id: str - path: str - mtime: int - size: int - hash: str - dependencies: list[str] # names of imported modules - data_mtime: int # mtime of data_json - data_json: str # path of .data.json - suppressed: list[str] # dependencies that weren't imported - options: dict[str, object] | None # build options - # dep_prios and dep_lines are in parallel with dependencies + suppressed - dep_prios: list[int] - dep_lines: list[int] - dep_hashes: list[str] - interface_hash: str # hash representing the public interface - error_lines: list[str] - version_id: str # mypy version for cache invalidation - ignore_all: bool # if errors were ignored - plugin_data: Any # config data from plugins - - # NOTE: dependencies + suppressed == all reachable imports; # suppressed contains those reachable imports that were prevented by # silent mode or simply not found. @@ -382,36 +351,6 @@ class FgDepMeta(TypedDict): mtime: int -def cache_meta_from_dict(meta: dict[str, Any], data_json: str) -> CacheMeta: - """Build a CacheMeta object from a json metadata dictionary - - Args: - meta: JSON metadata read from the metadata cache file - data_json: Path to the .data.json file containing the AST trees - """ - sentinel: Any = None # Values to be validated by the caller - return CacheMeta( - meta.get("id", sentinel), - meta.get("path", sentinel), - int(meta["mtime"]) if "mtime" in meta else sentinel, - meta.get("size", sentinel), - meta.get("hash", sentinel), - meta.get("dependencies", []), - int(meta["data_mtime"]) if "data_mtime" in meta else sentinel, - data_json, - meta.get("suppressed", []), - meta.get("options"), - meta.get("dep_prios", []), - meta.get("dep_lines", []), - meta.get("dep_hashes", []), - meta.get("interface_hash", ""), - meta.get("error_lines", []), - meta.get("version_id", sentinel), - meta.get("ignore_all", True), - meta.get("plugin_data", None), - ) - - # Priorities used for imports. (Here, top-level includes inside a class.) # These are used to determine a more predictable order in which the # nodes in an import cycle are processed. @@ -1234,7 +1173,7 @@ def _load_json_file( try: t1 = time.time() result = json_loads(data) - manager.add_stats(data_json_load_time=time.time() - t1) + manager.add_stats(data_file_load_time=time.time() - t1) except json.JSONDecodeError: manager.errors.set_file(file, None, manager.options) manager.errors.report( @@ -1313,8 +1252,8 @@ def get_cache_names(id: str, path: str, options: Options) -> tuple[str, str, str options: build options Returns: - A tuple with the file names to be used for the meta JSON, the - data JSON, and the fine-grained deps JSON, respectively. + A tuple with the file names to be used for the meta file, the + data file, and the fine-grained deps JSON, respectively. """ if options.cache_map: pair = options.cache_map.get(normpath(path, options)) @@ -1338,9 +1277,11 @@ def get_cache_names(id: str, path: str, options: Options) -> tuple[str, str, str deps_json = prefix + ".deps.json" if options.fixed_format_cache: data_suffix = ".data.ff" + meta_suffix = ".meta.ff" else: data_suffix = ".data.json" - return prefix + ".meta.json", prefix + data_suffix, deps_json + meta_suffix = ".meta.json" + return prefix + meta_suffix, prefix + data_suffix, deps_json def options_snapshot(id: str, manager: BuildManager) -> dict[str, object]: @@ -1369,47 +1310,50 @@ def find_cache_meta(id: str, path: str, manager: BuildManager) -> CacheMeta | No valid; otherwise None. """ # TODO: May need to take more build options into account - meta_json, data_json, _ = get_cache_names(id, path, manager.options) - manager.trace(f"Looking for {id} at {meta_json}") + meta_file, data_file, _ = get_cache_names(id, path, manager.options) + manager.trace(f"Looking for {id} at {meta_file}") + meta: bytes | dict[str, Any] | None t0 = time.time() - meta = _load_json_file( - meta_json, manager, log_success=f"Meta {id} ", log_error=f"Could not load cache for {id}: " - ) + if manager.options.fixed_format_cache: + meta = _load_ff_file(meta_file, manager, log_error=f"Could not load cache for {id}: ") + if meta is None: + return None + else: + meta = _load_json_file( + meta_file, + manager, + log_success=f"Meta {id} ", + log_error=f"Could not load cache for {id}: ", + ) + if meta is None: + return None + if not isinstance(meta, dict): + manager.log( # type: ignore[unreachable] + f"Could not load cache for {id}: meta cache is not a dict: {repr(meta)}" + ) + return None t1 = time.time() - if meta is None: - return None - if not isinstance(meta, dict): - manager.log(f"Could not load cache for {id}: meta cache is not a dict: {repr(meta)}") # type: ignore[unreachable] + if isinstance(meta, bytes): + data_io = Buffer(meta) + m = CacheMeta.read(data_io, data_file) + else: + m = CacheMeta.deserialize(meta, data_file) + if m is None: + manager.log(f"Metadata abandoned for {id}: attributes are missing") return None - m = cache_meta_from_dict(meta, data_json) t2 = time.time() manager.add_stats( load_meta_time=t2 - t0, load_meta_load_time=t1 - t0, load_meta_from_dict_time=t2 - t1 ) - # Don't check for path match, that is dealt with in validate_meta(). - # - # TODO: these `type: ignore`s wouldn't be necessary - # if the type annotations for CacheMeta were more accurate - # (all of these attributes can be `None`) - if ( - m.id != id - or m.mtime is None # type: ignore[redundant-expr] - or m.size is None # type: ignore[redundant-expr] - or m.dependencies is None # type: ignore[redundant-expr] - or m.data_mtime is None - ): - manager.log(f"Metadata abandoned for {id}: attributes are missing") + # Ignore cache if generated by an older mypy version. + if m.version_id != manager.version_id and not manager.options.skip_version_check: + manager.log(f"Metadata abandoned for {id}: different mypy version") return None - # Ignore cache if generated by an older mypy version. - if ( - (m.version_id != manager.version_id and not manager.options.skip_version_check) - or m.options is None - or len(m.dependencies) + len(m.suppressed) != len(m.dep_prios) - or len(m.dependencies) + len(m.suppressed) != len(m.dep_lines) - ): - manager.log(f"Metadata abandoned for {id}: new attributes are missing") + total_deps = len(m.dependencies) + len(m.suppressed) + if len(m.dep_prios) != total_deps or len(m.dep_lines) != total_deps: + manager.log(f"Metadata abandoned for {id}: broken dependencies") return None # Ignore cache if (relevant) options aren't the same. @@ -1479,11 +1423,11 @@ def validate_meta( bazel = manager.options.bazel assert path is not None, "Internal error: meta was provided without a path" if not manager.options.skip_cache_mtime_checks: - # Check data_json; assume if its mtime matches it's good. + # Check data_file; assume if its mtime matches it's good. try: - data_mtime = manager.getmtime(meta.data_json) + data_mtime = manager.getmtime(meta.data_file) except OSError: - manager.log(f"Metadata abandoned for {id}: failed to stat data_json") + manager.log(f"Metadata abandoned for {id}: failed to stat data_file") return None if data_mtime != meta.data_mtime: manager.log(f"Metadata abandoned for {id}: data cache is modified") @@ -1534,7 +1478,8 @@ def validate_meta( qmtime, qsize, qhash = manager.quickstart_state[path] if int(qmtime) == mtime and qsize == size and qhash == meta.hash: manager.log(f"Metadata fresh (by quickstart) for {id}: file {path}") - meta = meta._replace(mtime=mtime, path=path) + meta.mtime = mtime + meta.path = path return meta t0 = time.time() @@ -1557,36 +1502,18 @@ def validate_meta( else: t0 = time.time() # Optimization: update mtime and path (otherwise, this mismatch will reappear). - meta = meta._replace(mtime=mtime, path=path) - # Construct a dict we can pass to json.dumps() (compare to write_cache()). - meta_dict = { - "id": id, - "path": path, - "mtime": mtime, - "size": size, - "hash": source_hash, - "data_mtime": meta.data_mtime, - "dependencies": meta.dependencies, - "suppressed": meta.suppressed, - "options": options_snapshot(id, manager), - "dep_prios": meta.dep_prios, - "dep_lines": meta.dep_lines, - "dep_hashes": meta.dep_hashes, - "interface_hash": meta.interface_hash, - "error_lines": meta.error_lines, - "version_id": manager.version_id, - "ignore_all": meta.ignore_all, - "plugin_data": meta.plugin_data, - } - meta_bytes = json_dumps(meta_dict, manager.options.debug_cache) - meta_json, _, _ = get_cache_names(id, path, manager.options) + meta.mtime = mtime + meta.path = path + meta.size = size + meta.options = options_snapshot(id, manager) + meta_file, _, _ = get_cache_names(id, path, manager.options) manager.log( "Updating mtime for {}: file {}, meta {}, mtime {}".format( - id, path, meta_json, meta.mtime + id, path, meta_file, meta.mtime ) ) + write_cache_meta(meta, manager, meta_file) t1 = time.time() - manager.metastore.write(meta_json, meta_bytes) # Ignore errors, just an optimization. manager.add_stats(validate_update_time=time.time() - t1, validate_munging_time=t1 - t0) return meta @@ -1612,11 +1539,11 @@ def write_cache( suppressed: list[str], dep_prios: list[int], dep_lines: list[int], - old_interface_hash: str, + old_interface_hash: bytes, source_hash: str, ignore_all: bool, manager: BuildManager, -) -> tuple[str, tuple[dict[str, Any], str] | None]: +) -> tuple[bytes, tuple[CacheMeta, str] | None]: """Write cache files for a module. Note that this mypy's behavior is still correct when any given @@ -1637,7 +1564,7 @@ def write_cache( manager: the build manager (for pyversion, log/trace) Returns: - A tuple containing the interface hash and inner tuple with cache meta JSON + A tuple containing the interface hash and inner tuple with CacheMeta that should be written and path to cache file (inner tuple may be None, if the cache data could not be written). """ @@ -1646,8 +1573,8 @@ def write_cache( bazel = manager.options.bazel # Obtain file paths. - meta_json, data_json, _ = get_cache_names(id, path, manager.options) - manager.log(f"Writing {id} {path} {meta_json} {data_json}") + meta_file, data_file, _ = get_cache_names(id, path, manager.options) + manager.log(f"Writing {id} {path} {meta_file} {data_file}") # Update tree.path so that in bazel mode it's made relative (since # sometimes paths leak out). @@ -1664,7 +1591,7 @@ def write_cache( else: data = tree.serialize() data_bytes = json_dumps(data, manager.options.debug_cache) - interface_hash = hash_digest(data_bytes + json_dumps(plugin_data)) + interface_hash = hash_digest_bytes(data_bytes + json_dumps(plugin_data)) # Obtain and set up metadata st = manager.get_stat(path) @@ -1672,7 +1599,7 @@ def write_cache( manager.log(f"Cannot get stat for {path}") # Remove apparently-invalid cache files. # (This is purely an optimization.) - for filename in [data_json, meta_json]: + for filename in [data_file, meta_file]: try: os.remove(filename) except OSError: @@ -1686,10 +1613,10 @@ def write_cache( manager.trace(f"Interface for {id} is unchanged") else: manager.trace(f"Interface for {id} has changed") - if not metastore.write(data_json, data_bytes): + if not metastore.write(data_file, data_bytes): # Most likely the error is the replace() call # (see https://github.com/python/mypy/issues/3215). - manager.log(f"Error writing data JSON file {data_json}") + manager.log(f"Error writing cache data file {data_file}") # Let's continue without writing the meta file. Analysis: # If the replace failed, we've changed nothing except left # behind an extraneous temporary file; if the replace @@ -1701,9 +1628,9 @@ def write_cache( return interface_hash, None try: - data_mtime = manager.getmtime(data_json) + data_mtime = manager.getmtime(data_file) except OSError: - manager.log(f"Error in os.stat({data_json!r}), skipping cache write") + manager.log(f"Error in os.stat({data_file!r}), skipping cache write") return interface_hash, None mtime = 0 if bazel else int(st.st_mtime) @@ -1714,35 +1641,45 @@ def write_cache( # important, or otherwise the options would never match when # verifying the cache. assert source_hash is not None - meta = { - "id": id, - "path": path, - "mtime": mtime, - "size": size, - "hash": source_hash, - "data_mtime": data_mtime, - "dependencies": dependencies, - "suppressed": suppressed, - "options": options_snapshot(id, manager), - "dep_prios": dep_prios, - "dep_lines": dep_lines, - "interface_hash": interface_hash, - "version_id": manager.version_id, - "ignore_all": ignore_all, - "plugin_data": plugin_data, - } - return interface_hash, (meta, meta_json) + meta = CacheMeta( + id=id, + path=path, + mtime=mtime, + size=size, + hash=source_hash, + dependencies=dependencies, + data_mtime=data_mtime, + data_file=data_file, + suppressed=suppressed, + options=options_snapshot(id, manager), + dep_prios=dep_prios, + dep_lines=dep_lines, + interface_hash=interface_hash, + version_id=manager.version_id, + ignore_all=ignore_all, + plugin_data=plugin_data, + # These two will be filled by the caller. + dep_hashes=[], + error_lines=[], + ) + return interface_hash, (meta, meta_file) -def write_cache_meta(meta: dict[str, Any], manager: BuildManager, meta_json: str) -> None: +def write_cache_meta(meta: CacheMeta, manager: BuildManager, meta_file: str) -> None: # Write meta cache file metastore = manager.metastore - meta_str = json_dumps(meta, manager.options.debug_cache) - if not metastore.write(meta_json, meta_str): + if manager.options.fixed_format_cache: + data_io = Buffer() + meta.write(data_io) + meta_bytes = data_io.getvalue() + else: + meta_dict = meta.serialize() + meta_bytes = json_dumps(meta_dict, manager.options.debug_cache) + if not metastore.write(meta_file, meta_bytes): # Most likely the error is the replace() call # (see https://github.com/python/mypy/issues/3215). # The next run will simply find the cache entry out of date. - manager.log(f"Error writing meta JSON file {meta_json}") + manager.log(f"Error writing cache meta file {meta_file}") """Dependency manager. @@ -1918,7 +1855,7 @@ class State: dep_line_map: dict[str, int] # Map from dependency id to its last observed interface hash - dep_hashes: dict[str, str] = {} + dep_hashes: dict[str, bytes] = {} # List of errors reported for this file last time. error_lines: list[str] = [] @@ -1933,7 +1870,7 @@ class State: caller_line = 0 # Contains a hash of the public interface in incremental mode - interface_hash: str = "" + interface_hash: bytes = b"" # Options, specialized for this file options: Options @@ -2152,10 +2089,10 @@ def load_tree(self, temporary: bool = False) -> None: data: bytes | dict[str, Any] | None if self.options.fixed_format_cache: - data = _load_ff_file(self.meta.data_json, self.manager, "Could not load tree: ") + data = _load_ff_file(self.meta.data_file, self.manager, "Could not load tree: ") else: data = _load_json_file( - self.meta.data_json, self.manager, "Load tree ", "Could not load tree: " + self.meta.data_file, self.manager, "Load tree ", "Could not load tree: " ) if data is None: return @@ -2525,7 +2462,7 @@ def update_fine_grained_deps(self, deps: dict[str, set[str]]) -> None: merge_dependencies(self.compute_fine_grained_deps(), deps) type_state.update_protocol_deps(deps) - def write_cache(self) -> tuple[dict[str, Any], str] | None: + def write_cache(self) -> tuple[CacheMeta, str] | None: assert self.tree is not None, "Internal error: method must be called on parsed file only" # We don't support writing cache files in fine-grained incremental mode. if ( @@ -3554,10 +3491,10 @@ def process_stale_scc(graph: Graph, ascc: SCC, manager: BuildManager) -> None: meta_tuple = meta_tuples[id] if meta_tuple is None: continue - meta, meta_json = meta_tuple - meta["dep_hashes"] = [graph[dep].interface_hash for dep in graph[id].dependencies] - meta["error_lines"] = errors_by_id.get(id, []) - write_cache_meta(meta, manager, meta_json) + meta, meta_file = meta_tuple + meta.dep_hashes = [graph[dep].interface_hash for dep in graph[id].dependencies] + meta.error_lines = errors_by_id.get(id, []) + write_cache_meta(meta, manager, meta_file) manager.done_sccs.add(ascc.id) diff --git a/mypy/cache.py b/mypy/cache.py index f8d3e6a05ebac..aeb0e8810fd65 100644 --- a/mypy/cache.py +++ b/mypy/cache.py @@ -1,16 +1,18 @@ from __future__ import annotations from collections.abc import Sequence -from typing import Final +from typing import Any, Final from librt.internal import ( Buffer as Buffer, read_bool as read_bool, + read_bytes as read_bytes, read_float as read_float, read_int as read_int, read_str as read_str, read_tag as read_tag, write_bool as write_bool, + write_bytes as write_bytes, write_float as write_float, write_int as write_int, write_str as write_str, @@ -18,6 +20,148 @@ ) from mypy_extensions import u8 +from mypy.util import json_dumps, json_loads + + +class CacheMeta: + """Class representing cache metadata for a module.""" + + def __init__( + self, + *, + id: str, + path: str, + mtime: int, + size: int, + hash: str, + dependencies: list[str], + data_mtime: int, + data_file: str, + suppressed: list[str], + options: dict[str, object], + dep_prios: list[int], + dep_lines: list[int], + dep_hashes: list[bytes], + interface_hash: bytes, + error_lines: list[str], + version_id: str, + ignore_all: bool, + plugin_data: Any, + ) -> None: + self.id = id + self.path = path + self.mtime = mtime # source file mtime + self.size = size # source file size + self.hash = hash # source file hash (as a hex string for historical reasons) + self.dependencies = dependencies # names of imported modules + self.data_mtime = data_mtime # mtime of data_file + self.data_file = data_file # path of .data.json or .data.ff + self.suppressed = suppressed # dependencies that weren't imported + self.options = options # build options snapshot + # dep_prios and dep_lines are both aligned with dependencies + suppressed + self.dep_prios = dep_prios + self.dep_lines = dep_lines + # dep_hashes list is aligned with dependencies only + self.dep_hashes = dep_hashes # list of interface_hash for dependencies + self.interface_hash = interface_hash # hash representing the public interface + self.error_lines = error_lines + self.version_id = version_id # mypy version for cache invalidation + self.ignore_all = ignore_all # if errors were ignored + self.plugin_data = plugin_data # config data from plugins + + def serialize(self) -> dict[str, Any]: + return { + "id": self.id, + "path": self.path, + "mtime": self.mtime, + "size": self.size, + "hash": self.hash, + "data_mtime": self.data_mtime, + "dependencies": self.dependencies, + "suppressed": self.suppressed, + "options": self.options, + "dep_prios": self.dep_prios, + "dep_lines": self.dep_lines, + "dep_hashes": [dep.hex() for dep in self.dep_hashes], + "interface_hash": self.interface_hash.hex(), + "error_lines": self.error_lines, + "version_id": self.version_id, + "ignore_all": self.ignore_all, + "plugin_data": self.plugin_data, + } + + @classmethod + def deserialize(cls, meta: dict[str, Any], data_file: str) -> CacheMeta | None: + try: + return CacheMeta( + id=meta["id"], + path=meta["path"], + mtime=meta["mtime"], + size=meta["size"], + hash=meta["hash"], + dependencies=meta["dependencies"], + data_mtime=meta["data_mtime"], + data_file=data_file, + suppressed=meta["suppressed"], + options=meta["options"], + dep_prios=meta["dep_prios"], + dep_lines=meta["dep_lines"], + dep_hashes=[bytes.fromhex(dep) for dep in meta["dep_hashes"]], + interface_hash=bytes.fromhex(meta["interface_hash"]), + error_lines=meta["error_lines"], + version_id=meta["version_id"], + ignore_all=meta["ignore_all"], + plugin_data=meta["plugin_data"], + ) + except (KeyError, ValueError): + return None + + def write(self, data: Buffer) -> None: + write_str(data, self.id) + write_str(data, self.path) + write_int(data, self.mtime) + write_int(data, self.size) + write_str(data, self.hash) + write_str_list(data, self.dependencies) + write_int(data, self.data_mtime) + write_str_list(data, self.suppressed) + write_bytes(data, json_dumps(self.options)) + write_int_list(data, self.dep_prios) + write_int_list(data, self.dep_lines) + write_bytes_list(data, self.dep_hashes) + write_bytes(data, self.interface_hash) + write_str_list(data, self.error_lines) + write_str(data, self.version_id) + write_bool(data, self.ignore_all) + write_bytes(data, json_dumps(self.plugin_data)) + + @classmethod + def read(cls, data: Buffer, data_file: str) -> CacheMeta | None: + try: + return CacheMeta( + id=read_str(data), + path=read_str(data), + mtime=read_int(data), + size=read_int(data), + hash=read_str(data), + dependencies=read_str_list(data), + data_mtime=read_int(data), + data_file=data_file, + suppressed=read_str_list(data), + options=json_loads(read_bytes(data)), + dep_prios=read_int_list(data), + dep_lines=read_int_list(data), + dep_hashes=read_bytes_list(data), + interface_hash=read_bytes(data), + error_lines=read_str_list(data), + version_id=read_str(data), + ignore_all=read_bool(data), + plugin_data=json_loads(read_bytes(data)), + ) + except ValueError: + return None + + # Always use this type alias to refer to type tags. Tag = u8 @@ -112,6 +256,17 @@ def write_str_list(data: Buffer, value: Sequence[str]) -> None: write_str(data, item) +def read_bytes_list(data: Buffer) -> list[bytes]: + size = read_int(data) + return [read_bytes(data) for _ in range(size)] + + +def write_bytes_list(data: Buffer, value: Sequence[bytes]) -> None: + write_int(data, len(value)) + for item in value: + write_bytes(data, item) + + def read_str_opt_list(data: Buffer) -> list[str | None]: size = read_int(data) return [read_str_opt(data) for _ in range(size)] diff --git a/mypy/util.py b/mypy/util.py index d7ff2a367fa2d..c919ff87f5b02 100644 --- a/mypy/util.py +++ b/mypy/util.py @@ -569,6 +569,14 @@ def hash_digest(data: bytes) -> str: return hashlib.sha1(data).hexdigest() +def hash_digest_bytes(data: bytes) -> bytes: + """Compute a hash digest of some data. + + Similar to above but returns a bytes object. + """ + return hashlib.sha1(data).digest() + + def parse_gray_color(cup: bytes) -> str: """Reproduce a gray color in ANSI escape sequence""" assert sys.platform != "win32", "curses is not available on Windows" diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 996ec4c52b08f..da60e145a7903 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -341,7 +341,8 @@ def compile_ir_to_c( def get_ir_cache_name(id: str, path: str, options: Options) -> str: meta_path, _, _ = get_cache_names(id, path, options) - return meta_path.replace(".meta.json", ".ir.json") + # Mypy uses JSON cache even with --fixed-format-cache (for now). + return meta_path.replace(".meta.json", ".ir.json").replace(".meta.ff", ".ir.json") def get_state_ir_cache_name(state: State) -> str: diff --git a/pyproject.toml b/pyproject.toml index 96b05ba459b19..f9f6c01b5c1cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ requires = [ "mypy_extensions>=1.0.0", "pathspec>=0.9.0", "tomli>=1.1.0; python_version<'3.11'", - "librt>=0.2.1", + "librt>=0.3.0", # the following is from build-requirements.txt "types-psutil", "types-setuptools", @@ -54,7 +54,7 @@ dependencies = [ "mypy_extensions>=1.0.0", "pathspec>=0.9.0", "tomli>=1.1.0; python_version<'3.11'", - "librt>=0.2.1", + "librt>=0.3.0", ] dynamic = ["version"] diff --git a/test-requirements.txt b/test-requirements.txt index 0dc2a4cf8f189..b9ff4ffe085b7 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -22,7 +22,7 @@ identify==2.6.15 # via pre-commit iniconfig==2.1.0 # via pytest -librt==0.2.3 +librt==0.3.0 # via -r mypy-requirements.txt lxml==6.0.2 ; python_version < "3.15" # via -r test-requirements.in From cb0da626b15e30c0bfb0ca9f7b3867fb10fdaedb Mon Sep 17 00:00:00 2001 From: Michael Carlstrom Date: Thu, 23 Oct 2025 15:09:57 -0700 Subject: [PATCH 344/424] Use Pybind11 3.0.0+ (#20095) Pybind11 had a new major version bump to [3.0.0](https://github.com/pybind/pybind11/releases/tag/v3.0.0). Also fix bug in workflow to run with the new folder name. Signed-off-by: Michael Carlstrom --- .github/workflows/test_stubgenc.yml | 2 +- .../pybind11_fixtures/__init__.pyi | 11 ++-- .../pybind11_fixtures/demo.pyi | 15 ++--- .../pybind11_fixtures/__init__.pyi | 29 ++++----- .../pybind11_fixtures/demo.pyi | 63 ++++++++++--------- test-data/pybind11_fixtures/pyproject.toml | 2 +- 6 files changed, 63 insertions(+), 59 deletions(-) diff --git a/.github/workflows/test_stubgenc.yml b/.github/workflows/test_stubgenc.yml index 4676acf8695b0..6cf3cb71c3fff 100644 --- a/.github/workflows/test_stubgenc.yml +++ b/.github/workflows/test_stubgenc.yml @@ -11,7 +11,7 @@ on: - 'mypy/stubgenc.py' - 'mypy/stubdoc.py' - 'mypy/stubutil.py' - - 'test-data/stubgen/**' + - 'test-data/pybind11_fixtures/**' permissions: contents: read diff --git a/test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/__init__.pyi b/test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/__init__.pyi index 90afb46d6d94a..c841b207c130c 100644 --- a/test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/__init__.pyi +++ b/test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/__init__.pyi @@ -1,4 +1,5 @@ -import os +import pathlib +import typing from . import demo as demo from typing import overload @@ -6,12 +7,12 @@ class StaticMethods: def __init__(self, *args, **kwargs) -> None: ... @overload @staticmethod - def overloaded_static_method(value: int) -> int: ... + def overloaded_static_method(value: typing.SupportsInt) -> int: ... @overload @staticmethod - def overloaded_static_method(value: float) -> float: ... + def overloaded_static_method(value: typing.SupportsFloat) -> float: ... @staticmethod - def some_static_method(a: int, b: int) -> int: ... + def some_static_method(a: typing.SupportsInt, b: typing.SupportsInt) -> int: ... class TestStruct: field_readwrite: int @@ -23,5 +24,5 @@ class TestStruct: def func_incomplete_signature(*args, **kwargs): ... def func_returning_optional() -> int | None: ... def func_returning_pair() -> tuple[int, float]: ... -def func_returning_path() -> os.PathLike: ... +def func_returning_path() -> pathlib.Path: ... def func_returning_vector() -> list[float]: ... diff --git a/test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/demo.pyi b/test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/demo.pyi index 87b8ec0e4ad6d..09e75e1ad4aa1 100644 --- a/test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/demo.pyi +++ b/test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/demo.pyi @@ -1,3 +1,4 @@ +import typing from typing import ClassVar, overload PI: float @@ -9,7 +10,7 @@ class Point: __entries: ClassVar[dict] = ... degree: ClassVar[Point.AngleUnit] = ... radian: ClassVar[Point.AngleUnit] = ... - def __init__(self, value: int) -> None: ... + def __init__(self, value: typing.SupportsInt) -> None: ... def __eq__(self, other: object) -> bool: ... def __hash__(self) -> int: ... def __index__(self) -> int: ... @@ -26,7 +27,7 @@ class Point: inch: ClassVar[Point.LengthUnit] = ... mm: ClassVar[Point.LengthUnit] = ... pixel: ClassVar[Point.LengthUnit] = ... - def __init__(self, value: int) -> None: ... + def __init__(self, value: typing.SupportsInt) -> None: ... def __eq__(self, other: object) -> bool: ... def __hash__(self) -> int: ... def __index__(self) -> int: ... @@ -46,16 +47,16 @@ class Point: @overload def __init__(self) -> None: ... @overload - def __init__(self, x: float, y: float) -> None: ... + def __init__(self, x: typing.SupportsFloat, y: typing.SupportsFloat) -> None: ... def as_list(self) -> list[float]: ... @overload - def distance_to(self, x: float, y: float) -> float: ... + def distance_to(self, x: typing.SupportsFloat, y: typing.SupportsFloat) -> float: ... @overload def distance_to(self, other: Point) -> float: ... @property def length(self) -> float: ... def answer() -> int: ... -def midpoint(left: float, right: float) -> float: ... -def sum(arg0: int, arg1: int) -> int: ... -def weighted_midpoint(left: float, right: float, alpha: float = ...) -> float: ... +def midpoint(left: typing.SupportsFloat, right: typing.SupportsFloat) -> float: ... +def sum(arg0: typing.SupportsInt, arg1: typing.SupportsInt) -> int: ... +def weighted_midpoint(left: typing.SupportsFloat, right: typing.SupportsFloat, alpha: typing.SupportsFloat = ...) -> float: ... diff --git a/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/__init__.pyi b/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/__init__.pyi index 0eeb788d42784..701a837580b44 100644 --- a/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/__init__.pyi +++ b/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/__init__.pyi @@ -1,4 +1,5 @@ -import os +import pathlib +import typing from . import demo as demo from typing import overload @@ -7,27 +8,27 @@ class StaticMethods: """Initialize self. See help(type(self)) for accurate signature.""" @overload @staticmethod - def overloaded_static_method(value: int) -> int: + def overloaded_static_method(value: typing.SupportsInt) -> int: """overloaded_static_method(*args, **kwargs) Overloaded function. - 1. overloaded_static_method(value: int) -> int + 1. overloaded_static_method(value: typing.SupportsInt) -> int - 2. overloaded_static_method(value: float) -> float + 2. overloaded_static_method(value: typing.SupportsFloat) -> float """ @overload @staticmethod - def overloaded_static_method(value: float) -> float: + def overloaded_static_method(value: typing.SupportsFloat) -> float: """overloaded_static_method(*args, **kwargs) Overloaded function. - 1. overloaded_static_method(value: int) -> int + 1. overloaded_static_method(value: typing.SupportsInt) -> int - 2. overloaded_static_method(value: float) -> float + 2. overloaded_static_method(value: typing.SupportsFloat) -> float """ @staticmethod - def some_static_method(a: int, b: int) -> int: - """some_static_method(a: int, b: int) -> int + def some_static_method(a: typing.SupportsInt, b: typing.SupportsInt) -> int: + """some_static_method(a: typing.SupportsInt, b: typing.SupportsInt) -> int None """ @@ -46,10 +47,10 @@ class TestStruct: def func_incomplete_signature(*args, **kwargs): """func_incomplete_signature() -> dummy_sub_namespace::HasNoBinding""" def func_returning_optional() -> int | None: - """func_returning_optional() -> Optional[int]""" + """func_returning_optional() -> int | None""" def func_returning_pair() -> tuple[int, float]: - """func_returning_pair() -> Tuple[int, float]""" -def func_returning_path() -> os.PathLike: - """func_returning_path() -> os.PathLike""" + """func_returning_pair() -> tuple[int, float]""" +def func_returning_path() -> pathlib.Path: + """func_returning_path() -> pathlib.Path""" def func_returning_vector() -> list[float]: - """func_returning_vector() -> List[float]""" + """func_returning_vector() -> list[float]""" diff --git a/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/demo.pyi b/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/demo.pyi index 6e285f202f1a3..580aa27001784 100644 --- a/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/demo.pyi +++ b/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/demo.pyi @@ -1,3 +1,4 @@ +import typing from typing import ClassVar, overload PI: float @@ -14,23 +15,23 @@ class Point: __entries: ClassVar[dict] = ... degree: ClassVar[Point.AngleUnit] = ... radian: ClassVar[Point.AngleUnit] = ... - def __init__(self, value: int) -> None: - """__init__(self: pybind11_fixtures.demo.Point.AngleUnit, value: int) -> None""" + def __init__(self, value: typing.SupportsInt) -> None: + """__init__(self: pybind11_fixtures.demo.Point.AngleUnit, value: typing.SupportsInt) -> None""" def __eq__(self, other: object) -> bool: - """__eq__(self: object, other: object) -> bool""" + """__eq__(self: object, other: object, /) -> bool""" def __hash__(self) -> int: - """__hash__(self: object) -> int""" + """__hash__(self: object, /) -> int""" def __index__(self) -> int: - """__index__(self: pybind11_fixtures.demo.Point.AngleUnit) -> int""" + """__index__(self: pybind11_fixtures.demo.Point.AngleUnit, /) -> int""" def __int__(self) -> int: - """__int__(self: pybind11_fixtures.demo.Point.AngleUnit) -> int""" + """__int__(self: pybind11_fixtures.demo.Point.AngleUnit, /) -> int""" def __ne__(self, other: object) -> bool: - """__ne__(self: object, other: object) -> bool""" + """__ne__(self: object, other: object, /) -> bool""" @property def name(self) -> str: - """name(self: handle) -> str + """name(self: object, /) -> str - name(self: handle) -> str + name(self: object, /) -> str """ @property def value(self) -> int: @@ -49,23 +50,23 @@ class Point: inch: ClassVar[Point.LengthUnit] = ... mm: ClassVar[Point.LengthUnit] = ... pixel: ClassVar[Point.LengthUnit] = ... - def __init__(self, value: int) -> None: - """__init__(self: pybind11_fixtures.demo.Point.LengthUnit, value: int) -> None""" + def __init__(self, value: typing.SupportsInt) -> None: + """__init__(self: pybind11_fixtures.demo.Point.LengthUnit, value: typing.SupportsInt) -> None""" def __eq__(self, other: object) -> bool: - """__eq__(self: object, other: object) -> bool""" + """__eq__(self: object, other: object, /) -> bool""" def __hash__(self) -> int: - """__hash__(self: object) -> int""" + """__hash__(self: object, /) -> int""" def __index__(self) -> int: - """__index__(self: pybind11_fixtures.demo.Point.LengthUnit) -> int""" + """__index__(self: pybind11_fixtures.demo.Point.LengthUnit, /) -> int""" def __int__(self) -> int: - """__int__(self: pybind11_fixtures.demo.Point.LengthUnit) -> int""" + """__int__(self: pybind11_fixtures.demo.Point.LengthUnit, /) -> int""" def __ne__(self, other: object) -> bool: - """__ne__(self: object, other: object) -> bool""" + """__ne__(self: object, other: object, /) -> bool""" @property def name(self) -> str: - """name(self: handle) -> str + """name(self: object, /) -> str - name(self: handle) -> str + name(self: object, /) -> str """ @property def value(self) -> int: @@ -84,25 +85,25 @@ class Point: 1. __init__(self: pybind11_fixtures.demo.Point) -> None - 2. __init__(self: pybind11_fixtures.demo.Point, x: float, y: float) -> None + 2. __init__(self: pybind11_fixtures.demo.Point, x: typing.SupportsFloat, y: typing.SupportsFloat) -> None """ @overload - def __init__(self, x: float, y: float) -> None: + def __init__(self, x: typing.SupportsFloat, y: typing.SupportsFloat) -> None: """__init__(*args, **kwargs) Overloaded function. 1. __init__(self: pybind11_fixtures.demo.Point) -> None - 2. __init__(self: pybind11_fixtures.demo.Point, x: float, y: float) -> None + 2. __init__(self: pybind11_fixtures.demo.Point, x: typing.SupportsFloat, y: typing.SupportsFloat) -> None """ def as_list(self) -> list[float]: - """as_list(self: pybind11_fixtures.demo.Point) -> List[float]""" + """as_list(self: pybind11_fixtures.demo.Point) -> list[float]""" @overload - def distance_to(self, x: float, y: float) -> float: + def distance_to(self, x: typing.SupportsFloat, y: typing.SupportsFloat) -> float: """distance_to(*args, **kwargs) Overloaded function. - 1. distance_to(self: pybind11_fixtures.demo.Point, x: float, y: float) -> float + 1. distance_to(self: pybind11_fixtures.demo.Point, x: typing.SupportsFloat, y: typing.SupportsFloat) -> float 2. distance_to(self: pybind11_fixtures.demo.Point, other: pybind11_fixtures.demo.Point) -> float """ @@ -111,7 +112,7 @@ class Point: """distance_to(*args, **kwargs) Overloaded function. - 1. distance_to(self: pybind11_fixtures.demo.Point, x: float, y: float) -> float + 1. distance_to(self: pybind11_fixtures.demo.Point, x: typing.SupportsFloat, y: typing.SupportsFloat) -> float 2. distance_to(self: pybind11_fixtures.demo.Point, other: pybind11_fixtures.demo.Point) -> float """ @@ -124,12 +125,12 @@ def answer() -> int: answer docstring, with end quote" ''' -def midpoint(left: float, right: float) -> float: - """midpoint(left: float, right: float) -> float""" -def sum(arg0: int, arg1: int) -> int: - '''sum(arg0: int, arg1: int) -> int +def midpoint(left: typing.SupportsFloat, right: typing.SupportsFloat) -> float: + """midpoint(left: typing.SupportsFloat, right: typing.SupportsFloat) -> float""" +def sum(arg0: typing.SupportsInt, arg1: typing.SupportsInt) -> int: + '''sum(arg0: typing.SupportsInt, arg1: typing.SupportsInt) -> int multiline docstring test, edge case quotes """\'\'\' ''' -def weighted_midpoint(left: float, right: float, alpha: float = ...) -> float: - """weighted_midpoint(left: float, right: float, alpha: float = 0.5) -> float""" +def weighted_midpoint(left: typing.SupportsFloat, right: typing.SupportsFloat, alpha: typing.SupportsFloat = ...) -> float: + """weighted_midpoint(left: typing.SupportsFloat, right: typing.SupportsFloat, alpha: typing.SupportsFloat = 0.5) -> float""" diff --git a/test-data/pybind11_fixtures/pyproject.toml b/test-data/pybind11_fixtures/pyproject.toml index 773d036e62f56..56eaa50720654 100644 --- a/test-data/pybind11_fixtures/pyproject.toml +++ b/test-data/pybind11_fixtures/pyproject.toml @@ -4,7 +4,7 @@ requires = [ "wheel", # Officially supported pybind11 version. This is pinned to guarantee 100% reproducible CI. # As a result, the version needs to be bumped manually at will. - "pybind11==2.9.2", + "pybind11==3.0.1", ] build-backend = "setuptools.build_meta" From fb16e938d12f6b24b5edf8023d4adf8e0e36abb4 Mon Sep 17 00:00:00 2001 From: Randolf Scholz Date: Fri, 24 Oct 2025 00:21:48 +0200 Subject: [PATCH 345/424] [match-case] Fix narrowing of class pattern with union-argument. (#19517) Fixes #19468 Based on earlier PR #19473 - refactored the `conditional_types` function. - return `UninhabitedType(), default` when no ranges are given. This corresponds to `isinstance(x, ())`, with an empty tuple, which always returns `False` at runtime. - modified #19473 to change the proposed type, rather than directly returning. This is essential to maintain the behavior of the unit test `testIsinstanceWithOverlappingPromotionTypes` - Added special casing in `restrict_subtype_away`: if the second argument is a `TypeVar`, replace it with its upper bound (crucial to get correct result in `testNarrowSelfType`) - Allow `TypeChecker.get_isinstance_type` to return empty list (fixes `isinstance(x, ())` behavior). ## Modified tests - `testIsInstanceWithEmtpy2ndArg` now correctly infers unreachable for `isinstance(x, ())`. - `testNarrowingUnionMixins` now predicts the same results as pyright playground ## New Tests - `testMatchNarrowDownUnionUsingClassPattern` (https://mypy-play.net/?mypy=1.17.0&python=3.12&gist=e9ec514f49903022bd32a82ae1774abd) --- mypy/checker.py | 130 ++++++++++++++++----------- test-data/unit/check-isinstance.test | 3 +- test-data/unit/check-narrowing.test | 2 +- test-data/unit/check-python310.test | 16 ++++ 4 files changed, 95 insertions(+), 56 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index b6a9bb3b22cd5..2dd7f10e6f354 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -7905,6 +7905,10 @@ def is_writable_attribute(self, node: Node) -> bool: return False def get_isinstance_type(self, expr: Expression) -> list[TypeRange] | None: + """Get the type(s) resulting from an isinstance check. + + Returns an empty list for isinstance(x, ()). + """ if isinstance(expr, OpExpr) and expr.op == "|": left = self.get_isinstance_type(expr.left) if left is None and is_literal_none(expr.left): @@ -7944,11 +7948,6 @@ def get_isinstance_type(self, expr: Expression) -> list[TypeRange] | None: types.append(TypeRange(typ, is_upper_bound=False)) else: # we didn't see an actual type, but rather a variable with unknown value return None - if not types: - # this can happen if someone has empty tuple as 2nd argument to isinstance - # strictly speaking, we should return UninhabitedType but for simplicity we will simply - # refuse to do any type inference for now - return None return types def is_literal_enum(self, n: Expression) -> bool: @@ -8185,59 +8184,82 @@ def conditional_types( UninhabitedType means unreachable. None means no new information can be inferred. """ - if proposed_type_ranges: - if len(proposed_type_ranges) == 1: - target = proposed_type_ranges[0].item - target = get_proper_type(target) - if isinstance(target, LiteralType) and ( - target.is_enum_literal() or isinstance(target.value, bool) - ): - enum_name = target.fallback.type.fullname - current_type = try_expanding_sum_type_to_union(current_type, enum_name) - proposed_items = [type_range.item for type_range in proposed_type_ranges] - proposed_type = make_simplified_union(proposed_items) - if isinstance(get_proper_type(current_type), AnyType): - return proposed_type, current_type - elif isinstance(proposed_type, AnyType): - # We don't really know much about the proposed type, so we shouldn't - # attempt to narrow anything. Instead, we broaden the expr to Any to - # avoid false positives - return proposed_type, default - elif not any(type_range.is_upper_bound for type_range in proposed_type_ranges) and ( - # concrete subtypes - is_proper_subtype(current_type, proposed_type, ignore_promotions=True) - # structural subtypes - or ( - ( - isinstance(proposed_type, CallableType) - or (isinstance(proposed_type, Instance) and proposed_type.type.is_protocol) - ) - and is_subtype(current_type, proposed_type, ignore_promotions=True) - ) + if proposed_type_ranges is None: + # An isinstance check, but we don't understand the type + return current_type, default + + if not proposed_type_ranges: + # This is the case for `if isinstance(x, ())` which always returns False. + return UninhabitedType(), default + + if len(proposed_type_ranges) == 1: + # expand e.g. bool -> Literal[True] | Literal[False] + target = proposed_type_ranges[0].item + target = get_proper_type(target) + if isinstance(target, LiteralType) and ( + target.is_enum_literal() or isinstance(target.value, bool) ): - # Expression is always of one of the types in proposed_type_ranges - return default, UninhabitedType() - elif not is_overlapping_types(current_type, proposed_type, ignore_promotions=True): - # Expression is never of any type in proposed_type_ranges - return UninhabitedType(), default - else: - # we can only restrict when the type is precise, not bounded - proposed_precise_type = UnionType.make_union( - [ - type_range.item - for type_range in proposed_type_ranges - if not type_range.is_upper_bound - ] - ) - remaining_type = restrict_subtype_away( - current_type, - proposed_precise_type, + enum_name = target.fallback.type.fullname + current_type = try_expanding_sum_type_to_union(current_type, enum_name) + + proper_type = get_proper_type(current_type) + # factorize over union types: isinstance(A|B, C) -> yes = A_yes | B_yes + if isinstance(proper_type, UnionType): + result: list[tuple[Type | None, Type | None]] = [ + conditional_types( + union_item, + proposed_type_ranges, + default=union_item, consider_runtime_isinstance=consider_runtime_isinstance, ) - return proposed_type, remaining_type + for union_item in get_proper_types(proper_type.items) + ] + # separate list of tuples into two lists + yes_types, no_types = zip(*result) + proposed_type = make_simplified_union([t for t in yes_types if t is not None]) else: - # An isinstance check, but we don't understand the type - return current_type, default + proposed_items = [type_range.item for type_range in proposed_type_ranges] + proposed_type = make_simplified_union(proposed_items) + + if isinstance(proper_type, AnyType): + return proposed_type, current_type + elif isinstance(proposed_type, AnyType): + # We don't really know much about the proposed type, so we shouldn't + # attempt to narrow anything. Instead, we broaden the expr to Any to + # avoid false positives + return proposed_type, default + elif not any(type_range.is_upper_bound for type_range in proposed_type_ranges) and ( + # concrete subtypes + is_proper_subtype(current_type, proposed_type, ignore_promotions=True) + # structural subtypes + or ( + ( + isinstance(proposed_type, CallableType) + or (isinstance(proposed_type, Instance) and proposed_type.type.is_protocol) + ) + and is_subtype(current_type, proposed_type, ignore_promotions=True) + ) + ): + # Expression is always of one of the types in proposed_type_ranges + return default, UninhabitedType() + elif not is_overlapping_types(current_type, proposed_type, ignore_promotions=True): + # Expression is never of any type in proposed_type_ranges + return UninhabitedType(), default + else: + # we can only restrict when the type is precise, not bounded + proposed_precise_type = UnionType.make_union( + [ + type_range.item + for type_range in proposed_type_ranges + if not type_range.is_upper_bound + ] + ) + remaining_type = restrict_subtype_away( + current_type, + proposed_precise_type, + consider_runtime_isinstance=consider_runtime_isinstance, + ) + return proposed_type, remaining_type def conditional_types_to_typemaps( diff --git a/test-data/unit/check-isinstance.test b/test-data/unit/check-isinstance.test index 5043d54221086..acd4b588f98ca 100644 --- a/test-data/unit/check-isinstance.test +++ b/test-data/unit/check-isinstance.test @@ -1483,11 +1483,12 @@ def f(x: Union[int, A], a: Type[A]) -> None: [builtins fixtures/isinstancelist.pyi] [case testIsInstanceWithEmtpy2ndArg] +# flags: --warn-unreachable from typing import Union def f(x: Union[int, str]) -> None: if isinstance(x, ()): - reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + reveal_type(x) # E: Statement is unreachable else: reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" [builtins fixtures/isinstancelist.pyi] diff --git a/test-data/unit/check-narrowing.test b/test-data/unit/check-narrowing.test index 7fffd3ce94e54..00d33c86414fe 100644 --- a/test-data/unit/check-narrowing.test +++ b/test-data/unit/check-narrowing.test @@ -2639,7 +2639,7 @@ def baz(item: Base) -> None: reveal_type(item) # N: Revealed type is "Union[__main__., __main__.]" if isinstance(item, FooMixin): - reveal_type(item) # N: Revealed type is "__main__.FooMixin" + reveal_type(item) # N: Revealed type is "__main__." item.foo() else: reveal_type(item) # N: Revealed type is "__main__." diff --git a/test-data/unit/check-python310.test b/test-data/unit/check-python310.test index 3a9f12e8a5500..1e27e30d4b041 100644 --- a/test-data/unit/check-python310.test +++ b/test-data/unit/check-python310.test @@ -1936,6 +1936,22 @@ def union(x: str | bool) -> None: reveal_type(x) # N: Revealed type is "Union[builtins.str, Literal[False]]" [builtins fixtures/tuple.pyi] +[case testMatchNarrowDownUnionUsingClassPattern] + +class Foo: ... +class Bar(Foo): ... + +def test_1(bar: Bar) -> None: + match bar: + case Foo() as foo: + reveal_type(foo) # N: Revealed type is "__main__.Bar" + +def test_2(bar: Bar | str) -> None: + match bar: + case Foo() as foo: + reveal_type(foo) # N: Revealed type is "__main__.Bar" + + [case testMatchAssertFalseToSilenceFalsePositives] class C: a: int | str From 0ece662da24111c25345fdba5e69346bbd5c287f Mon Sep 17 00:00:00 2001 From: Randolf Scholz Date: Mon, 27 Oct 2025 17:41:46 +0100 Subject: [PATCH 346/424] [PEP 696] Fix swapping TypeVars with defaults. (#19449) - Fixes #19444. (added `testTypeVarDefaultsSwap`) - Fixes #19362 (added `testTypeVarDefaultsSwap2`) Changed the logic for recursion guards of `TypeVarType`: Instead of always substituting `repl = repl.accept(self)`, and situationally updating `repl.default = repl.default.accept(self)` if the result is a `TypeVarType`, we now always update `repl.default = repl.default.accept(self)` a priori and then only choose the expanded `repl.accept(self)` if the result is a concrete type. ## New Tests - `testTypeVarDefaultsSwap` (https://mypy-play.net/?mypy=1.17.0&python=3.12&gist=d5a025a31ae3c8b9e2a36f4738aa1991) - `testTypeVarDefaultsSwap2` (https://mypy-play.net/?mypy=1.17.0&python=3.12&gist=d3ed42c82f7144967c97d846c4c041ef) PS: closed earlier PRs #19447, since it contained debugging changes, and #19448 because it didn't solve #19362. --- mypy/expandtype.py | 6 ++-- test-data/unit/check-typevar-defaults.test | 32 ++++++++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/mypy/expandtype.py b/mypy/expandtype.py index e2a42317141f4..be1c2dd91e77c 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -251,9 +251,9 @@ def visit_type_var(self, t: TypeVarType) -> Type: if (tvar_id := repl.id) in self.recursive_tvar_guard: return self.recursive_tvar_guard[tvar_id] or repl self.recursive_tvar_guard[tvar_id] = None - repl = repl.accept(self) - if isinstance(repl, TypeVarType): - repl.default = repl.default.accept(self) + repl.default = repl.default.accept(self) + expanded = repl.accept(self) # Note: `expanded is repl` may be true. + repl = repl if isinstance(expanded, TypeVarType) else expanded self.recursive_tvar_guard[tvar_id] = repl return repl diff --git a/test-data/unit/check-typevar-defaults.test b/test-data/unit/check-typevar-defaults.test index 22270e17787e9..103c0e782797d 100644 --- a/test-data/unit/check-typevar-defaults.test +++ b/test-data/unit/check-typevar-defaults.test @@ -416,6 +416,38 @@ def func_c4( reveal_type(m) # N: Revealed type is "__main__.ClassC4[builtins.int, builtins.float]" [builtins fixtures/tuple.pyi] +[case testTypeVarDefaultsSwap] +from typing import TypeVar, Generic + +T = TypeVar("T") +X = TypeVar("X", default=object) +Y = TypeVar("Y", default=object) + + +class Foo(Generic[T, Y]): + def test(self) -> None: + reveal_type( Foo[Y, T]() ) # N: Revealed type is "__main__.Foo[Y`2 = builtins.object, T`1]" + + +class Bar(Generic[X, Y]): + def test(self) -> None: + reveal_type( Bar[Y, X]() ) # N: Revealed type is "__main__.Bar[Y`2 = builtins.object, X`1 = builtins.object]" + + +[case testTypeVarDefaultsSwap2] +from typing import TypeVar, Generic + +X = TypeVar("X", default=object) +Y = TypeVar("Y", default=object) +U = TypeVar("U", default=object) +V = TypeVar("V", default=object) + +class Transform(Generic[X, Y]): + def invert(self) -> "Transform[Y, X]": ... + +class Foo(Transform[U, V], Generic[U, V]): + def invert(self) -> "Foo[V, U]": ... + [case testTypeVarDefaultsClassRecursive1] # flags: --disallow-any-generics from typing import Generic, TypeVar, List From 00df9a65e196f775d98947a2ca0c377afa4cb236 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Mon, 27 Oct 2025 11:11:41 -0600 Subject: [PATCH 347/424] Fix an INTERNAL ERROR when creating cobertura output for namespace package (#20112) Fixes https://github.com/python/mypy/issues/19843 This fixes an Internal Error where namespace packages were not supported properly. We have to use `os.path.isdir(path)` instead of trying to catch an `IsADirectoryError` exception because of a bug on Windows which causes it to throw a `PermissionError` instead in [the relevant situation](https://discuss.python.org/t/permissionerror-errno-13-permission-denied-python-2023/22360/8), which makes `except IsADirectoryError` unreliable. (We also can't just `except (IsADirectoryError, PermissionError)` because what if there is an actual permission error?) --- mypy/report.py | 5 +---- test-data/unit/reports.test | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/mypy/report.py b/mypy/report.py index 39cd80ed38bf0..90a62c8a4c7c5 100644 --- a/mypy/report.py +++ b/mypy/report.py @@ -141,12 +141,9 @@ def should_skip_path(path: str) -> bool: def iterate_python_lines(path: str) -> Iterator[tuple[int, str]]: """Return an iterator over (line number, line text) from a Python file.""" - try: + if not os.path.isdir(path): # can happen with namespace packages with tokenize.open(path) as input_file: yield from enumerate(input_file, 1) - except IsADirectoryError: - # can happen with namespace packages - pass class FuncCounterVisitor(TraverserVisitor): diff --git a/test-data/unit/reports.test b/test-data/unit/reports.test index 82c3869bb855a..714610314c883 100644 --- a/test-data/unit/reports.test +++ b/test-data/unit/reports.test @@ -547,3 +547,37 @@ namespace_packages = True + +[case testReportCoberturaCrashOnNamespacePackages] +# cmd: mypy --cobertura-xml-report report -p folder +-- Regression test for https://github.com/python/mypy/issues/19843 +[file folder/subfolder/something.py] +-- This output is not important, but due to the way tests are run we need to check it. +[outfile report/cobertura.xml] + + + $PWD + + + + + + + + + + + + + + + + + + + + + + + + From 28536b5338df20465a8bc98fd34859e0714cba46 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Mon, 27 Oct 2025 11:46:20 -0600 Subject: [PATCH 348/424] Fix IsADirectoryError for namespace packages when using --linecoverage-report (#20109) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #18128. This fixes an Internal Error where namespace packages were not supported properly. This fix was inspired by @sterliakov noticing that this bug was very similar to #19843, which has a similar fix. Note that we use `os.path.isdir(tree.path)` instead of trying to catch an `IsADirectoryError` exception because of a bug on Windows which causes it to throw a `PermissionError` instead in [the relevant situation](https://discuss.python.org/t/permissionerror-errno-13-permission-denied-python-2023/22360/8), which makes `except IsADirectoryError` unreliable. (We also can't just `except (IsADirectoryError, PermissionError)` because what if there is an actual permission error?) Anyway, we just early-return — which, based on my manual testing and reading the code, seems to be the right thing to do here. --------- Co-authored-by: Ivan Levkivskyi --- mypy/report.py | 3 +++ test-data/unit/reports.test | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/mypy/report.py b/mypy/report.py index 90a62c8a4c7c5..398127a33026f 100644 --- a/mypy/report.py +++ b/mypy/report.py @@ -421,6 +421,9 @@ def on_file( type_map: dict[Expression, Type], options: Options, ) -> None: + if os.path.isdir(tree.path): # can happen with namespace packages + return + with open(tree.path) as f: tree_source = f.readlines() diff --git a/test-data/unit/reports.test b/test-data/unit/reports.test index 714610314c883..3964145759480 100644 --- a/test-data/unit/reports.test +++ b/test-data/unit/reports.test @@ -548,6 +548,14 @@ namespace_packages = True +[case testReportIsADirectoryErrorCrashOnNamespacePackages] +# cmd: mypy --linecoverage-report report -p folder +-- Regression test for https://github.com/python/mypy/issues/18128 +-- "IsADirectoryError for namespace packages when using --linecoverage-report" +[file folder/subfolder/something.py] +class Something: + pass + [case testReportCoberturaCrashOnNamespacePackages] # cmd: mypy --cobertura-xml-report report -p folder -- Regression test for https://github.com/python/mypy/issues/19843 From 31a915afe5292e25fb627a73ab94b45f23eceb02 Mon Sep 17 00:00:00 2001 From: bzoracler <50305397+bzoracler@users.noreply.github.com> Date: Tue, 28 Oct 2025 06:57:44 +1300 Subject: [PATCH 349/424] Use dummy concrete type instead of `Any` when checking protocol variance (#20110) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #20108. Variance checks for protocols follow a procedure roughly equivalent to that described in [typing.python.org - Variance Inference](https://typing.python.org/en/latest/spec/generics.html#variance-inference). A major difference in mypy's current implementation is in Step 3: > Create two specialized versions of the class. We’ll refer to these as `upper` and `lower` specializations. In both of these specializations, replace all type parameters other than the one being inferred by a dummy type instance (a concrete anonymous class that is assumed to meet the bounds or constraints of the type parameter). Mypy currently uses `Any` rather than a concrete dummy type. This causes issues during overload subtype checks in the example reported in the original issue, as the specialisations when checking variance suitability of `_T2_contra` look like: ```python from typing import TypeVar, Protocol, overload _T1_contra = TypeVar("_T1_contra", contravariant=True) _T2_contra = TypeVar("_T2_contra", contravariant=True) class A(Protocol[<_T1_contra=Any>, _T2_contra]): @overload def method(self, a: <_T1_contra=Any>) -> None: ... @overload def method(self, a: _T2_contra) -> None: ... ``` This PR replaces the use of `Any` with a dummy concrete type in the entire protocol variance check to more closely follow the variance inference algorithm in the spec and fixes this overload issue. --- mypy/checker.py | 13 +++++++++---- test-data/unit/check-protocols.test | 13 +++++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 2dd7f10e6f354..c1cb29652e88f 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -395,6 +395,8 @@ class TypeChecker(NodeVisitor[None], TypeCheckerSharedApi): # A helper state to produce unique temporary names on demand. _unique_id: int + # Fake concrete type used when checking variance + _variance_dummy_type: Instance | None def __init__( self, @@ -469,6 +471,7 @@ def __init__( self.pattern_checker = PatternChecker(self, self.msg, self.plugin, options) self._unique_id = 0 + self._variance_dummy_type = None @property def expr_checker(self) -> mypy.checkexpr.ExpressionChecker: @@ -2918,17 +2921,19 @@ def check_protocol_variance(self, defn: ClassDef) -> None: info = defn.info object_type = Instance(info.mro[-1], []) tvars = info.defn.type_vars + if self._variance_dummy_type is None: + _, dummy_info = self.make_fake_typeinfo("", "Dummy", "Dummy", []) + self._variance_dummy_type = Instance(dummy_info, []) + dummy = self._variance_dummy_type for i, tvar in enumerate(tvars): if not isinstance(tvar, TypeVarType): # Variance of TypeVarTuple and ParamSpec is underspecified by PEPs. continue up_args: list[Type] = [ - object_type if i == j else AnyType(TypeOfAny.special_form) - for j, _ in enumerate(tvars) + object_type if i == j else dummy.copy_modified() for j, _ in enumerate(tvars) ] down_args: list[Type] = [ - UninhabitedType() if i == j else AnyType(TypeOfAny.special_form) - for j, _ in enumerate(tvars) + UninhabitedType() if i == j else dummy.copy_modified() for j, _ in enumerate(tvars) ] up, down = Instance(info, up_args), Instance(info, down_args) # TODO: add advanced variance checks for recursive protocols diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index ae6f603555127..e7971cd5b5d83 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -1373,6 +1373,19 @@ main:16: note: def meth(self, x: int) -> int main:16: note: @overload main:16: note: def meth(self, x: bytes) -> str +[case testProtocolWithMultiContravariantTypeVarOverloads] +from typing import overload, Protocol, TypeVar + +T1 = TypeVar("T1", contravariant=True) +T2 = TypeVar("T2", contravariant=True) + +class A(Protocol[T1, T2]): + @overload + def method(self, a: T1) -> None: ... + @overload + def method(self, a: T2) -> None: ... + + -- Join and meet with protocol types -- --------------------------------- From 842a8fdcaa711c92c1067d373098844eb7d1ab57 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Mon, 27 Oct 2025 12:16:15 -0600 Subject: [PATCH 350/424] Add a regression test for #15979, and fix linecount-report for Windows (#20111) https://github.com/python/mypy/pull/16019, which fixed this issue (for linux) was not accompanied by a regression test. Thus, nobody noticed that it doesn't work on Windows! This fixes an Internal Error where namespace packages were not supported properly. This fix was inspired by @sterliakov noticing that https://github.com/python/mypy/issues/18128 was very similar to #19843, which has a similar fix. Note that we use `os.path.isdir(tree.path)` instead of trying to catch an `IsADirectoryError` exception because of a bug on Windows which causes it to throw a `PermissionError` instead in [the relevant situation](https://discuss.python.org/t/permissionerror-errno-13-permission-denied-python-2023/22360/8), which makes `except IsADirectoryError` unreliable. (We also can't just `except (IsADirectoryError, PermissionError)` because what if there is an actual permission error?) --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Ivan Levkivskyi --- mypy/report.py | 5 ++--- test-data/unit/reports.test | 7 +++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/mypy/report.py b/mypy/report.py index 398127a33026f..4a0b965077f6a 100644 --- a/mypy/report.py +++ b/mypy/report.py @@ -169,11 +169,10 @@ def on_file( ) -> None: # Count physical lines. This assumes the file's encoding is a # superset of ASCII (or at least uses \n in its line endings). - try: + if not os.path.isdir(tree.path): # can happen with namespace packages with open(tree.path, "rb") as f: physical_lines = len(f.readlines()) - except IsADirectoryError: - # can happen with namespace packages + else: physical_lines = 0 func_counter = FuncCounterVisitor() diff --git a/test-data/unit/reports.test b/test-data/unit/reports.test index 3964145759480..cce2f7295e3bf 100644 --- a/test-data/unit/reports.test +++ b/test-data/unit/reports.test @@ -548,6 +548,13 @@ namespace_packages = True +[case testLinecountReportCrashOnNamespacePackages] +# cmd: mypy --linecount-report report -p folder +-- Regression test for https://github.com/python/mypy/issues/15979 +[file folder/subfolder/something.py] +class Something: + pass + [case testReportIsADirectoryErrorCrashOnNamespacePackages] # cmd: mypy --linecoverage-report report -p folder -- Regression test for https://github.com/python/mypy/issues/18128 From 841db1f7e537b1ac15c8866bec574c04ce125554 Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Tue, 28 Oct 2025 12:57:53 +0100 Subject: [PATCH 351/424] Remember the pair in `is_overlapping_types` if at least one of them is an alias (#20127) Fixes #20107. One recursive alias is enough to trigger infinite recursion --- mypy/meet.py | 2 +- test-data/unit/check-recursive-types.test | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/mypy/meet.py b/mypy/meet.py index 63305c2bb236e..1cb291ff90d50 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -346,7 +346,7 @@ def is_overlapping_types( seen_types = set() elif (left, right) in seen_types: return True - if isinstance(left, TypeAliasType) and isinstance(right, TypeAliasType): + if is_recursive_pair(left, right): seen_types.add((left, right)) left, right = get_proper_types((left, right)) diff --git a/test-data/unit/check-recursive-types.test b/test-data/unit/check-recursive-types.test index c82111322fe17..c09f1e6b90c0b 100644 --- a/test-data/unit/check-recursive-types.test +++ b/test-data/unit/check-recursive-types.test @@ -1024,3 +1024,19 @@ L = list[T] A = L[A] a: A = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "A") + +[case testRecursiveAliasInstanceOverlapCheck] +# flags: --warn-unreachable +from typing_extensions import TypeAlias + +OneClass: TypeAlias = 'list[OneClass]' + +class TwoClass(list['TwoClass']): + pass + +def f(obj: OneClass) -> None: + if isinstance(obj, TwoClass): + reveal_type(obj) # N: Revealed type is "__main__.TwoClass" + else: + reveal_type(obj) # N: Revealed type is "builtins.list[...]" +[builtins fixtures/isinstancelist.pyi] From 054f7212d8c79d7b8e672f1cd5f207d6f9ec7fba Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Wed, 29 Oct 2025 00:32:30 +0100 Subject: [PATCH 352/424] Discard partials remaining after inference failure (#20126) Fixes #16573. Fixes #3031. When we infer something with Never as a replacement for a partial type, we cannot ignore that result entirely: if we do that, any use of that variable down the road will still refer to `partial`, causing reachability issues and unexpected partials leaks. --- mypy/checker.py | 16 ++++-- test-data/unit/check-inference.test | 77 ++++++++++++++++++++++++++--- 2 files changed, 84 insertions(+), 9 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index c1cb29652e88f..63e128f78310a 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -3299,9 +3299,19 @@ def check_assignment( del partial_types[var] lvalue_type = var.type else: - # Try to infer a partial type. No need to check the return value, as - # an error will be reported elsewhere. - self.infer_partial_type(lvalue_type.var, lvalue, rvalue_type) + # Try to infer a partial type. + if not self.infer_partial_type(var, lvalue, rvalue_type): + # If that also failed, give up and let the caller know that we + # cannot read their mind. The definition site will be reported later. + # Calling .put() directly because the newly inferred type is + # not a subtype of None - we are not looking for narrowing + fallback = self.inference_error_fallback_type(rvalue_type) + self.binder.put(lvalue, fallback) + # Same as self.set_inference_error_fallback_type but inlined + # to avoid computing fallback twice. + # We are replacing partial now, so the variable type + # should remain optional. + self.set_inferred_type(var, lvalue, make_optional_type(fallback)) elif ( is_literal_none(rvalue) and isinstance(lvalue, NameExpr) diff --git a/test-data/unit/check-inference.test b/test-data/unit/check-inference.test index 24ea61f2c7156..9ed9c5e9ec787 100644 --- a/test-data/unit/check-inference.test +++ b/test-data/unit/check-inference.test @@ -2522,7 +2522,6 @@ class C: def __init__(self) -> None: self.x = [] # E: Need type annotation for "x" (hint: "x: list[] = ...") [builtins fixtures/list.pyi] -[out] [case testNoCrashOnPartialVariable] from typing import Tuple, TypeVar @@ -2534,7 +2533,6 @@ x = None (x,) = f('') reveal_type(x) # N: Revealed type is "builtins.str" [builtins fixtures/tuple.pyi] -[out] [case testNoCrashOnPartialVariable2] # flags: --no-local-partial-types @@ -2543,11 +2541,10 @@ T = TypeVar('T', bound=str) def f() -> Tuple[T]: ... -x = None +x = None # E: Need type annotation for "x" if int(): (x,) = f() [builtins fixtures/tuple.pyi] -[out] [case testNoCrashOnPartialVariable3] from typing import Tuple, TypeVar @@ -2559,7 +2556,76 @@ x = None (x, x) = f('') reveal_type(x) # N: Revealed type is "builtins.str" [builtins fixtures/tuple.pyi] -[out] + +[case testRejectsPartialWithUninhabited] +from typing import Generic, TypeVar +T = TypeVar('T') + +class Foo(Generic[T]): ... + +def check() -> None: + x = None # E: Need type annotation for "x" + if int(): + x = Foo() + reveal_type(x) # N: Revealed type is "__main__.Foo[Any]" + reveal_type(x) # N: Revealed type is "Union[__main__.Foo[Any], None]" + +[case testRejectsPartialWithUninhabited2] +from typing import Generic, TypeVar +T = TypeVar('T') + +class Foo(Generic[T]): ... + +x = None # E: Need type annotation for "x" + +def check() -> None: + global x + x = Foo() + reveal_type(x) # N: Revealed type is "__main__.Foo[Any]" + +reveal_type(x) # N: Revealed type is "Union[__main__.Foo[Any], None]" + +[case testRejectsPartialWithUninhabited3] +# Without force-rejecting Partial, this crashes: +# https://github.com/python/mypy/issues/16573 +from typing import Generic, TypeVar +T = TypeVar('T') + +class Foo(Generic[T]): ... + +def check() -> None: + client = None # E: Need type annotation for "client" + + if client := Foo(): + reveal_type(client) # N: Revealed type is "__main__.Foo[Any]" + + reveal_type(client) # N: Revealed type is "Union[__main__.Foo[Any], None]" + + client = 0 # E: Incompatible types in assignment (expression has type "int", variable has type "Optional[Foo[Any]]") + reveal_type(client) # N: Revealed type is "Union[__main__.Foo[Any], None]" + +[case testRejectsPartialWithUninhabitedIndependently] +from typing import Generic, TypeVar +T = TypeVar('T') + +class Foo(Generic[T]): ... + +client = None # E: Need type annotation for "client" + +def bad() -> None: + global client + client = Foo() + reveal_type(client) # N: Revealed type is "__main__.Foo[Any]" + +def good() -> None: + global client + client = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "Optional[Foo[Any]]") + reveal_type(client) # N: Revealed type is "Union[__main__.Foo[Any], None]" + +def bad2() -> None: + global client + client = Foo() + reveal_type(client) # N: Revealed type is "__main__.Foo[Any]" [case testInferenceNestedTuplesFromGenericIterable] from typing import Tuple, TypeVar @@ -2574,7 +2640,6 @@ def main() -> None: reveal_type(a) # N: Revealed type is "builtins.int" reveal_type(b) # N: Revealed type is "builtins.int" [builtins fixtures/tuple.pyi] -[out] [case testDontMarkUnreachableAfterInferenceUninhabited] from typing import TypeVar From 16984327a45300a0563498f2823aac5c3d2b11af Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Thu, 30 Oct 2025 06:56:51 -0700 Subject: [PATCH 353/424] Sync pythoncapi_compat.h (#20143) --- mypyc/lib-rt/exc_ops.c | 4 +- mypyc/lib-rt/librt_internal.c | 4 +- mypyc/lib-rt/misc_ops.c | 4 +- mypyc/lib-rt/pythoncapi_compat.h | 407 ++++++++++++++++++++++++++++++- mypyc/lib-rt/str_ops.c | 4 +- 5 files changed, 410 insertions(+), 13 deletions(-) diff --git a/mypyc/lib-rt/exc_ops.c b/mypyc/lib-rt/exc_ops.c index d8307ecf21f85..85498420d2af4 100644 --- a/mypyc/lib-rt/exc_ops.c +++ b/mypyc/lib-rt/exc_ops.c @@ -1,3 +1,5 @@ +#include "pythoncapi_compat.h" + // Exception related primitive operations // // These are registered in mypyc.primitives.exc_ops. @@ -24,7 +26,7 @@ void CPy_Reraise(void) { } void CPyErr_SetObjectAndTraceback(PyObject *type, PyObject *value, PyObject *traceback) { - if (!PyType_Check(type) && value == Py_None) { + if (!PyType_Check(type) && Py_IsNone(value)) { // The first argument must be an exception instance value = type; type = (PyObject *)Py_TYPE(value); diff --git a/mypyc/lib-rt/librt_internal.c b/mypyc/lib-rt/librt_internal.c index 6f6a110446ade..af8dcbccaa434 100644 --- a/mypyc/lib-rt/librt_internal.c +++ b/mypyc/lib-rt/librt_internal.c @@ -1,3 +1,5 @@ +#include "pythoncapi_compat.h" + #define PY_SSIZE_T_CLEAN #include #include @@ -245,7 +247,7 @@ write_bool(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwname PyErr_SetString(PyExc_TypeError, "value must be a bool"); return NULL; } - if (unlikely(write_bool_internal(data, value == Py_True) == CPY_NONE_ERROR)) { + if (unlikely(write_bool_internal(data, Py_IsTrue(value)) == CPY_NONE_ERROR)) { return NULL; } Py_INCREF(Py_None); diff --git a/mypyc/lib-rt/misc_ops.c b/mypyc/lib-rt/misc_ops.c index ca09c347b4ffe..8e5bfffba7594 100644 --- a/mypyc/lib-rt/misc_ops.c +++ b/mypyc/lib-rt/misc_ops.c @@ -1,3 +1,5 @@ +#include "pythoncapi_compat.h" + // Misc primitive operations + C helpers // // These are registered in mypyc.primitives.misc_ops. @@ -750,7 +752,7 @@ CPy_Super(PyObject *builtins, PyObject *self) { static bool import_single(PyObject *mod_id, PyObject **mod_static, PyObject *globals_id, PyObject *globals_name, PyObject *globals) { - if (*mod_static == Py_None) { + if (Py_IsNone(*mod_static)) { CPyModule *mod = PyImport_Import(mod_id); if (mod == NULL) { return false; diff --git a/mypyc/lib-rt/pythoncapi_compat.h b/mypyc/lib-rt/pythoncapi_compat.h index f94e50a3479f3..b16075fcc9b86 100644 --- a/mypyc/lib-rt/pythoncapi_compat.h +++ b/mypyc/lib-rt/pythoncapi_compat.h @@ -34,16 +34,16 @@ extern "C" { # define _Py_CAST(type, expr) ((type)(expr)) #endif -#ifndef _Py_NULL // Static inline functions should use _Py_NULL rather than using directly NULL // to prevent C++ compiler warnings. On C23 and newer and on C++11 and newer, // _Py_NULL is defined as nullptr. -#if (defined (__STDC_VERSION__) && __STDC_VERSION__ > 201710L) \ - || (defined(__cplusplus) && __cplusplus >= 201103) -# define _Py_NULL nullptr -#else -# define _Py_NULL NULL -#endif +#ifndef _Py_NULL +# if (defined (__STDC_VERSION__) && __STDC_VERSION__ > 201710L) \ + || (defined(__cplusplus) && __cplusplus >= 201103) +# define _Py_NULL nullptr +# else +# define _Py_NULL NULL +# endif #endif // Cast argument to PyObject* type. @@ -1456,6 +1456,18 @@ PyUnicodeWriter_WriteUTF8(PyUnicodeWriter *writer, return res; } +static inline int +PyUnicodeWriter_WriteASCII(PyUnicodeWriter *writer, + const char *str, Py_ssize_t size) +{ + if (size < 0) { + size = (Py_ssize_t)strlen(str); + } + + return _PyUnicodeWriter_WriteASCIIString((_PyUnicodeWriter*)writer, + str, size); +} + static inline int PyUnicodeWriter_WriteWideChar(PyUnicodeWriter *writer, const wchar_t *str, Py_ssize_t size) @@ -1479,7 +1491,8 @@ PyUnicodeWriter_WriteSubstring(PyUnicodeWriter *writer, PyObject *str, Py_ssize_t start, Py_ssize_t end) { if (!PyUnicode_Check(str)) { - PyErr_Format(PyExc_TypeError, "expect str, not %T", str); + PyErr_Format(PyExc_TypeError, "expect str, not %s", + Py_TYPE(str)->tp_name); return -1; } if (start < 0 || start > end) { @@ -1978,7 +1991,7 @@ static inline int Py_fclose(FILE *file) #endif -#if 0x03090000 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x030E0000 && !defined(PYPY_VERSION) +#if 0x03080000 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x030E0000 && !defined(PYPY_VERSION) static inline PyObject* PyConfig_Get(const char *name) { @@ -2019,7 +2032,9 @@ PyConfig_Get(const char *name) PYTHONCAPI_COMPAT_SPEC(module_search_paths, WSTR_LIST, "path"), PYTHONCAPI_COMPAT_SPEC(optimization_level, UINT, _Py_NULL), PYTHONCAPI_COMPAT_SPEC(parser_debug, BOOL, _Py_NULL), +#if 0x03090000 <= PY_VERSION_HEX PYTHONCAPI_COMPAT_SPEC(platlibdir, WSTR, "platlibdir"), +#endif PYTHONCAPI_COMPAT_SPEC(prefix, WSTR_OPT, "prefix"), PYTHONCAPI_COMPAT_SPEC(pycache_prefix, WSTR_OPT, "pycache_prefix"), PYTHONCAPI_COMPAT_SPEC(quiet, BOOL, _Py_NULL), @@ -2198,6 +2213,380 @@ PyConfig_GetInt(const char *name, int *value) } #endif // PY_VERSION_HEX > 0x03090000 && !defined(PYPY_VERSION) +// gh-133144 added PyUnstable_Object_IsUniquelyReferenced() to Python 3.14.0b1. +// Adapted from _PyObject_IsUniquelyReferenced() implementation. +#if PY_VERSION_HEX < 0x030E00B0 +static inline int PyUnstable_Object_IsUniquelyReferenced(PyObject *obj) +{ +#if !defined(Py_GIL_DISABLED) + return Py_REFCNT(obj) == 1; +#else + // NOTE: the entire ob_ref_shared field must be zero, including flags, to + // ensure that other threads cannot concurrently create new references to + // this object. + return (_Py_IsOwnedByCurrentThread(obj) && + _Py_atomic_load_uint32_relaxed(&obj->ob_ref_local) == 1 && + _Py_atomic_load_ssize_relaxed(&obj->ob_ref_shared) == 0); +#endif +} +#endif + + +#if PY_VERSION_HEX < 0x030F0000 +static inline PyObject* +PySys_GetAttrString(const char *name) +{ +#if PY_VERSION_HEX >= 0x03000000 + PyObject *value = Py_XNewRef(PySys_GetObject(name)); +#else + PyObject *value = Py_XNewRef(PySys_GetObject((char*)name)); +#endif + if (value != NULL) { + return value; + } + if (!PyErr_Occurred()) { + PyErr_Format(PyExc_RuntimeError, "lost sys.%s", name); + } + return NULL; +} + +static inline PyObject* +PySys_GetAttr(PyObject *name) +{ +#if PY_VERSION_HEX >= 0x03000000 + const char *name_str = PyUnicode_AsUTF8(name); +#else + const char *name_str = PyString_AsString(name); +#endif + if (name_str == NULL) { + return NULL; + } + + return PySys_GetAttrString(name_str); +} + +static inline int +PySys_GetOptionalAttrString(const char *name, PyObject **value) +{ +#if PY_VERSION_HEX >= 0x03000000 + *value = Py_XNewRef(PySys_GetObject(name)); +#else + *value = Py_XNewRef(PySys_GetObject((char*)name)); +#endif + if (*value != NULL) { + return 1; + } + return 0; +} + +static inline int +PySys_GetOptionalAttr(PyObject *name, PyObject **value) +{ +#if PY_VERSION_HEX >= 0x03000000 + const char *name_str = PyUnicode_AsUTF8(name); +#else + const char *name_str = PyString_AsString(name); +#endif + if (name_str == NULL) { + *value = NULL; + return -1; + } + + return PySys_GetOptionalAttrString(name_str, value); +} +#endif // PY_VERSION_HEX < 0x030F00A1 + + +#if PY_VERSION_HEX < 0x030F00A1 +typedef struct PyBytesWriter { + char small_buffer[256]; + PyObject *obj; + Py_ssize_t size; +} PyBytesWriter; + +static inline Py_ssize_t +_PyBytesWriter_GetAllocated(PyBytesWriter *writer) +{ + if (writer->obj == NULL) { + return sizeof(writer->small_buffer); + } + else { + return PyBytes_GET_SIZE(writer->obj); + } +} + + +static inline int +_PyBytesWriter_Resize_impl(PyBytesWriter *writer, Py_ssize_t size, + int resize) +{ + int overallocate = resize; + assert(size >= 0); + + if (size <= _PyBytesWriter_GetAllocated(writer)) { + return 0; + } + + if (overallocate) { +#ifdef MS_WINDOWS + /* On Windows, overallocate by 50% is the best factor */ + if (size <= (PY_SSIZE_T_MAX - size / 2)) { + size += size / 2; + } +#else + /* On Linux, overallocate by 25% is the best factor */ + if (size <= (PY_SSIZE_T_MAX - size / 4)) { + size += size / 4; + } +#endif + } + + if (writer->obj != NULL) { + if (_PyBytes_Resize(&writer->obj, size)) { + return -1; + } + assert(writer->obj != NULL); + } + else { + writer->obj = PyBytes_FromStringAndSize(NULL, size); + if (writer->obj == NULL) { + return -1; + } + + if (resize) { + assert((size_t)size > sizeof(writer->small_buffer)); + memcpy(PyBytes_AS_STRING(writer->obj), + writer->small_buffer, + sizeof(writer->small_buffer)); + } + } + return 0; +} + +static inline void* +PyBytesWriter_GetData(PyBytesWriter *writer) +{ + if (writer->obj == NULL) { + return writer->small_buffer; + } + else { + return PyBytes_AS_STRING(writer->obj); + } +} + +static inline Py_ssize_t +PyBytesWriter_GetSize(PyBytesWriter *writer) +{ + return writer->size; +} + +static inline void +PyBytesWriter_Discard(PyBytesWriter *writer) +{ + if (writer == NULL) { + return; + } + + Py_XDECREF(writer->obj); + PyMem_Free(writer); +} + +static inline PyBytesWriter* +PyBytesWriter_Create(Py_ssize_t size) +{ + if (size < 0) { + PyErr_SetString(PyExc_ValueError, "size must be >= 0"); + return NULL; + } + + PyBytesWriter *writer = (PyBytesWriter*)PyMem_Malloc(sizeof(PyBytesWriter)); + if (writer == NULL) { + PyErr_NoMemory(); + return NULL; + } + + writer->obj = NULL; + writer->size = 0; + + if (size >= 1) { + if (_PyBytesWriter_Resize_impl(writer, size, 0) < 0) { + PyBytesWriter_Discard(writer); + return NULL; + } + writer->size = size; + } + return writer; +} + +static inline PyObject* +PyBytesWriter_FinishWithSize(PyBytesWriter *writer, Py_ssize_t size) +{ + PyObject *result; + if (size == 0) { + result = PyBytes_FromStringAndSize("", 0); + } + else if (writer->obj != NULL) { + if (size != PyBytes_GET_SIZE(writer->obj)) { + if (_PyBytes_Resize(&writer->obj, size)) { + goto error; + } + } + result = writer->obj; + writer->obj = NULL; + } + else { + result = PyBytes_FromStringAndSize(writer->small_buffer, size); + } + PyBytesWriter_Discard(writer); + return result; + +error: + PyBytesWriter_Discard(writer); + return NULL; +} + +static inline PyObject* +PyBytesWriter_Finish(PyBytesWriter *writer) +{ + return PyBytesWriter_FinishWithSize(writer, writer->size); +} + +static inline PyObject* +PyBytesWriter_FinishWithPointer(PyBytesWriter *writer, void *buf) +{ + Py_ssize_t size = (char*)buf - (char*)PyBytesWriter_GetData(writer); + if (size < 0 || size > _PyBytesWriter_GetAllocated(writer)) { + PyBytesWriter_Discard(writer); + PyErr_SetString(PyExc_ValueError, "invalid end pointer"); + return NULL; + } + + return PyBytesWriter_FinishWithSize(writer, size); +} + +static inline int +PyBytesWriter_Resize(PyBytesWriter *writer, Py_ssize_t size) +{ + if (size < 0) { + PyErr_SetString(PyExc_ValueError, "size must be >= 0"); + return -1; + } + if (_PyBytesWriter_Resize_impl(writer, size, 1) < 0) { + return -1; + } + writer->size = size; + return 0; +} + +static inline int +PyBytesWriter_Grow(PyBytesWriter *writer, Py_ssize_t size) +{ + if (size < 0 && writer->size + size < 0) { + PyErr_SetString(PyExc_ValueError, "invalid size"); + return -1; + } + if (size > PY_SSIZE_T_MAX - writer->size) { + PyErr_NoMemory(); + return -1; + } + size = writer->size + size; + + if (_PyBytesWriter_Resize_impl(writer, size, 1) < 0) { + return -1; + } + writer->size = size; + return 0; +} + +static inline void* +PyBytesWriter_GrowAndUpdatePointer(PyBytesWriter *writer, + Py_ssize_t size, void *buf) +{ + Py_ssize_t pos = (char*)buf - (char*)PyBytesWriter_GetData(writer); + if (PyBytesWriter_Grow(writer, size) < 0) { + return NULL; + } + return (char*)PyBytesWriter_GetData(writer) + pos; +} + +static inline int +PyBytesWriter_WriteBytes(PyBytesWriter *writer, + const void *bytes, Py_ssize_t size) +{ + if (size < 0) { + size_t len = strlen((const char*)bytes); + if (len > (size_t)PY_SSIZE_T_MAX) { + PyErr_NoMemory(); + return -1; + } + size = (Py_ssize_t)len; + } + + Py_ssize_t pos = writer->size; + if (PyBytesWriter_Grow(writer, size) < 0) { + return -1; + } + char *buf = (char*)PyBytesWriter_GetData(writer); + memcpy(buf + pos, bytes, (size_t)size); + return 0; +} + +static inline int +PyBytesWriter_Format(PyBytesWriter *writer, const char *format, ...) + Py_GCC_ATTRIBUTE((format(printf, 2, 3))); + +static inline int +PyBytesWriter_Format(PyBytesWriter *writer, const char *format, ...) +{ + va_list vargs; + va_start(vargs, format); + PyObject *str = PyBytes_FromFormatV(format, vargs); + va_end(vargs); + + if (str == NULL) { + return -1; + } + int res = PyBytesWriter_WriteBytes(writer, + PyBytes_AS_STRING(str), + PyBytes_GET_SIZE(str)); + Py_DECREF(str); + return res; +} +#endif // PY_VERSION_HEX < 0x030F00A1 + + +#if PY_VERSION_HEX < 0x030F00A1 +static inline PyObject* +PyTuple_FromArray(PyObject *const *array, Py_ssize_t size) +{ + PyObject *tuple = PyTuple_New(size); + if (tuple == NULL) { + return NULL; + } + for (Py_ssize_t i=0; i < size; i++) { + PyObject *item = array[i]; + PyTuple_SET_ITEM(tuple, i, Py_NewRef(item)); + } + return tuple; +} +#endif + + +#if PY_VERSION_HEX < 0x030F00A1 +static inline Py_hash_t +PyUnstable_Unicode_GET_CACHED_HASH(PyObject *op) +{ +#ifdef PYPY_VERSION + (void)op; // unused argument + return -1; +#elif PY_VERSION_HEX >= 0x03000000 + return ((PyASCIIObject*)op)->hash; +#else + return ((PyUnicodeObject*)op)->hash; +#endif +} +#endif + #ifdef __cplusplus } diff --git a/mypyc/lib-rt/str_ops.c b/mypyc/lib-rt/str_ops.c index f16e99bb4159b..721a2bbb10b98 100644 --- a/mypyc/lib-rt/str_ops.c +++ b/mypyc/lib-rt/str_ops.c @@ -1,3 +1,5 @@ +#include "pythoncapi_compat.h" + // String primitive operations // // These are registered in mypyc.primitives.str_ops. @@ -321,7 +323,7 @@ static PyObject *_PyStr_XStrip(PyObject *self, int striptype, PyObject *sepobj) // Copied from do_strip function in cpython.git/Objects/unicodeobject.c@0ef4ffeefd1737c18dc9326133c7894d58108c2e. PyObject *_CPyStr_Strip(PyObject *self, int strip_type, PyObject *sep) { - if (sep == NULL || sep == Py_None) { + if (sep == NULL || Py_IsNone(sep)) { Py_ssize_t len, i, j; // This check is needed from Python 3.9 and earlier. From 49e9a9b369a2d63c946dbc0a97ddc9e0b7cc2546 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 30 Oct 2025 23:07:58 +0000 Subject: [PATCH 354/424] Expose buffer cache layout version (#20145) This idea is quite simple: each time we change how an object is cached in the buffer we bump this version (even if we don't change the ABI version). Notes: * This is *not* the same as ABI version, if the ABI version is different, the capsule import will fail and you can't even use this version of `librt`. We can however use buffer cache layout version to abandon cache files that would result in reading garbage data. * I can't actually start using this in cache metas in the same PR, to avoid breaking self-check in CI. If there are no comments objections, I will merge this soon. Then release a new `librt` version, and make second PR that actually uses this feature. While I am at it I fix a bug in the float deserialization (magic value was not handled). --- mypy/typeshed/stubs/librt/librt/internal.pyi | 1 + mypyc/lib-rt/librt_internal.c | 14 +++++++++++++- mypyc/lib-rt/librt_internal.h | 2 ++ mypyc/primitives/misc_ops.py | 10 +++++++++- mypyc/test-data/irbuild-classes.test | 6 +++++- mypyc/test-data/run-classes.test | 6 +++++- 6 files changed, 35 insertions(+), 4 deletions(-) diff --git a/mypy/typeshed/stubs/librt/librt/internal.pyi b/mypy/typeshed/stubs/librt/librt/internal.pyi index 8a5fc262931e7..8654e31c100e1 100644 --- a/mypy/typeshed/stubs/librt/librt/internal.pyi +++ b/mypy/typeshed/stubs/librt/librt/internal.pyi @@ -16,3 +16,4 @@ def write_int(data: Buffer, value: int) -> None: ... def read_int(data: Buffer) -> int: ... def write_tag(data: Buffer, value: u8) -> None: ... def read_tag(data: Buffer) -> u8: ... +def cache_version() -> u8: ... diff --git a/mypyc/lib-rt/librt_internal.c b/mypyc/lib-rt/librt_internal.c index af8dcbccaa434..b37136be0784a 100644 --- a/mypyc/lib-rt/librt_internal.c +++ b/mypyc/lib-rt/librt_internal.c @@ -656,6 +656,16 @@ write_tag(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames return Py_None; } +static uint8_t +cache_version_internal(void) { + return 0; +} + +static PyObject* +cache_version(PyObject *self, PyObject *Py_UNUSED(ignored)) { + return PyLong_FromLong(cache_version_internal()); +} + static PyMethodDef librt_internal_module_methods[] = { {"write_bool", (PyCFunction)write_bool, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("write a bool")}, {"read_bool", (PyCFunction)read_bool, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("read a bool")}, @@ -669,6 +679,7 @@ static PyMethodDef librt_internal_module_methods[] = { {"read_int", (PyCFunction)read_int, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("read an int")}, {"write_tag", (PyCFunction)write_tag, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("write a short int")}, {"read_tag", (PyCFunction)read_tag, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("read a short int")}, + {"cache_version", (PyCFunction)cache_version, METH_NOARGS, PyDoc_STR("cache format version")}, {NULL, NULL, 0, NULL} }; @@ -688,7 +699,7 @@ librt_internal_module_exec(PyObject *m) } // Export mypy internal C API, be careful with the order! - static void *NativeInternal_API[16] = { + static void *NativeInternal_API[17] = { (void *)Buffer_internal, (void *)Buffer_internal_empty, (void *)Buffer_getvalue_internal, @@ -705,6 +716,7 @@ librt_internal_module_exec(PyObject *m) (void *)NativeInternal_ABI_Version, (void *)write_bytes_internal, (void *)read_bytes_internal, + (void *)cache_version_internal, }; PyObject *c_api_object = PyCapsule_New((void *)NativeInternal_API, "librt.internal._C_API", NULL); if (PyModule_Add(m, "_C_API", c_api_object) < 0) { diff --git a/mypyc/lib-rt/librt_internal.h b/mypyc/lib-rt/librt_internal.h index d996b8fd95c1e..1d16e1cb127f5 100644 --- a/mypyc/lib-rt/librt_internal.h +++ b/mypyc/lib-rt/librt_internal.h @@ -21,6 +21,7 @@ static uint8_t read_tag_internal(PyObject *data); static int NativeInternal_ABI_Version(void); static char write_bytes_internal(PyObject *data, PyObject *value); static PyObject *read_bytes_internal(PyObject *data); +static uint8_t cache_version_internal(void); #else @@ -42,6 +43,7 @@ static void **NativeInternal_API; #define NativeInternal_ABI_Version (*(int (*)(void)) NativeInternal_API[13]) #define write_bytes_internal (*(char (*)(PyObject *source, PyObject *value)) NativeInternal_API[14]) #define read_bytes_internal (*(PyObject* (*)(PyObject *source)) NativeInternal_API[15]) +#define cache_version_internal (*(uint8_t (*)(void)) NativeInternal_API[16]) static int import_librt_internal(void) diff --git a/mypyc/primitives/misc_ops.py b/mypyc/primitives/misc_ops.py index c12172875e8ba..10f4bc001e293 100644 --- a/mypyc/primitives/misc_ops.py +++ b/mypyc/primitives/misc_ops.py @@ -422,7 +422,7 @@ arg_types=[object_rprimitive], return_type=float_rprimitive, c_function_name="read_float_internal", - error_kind=ERR_MAGIC, + error_kind=ERR_MAGIC_OVERLAPPING, ) function_op( @@ -456,3 +456,11 @@ c_function_name="read_tag_internal", error_kind=ERR_MAGIC_OVERLAPPING, ) + +function_op( + name="librt.internal.cache_version", + arg_types=[], + return_type=uint8_rprimitive, + c_function_name="cache_version_internal", + error_kind=ERR_NEVER, +) diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index 27ffba45ba396..a8ee7213ef965 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -1455,6 +1455,7 @@ from mypy_extensions import u8 from librt.internal import ( Buffer, write_bool, read_bool, write_str, read_str, write_float, read_float, write_int, read_int, write_tag, read_tag, write_bytes, read_bytes, + cache_version, ) Tag = u8 @@ -1476,6 +1477,7 @@ def foo() -> None: z = read_float(b) t = read_int(b) u = read_tag(b) + v = cache_version() [out] def foo(): r0, b :: librt.internal.Buffer @@ -1490,7 +1492,7 @@ def foo(): r13, y :: bool r14, z :: float r15, t :: int - r16, u :: u8 + r16, u, r17, v :: u8 L0: r0 = Buffer_internal_empty() b = r0 @@ -1517,6 +1519,8 @@ L0: t = r15 r16 = read_tag_internal(b) u = r16 + r17 = cache_version_internal() + v = r17 return 1 [case testEnumFastPath] diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index efa6c225ecabd..e08f2fd7007d8 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -2715,7 +2715,8 @@ from typing import Final from mypy_extensions import u8 from librt.internal import ( Buffer, write_bool, read_bool, write_str, read_str, write_float, read_float, - write_int, read_int, write_tag, read_tag, write_bytes, read_bytes + write_int, read_int, write_tag, read_tag, write_bytes, read_bytes, + cache_version, ) Tag = u8 @@ -2724,6 +2725,7 @@ TAG_B: Final[Tag] = 255 TAG_SPECIAL: Final[Tag] = 239 def test_buffer_basic() -> None: + assert cache_version() == 0 b = Buffer(b"foo") assert b.getvalue() == b"foo" @@ -2739,6 +2741,7 @@ def test_buffer_roundtrip() -> None: write_bytes(b, b"a" * 127) write_bytes(b, b"a" * 128) write_float(b, 0.1) + write_float(b, -113.0) write_int(b, 0) write_int(b, 1) write_tag(b, TAG_A) @@ -2763,6 +2766,7 @@ def test_buffer_roundtrip() -> None: assert read_bytes(b) == b"a" * 127 assert read_bytes(b) == b"a" * 128 assert read_float(b) == 0.1 + assert read_float(b) == -113.0 assert read_int(b) == 0 assert read_int(b) == 1 assert read_tag(b) == TAG_A From 99bc45f68839c798273e7799c9487f3f3e2bc2b6 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 30 Oct 2025 23:08:18 +0000 Subject: [PATCH 355/424] Use self-descriptive cache with type tags (#20137) This should make our cache format more future proof w.r.t lazy deserialization and other features: * This makes cache ~5% larger (data + meta) * This makes interpreted performance ~5% worse (just the serialization steps, *not* overall time) * No visible performance impact when compiled (probably because all extra indirections like `read_str()` -> `read_str_bare()` etc are C calls). Note that I now serialize various little arbitrary JSON blobs (like plugin data etc) using fixed format, since with the tags we can do this. --- mypy/cache.py | 290 +++++++++++++++++++++++++++------- mypy/nodes.py | 200 ++++++++++++++--------- mypy/types.py | 245 ++++++++++++++++++---------- mypyc/lib-rt/librt_internal.c | 4 + 4 files changed, 529 insertions(+), 210 deletions(-) diff --git a/mypy/cache.py b/mypy/cache.py index aeb0e8810fd65..0d2db67fac948 100644 --- a/mypy/cache.py +++ b/mypy/cache.py @@ -1,27 +1,70 @@ +""" +This module contains high-level logic for fixed format serialization. + +Lower-level parts are implemented in C in mypyc/lib-rt/librt_internal.c +Short summary of low-level functionality: +* integers are automatically serialized as 1, 2, or 4 bytes, or arbitrary length. +* str/bytes are serialized as size (1, 2, or 4 bytes) followed by bytes buffer. +* floats are serialized as C doubles. + +At high-level we add type tags as needed so that our format is self-descriptive. +More precisely: +* False, True, and None are stored as just a tag: 0, 1, 2 correspondingly. +* builtin primitives like int/str/bytes/float are stored as their type tag followed + by bare (low-level) representation of the value. Reserved tag range for primitives is + 3 ... 19. +* generic (heterogeneous) list are stored as tag, followed by bare size, followed by + sequence of tagged values. +* homogeneous lists of primitives are stored as tag, followed by bare size, followed + by sequence of bare values. +* reserved tag range for sequence-like builtins is 20 ... 29 +* currently we have only one mapping-like format: string-keyed dictionary with heterogeneous + values. It is stored as tag, followed by bare size, followed by sequence of pairs: bare + string key followed by tagged value. +* reserved tag range for mapping-like builtins is 30 ... 39 +* there is an additional reserved tag range 40 ... 49 for any other builtin collections. +* custom classes (like types, symbols etc.) are stored as tag, followed by a sequence of + tagged field values, followed by a special end tag 255. Names of class fields are + *not* stored, the caller should know the field names and order for the given class tag. +* reserved tag range for symbols (TypeInfo, Var, etc) is 50 ... 79. +* class Instance is the only exception from the above format (since it is the most common one). + It has two extra formats: few most common instances like "builtins.object" are stored as + instance tag followed by a secondary tag, other plain non-generic instances are stored as + instance tag followed by secondary tag followed by fullname as bare string. All generic + readers must handle these. +* reserved tag range for Instance type formats is 80 ... 99, for other types it is 100 ... 149. +* tag 254 is reserved for if we would ever need to extend the tag range to indicated second tag + page. Tags 150 ... 253 are free for everything else (e.g. AST nodes etc). + +General convention is that custom classes implement write() and read() methods for FF +serialization. The write method should write both class tag and end tag. The read method +conventionally *does not* read the start tag (to simplify logic for unions). Known exceptions +are MypyFile.read() and SymbolTableNode.read(), since those two never appear in a union. +""" + from __future__ import annotations from collections.abc import Sequence -from typing import Any, Final +from typing import Any, Final, Union +from typing_extensions import TypeAlias as _TypeAlias from librt.internal import ( Buffer as Buffer, read_bool as read_bool, - read_bytes as read_bytes, - read_float as read_float, - read_int as read_int, - read_str as read_str, + read_bytes as read_bytes_bare, + read_float as read_float_bare, + read_int as read_int_bare, + read_str as read_str_bare, read_tag as read_tag, write_bool as write_bool, - write_bytes as write_bytes, - write_float as write_float, - write_int as write_int, - write_str as write_str, + write_bytes as write_bytes_bare, + write_float as write_float_bare, + write_int as write_int_bare, + write_str as write_str_bare, write_tag as write_tag, ) from mypy_extensions import u8 -from mypy.util import json_dumps, json_loads - class CacheMeta: """Class representing cache metadata for a module.""" @@ -125,7 +168,7 @@ def write(self, data: Buffer) -> None: write_str_list(data, self.dependencies) write_int(data, self.data_mtime) write_str_list(data, self.suppressed) - write_bytes(data, json_dumps(self.options)) + write_json(data, self.options) write_int_list(data, self.dep_prios) write_int_list(data, self.dep_lines) write_bytes_list(data, self.dep_hashes) @@ -133,7 +176,9 @@ def write(self, data: Buffer) -> None: write_str_list(data, self.error_lines) write_str(data, self.version_id) write_bool(data, self.ignore_all) - write_bytes(data, json_dumps(self.plugin_data)) + # Plugin data may be not a dictionary, so we use + # a more generic write_json_value() here. + write_json_value(data, self.plugin_data) @classmethod def read(cls, data: Buffer, data_file: str) -> CacheMeta | None: @@ -148,7 +193,7 @@ def read(cls, data: Buffer, data_file: str) -> CacheMeta | None: data_mtime=read_int(data), data_file=data_file, suppressed=read_str_list(data), - options=json_loads(read_bytes(data)), + options=read_json(data), dep_prios=read_int_list(data), dep_lines=read_int_list(data), dep_hashes=read_bytes_list(data), @@ -156,7 +201,7 @@ def read(cls, data: Buffer, data_file: str) -> CacheMeta | None: error_lines=read_str_list(data), version_id=read_str(data), ignore_all=read_bool(data), - plugin_data=json_loads(read_bytes(data)), + plugin_data=read_json_value(data), ) except ValueError: return None @@ -165,114 +210,243 @@ def read(cls, data: Buffer, data_file: str) -> CacheMeta | None: # Always use this type alias to refer to type tags. Tag = u8 -LITERAL_INT: Final[Tag] = 1 -LITERAL_STR: Final[Tag] = 2 -LITERAL_BOOL: Final[Tag] = 3 -LITERAL_FLOAT: Final[Tag] = 4 -LITERAL_COMPLEX: Final[Tag] = 5 -LITERAL_NONE: Final[Tag] = 6 +# Primitives. +LITERAL_FALSE: Final[Tag] = 0 +LITERAL_TRUE: Final[Tag] = 1 +LITERAL_NONE: Final[Tag] = 2 +LITERAL_INT: Final[Tag] = 3 +LITERAL_STR: Final[Tag] = 4 +LITERAL_BYTES: Final[Tag] = 5 +LITERAL_FLOAT: Final[Tag] = 6 +LITERAL_COMPLEX: Final[Tag] = 7 + +# Collections. +LIST_GEN: Final[Tag] = 20 +LIST_INT: Final[Tag] = 21 +LIST_STR: Final[Tag] = 22 +LIST_BYTES: Final[Tag] = 23 +DICT_STR_GEN: Final[Tag] = 30 + +# Misc classes. +EXTRA_ATTRS: Final[Tag] = 150 +DT_SPEC: Final[Tag] = 151 + +END_TAG: Final[Tag] = 255 def read_literal(data: Buffer, tag: Tag) -> int | str | bool | float: if tag == LITERAL_INT: - return read_int(data) + return read_int_bare(data) elif tag == LITERAL_STR: - return read_str(data) - elif tag == LITERAL_BOOL: - return read_bool(data) + return read_str_bare(data) + elif tag == LITERAL_FALSE: + return False + elif tag == LITERAL_TRUE: + return True elif tag == LITERAL_FLOAT: - return read_float(data) + return read_float_bare(data) assert False, f"Unknown literal tag {tag}" +# There is an intentional asymmetry between read and write for literals because +# None and/or complex values are only allowed in some contexts but not in others. def write_literal(data: Buffer, value: int | str | bool | float | complex | None) -> None: if isinstance(value, bool): - write_tag(data, LITERAL_BOOL) write_bool(data, value) elif isinstance(value, int): write_tag(data, LITERAL_INT) - write_int(data, value) + write_int_bare(data, value) elif isinstance(value, str): write_tag(data, LITERAL_STR) - write_str(data, value) + write_str_bare(data, value) elif isinstance(value, float): write_tag(data, LITERAL_FLOAT) - write_float(data, value) + write_float_bare(data, value) elif isinstance(value, complex): write_tag(data, LITERAL_COMPLEX) - write_float(data, value.real) - write_float(data, value.imag) + write_float_bare(data, value.real) + write_float_bare(data, value.imag) else: write_tag(data, LITERAL_NONE) +def read_int(data: Buffer) -> int: + assert read_tag(data) == LITERAL_INT + return read_int_bare(data) + + +def write_int(data: Buffer, value: int) -> None: + write_tag(data, LITERAL_INT) + write_int_bare(data, value) + + +def read_str(data: Buffer) -> str: + assert read_tag(data) == LITERAL_STR + return read_str_bare(data) + + +def write_str(data: Buffer, value: str) -> None: + write_tag(data, LITERAL_STR) + write_str_bare(data, value) + + +def read_bytes(data: Buffer) -> bytes: + assert read_tag(data) == LITERAL_BYTES + return read_bytes_bare(data) + + +def write_bytes(data: Buffer, value: bytes) -> None: + write_tag(data, LITERAL_BYTES) + write_bytes_bare(data, value) + + def read_int_opt(data: Buffer) -> int | None: - if read_bool(data): - return read_int(data) - return None + tag = read_tag(data) + if tag == LITERAL_NONE: + return None + assert tag == LITERAL_INT + return read_int_bare(data) def write_int_opt(data: Buffer, value: int | None) -> None: if value is not None: - write_bool(data, True) - write_int(data, value) + write_tag(data, LITERAL_INT) + write_int_bare(data, value) else: - write_bool(data, False) + write_tag(data, LITERAL_NONE) def read_str_opt(data: Buffer) -> str | None: - if read_bool(data): - return read_str(data) - return None + tag = read_tag(data) + if tag == LITERAL_NONE: + return None + assert tag == LITERAL_STR + return read_str_bare(data) def write_str_opt(data: Buffer, value: str | None) -> None: if value is not None: - write_bool(data, True) - write_str(data, value) + write_tag(data, LITERAL_STR) + write_str_bare(data, value) else: - write_bool(data, False) + write_tag(data, LITERAL_NONE) def read_int_list(data: Buffer) -> list[int]: - size = read_int(data) - return [read_int(data) for _ in range(size)] + assert read_tag(data) == LIST_INT + size = read_int_bare(data) + return [read_int_bare(data) for _ in range(size)] def write_int_list(data: Buffer, value: list[int]) -> None: - write_int(data, len(value)) + write_tag(data, LIST_INT) + write_int_bare(data, len(value)) for item in value: - write_int(data, item) + write_int_bare(data, item) def read_str_list(data: Buffer) -> list[str]: - size = read_int(data) - return [read_str(data) for _ in range(size)] + assert read_tag(data) == LIST_STR + size = read_int_bare(data) + return [read_str_bare(data) for _ in range(size)] def write_str_list(data: Buffer, value: Sequence[str]) -> None: - write_int(data, len(value)) + write_tag(data, LIST_STR) + write_int_bare(data, len(value)) for item in value: - write_str(data, item) + write_str_bare(data, item) def read_bytes_list(data: Buffer) -> list[bytes]: - size = read_int(data) - return [read_bytes(data) for _ in range(size)] + assert read_tag(data) == LIST_BYTES + size = read_int_bare(data) + return [read_bytes_bare(data) for _ in range(size)] def write_bytes_list(data: Buffer, value: Sequence[bytes]) -> None: - write_int(data, len(value)) + write_tag(data, LIST_BYTES) + write_int_bare(data, len(value)) for item in value: - write_bytes(data, item) + write_bytes_bare(data, item) def read_str_opt_list(data: Buffer) -> list[str | None]: - size = read_int(data) + assert read_tag(data) == LIST_GEN + size = read_int_bare(data) return [read_str_opt(data) for _ in range(size)] def write_str_opt_list(data: Buffer, value: list[str | None]) -> None: - write_int(data, len(value)) + write_tag(data, LIST_GEN) + write_int_bare(data, len(value)) for item in value: write_str_opt(data, item) + + +JsonValue: _TypeAlias = Union[None, int, str, bool, list["JsonValue"], dict[str, "JsonValue"]] + + +def read_json_value(data: Buffer) -> JsonValue: + tag = read_tag(data) + if tag == LITERAL_NONE: + return None + if tag == LITERAL_FALSE: + return False + if tag == LITERAL_TRUE: + return True + if tag == LITERAL_INT: + return read_int_bare(data) + if tag == LITERAL_STR: + return read_str_bare(data) + if tag == LIST_GEN: + size = read_int_bare(data) + return [read_json_value(data) for _ in range(size)] + if tag == DICT_STR_GEN: + size = read_int_bare(data) + return {read_str_bare(data): read_json_value(data) for _ in range(size)} + assert False, f"Invalid JSON tag: {tag}" + + +# Currently tuples are used by mypyc plugin. They will be normalized to +# JSON lists after a roundtrip. +def write_json_value(data: Buffer, value: JsonValue | tuple[JsonValue, ...]) -> None: + if value is None: + write_tag(data, LITERAL_NONE) + elif isinstance(value, bool): + write_bool(data, value) + elif isinstance(value, int): + write_tag(data, LITERAL_INT) + write_int_bare(data, value) + elif isinstance(value, str): + write_tag(data, LITERAL_STR) + write_str_bare(data, value) + elif isinstance(value, (list, tuple)): + write_tag(data, LIST_GEN) + write_int_bare(data, len(value)) + for val in value: + write_json_value(data, val) + elif isinstance(value, dict): + write_tag(data, DICT_STR_GEN) + write_int_bare(data, len(value)) + for key in sorted(value): + write_str_bare(data, key) + write_json_value(data, value[key]) + else: + assert False, f"Invalid JSON value: {value}" + + +# These are functions for JSON *dictionaries* specifically. Unfortunately, we +# must use imprecise types here, because the callers use imprecise types. +def read_json(data: Buffer) -> dict[str, Any]: + assert read_tag(data) == DICT_STR_GEN + size = read_int_bare(data) + return {read_str_bare(data): read_json_value(data) for _ in range(size)} + + +def write_json(data: Buffer, value: dict[str, Any]) -> None: + write_tag(data, DICT_STR_GEN) + write_int_bare(data, len(value)) + for key in sorted(value): + write_str_bare(data, key) + write_json_value(data, value[key]) diff --git a/mypy/nodes.py b/mypy/nodes.py index 040f3fc28dce0..6cf984e5a2185 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -2,7 +2,6 @@ from __future__ import annotations -import json import os from abc import abstractmethod from collections import defaultdict @@ -11,19 +10,31 @@ from typing import TYPE_CHECKING, Any, Callable, Final, Optional, TypeVar, Union, cast from typing_extensions import TypeAlias as _TypeAlias, TypeGuard +from librt.internal import ( + read_float as read_float_bare, + read_int as read_int_bare, + read_str as read_str_bare, + write_int as write_int_bare, + write_str as write_str_bare, +) from mypy_extensions import trait import mypy.strconv from mypy.cache import ( + DICT_STR_GEN, + DT_SPEC, + END_TAG, + LIST_GEN, + LIST_STR, LITERAL_COMPLEX, LITERAL_NONE, Buffer, Tag, read_bool, - read_float, read_int, read_int_list, read_int_opt, + read_json, read_literal, read_str, read_str_list, @@ -34,6 +45,7 @@ write_int, write_int_list, write_int_opt, + write_json, write_literal, write_str, write_str_list, @@ -432,6 +444,7 @@ def write(self, data: Buffer) -> None: write_str(data, self.path) write_bool(data, self.is_partial_stub_package) write_str_list(data, sorted(self.future_import_flags)) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> MypyFile: @@ -444,6 +457,7 @@ def read(cls, data: Buffer) -> MypyFile: tree.is_partial_stub_package = read_bool(data) tree.future_import_flags = set(read_str_list(data)) tree.is_cache_skeleton = True + assert read_tag(data) == END_TAG return tree @@ -720,30 +734,33 @@ def deserialize(cls, data: JsonDict) -> OverloadedFuncDef: def write(self, data: Buffer) -> None: write_tag(data, OVERLOADED_FUNC_DEF) - write_int(data, len(self.items)) + write_tag(data, LIST_GEN) + write_int_bare(data, len(self.items)) for item in self.items: item.write(data) mypy.types.write_type_opt(data, self.type) write_str(data, self._fullname) if self.impl is None: - write_bool(data, False) + write_tag(data, LITERAL_NONE) else: - write_bool(data, True) self.impl.write(data) write_flags(data, self, FUNCBASE_FLAGS) write_str_opt(data, self.deprecated) write_int_opt(data, self.setter_index) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> OverloadedFuncDef: - res = OverloadedFuncDef([read_overload_part(data) for _ in range(read_int(data))]) + assert read_tag(data) == LIST_GEN + res = OverloadedFuncDef([read_overload_part(data) for _ in range(read_int_bare(data))]) typ = mypy.types.read_type_opt(data) if typ is not None: assert isinstance(typ, mypy.types.ProperType) res.type = typ res._fullname = read_str(data) - if read_bool(data): - res.impl = read_overload_part(data) + tag = read_tag(data) + if tag != LITERAL_NONE: + res.impl = read_overload_part(data, tag) # set line for empty overload items, as not set in __init__ if len(res.items) > 0: res.set_line(res.impl.line) @@ -751,6 +768,7 @@ def read(cls, data: Buffer) -> OverloadedFuncDef: res.deprecated = read_str_opt(data) res.setter_index = read_int_opt(data) # NOTE: res.info will be set in the fixup phase. + assert read_tag(data) == END_TAG return res def is_dynamic(self) -> bool: @@ -1039,19 +1057,20 @@ def write(self, data: Buffer) -> None: write_int_list(data, [int(ak.value) for ak in self.arg_kinds]) write_int(data, self.abstract_status) if self.dataclass_transform_spec is None: - write_bool(data, False) + write_tag(data, LITERAL_NONE) else: - write_bool(data, True) self.dataclass_transform_spec.write(data) write_str_opt(data, self.deprecated) write_str_opt(data, self.original_first_arg) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> FuncDef: name = read_str(data) typ: mypy.types.FunctionLike | None = None - if read_bool(data): - typ = mypy.types.read_function_like(data) + tag = read_tag(data) + if tag != LITERAL_NONE: + typ = mypy.types.read_function_like(data, tag) ret = FuncDef(name, [], Block([]), typ) ret._fullname = read_str(data) read_flags(data, ret, FUNCDEF_FLAGS) @@ -1059,7 +1078,9 @@ def read(cls, data: Buffer) -> FuncDef: ret.arg_names = read_str_opt_list(data) ret.arg_kinds = [ARG_KINDS[ak] for ak in read_int_list(data)] ret.abstract_status = read_int(data) - if read_bool(data): + tag = read_tag(data) + if tag != LITERAL_NONE: + assert tag == DT_SPEC ret.dataclass_transform_spec = DataclassTransformSpec.read(data) ret.deprecated = read_str_opt(data) ret.original_first_arg = read_str_opt(data) @@ -1067,6 +1088,7 @@ def read(cls, data: Buffer) -> FuncDef: del ret.arguments del ret.max_pos del ret.min_args + assert read_tag(data) == END_TAG return ret @@ -1146,6 +1168,7 @@ def write(self, data: Buffer) -> None: self.func.write(data) self.var.write(data) write_bool(data, self.is_overload) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> Decorator: @@ -1155,6 +1178,7 @@ def read(cls, data: Buffer) -> Decorator: var = Var.read(data) dec = Decorator(func, [], var) dec.is_overload = read_bool(data) + assert read_tag(data) == END_TAG return dec def is_dynamic(self) -> bool: @@ -1341,6 +1365,7 @@ def write(self, data: Buffer) -> None: write_str(data, self._fullname) write_flags(data, self, VAR_FLAGS) write_literal(data, self.final_value) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> Var: @@ -1348,18 +1373,20 @@ def read(cls, data: Buffer) -> Var: typ = mypy.types.read_type_opt(data) v = Var(name, typ) setter_type: mypy.types.CallableType | None = None - if read_bool(data): - assert read_tag(data) == mypy.types.CALLABLE_TYPE + tag = read_tag(data) + if tag != LITERAL_NONE: + assert tag == mypy.types.CALLABLE_TYPE setter_type = mypy.types.CallableType.read(data) v.setter_type = setter_type v.is_ready = False # Override True default set in __init__ v._fullname = read_str(data) read_flags(data, v, VAR_FLAGS) - marker = read_tag(data) - if marker == LITERAL_COMPLEX: - v.final_value = complex(read_float(data), read_float(data)) - elif marker != LITERAL_NONE: - v.final_value = read_literal(data, marker) + tag = read_tag(data) + if tag == LITERAL_COMPLEX: + v.final_value = complex(read_float_bare(data), read_float_bare(data)) + elif tag != LITERAL_NONE: + v.final_value = read_literal(data, tag) + assert read_tag(data) == END_TAG return v @@ -1477,15 +1504,13 @@ def write(self, data: Buffer) -> None: write_str(data, self.name) mypy.types.write_type_list(data, self.type_vars) write_str(data, self.fullname) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> ClassDef: - res = ClassDef( - read_str(data), - Block([]), - [mypy.types.read_type_var_like(data) for _ in range(read_int(data))], - ) + res = ClassDef(read_str(data), Block([]), mypy.types.read_type_var_likes(data)) res.fullname = read_str(data) + assert read_tag(data) == END_TAG return res @@ -2913,10 +2938,11 @@ def write(self, data: Buffer) -> None: self.upper_bound.write(data) self.default.write(data) write_int(data, self.variance) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> TypeVarExpr: - return TypeVarExpr( + ret = TypeVarExpr( read_str(data), read_str(data), mypy.types.read_type_list(data), @@ -2924,6 +2950,8 @@ def read(cls, data: Buffer) -> TypeVarExpr: mypy.types.read_type(data), read_int(data), ) + assert read_tag(data) == END_TAG + return ret class ParamSpecExpr(TypeVarLikeExpr): @@ -2962,16 +2990,19 @@ def write(self, data: Buffer) -> None: self.upper_bound.write(data) self.default.write(data) write_int(data, self.variance) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> ParamSpecExpr: - return ParamSpecExpr( + ret = ParamSpecExpr( read_str(data), read_str(data), mypy.types.read_type(data), mypy.types.read_type(data), read_int(data), ) + assert read_tag(data) == END_TAG + return ret class TypeVarTupleExpr(TypeVarLikeExpr): @@ -3031,12 +3062,13 @@ def write(self, data: Buffer) -> None: self.upper_bound.write(data) self.default.write(data) write_int(data, self.variance) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> TypeVarTupleExpr: assert read_tag(data) == mypy.types.INSTANCE fallback = mypy.types.Instance.read(data) - return TypeVarTupleExpr( + ret = TypeVarTupleExpr( read_str(data), read_str(data), mypy.types.read_type(data), @@ -3044,6 +3076,8 @@ def read(cls, data: Buffer) -> TypeVarTupleExpr: mypy.types.read_type(data), read_int(data), ) + assert read_tag(data) == END_TAG + return ret class TypeAliasExpr(Expression): @@ -3934,20 +3968,19 @@ def write(self, data: Buffer) -> None: mypy.types.write_type_opt(data, self.tuple_type) mypy.types.write_type_opt(data, self.typeddict_type) write_flags(data, self, TypeInfo.FLAGS) - write_str(data, json.dumps(self.metadata)) + write_json(data, self.metadata) if self.slots is None: - write_bool(data, False) + write_tag(data, LITERAL_NONE) else: - write_bool(data, True) write_str_list(data, sorted(self.slots)) write_str_list(data, self.deletable_attributes) mypy.types.write_type_opt(data, self.self_type) if self.dataclass_transform_spec is None: - write_bool(data, False) + write_tag(data, LITERAL_NONE) else: - write_bool(data, True) self.dataclass_transform_spec.write(data) write_str_opt(data, self.deprecated) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> TypeInfo: @@ -3963,7 +3996,8 @@ def read(cls, data: Buffer) -> TypeInfo: ti.type_vars = read_str_list(data) ti.has_param_spec_type = read_bool(data) ti.bases = [] - for _ in range(read_int(data)): + assert read_tag(data) == LIST_GEN + for _ in range(read_int_bare(data)): assert read_tag(data) == mypy.types.INSTANCE ti.bases.append(mypy.types.Instance.read(data)) # NOTE: ti.mro will be set in the fixup phase based on these @@ -3978,34 +4012,37 @@ def read(cls, data: Buffer) -> TypeInfo: # rechecked, it can tell that the mro has changed. ti._mro_refs = read_str_list(data) ti._promote = cast(list[mypy.types.ProperType], mypy.types.read_type_list(data)) - if read_bool(data): - assert read_tag(data) == mypy.types.INSTANCE + if (tag := read_tag(data)) != LITERAL_NONE: + assert tag == mypy.types.INSTANCE ti.alt_promote = mypy.types.Instance.read(data) - if read_bool(data): - assert read_tag(data) == mypy.types.INSTANCE + if (tag := read_tag(data)) != LITERAL_NONE: + assert tag == mypy.types.INSTANCE ti.declared_metaclass = mypy.types.Instance.read(data) - if read_bool(data): - assert read_tag(data) == mypy.types.INSTANCE + if (tag := read_tag(data)) != LITERAL_NONE: + assert tag == mypy.types.INSTANCE ti.metaclass_type = mypy.types.Instance.read(data) - if read_bool(data): - assert read_tag(data) == mypy.types.TUPLE_TYPE + if (tag := read_tag(data)) != LITERAL_NONE: + assert tag == mypy.types.TUPLE_TYPE ti.tuple_type = mypy.types.TupleType.read(data) - if read_bool(data): - assert read_tag(data) == mypy.types.TYPED_DICT_TYPE + if (tag := read_tag(data)) != LITERAL_NONE: + assert tag == mypy.types.TYPED_DICT_TYPE ti.typeddict_type = mypy.types.TypedDictType.read(data) read_flags(data, ti, TypeInfo.FLAGS) - metadata = read_str(data) - if metadata != "{}": - ti.metadata = json.loads(metadata) - if read_bool(data): - ti.slots = set(read_str_list(data)) + ti.metadata = read_json(data) + tag = read_tag(data) + if tag != LITERAL_NONE: + assert tag == LIST_STR + ti.slots = {read_str_bare(data) for _ in range(read_int_bare(data))} ti.deletable_attributes = read_str_list(data) - if read_bool(data): - assert read_tag(data) == mypy.types.TYPE_VAR_TYPE + if (tag := read_tag(data)) != LITERAL_NONE: + assert tag == mypy.types.TYPE_VAR_TYPE ti.self_type = mypy.types.TypeVarType.read(data) - if read_bool(data): + tag = read_tag(data) + if tag != LITERAL_NONE: + assert tag == DT_SPEC ti.dataclass_transform_spec = DataclassTransformSpec.read(data) ti.deprecated = read_str_opt(data) + assert read_tag(data) == END_TAG return ti @@ -4290,14 +4327,15 @@ def write(self, data: Buffer) -> None: write_bool(data, self.no_args) write_bool(data, self.normalized) write_bool(data, self.python_3_12_type_alias) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> TypeAlias: fullname = read_str(data) module = read_str(data) target = mypy.types.read_type(data) - alias_tvars = [mypy.types.read_type_var_like(data) for _ in range(read_int(data))] - return TypeAlias( + alias_tvars = mypy.types.read_type_var_likes(data) + ret = TypeAlias( target, fullname, module, @@ -4308,6 +4346,8 @@ def read(cls, data: Buffer) -> TypeAlias: normalized=read_bool(data), python_3_12_type_alias=read_bool(data), ) + assert read_tag(data) == END_TAG + return ret class PlaceholderNode(SymbolNode): @@ -4568,6 +4608,7 @@ def deserialize(cls, data: JsonDict) -> SymbolTableNode: return stnode def write(self, data: Buffer, prefix: str, name: str) -> None: + write_tag(data, SYMBOL_TABLE_NODE) write_int(data, self.kind) write_bool(data, self.module_hidden) write_bool(data, self.module_public) @@ -4595,9 +4636,11 @@ def write(self, data: Buffer, prefix: str, name: str) -> None: if cross_ref is None: assert self.node is not None self.node.write(data) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> SymbolTableNode: + assert read_tag(data) == SYMBOL_TABLE_NODE sym = SymbolTableNode(read_int(data), None) sym.module_hidden = read_bool(data) sym.module_public = read_bool(data) @@ -4608,6 +4651,7 @@ def read(cls, data: Buffer) -> SymbolTableNode: sym.node = read_symbol(data) else: sym.cross_ref = cross_ref + assert read_tag(data) == END_TAG return sym @@ -4671,18 +4715,23 @@ def write(self, data: Buffer, fullname: str) -> None: if key == "__builtins__" or value.no_serialize: continue size += 1 - write_int(data, size) + # We intentionally tag SymbolTable as a simple dictionary str -> SymbolTableNode. + write_tag(data, DICT_STR_GEN) + write_int_bare(data, size) for key in sorted(self): value = self[key] if key == "__builtins__" or value.no_serialize: continue - write_str(data, key) + write_str_bare(data, key) value.write(data, fullname, key) @classmethod def read(cls, data: Buffer) -> SymbolTable: - size = read_int(data) - return SymbolTable([(read_str(data), SymbolTableNode.read(data)) for _ in range(size)]) + assert read_tag(data) == DICT_STR_GEN + size = read_int_bare(data) + return SymbolTable( + [(read_str_bare(data), SymbolTableNode.read(data)) for _ in range(size)] + ) class DataclassTransformSpec: @@ -4735,21 +4784,25 @@ def deserialize(cls, data: JsonDict) -> DataclassTransformSpec: ) def write(self, data: Buffer) -> None: + write_tag(data, DT_SPEC) write_bool(data, self.eq_default) write_bool(data, self.order_default) write_bool(data, self.kw_only_default) write_bool(data, self.frozen_default) write_str_list(data, self.field_specifiers) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> DataclassTransformSpec: - return DataclassTransformSpec( + ret = DataclassTransformSpec( eq_default=read_bool(data), order_default=read_bool(data), kw_only_default=read_bool(data), frozen_default=read_bool(data), field_specifiers=tuple(read_str_list(data)), ) + assert read_tag(data) == END_TAG + return ret def get_flags(node: Node, names: list[str]) -> list[str]: @@ -4889,17 +4942,19 @@ def local_definitions( yield from local_definitions(node.names, fullname, node) -MYPY_FILE: Final[Tag] = 0 -OVERLOADED_FUNC_DEF: Final[Tag] = 1 -FUNC_DEF: Final[Tag] = 2 -DECORATOR: Final[Tag] = 3 -VAR: Final[Tag] = 4 -TYPE_VAR_EXPR: Final[Tag] = 5 -PARAM_SPEC_EXPR: Final[Tag] = 6 -TYPE_VAR_TUPLE_EXPR: Final[Tag] = 7 -TYPE_INFO: Final[Tag] = 8 -TYPE_ALIAS: Final[Tag] = 9 -CLASS_DEF: Final[Tag] = 10 +# See docstring for mypy/cache.py for reserved tag ranges. +MYPY_FILE: Final[Tag] = 50 +OVERLOADED_FUNC_DEF: Final[Tag] = 51 +FUNC_DEF: Final[Tag] = 52 +DECORATOR: Final[Tag] = 53 +VAR: Final[Tag] = 54 +TYPE_VAR_EXPR: Final[Tag] = 55 +PARAM_SPEC_EXPR: Final[Tag] = 56 +TYPE_VAR_TUPLE_EXPR: Final[Tag] = 57 +TYPE_INFO: Final[Tag] = 58 +TYPE_ALIAS: Final[Tag] = 59 +CLASS_DEF: Final[Tag] = 60 +SYMBOL_TABLE_NODE: Final[Tag] = 61 def read_symbol(data: Buffer) -> mypy.nodes.SymbolNode: @@ -4926,8 +4981,9 @@ def read_symbol(data: Buffer) -> mypy.nodes.SymbolNode: assert False, f"Unknown symbol tag {tag}" -def read_overload_part(data: Buffer) -> OverloadPart: - tag = read_tag(data) +def read_overload_part(data: Buffer, tag: Tag | None = None) -> OverloadPart: + if tag is None: + tag = read_tag(data) if tag == DECORATOR: return Decorator.read(data) if tag == FUNC_DEF: diff --git a/mypy/types.py b/mypy/types.py index 426d560c2bf7e..6013f4c252988 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -8,9 +8,21 @@ from typing import TYPE_CHECKING, Any, ClassVar, Final, NewType, TypeVar, Union, cast, overload from typing_extensions import Self, TypeAlias as _TypeAlias, TypeGuard +from librt.internal import ( + read_int as read_int_bare, + read_str as read_str_bare, + write_int as write_int_bare, + write_str as write_str_bare, +) + import mypy.nodes from mypy.bogus_type import Bogus from mypy.cache import ( + DICT_STR_GEN, + END_TAG, + EXTRA_ATTRS, + LIST_GEN, + LITERAL_NONE, Buffer, Tag, read_bool, @@ -440,11 +452,13 @@ def write(self, data: Buffer) -> None: write_type_list(data, self.args) assert self.alias is not None write_str(data, self.alias.fullname) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> TypeAliasType: alias = TypeAliasType(None, read_type_list(data)) alias.type_ref = read_str(data) + assert read_tag(data) == END_TAG return alias @@ -724,10 +738,11 @@ def write(self, data: Buffer) -> None: self.upper_bound.write(data) self.default.write(data) write_int(data, self.variance) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> TypeVarType: - return TypeVarType( + ret = TypeVarType( read_str(data), read_str(data), TypeVarId(read_int(data), namespace=read_str(data)), @@ -736,6 +751,8 @@ def read(cls, data: Buffer) -> TypeVarType: read_type(data), read_int(data), ) + assert read_tag(data) == END_TAG + return ret class ParamSpecFlavor: @@ -876,12 +893,13 @@ def write(self, data: Buffer) -> None: write_int(data, self.flavor) self.upper_bound.write(data) self.default.write(data) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> ParamSpecType: assert read_tag(data) == PARAMETERS prefix = Parameters.read(data) - return ParamSpecType( + ret = ParamSpecType( read_str(data), read_str(data), TypeVarId(read_int(data), namespace=read_str(data)), @@ -890,6 +908,8 @@ def read(cls, data: Buffer) -> ParamSpecType: read_type(data), prefix=prefix, ) + assert read_tag(data) == END_TAG + return ret class TypeVarTupleType(TypeVarLikeType): @@ -956,12 +976,13 @@ def write(self, data: Buffer) -> None: self.upper_bound.write(data) self.default.write(data) write_int(data, self.min_len) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> TypeVarTupleType: assert read_tag(data) == INSTANCE fallback = Instance.read(data) - return TypeVarTupleType( + ret = TypeVarTupleType( read_str(data), read_str(data), TypeVarId(read_int(data), namespace=read_str(data)), @@ -970,6 +991,8 @@ def read(cls, data: Buffer) -> TypeVarTupleType: read_type(data), min_len=read_int(data), ) + assert read_tag(data) == END_TAG + return ret def accept(self, visitor: TypeVisitor[T]) -> T: return visitor.visit_type_var_tuple(self) @@ -1108,15 +1131,18 @@ def write(self, data: Buffer) -> None: write_type_list(data, self.args) write_str_opt(data, self.original_str_expr) write_str_opt(data, self.original_str_fallback) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> UnboundType: - return UnboundType( + ret = UnboundType( read_str(data), read_type_list(data), original_str_expr=read_str_opt(data), original_str_fallback=read_str_opt(data), ) + assert read_tag(data) == END_TAG + return ret class CallableArgument(ProperType): @@ -1215,10 +1241,13 @@ def serialize(self) -> JsonDict: def write(self, data: Buffer) -> None: write_tag(data, UNPACK_TYPE) self.type.write(data) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> UnpackType: - return UnpackType(read_type(data)) + ret = UnpackType(read_type(data)) + assert read_tag(data) == END_TAG + return ret @classmethod def deserialize(cls, data: JsonDict) -> UnpackType: @@ -1326,15 +1355,19 @@ def write(self, data: Buffer) -> None: write_type_opt(data, self.source_any) write_int(data, self.type_of_any) write_str_opt(data, self.missing_import_name) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> AnyType: - if read_bool(data): - assert read_tag(data) == ANY_TYPE + tag = read_tag(data) + if tag != LITERAL_NONE: + assert tag == ANY_TYPE source_any = AnyType.read(data) else: source_any = None - return AnyType(read_int(data), source_any, read_str_opt(data)) + ret = AnyType(read_int(data), source_any, read_str_opt(data)) + assert read_tag(data) == END_TAG + return ret class UninhabitedType(ProperType): @@ -1384,9 +1417,11 @@ def deserialize(cls, data: JsonDict) -> UninhabitedType: def write(self, data: Buffer) -> None: write_tag(data, UNINHABITED_TYPE) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> UninhabitedType: + assert read_tag(data) == END_TAG return UninhabitedType() @@ -1423,9 +1458,11 @@ def deserialize(cls, data: JsonDict) -> NoneType: def write(self, data: Buffer) -> None: write_tag(data, NONE_TYPE) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> NoneType: + assert read_tag(data) == END_TAG return NoneType() def is_singleton_type(self) -> bool: @@ -1478,10 +1515,13 @@ def deserialize(cls, data: JsonDict) -> DeletedType: def write(self, data: Buffer) -> None: write_tag(data, DELETED_TYPE) write_str_opt(data, self.source) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> DeletedType: - return DeletedType(read_str_opt(data)) + ret = DeletedType(read_str_opt(data)) + assert read_tag(data) == END_TAG + return ret # Fake TypeInfo to be used as a placeholder during Instance de-serialization. @@ -1539,13 +1579,17 @@ def deserialize(cls, data: JsonDict) -> ExtraAttrs: ) def write(self, data: Buffer) -> None: + write_tag(data, EXTRA_ATTRS) write_type_map(data, self.attrs) write_str_list(data, sorted(self.immutable)) write_str_opt(data, self.mod_name) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> ExtraAttrs: - return ExtraAttrs(read_type_map(data), set(read_str_list(data)), read_str_opt(data)) + ret = ExtraAttrs(read_type_map(data), set(read_str_list(data)), read_str_opt(data)) + assert read_tag(data) == END_TAG + return ret class Instance(ProperType): @@ -1699,17 +1743,17 @@ def write(self, data: Buffer) -> None: write_tag(data, INSTANCE_OBJECT) else: write_tag(data, INSTANCE_SIMPLE) - write_str(data, type_ref) + write_str_bare(data, type_ref) return write_tag(data, INSTANCE_GENERIC) write_str(data, self.type.fullname) write_type_list(data, self.args) write_type_opt(data, self.last_known_value) if self.extra_attrs is None: - write_bool(data, False) + write_tag(data, LITERAL_NONE) else: - write_bool(data, True) self.extra_attrs.write(data) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> Instance: @@ -1743,17 +1787,21 @@ def read(cls, data: Buffer) -> Instance: return instance_cache.object_type if tag == INSTANCE_SIMPLE: inst = Instance(NOT_READY, []) - inst.type_ref = read_str(data) + inst.type_ref = read_str_bare(data) return inst assert tag == INSTANCE_GENERIC type_ref = read_str(data) inst = Instance(NOT_READY, read_type_list(data)) inst.type_ref = type_ref - if read_bool(data): - assert read_tag(data) == LITERAL_TYPE + tag = read_tag(data) + if tag != LITERAL_NONE: + assert tag == LITERAL_TYPE inst.last_known_value = LiteralType.read(data) - if read_bool(data): + tag = read_tag(data) + if tag != LITERAL_NONE: + assert tag == EXTRA_ATTRS inst.extra_attrs = ExtraAttrs.read(data) + assert read_tag(data) == END_TAG return inst def copy_modified( @@ -2057,18 +2105,21 @@ def write(self, data: Buffer) -> None: write_str_opt_list(data, self.arg_names) write_type_list(data, self.variables) write_bool(data, self.imprecise_arg_kinds) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> Parameters: - return Parameters( + ret = Parameters( read_type_list(data), # This is a micro-optimization until mypyc gets dedicated enum support. Otherwise, # we would spend ~20% of types deserialization time in Enum.__call__(). [ARG_KINDS[ak] for ak in read_int_list(data)], read_str_opt_list(data), - variables=[read_type_var_like(data) for _ in range(read_int(data))], + variables=read_type_var_likes(data), imprecise_arg_kinds=read_bool(data), ) + assert read_tag(data) == END_TAG + return ret def __hash__(self) -> int: return hash( @@ -2593,19 +2644,20 @@ def write(self, data: Buffer) -> None: write_bool(data, self.from_concatenate) write_bool(data, self.imprecise_arg_kinds) write_bool(data, self.unpack_kwargs) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> CallableType: assert read_tag(data) == INSTANCE fallback = Instance.read(data) - return CallableType( + ret = CallableType( read_type_list(data), [ARG_KINDS[ak] for ak in read_int_list(data)], read_str_opt_list(data), read_type(data), fallback, name=read_str_opt(data), - variables=[read_type_var_like(data) for _ in range(read_int(data))], + variables=read_type_var_likes(data), is_ellipsis_args=read_bool(data), implicit=read_bool(data), is_bound=read_bool(data), @@ -2615,6 +2667,8 @@ def read(cls, data: Buffer) -> CallableType: imprecise_arg_kinds=read_bool(data), unpack_kwargs=read_bool(data), ) + assert read_tag(data) == END_TAG + return ret # This is a little safety net to prevent reckless special-casing of callables @@ -2694,13 +2748,16 @@ def deserialize(cls, data: JsonDict) -> Overloaded: def write(self, data: Buffer) -> None: write_tag(data, OVERLOADED) write_type_list(data, self.items) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> Overloaded: items = [] - for _ in range(read_int(data)): + assert read_tag(data) == LIST_GEN + for _ in range(read_int_bare(data)): assert read_tag(data) == CALLABLE_TYPE items.append(CallableType.read(data)) + assert read_tag(data) == END_TAG return Overloaded(items) @@ -2804,12 +2861,15 @@ def write(self, data: Buffer) -> None: self.partial_fallback.write(data) write_type_list(data, self.items) write_bool(data, self.implicit) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> TupleType: assert read_tag(data) == INSTANCE fallback = Instance.read(data) - return TupleType(read_type_list(data), fallback, implicit=read_bool(data)) + ret = TupleType(read_type_list(data), fallback, implicit=read_bool(data)) + assert read_tag(data) == END_TAG + return ret def copy_modified( self, *, fallback: Instance | None = None, items: list[Type] | None = None @@ -2987,14 +3047,17 @@ def write(self, data: Buffer) -> None: write_type_map(data, self.items) write_str_list(data, sorted(self.required_keys)) write_str_list(data, sorted(self.readonly_keys)) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> TypedDictType: assert read_tag(data) == INSTANCE fallback = Instance.read(data) - return TypedDictType( + ret = TypedDictType( read_type_map(data), set(read_str_list(data)), set(read_str_list(data)), fallback ) + assert read_tag(data) == END_TAG + return ret @property def is_final(self) -> bool: @@ -3248,13 +3311,16 @@ def write(self, data: Buffer) -> None: write_tag(data, LITERAL_TYPE) self.fallback.write(data) write_literal(data, self.value) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> LiteralType: assert read_tag(data) == INSTANCE fallback = Instance.read(data) tag = read_tag(data) - return LiteralType(read_literal(data, tag), fallback) + ret = LiteralType(read_literal(data, tag), fallback) + assert read_tag(data) == END_TAG + return ret def is_singleton_type(self) -> bool: return self.is_enum_literal() or isinstance(self.value, bool) @@ -3361,10 +3427,13 @@ def write(self, data: Buffer) -> None: write_tag(data, UNION_TYPE) write_type_list(data, self.items) write_bool(data, self.uses_pep604_syntax) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> UnionType: - return UnionType(read_type_list(data), uses_pep604_syntax=read_bool(data)) + ret = UnionType(read_type_list(data), uses_pep604_syntax=read_bool(data)) + assert read_tag(data) == END_TAG + return ret class PartialType(ProperType): @@ -3505,10 +3574,13 @@ def deserialize(cls, data: JsonDict) -> Type: def write(self, data: Buffer) -> None: write_tag(data, TYPE_TYPE) self.item.write(data) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> Type: - return TypeType.make_normalized(read_type(data)) + ret = TypeType.make_normalized(read_type(data)) + assert read_tag(data) == END_TAG + return ret class PlaceholderType(ProperType): @@ -4172,37 +4244,41 @@ def type_vars_as_args(type_vars: Sequence[TypeVarLikeType]) -> tuple[Type, ...]: return tuple(args) -TYPE_ALIAS_TYPE: Final[Tag] = 1 -TYPE_VAR_TYPE: Final[Tag] = 2 -PARAM_SPEC_TYPE: Final[Tag] = 3 -TYPE_VAR_TUPLE_TYPE: Final[Tag] = 4 -UNBOUND_TYPE: Final[Tag] = 5 -UNPACK_TYPE: Final[Tag] = 6 -ANY_TYPE: Final[Tag] = 7 -UNINHABITED_TYPE: Final[Tag] = 8 -NONE_TYPE: Final[Tag] = 9 -DELETED_TYPE: Final[Tag] = 10 -INSTANCE: Final[Tag] = 11 -CALLABLE_TYPE: Final[Tag] = 12 -OVERLOADED: Final[Tag] = 13 -TUPLE_TYPE: Final[Tag] = 14 -TYPED_DICT_TYPE: Final[Tag] = 15 -LITERAL_TYPE: Final[Tag] = 16 -UNION_TYPE: Final[Tag] = 17 -TYPE_TYPE: Final[Tag] = 18 -PARAMETERS: Final[Tag] = 19 - -INSTANCE_STR: Final[Tag] = 101 -INSTANCE_FUNCTION: Final[Tag] = 102 -INSTANCE_INT: Final[Tag] = 103 -INSTANCE_BOOL: Final[Tag] = 104 -INSTANCE_OBJECT: Final[Tag] = 105 -INSTANCE_SIMPLE: Final[Tag] = 106 -INSTANCE_GENERIC: Final[Tag] = 107 - - -def read_type(data: Buffer) -> Type: - tag = read_tag(data) +# See docstring for mypy/cache.py for reserved tag ranges. +# Instance-related tags. +INSTANCE: Final[Tag] = 80 +INSTANCE_SIMPLE: Final[Tag] = 81 +INSTANCE_GENERIC: Final[Tag] = 82 +INSTANCE_STR: Final[Tag] = 83 +INSTANCE_FUNCTION: Final[Tag] = 84 +INSTANCE_INT: Final[Tag] = 85 +INSTANCE_BOOL: Final[Tag] = 86 +INSTANCE_OBJECT: Final[Tag] = 87 + +# Other type tags. +TYPE_ALIAS_TYPE: Final[Tag] = 100 +TYPE_VAR_TYPE: Final[Tag] = 101 +PARAM_SPEC_TYPE: Final[Tag] = 102 +TYPE_VAR_TUPLE_TYPE: Final[Tag] = 103 +UNBOUND_TYPE: Final[Tag] = 104 +UNPACK_TYPE: Final[Tag] = 105 +ANY_TYPE: Final[Tag] = 106 +UNINHABITED_TYPE: Final[Tag] = 107 +NONE_TYPE: Final[Tag] = 108 +DELETED_TYPE: Final[Tag] = 109 +CALLABLE_TYPE: Final[Tag] = 110 +OVERLOADED: Final[Tag] = 111 +TUPLE_TYPE: Final[Tag] = 112 +TYPED_DICT_TYPE: Final[Tag] = 113 +LITERAL_TYPE: Final[Tag] = 114 +UNION_TYPE: Final[Tag] = 115 +TYPE_TYPE: Final[Tag] = 116 +PARAMETERS: Final[Tag] = 117 + + +def read_type(data: Buffer, tag: Tag | None = None) -> Type: + if tag is None: + tag = read_tag(data) # The branches here are ordered manually by type "popularity". if tag == INSTANCE: return Instance.read(data) @@ -4245,8 +4321,7 @@ def read_type(data: Buffer) -> Type: assert False, f"Unknown type tag {tag}" -def read_function_like(data: Buffer) -> FunctionLike: - tag = read_tag(data) +def read_function_like(data: Buffer, tag: Tag) -> FunctionLike: if tag == CALLABLE_TYPE: return CallableType.read(data) if tag == OVERLOADED: @@ -4254,51 +4329,61 @@ def read_function_like(data: Buffer) -> FunctionLike: assert False, f"Invalid type tag for FunctionLike {tag}" -def read_type_var_like(data: Buffer) -> TypeVarLikeType: - tag = read_tag(data) - if tag == TYPE_VAR_TYPE: - return TypeVarType.read(data) - if tag == PARAM_SPEC_TYPE: - return ParamSpecType.read(data) - if tag == TYPE_VAR_TUPLE_TYPE: - return TypeVarTupleType.read(data) - assert False, f"Invalid type tag for TypeVarLikeType {tag}" +def read_type_var_likes(data: Buffer) -> list[TypeVarLikeType]: + """Specialized version of read_type_list() for lists of type variables.""" + assert read_tag(data) == LIST_GEN + ret: list[TypeVarLikeType] = [] + for _ in range(read_int_bare(data)): + tag = read_tag(data) + if tag == TYPE_VAR_TYPE: + ret.append(TypeVarType.read(data)) + elif tag == PARAM_SPEC_TYPE: + ret.append(ParamSpecType.read(data)) + elif tag == TYPE_VAR_TUPLE_TYPE: + ret.append(TypeVarTupleType.read(data)) + else: + assert False, f"Invalid type tag for TypeVarLikeType {tag}" + return ret def read_type_opt(data: Buffer) -> Type | None: - if read_bool(data): - return read_type(data) - return None + tag = read_tag(data) + if tag == LITERAL_NONE: + return None + return read_type(data, tag) def write_type_opt(data: Buffer, value: Type | None) -> None: if value is not None: - write_bool(data, True) value.write(data) else: - write_bool(data, False) + write_tag(data, LITERAL_NONE) def read_type_list(data: Buffer) -> list[Type]: - size = read_int(data) + assert read_tag(data) == LIST_GEN + size = read_int_bare(data) return [read_type(data) for _ in range(size)] def write_type_list(data: Buffer, value: Sequence[Type]) -> None: - write_int(data, len(value)) + write_tag(data, LIST_GEN) + write_int_bare(data, len(value)) for item in value: item.write(data) def read_type_map(data: Buffer) -> dict[str, Type]: - size = read_int(data) - return {read_str(data): read_type(data) for _ in range(size)} + assert read_tag(data) == DICT_STR_GEN + size = read_int_bare(data) + return {read_str_bare(data): read_type(data) for _ in range(size)} def write_type_map(data: Buffer, value: dict[str, Type]) -> None: - write_int(data, len(value)) + write_tag(data, DICT_STR_GEN) + write_int_bare(data, len(value)) for key in sorted(value): - write_str(data, key) + write_str_bare(data, key) value[key].write(data) diff --git a/mypyc/lib-rt/librt_internal.c b/mypyc/lib-rt/librt_internal.c index b37136be0784a..8599017c31a8f 100644 --- a/mypyc/lib-rt/librt_internal.c +++ b/mypyc/lib-rt/librt_internal.c @@ -206,6 +206,10 @@ read_bool_internal(PyObject *data) { _CHECK_BUFFER(data, CPY_BOOL_ERROR) _CHECK_READ(data, 1, CPY_BOOL_ERROR) char res = _READ(data, char) + if (unlikely((res != 0) & (res != 1))) { + PyErr_SetString(PyExc_ValueError, "invalid bool value"); + return CPY_BOOL_ERROR; + } return res; } From 3f03755c05f6e8506fb5a0b9b83d1a5f279b377f Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Fri, 31 Oct 2025 03:28:00 +0100 Subject: [PATCH 356/424] Do not store deferred NamedTuple fields as redefinitions (#20147) Fixes #17059. When we encounter a field with a Placeholder as a default, do not record it in the new symtable as redefinition - it becomes orphan immediately and remains there, crashing suring serialization. The test cases are both crashing on current master. Ideally we hould emit name-defined in all cases, but it is another unrelated issue (#17610, and likely some others; this is not specific to named tuples - plain classes also allow referencing variables that are not defined yet). --- mypy/semanal_namedtuple.py | 13 +++++++++---- test-data/unit/check-namedtuple.test | 25 +++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/mypy/semanal_namedtuple.py b/mypy/semanal_namedtuple.py index 37a650f1b6644..f27c89e34fdf8 100644 --- a/mypy/semanal_namedtuple.py +++ b/mypy/semanal_namedtuple.py @@ -34,6 +34,7 @@ NamedTupleExpr, NameExpr, PassStmt, + PlaceholderNode, RefExpr, Statement, StrExpr, @@ -697,10 +698,14 @@ def save_namedtuple_body(self, named_tuple_info: TypeInfo) -> Iterator[None]: if isinstance(sym.node, (FuncBase, Decorator)) and not sym.plugin_generated: # Keep user-defined methods as is. continue - # Keep existing (user-provided) definitions under mangled names, so they - # get semantically analyzed. - r_key = get_unique_redefinition_name(key, named_tuple_info.names) - named_tuple_info.names[r_key] = sym + # Do not retain placeholders - we'll get back here if they cease to + # be placeholders later. If we keep placeholders alive, they may never + # be reached again, making it to cacheable symtable. + if not isinstance(sym.node, PlaceholderNode): + # Keep existing (user-provided) definitions under mangled names, so they + # get semantically analyzed. + r_key = get_unique_redefinition_name(key, named_tuple_info.names) + named_tuple_info.names[r_key] = sym named_tuple_info.names[key] = value # Helpers diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 45de2a9e50aef..66eb555421f44 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -1530,3 +1530,28 @@ class Base: names = [name for name in namespace if fail] # E: Name "fail" is not defined self.n = namedtuple("n", names) # E: NamedTuple type as an attribute is not supported [builtins fixtures/tuple.pyi] + +[case testNamedTupleDefaultValueDefer] +# flags: --debug-serialize +from typing import NamedTuple + +class NT(NamedTuple): + foo: int = UNDEFINED # E: Name "UNDEFINED" is not defined +[builtins fixtures/tuple.pyi] + +[case testNamedTupleDefaultValueDefer2] +# flags: --debug-serialize +from typing import NamedTuple + +class NT(NamedTuple): + foo: int = DEFERRED_INT + +class NT2(NamedTuple): + foo: int = DEFERRED_STR # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +from foo import DEFERRED_INT, DEFERRED_STR + +[file foo.py] +DEFERRED_INT = 1 +DEFERRED_STR = "a" +[builtins fixtures/tuple.pyi] From f64bf112da27265e71419dba264de823c2d5f373 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 31 Oct 2025 10:03:04 +0000 Subject: [PATCH 357/424] Optimize serialization format for 2 and 4 bytes ints (#20120) This is important for serialized ASTs (think line numbers for every node). Also in this PR: * Re-use same integer logic for str/bytes length (it is slightly less optimal, but code re-use is good). * Remove unused field from `Buffer` type. * Make format the same on 32-bit and 64-bit platforms (we still assume little-endian platform). --- mypyc/lib-rt/librt_internal.c | 230 ++++++++++++++++++++----------- mypyc/test-data/run-classes.test | 96 ++++++++++--- 2 files changed, 220 insertions(+), 106 deletions(-) diff --git a/mypyc/lib-rt/librt_internal.c b/mypyc/lib-rt/librt_internal.c index 8599017c31a8f..4f6e138c96f9c 100644 --- a/mypyc/lib-rt/librt_internal.c +++ b/mypyc/lib-rt/librt_internal.c @@ -8,15 +8,23 @@ #include "librt_internal.h" #define START_SIZE 512 -#define MAX_SHORT_INT_TAGGED (255 << 1) -#define MAX_SHORT_LEN 127 -#define LONG_STR_TAG 1 +// See comment in read_int_internal() on motivation for these values. +#define MIN_ONE_BYTE_INT -10 +#define MAX_ONE_BYTE_INT 117 // 2 ** 7 - 1 - 10 +#define MIN_TWO_BYTES_INT -100 +#define MAX_TWO_BYTES_INT 16283 // 2 ** (8 + 6) - 1 - 100 +#define MIN_FOUR_BYTES_INT -10000 +#define MAX_FOUR_BYTES_INT 536860911 // 2 ** (3 * 8 + 5) - 1 - 10000 -#define MIN_SHORT_INT -10 -#define MAX_SHORT_INT 117 -#define MEDIUM_INT_TAG 1 -#define LONG_INT_TAG 3 +#define TWO_BYTES_INT_BIT 1 +#define FOUR_BYTES_INT_BIT 2 +#define LONG_INT_BIT 4 + +#define FOUR_BYTES_INT_TRAILER 3 +// We add one reserved bit here so that we can potentially support +// 8 bytes format in the future. +#define LONG_INT_TRAILER 15 #define CPY_BOOL_ERROR 2 #define CPY_NONE_ERROR 2 @@ -35,13 +43,22 @@ #define _WRITE(data, type, v) *(type *)(((BufferObject *)data)->buf + ((BufferObject *)data)->pos) = v; \ ((BufferObject *)data)->pos += sizeof(type); +#if PY_BIG_ENDIAN +uint16_t reverse_16(uint16_t number) { + return (number << 8) | (number >> 8); +} + +uint32_t reverse_32(uint32_t number) { + return ((number & 0xFF) << 24) | ((number & 0xFF00) << 8) | ((number & 0xFF0000) >> 8) | (number >> 24); +} +#endif + typedef struct { PyObject_HEAD Py_ssize_t pos; Py_ssize_t end; Py_ssize_t size; char *buf; - PyObject *source; } BufferObject; static PyTypeObject BufferType; @@ -259,26 +276,50 @@ write_bool(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwname } /* -str format: size followed by UTF-8 bytes - short strings (len <= 127): single byte for size as `(uint8_t)size << 1` - long strings: \x01 followed by size as Py_ssize_t +str format: size as int (see below) followed by UTF-8 bytes */ +static inline CPyTagged +_read_short_int(PyObject *data, uint8_t first) { + uint8_t second; + uint16_t two_more; + if ((first & TWO_BYTES_INT_BIT) == 0) { + // Note we use tagged ints since this function can return an error. + return ((Py_ssize_t)(first >> 1) + MIN_ONE_BYTE_INT) << 1; + } + if ((first & FOUR_BYTES_INT_BIT) == 0) { + _CHECK_READ(data, 1, CPY_INT_TAG) + second = _READ(data, uint8_t) + return ((((Py_ssize_t)second) << 6) + (Py_ssize_t)(first >> 2) + MIN_TWO_BYTES_INT) << 1; + } + // The caller is responsible to verify this is called only for short ints. + _CHECK_READ(data, 3, CPY_INT_TAG) + // TODO: check if compilers emit optimal code for these two reads, and tweak if needed. + second = _READ(data, uint8_t) + two_more = _READ(data, uint16_t) +#if PY_BIG_ENDIAN + two_more = reverse_16(two_more); +#endif + Py_ssize_t higher = (((Py_ssize_t)two_more) << 13) + (((Py_ssize_t)second) << 5); + return (higher + (Py_ssize_t)(first >> 3) + MIN_FOUR_BYTES_INT) << 1; +} + static PyObject* read_str_internal(PyObject *data) { _CHECK_BUFFER(data, NULL) // Read string length. - Py_ssize_t size; _CHECK_READ(data, 1, NULL) uint8_t first = _READ(data, uint8_t) - if (likely(first != LONG_STR_TAG)) { - // Common case: short string (len <= 127). - size = (Py_ssize_t)(first >> 1); - } else { - _CHECK_READ(data, sizeof(CPyTagged), NULL) - size = _READ(data, Py_ssize_t) + if (unlikely(first == LONG_INT_TRAILER)) { + // Fail fast for invalid/tampered data. + PyErr_SetString(PyExc_ValueError, "invalid str size"); + return NULL; } + CPyTagged tagged_size = _read_short_int(data, first); + if (tagged_size == CPY_INT_TAG) + return NULL; + Py_ssize_t size = tagged_size >> 1; // Read string content. char *buf = ((BufferObject *)data)->buf; _CHECK_READ(data, size, NULL) @@ -302,6 +343,35 @@ read_str(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames) return read_str_internal(data); } +// The caller *must* check that real_value is within allowed range (29 bits). +static inline char +_write_short_int(PyObject *data, Py_ssize_t real_value) { + if (real_value >= MIN_ONE_BYTE_INT && real_value <= MAX_ONE_BYTE_INT) { + _CHECK_SIZE(data, 1) + _WRITE(data, uint8_t, (uint8_t)(real_value - MIN_ONE_BYTE_INT) << 1) + ((BufferObject *)data)->end += 1; + } else if (real_value >= MIN_TWO_BYTES_INT && real_value <= MAX_TWO_BYTES_INT) { + _CHECK_SIZE(data, 2) +#if PY_BIG_ENDIAN + uint16_t to_write = ((uint16_t)(real_value - MIN_TWO_BYTES_INT) << 2) | TWO_BYTES_INT_BIT; + _WRITE(data, uint16_t, reverse_16(to_write)) +#else + _WRITE(data, uint16_t, ((uint16_t)(real_value - MIN_TWO_BYTES_INT) << 2) | TWO_BYTES_INT_BIT) +#endif + ((BufferObject *)data)->end += 2; + } else { + _CHECK_SIZE(data, 4) +#if PY_BIG_ENDIAN + uint32_t to_write = ((uint32_t)(real_value - MIN_FOUR_BYTES_INT) << 3) | FOUR_BYTES_INT_TRAILER; + _WRITE(data, uint32_t, reverse_32(to_write)) +#else + _WRITE(data, uint32_t, ((uint32_t)(real_value - MIN_FOUR_BYTES_INT) << 3) | FOUR_BYTES_INT_TRAILER) +#endif + ((BufferObject *)data)->end += 4; + } + return CPY_NONE; +} + static char write_str_internal(PyObject *data, PyObject *value) { _CHECK_BUFFER(data, CPY_NONE_ERROR) @@ -311,24 +381,20 @@ write_str_internal(PyObject *data, PyObject *value) { if (unlikely(chunk == NULL)) return CPY_NONE_ERROR; - Py_ssize_t need; // Write string length. - if (likely(size <= MAX_SHORT_LEN)) { - // Common case: short string (len <= 127) store as single byte. - need = size + 1; - _CHECK_SIZE(data, need) - _WRITE(data, uint8_t, (uint8_t)size << 1) + if (likely(size >= MIN_FOUR_BYTES_INT && size <= MAX_FOUR_BYTES_INT)) { + if (_write_short_int(data, size) == CPY_NONE_ERROR) + return CPY_NONE_ERROR; } else { - need = size + sizeof(Py_ssize_t) + 1; - _CHECK_SIZE(data, need) - _WRITE(data, uint8_t, LONG_STR_TAG) - _WRITE(data, Py_ssize_t, size) + PyErr_SetString(PyExc_ValueError, "str too long to serialize"); + return CPY_NONE_ERROR; } // Write string content. + _CHECK_SIZE(data, size) char *buf = ((BufferObject *)data)->buf; memcpy(buf + ((BufferObject *)data)->pos, chunk, size); ((BufferObject *)data)->pos += size; - ((BufferObject *)data)->end += need; + ((BufferObject *)data)->end += size; return CPY_NONE; } @@ -353,9 +419,7 @@ write_str(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames } /* -bytes format: size followed by bytes - short bytes (len <= 127): single byte for size as `(uint8_t)size << 1` - long bytes: \x01 followed by size as Py_ssize_t +bytes format: size as int (see below) followed by bytes */ static PyObject* @@ -363,16 +427,17 @@ read_bytes_internal(PyObject *data) { _CHECK_BUFFER(data, NULL) // Read length. - Py_ssize_t size; _CHECK_READ(data, 1, NULL) uint8_t first = _READ(data, uint8_t) - if (likely(first != LONG_STR_TAG)) { - // Common case: short bytes (len <= 127). - size = (Py_ssize_t)(first >> 1); - } else { - _CHECK_READ(data, sizeof(CPyTagged), NULL) - size = _READ(data, Py_ssize_t) + if (unlikely(first == LONG_INT_TRAILER)) { + // Fail fast for invalid/tampered data. + PyErr_SetString(PyExc_ValueError, "invalid bytes size"); + return NULL; } + CPyTagged tagged_size = _read_short_int(data, first); + if (tagged_size == CPY_INT_TAG) + return NULL; + Py_ssize_t size = tagged_size >> 1; // Read bytes content. char *buf = ((BufferObject *)data)->buf; _CHECK_READ(data, size, NULL) @@ -405,24 +470,20 @@ write_bytes_internal(PyObject *data, PyObject *value) { return CPY_NONE_ERROR; Py_ssize_t size = PyBytes_GET_SIZE(value); - Py_ssize_t need; // Write length. - if (likely(size <= MAX_SHORT_LEN)) { - // Common case: short bytes (len <= 127) store as single byte. - need = size + 1; - _CHECK_SIZE(data, need) - _WRITE(data, uint8_t, (uint8_t)size << 1) + if (likely(size >= MIN_FOUR_BYTES_INT && size <= MAX_FOUR_BYTES_INT)) { + if (_write_short_int(data, size) == CPY_NONE_ERROR) + return CPY_NONE_ERROR; } else { - need = size + sizeof(Py_ssize_t) + 1; - _CHECK_SIZE(data, need) - _WRITE(data, uint8_t, LONG_STR_TAG) - _WRITE(data, Py_ssize_t, size) + PyErr_SetString(PyExc_ValueError, "bytes too long to serialize"); + return CPY_NONE_ERROR; } // Write bytes content. + _CHECK_SIZE(data, size) char *buf = ((BufferObject *)data)->buf; memcpy(buf + ((BufferObject *)data)->pos, chunk, size); ((BufferObject *)data)->pos += size; - ((BufferObject *)data)->end += need; + ((BufferObject *)data)->end += size; return CPY_NONE; } @@ -455,7 +516,7 @@ static double read_float_internal(PyObject *data) { _CHECK_BUFFER(data, CPY_FLOAT_ERROR) _CHECK_READ(data, sizeof(double), CPY_FLOAT_ERROR) - double res = _READ(data, double); + double res = _READ(data, double) return res; } @@ -505,9 +566,13 @@ write_float(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnam /* int format: - most common values (-10 <= value <= 117): single byte as `(uint8_t)(value + 10) << 1` - medium values (fit in CPyTagged): \x01 followed by CPyTagged value - long values (very rare): \x03 followed by decimal string (see str format) + one byte: last bit 0, 7 bits used + two bytes: last two bits 01, 14 bits used + four bytes: last three bits 011, 29 bits used + everything else: 00001111 followed by serialized string representation + +Note: for fixed size formats we skew ranges towards more positive values, +since negative integers are much more rare. */ static CPyTagged @@ -516,22 +581,17 @@ read_int_internal(PyObject *data) { _CHECK_READ(data, 1, CPY_INT_TAG) uint8_t first = _READ(data, uint8_t) - if ((first & MEDIUM_INT_TAG) == 0) { - // Most common case: int that is small in absolute value. - return ((Py_ssize_t)(first >> 1) + MIN_SHORT_INT) << 1; - } - if (first == MEDIUM_INT_TAG) { - _CHECK_READ(data, sizeof(CPyTagged), CPY_INT_TAG) - CPyTagged ret = _READ(data, CPyTagged) - return ret; + if (likely(first != LONG_INT_TRAILER)) { + return _read_short_int(data, first); } - // People who have literal ints not fitting in size_t should be punished :-) PyObject *str_ret = read_str_internal(data); if (unlikely(str_ret == NULL)) return CPY_INT_TAG; PyObject* ret_long = PyLong_FromUnicodeObject(str_ret, 10); Py_DECREF(str_ret); - return ((CPyTagged)ret_long) | CPY_INT_TAG; + if (ret_long == NULL) + return CPY_INT_TAG; + return CPyTagged_StealFromObject(ret_long); } static PyObject* @@ -549,36 +609,38 @@ read_int(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames) return CPyTagged_StealAsObject(retval); } +static inline char +_write_long_int(PyObject *data, CPyTagged value) { + // TODO(jukka): write a more compact/optimal format for arbitrary length ints. + _CHECK_SIZE(data, 1) + _WRITE(data, uint8_t, LONG_INT_TRAILER) + ((BufferObject *)data)->end += 1; + PyObject* int_value = CPyTagged_AsObject(value); + if (unlikely(int_value == NULL)) + return CPY_NONE_ERROR; + PyObject *str_value = PyObject_Str(int_value); + Py_DECREF(int_value); + if (unlikely(str_value == NULL)) + return CPY_NONE_ERROR; + char res = write_str_internal(data, str_value); + Py_DECREF(str_value); + return res; +} + static char write_int_internal(PyObject *data, CPyTagged value) { _CHECK_BUFFER(data, CPY_NONE_ERROR) if (likely((value & CPY_INT_TAG) == 0)) { Py_ssize_t real_value = CPyTagged_ShortAsSsize_t(value); - if (real_value >= MIN_SHORT_INT && real_value <= MAX_SHORT_INT) { - // Most common case: int that is small in absolute value. - _CHECK_SIZE(data, 1) - _WRITE(data, uint8_t, (uint8_t)(real_value - MIN_SHORT_INT) << 1) - ((BufferObject *)data)->end += 1; + if (likely(real_value >= MIN_FOUR_BYTES_INT && real_value <= MAX_FOUR_BYTES_INT)) { + return _write_short_int(data, real_value); } else { - _CHECK_SIZE(data, sizeof(CPyTagged) + 1) - _WRITE(data, uint8_t, MEDIUM_INT_TAG) - _WRITE(data, CPyTagged, value) - ((BufferObject *)data)->end += sizeof(CPyTagged) + 1; + return _write_long_int(data, value); } } else { - _CHECK_SIZE(data, 1) - _WRITE(data, uint8_t, LONG_INT_TAG) - ((BufferObject *)data)->end += 1; - PyObject *str_value = PyObject_Str(CPyTagged_LongAsObject(value)); - if (unlikely(str_value == NULL)) - return CPY_NONE_ERROR; - char res = write_str_internal(data, str_value); - Py_DECREF(str_value); - if (unlikely(res == CPY_NONE_ERROR)) - return CPY_NONE_ERROR; + return _write_long_int(data, value); } - return CPY_NONE; } static PyObject* diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index e08f2fd7007d8..04bbed78b3189 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -2738,8 +2738,8 @@ def test_buffer_roundtrip() -> None: write_bytes(b, b"bar") write_bytes(b, b"bar" * 100) write_bytes(b, b"") - write_bytes(b, b"a" * 127) - write_bytes(b, b"a" * 128) + write_bytes(b, b"a" * 117) + write_bytes(b, b"a" * 118) write_float(b, 0.1) write_float(b, -113.0) write_int(b, 0) @@ -2752,8 +2752,9 @@ def test_buffer_roundtrip() -> None: write_int(b, 255) write_int(b, -1) write_int(b, -255) - write_int(b, 1234512344) - write_int(b, 1234512345) + write_int(b, 536860911) + write_int(b, 536860912) + write_int(b, 1234567891) b = Buffer(b.getvalue()) assert read_str(b) == "foo" @@ -2763,8 +2764,8 @@ def test_buffer_roundtrip() -> None: assert read_bytes(b) == b"bar" assert read_bytes(b) == b"bar" * 100 assert read_bytes(b) == b"" - assert read_bytes(b) == b"a" * 127 - assert read_bytes(b) == b"a" * 128 + assert read_bytes(b) == b"a" * 117 + assert read_bytes(b) == b"a" * 118 assert read_float(b) == 0.1 assert read_float(b) == -113.0 assert read_int(b) == 0 @@ -2777,8 +2778,9 @@ def test_buffer_roundtrip() -> None: assert read_int(b) == 255 assert read_int(b) == -1 assert read_int(b) == -255 - assert read_int(b) == 1234512344 - assert read_int(b) == 1234512345 + assert read_int(b) == 536860911 + assert read_int(b) == 536860912 + assert read_int(b) == 1234567891 def test_buffer_int_size() -> None: for i in (-10, -9, 0, 116, 117): @@ -2787,21 +2789,44 @@ def test_buffer_int_size() -> None: assert len(b.getvalue()) == 1 b = Buffer(b.getvalue()) assert read_int(b) == i - for i in (-12345, -12344, -11, 118, 12344, 12345): + for i in (-100, -11, 118, 12344, 16283): b = Buffer() write_int(b, i) - assert len(b.getvalue()) <= 9 # sizeof(size_t) + 1 + assert len(b.getvalue()) == 2 b = Buffer(b.getvalue()) assert read_int(b) == i + for i in (-10000, 16284, 123456789): + b = Buffer() + write_int(b, i) + assert len(b.getvalue()) == 4 + b = Buffer(b.getvalue()) + assert read_int(b) == i + +def test_buffer_int_powers() -> None: + # 0, 1, 2 are tested above + for p in range(2, 9): + b = Buffer() + write_int(b, 1 << p) + write_int(b, -1 << p) + b = Buffer(b.getvalue()) + assert read_int(b) == 1 << p + assert read_int(b) == -1 << p def test_buffer_str_size() -> None: - for s in ("", "a", "a" * 127): + for s in ("", "a", "a" * 117): b = Buffer() write_str(b, s) assert len(b.getvalue()) == len(s) + 1 b = Buffer(b.getvalue()) assert read_str(b) == s + for s in ("a" * 118, "a" * 16283): + b = Buffer() + write_str(b, s) + assert len(b.getvalue()) == len(s) + 2 + b = Buffer(b.getvalue()) + assert read_str(b) == s + [file driver.py] from native import * @@ -2809,6 +2834,7 @@ test_buffer_basic() test_buffer_roundtrip() test_buffer_int_size() test_buffer_str_size() +test_buffer_int_powers() def test_buffer_basic_interpreted() -> None: b = Buffer(b"foo") @@ -2823,8 +2849,8 @@ def test_buffer_roundtrip_interpreted() -> None: write_bytes(b, b"bar") write_bytes(b, b"bar" * 100) write_bytes(b, b"") - write_bytes(b, b"a" * 127) - write_bytes(b, b"a" * 128) + write_bytes(b, b"a" * 117) + write_bytes(b, b"a" * 118) write_float(b, 0.1) write_int(b, 0) write_int(b, 1) @@ -2836,8 +2862,9 @@ def test_buffer_roundtrip_interpreted() -> None: write_int(b, 255) write_int(b, -1) write_int(b, -255) - write_int(b, 1234512344) - write_int(b, 1234512345) + write_int(b, 536860911) + write_int(b, 536860912) + write_int(b, 1234567891) b = Buffer(b.getvalue()) assert read_str(b) == "foo" @@ -2847,8 +2874,8 @@ def test_buffer_roundtrip_interpreted() -> None: assert read_bytes(b) == b"bar" assert read_bytes(b) == b"bar" * 100 assert read_bytes(b) == b"" - assert read_bytes(b) == b"a" * 127 - assert read_bytes(b) == b"a" * 128 + assert read_bytes(b) == b"a" * 117 + assert read_bytes(b) == b"a" * 118 assert read_float(b) == 0.1 assert read_int(b) == 0 assert read_int(b) == 1 @@ -2860,8 +2887,9 @@ def test_buffer_roundtrip_interpreted() -> None: assert read_int(b) == 255 assert read_int(b) == -1 assert read_int(b) == -255 - assert read_int(b) == 1234512344 - assert read_int(b) == 1234512345 + assert read_int(b) == 536860911 + assert read_int(b) == 536860912 + assert read_int(b) == 1234567891 def test_buffer_int_size_interpreted() -> None: for i in (-10, -9, 0, 116, 117): @@ -2870,25 +2898,49 @@ def test_buffer_int_size_interpreted() -> None: assert len(b.getvalue()) == 1 b = Buffer(b.getvalue()) assert read_int(b) == i - for i in (-12345, -12344, -11, 118, 12344, 12345): + for i in (-100, -11, 118, 12344, 16283): b = Buffer() write_int(b, i) - assert len(b.getvalue()) <= 9 # sizeof(size_t) + 1 + assert len(b.getvalue()) == 2 b = Buffer(b.getvalue()) assert read_int(b) == i + for i in (-10000, 16284, 123456789): + b = Buffer() + write_int(b, i) + assert len(b.getvalue()) == 4 + b = Buffer(b.getvalue()) + assert read_int(b) == i + +def test_buffer_int_powers_interpreted() -> None: + # 0, 1, 2 are tested above + for p in range(2, 9): + b = Buffer() + write_int(b, 1 << p) + write_int(b, -1 << p) + b = Buffer(b.getvalue()) + assert read_int(b) == 1 << p + assert read_int(b) == -1 << p def test_buffer_str_size_interpreted() -> None: - for s in ("", "a", "a" * 127): + for s in ("", "a", "a" * 117): b = Buffer() write_str(b, s) assert len(b.getvalue()) == len(s) + 1 b = Buffer(b.getvalue()) assert read_str(b) == s + for s in ("a" * 118, "a" * 16283): + b = Buffer() + write_str(b, s) + assert len(b.getvalue()) == len(s) + 2 + b = Buffer(b.getvalue()) + assert read_str(b) == s + test_buffer_basic_interpreted() test_buffer_roundtrip_interpreted() test_buffer_int_size_interpreted() test_buffer_str_size_interpreted() +test_buffer_int_powers_interpreted() [case testBufferEmpty_librt_internal] from librt.internal import Buffer, write_int, read_int From c52b17afe2273cf4b6aee51c6e8b64f2c81f54e1 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 31 Oct 2025 12:32:45 +0000 Subject: [PATCH 358/424] More robust packing of flats in FF cache (#20150) This should fix issues with `librt` not working well on platforms with hardware float support, it should also make cache format endian-independent (since we ask for little-endian format using `1` as last argument independently of current endianness). More details in https://docs.python.org/3/c-api/float.html#pack-functions --- mypyc/lib-rt/librt_internal.c | 20 ++++++++++++++------ mypyc/test-data/run-classes.test | 2 ++ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/mypyc/lib-rt/librt_internal.c b/mypyc/lib-rt/librt_internal.c index 4f6e138c96f9c..eb864619d3983 100644 --- a/mypyc/lib-rt/librt_internal.c +++ b/mypyc/lib-rt/librt_internal.c @@ -509,14 +509,18 @@ write_bytes(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnam /* float format: - stored as a C double + stored using PyFloat helpers in little-endian format. */ static double read_float_internal(PyObject *data) { _CHECK_BUFFER(data, CPY_FLOAT_ERROR) - _CHECK_READ(data, sizeof(double), CPY_FLOAT_ERROR) - double res = _READ(data, double) + _CHECK_READ(data, 8, CPY_FLOAT_ERROR) + char *buf = ((BufferObject *)data)->buf; + double res = PyFloat_Unpack8(buf + ((BufferObject *)data)->pos, 1); + if (unlikely((res == -1.0) && PyErr_Occurred())) + return CPY_FLOAT_ERROR; + ((BufferObject *)data)->pos += 8; return res; } @@ -538,9 +542,13 @@ read_float(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwname static char write_float_internal(PyObject *data, double value) { _CHECK_BUFFER(data, CPY_NONE_ERROR) - _CHECK_SIZE(data, sizeof(double)) - _WRITE(data, double, value) - ((BufferObject *)data)->end += sizeof(double); + _CHECK_SIZE(data, 8) + char *buf = ((BufferObject *)data)->buf; + int res = PyFloat_Pack8(value, buf + ((BufferObject *)data)->pos, 1); + if (unlikely(res == -1)) + return CPY_NONE_ERROR; + ((BufferObject *)data)->pos += 8; + ((BufferObject *)data)->end += 8; return CPY_NONE; } diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 04bbed78b3189..9c2ba14c08732 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -2741,6 +2741,7 @@ def test_buffer_roundtrip() -> None: write_bytes(b, b"a" * 117) write_bytes(b, b"a" * 118) write_float(b, 0.1) + write_float(b, -1.0) write_float(b, -113.0) write_int(b, 0) write_int(b, 1) @@ -2767,6 +2768,7 @@ def test_buffer_roundtrip() -> None: assert read_bytes(b) == b"a" * 117 assert read_bytes(b) == b"a" * 118 assert read_float(b) == 0.1 + assert read_float(b) == -1.0 assert read_float(b) == -113.0 assert read_int(b) == 0 assert read_int(b) == 1 From 72131392483c19a6b8db812f4d4acfdb50a769dd Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 31 Oct 2025 16:01:27 +0000 Subject: [PATCH 359/424] Use more efficient serialization format for long integers in cache files (#20151) A long integer (one that doesn't fit in the 4-byte encoding) will now be encoded like this: * initial header byte * short integer (1-4 bytes) encoding the number of bytes of data and sign * variable-length number of data bytes (absolute value of the integer) -- all bits are used For example, a 32-bit integer can now always be encoded using at most 6 bytes (+ type tag). This is optimized for size efficiency, not performance, since large integers are not expected to be a performance bottleneck. Having an efficient format makes it easier to improve performance in the future, however, without changing the encoding. The header byte has a few unused bits which could be used to slightly improve efficiency, but I decided that it's not worth the extra complexity. --- mypyc/lib-rt/librt_internal.c | 108 +++++++++++++++++++++++++++---- mypyc/test-data/run-classes.test | 27 +++++++- 2 files changed, 120 insertions(+), 15 deletions(-) diff --git a/mypyc/lib-rt/librt_internal.c b/mypyc/lib-rt/librt_internal.c index eb864619d3983..6cae63cfadcb1 100644 --- a/mypyc/lib-rt/librt_internal.c +++ b/mypyc/lib-rt/librt_internal.c @@ -592,14 +592,35 @@ read_int_internal(PyObject *data) { if (likely(first != LONG_INT_TRAILER)) { return _read_short_int(data, first); } - PyObject *str_ret = read_str_internal(data); - if (unlikely(str_ret == NULL)) + + // Long integer encoding -- byte length and sign, followed by a byte array. + + // Read byte length and sign. + _CHECK_READ(data, 1, CPY_INT_TAG) + first = _READ(data, uint8_t) + Py_ssize_t size_and_sign = _read_short_int(data, first); + if (size_and_sign == CPY_INT_TAG) return CPY_INT_TAG; - PyObject* ret_long = PyLong_FromUnicodeObject(str_ret, 10); - Py_DECREF(str_ret); - if (ret_long == NULL) + bool sign = (size_and_sign >> 1) & 1; + Py_ssize_t size = size_and_sign >> 2; + + // Construct an int object from the byte array. + _CHECK_READ(data, size, CPY_INT_TAG) + char *buf = ((BufferObject *)data)->buf; + PyObject *num = _PyLong_FromByteArray( + (unsigned char *)(buf + ((BufferObject *)data)->pos), size, 1, 0); + if (num == NULL) return CPY_INT_TAG; - return CPyTagged_StealFromObject(ret_long); + ((BufferObject *)data)->pos += size; + if (sign) { + PyObject *old = num; + num = PyNumber_Negative(old); + Py_DECREF(old); + if (num == NULL) { + return CPY_INT_TAG; + } + } + return CPyTagged_StealFromObject(num); } static PyObject* @@ -617,22 +638,81 @@ read_int(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames) return CPyTagged_StealAsObject(retval); } + +static inline int hex_to_int(char c) { + if (c >= '0' && c <= '9') + return c - '0'; + else if (c >= 'a' && c <= 'f') + return c - 'a' + 10; + else + return c - 'A' + 10; // Assume valid hex digit +} + static inline char _write_long_int(PyObject *data, CPyTagged value) { - // TODO(jukka): write a more compact/optimal format for arbitrary length ints. _CHECK_SIZE(data, 1) _WRITE(data, uint8_t, LONG_INT_TRAILER) ((BufferObject *)data)->end += 1; + + PyObject *hex_str = NULL; PyObject* int_value = CPyTagged_AsObject(value); if (unlikely(int_value == NULL)) - return CPY_NONE_ERROR; - PyObject *str_value = PyObject_Str(int_value); + goto error; + + hex_str = PyNumber_ToBase(int_value, 16); + if (hex_str == NULL) + goto error; Py_DECREF(int_value); - if (unlikely(str_value == NULL)) - return CPY_NONE_ERROR; - char res = write_str_internal(data, str_value); - Py_DECREF(str_value); - return res; + int_value = NULL; + + const char *str = PyUnicode_AsUTF8(hex_str); + if (str == NULL) + goto error; + Py_ssize_t len = strlen(str); + bool neg; + if (str[0] == '-') { + str++; + len--; + neg = true; + } else { + neg = false; + } + // Skip the 0x hex prefix. + str += 2; + len -= 2; + + // Write bytes encoded length and sign. + Py_ssize_t size = (len + 1) / 2; + Py_ssize_t encoded_size = (size << 1) | neg; + if (encoded_size <= MAX_FOUR_BYTES_INT) { + if (_write_short_int(data, encoded_size) == CPY_NONE_ERROR) + goto error; + } else { + PyErr_SetString(PyExc_ValueError, "int too long to serialize"); + goto error; + } + + // Write absolute integer value as byte array in a variable-length little endian format. + int i; + for (i = len; i > 1; i -= 2) { + if (write_tag_internal( + data, hex_to_int(str[i - 1]) | (hex_to_int(str[i - 2]) << 4)) == CPY_NONE_ERROR) + goto error; + } + // The final byte may correspond to only one hex digit. + if (i == 1) { + if (write_tag_internal(data, hex_to_int(str[i - 1])) == CPY_NONE_ERROR) + goto error; + } + + Py_DECREF(hex_str); + return CPY_NONE; + + error: + + Py_XDECREF(int_value); + Py_XDECREF(hex_str); + return CPY_NONE_ERROR; } static char diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 9c2ba14c08732..b02d10446800c 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -2806,13 +2806,36 @@ def test_buffer_int_size() -> None: def test_buffer_int_powers() -> None: # 0, 1, 2 are tested above - for p in range(2, 9): + for p in range(2, 200): b = Buffer() write_int(b, 1 << p) + write_int(b, (1 << p) - 1) write_int(b, -1 << p) + write_int(b, (-1 << p) + 1) b = Buffer(b.getvalue()) assert read_int(b) == 1 << p + assert read_int(b) == (1 << p) - 1 assert read_int(b) == -1 << p + assert read_int(b) == (-1 << p) + 1 + +def test_positive_long_int_serialized_bytes() -> None: + b = Buffer() + n = 0x123456789ab + write_int(b, n) + x = b.getvalue() + # Two prefix bytes, followed by little endian encoded integer in variable-length format + assert x == b"\x0f\x2c\xab\x89\x67\x45\x23\x01" + b = Buffer(x) + assert read_int(b) == n + +def test_negative_long_int_serialized_bytes() -> None: + b = Buffer() + n = -0x123456789abcde + write_int(b, n) + x = b.getvalue() + assert x == b"\x0f\x32\xde\xbc\x9a\x78\x56\x34\x12" + b = Buffer(x) + assert read_int(b) == n def test_buffer_str_size() -> None: for s in ("", "a", "a" * 117): @@ -2837,6 +2860,8 @@ test_buffer_roundtrip() test_buffer_int_size() test_buffer_str_size() test_buffer_int_powers() +test_positive_long_int_serialized_bytes() +test_negative_long_int_serialized_bytes() def test_buffer_basic_interpreted() -> None: b = Buffer(b"foo") From c5301092ddf8fefab427f578ee399fbf7b273978 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Fri, 31 Oct 2025 15:35:26 -0700 Subject: [PATCH 360/424] Avoid running tests on macos-13 runner (#20155) --- .github/workflows/test.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index de3f8877ee676..07b4b3f030204 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -79,12 +79,18 @@ jobs: # # allow_failure: true # test_mypyc: true - - name: mypyc runtime tests with py39-macos - python: '3.9.21' - # TODO: macos-13 is the last one to support Python 3.9, change it to macos-latest when updating the Python version - os: macos-13 + - name: mypyc runtime tests with py313-macos + python: '3.13' + os: macos-latest toxenv: py tox_extra_args: "-n 3 mypyc/test/test_run.py mypyc/test/test_external.py" + + - name: mypyc runtime tests with py313-ubuntu + python: '3.13' + os: ubuntu-latest + toxenv: py + tox_extra_args: "-n 3 mypyc/test/test_run.py mypyc/test/test_external.py" + # This is broken. See # - https://github.com/python/mypy/issues/17819 # - https://github.com/python/mypy/pull/17822 From d6e9c31bc33d3796ece88a9e07d051101515bafd Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Fri, 31 Oct 2025 17:30:33 -0700 Subject: [PATCH 361/424] Upgrade ruff, black (#20158) --- .pre-commit-config.yaml | 6 +++--- pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1f4282e4f65b5..1466c8e0fda47 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,14 +6,14 @@ repos: - id: trailing-whitespace - id: end-of-file-fixer - repo: https://github.com/psf/black-pre-commit-mirror - rev: 25.1.0 + rev: 25.9.0 hooks: - id: black exclude: '^(test-data/)' - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.11.4 + rev: v0.14.3 hooks: - - id: ruff + - id: ruff-check args: [--exit-non-zero-on-fix] - repo: https://github.com/python-jsonschema/check-jsonschema rev: 0.32.1 diff --git a/pyproject.toml b/pyproject.toml index f9f6c01b5c1cd..42e10967cba29 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -107,7 +107,7 @@ mypyc = [ [tool.black] line-length = 99 -target-version = ["py39", "py310", "py311", "py312", "py313"] +target-version = ["py39", "py310", "py311", "py312", "py313", "py314"] skip-magic-trailing-comma = true force-exclude = ''' ^/mypy/typeshed| From 98d79300d2878dac037f8f468b8f9b0dd197e8c2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 31 Oct 2025 17:54:14 -0700 Subject: [PATCH 362/424] Sync typeshed (#20157) Source commit: https://github.com/python/typeshed/commit/bf7214784877c52638844c065360d4814fae4c65 --- mypy/typeshed/stdlib/builtins.pyi | 4 ++++ mypy/typeshed/stdlib/cmath.pyi | 2 +- mypy/typeshed/stdlib/contextlib.pyi | 21 +++++++++++++++------ mypy/typeshed/stdlib/enum.pyi | 1 + mypy/typeshed/stdlib/os/__init__.pyi | 3 +++ mypy/typeshed/stdlib/sys/__init__.pyi | 12 ++++++++++++ mypy/typeshed/stdlib/sysconfig.pyi | 8 +++++--- mypy/typeshed/stdlib/turtle.pyi | 8 ++++---- mypy/typeshed/stdlib/zlib.pyi | 4 ++-- 9 files changed, 47 insertions(+), 16 deletions(-) diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi index ddf81db181bfa..e03a92ce3d91d 100644 --- a/mypy/typeshed/stdlib/builtins.pyi +++ b/mypy/typeshed/stdlib/builtins.pyi @@ -1960,6 +1960,10 @@ class BaseException: def __new__(cls, *args: Any, **kwds: Any) -> Self: ... def __setstate__(self, state: dict[str, Any] | None, /) -> None: ... def with_traceback(self, tb: TracebackType | None, /) -> Self: ... + # Necessary for security-focused static analyzers (e.g, pysa) + # See https://github.com/python/typeshed/pull/14900 + def __str__(self) -> str: ... # noqa: Y029 + def __repr__(self) -> str: ... # noqa: Y029 if sys.version_info >= (3, 11): # only present after add_note() is called __notes__: list[str] diff --git a/mypy/typeshed/stdlib/cmath.pyi b/mypy/typeshed/stdlib/cmath.pyi index a08addcf54389..aed4c63862fe7 100644 --- a/mypy/typeshed/stdlib/cmath.pyi +++ b/mypy/typeshed/stdlib/cmath.pyi @@ -23,7 +23,7 @@ def exp(z: _C, /) -> complex: ... def isclose(a: _C, b: _C, *, rel_tol: SupportsFloat = 1e-09, abs_tol: SupportsFloat = 0.0) -> bool: ... def isinf(z: _C, /) -> bool: ... def isnan(z: _C, /) -> bool: ... -def log(x: _C, base: _C = ..., /) -> complex: ... +def log(z: _C, base: _C = ..., /) -> complex: ... def log10(z: _C, /) -> complex: ... def phase(z: _C, /) -> float: ... def polar(z: _C, /) -> tuple[float, float]: ... diff --git a/mypy/typeshed/stdlib/contextlib.pyi b/mypy/typeshed/stdlib/contextlib.pyi index 383a1b7f334b4..221102ee23956 100644 --- a/mypy/typeshed/stdlib/contextlib.pyi +++ b/mypy/typeshed/stdlib/contextlib.pyi @@ -4,7 +4,7 @@ from _typeshed import FileDescriptorOrPath, Unused from abc import ABC, abstractmethod from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Generator, Iterator from types import TracebackType -from typing import IO, Any, Generic, Protocol, TypeVar, overload, runtime_checkable, type_check_only +from typing import Any, Generic, Protocol, TypeVar, overload, runtime_checkable, type_check_only from typing_extensions import ParamSpec, Self, TypeAlias __all__ = [ @@ -30,7 +30,6 @@ if sys.version_info >= (3, 11): _T = TypeVar("_T") _T_co = TypeVar("_T_co", covariant=True) -_T_io = TypeVar("_T_io", bound=IO[str] | None) _ExitT_co = TypeVar("_ExitT_co", covariant=True, bound=bool | None, default=bool | None) _F = TypeVar("_F", bound=Callable[..., Any]) _G_co = TypeVar("_G_co", bound=Generator[Any, Any, Any] | AsyncGenerator[Any, Any], covariant=True) @@ -141,14 +140,24 @@ class suppress(AbstractContextManager[None, bool]): self, exctype: type[BaseException] | None, excinst: BaseException | None, exctb: TracebackType | None ) -> bool: ... -class _RedirectStream(AbstractContextManager[_T_io, None]): - def __init__(self, new_target: _T_io) -> None: ... +# This is trying to describe what is needed for (most?) uses +# of `redirect_stdout` and `redirect_stderr`. +# https://github.com/python/typeshed/issues/14903 +@type_check_only +class _SupportsRedirect(Protocol): + def write(self, s: str, /) -> int: ... + def flush(self) -> None: ... + +_SupportsRedirectT = TypeVar("_SupportsRedirectT", bound=_SupportsRedirect | None) + +class _RedirectStream(AbstractContextManager[_SupportsRedirectT, None]): + def __init__(self, new_target: _SupportsRedirectT) -> None: ... def __exit__( self, exctype: type[BaseException] | None, excinst: BaseException | None, exctb: TracebackType | None ) -> None: ... -class redirect_stdout(_RedirectStream[_T_io]): ... -class redirect_stderr(_RedirectStream[_T_io]): ... +class redirect_stdout(_RedirectStream[_SupportsRedirectT]): ... +class redirect_stderr(_RedirectStream[_SupportsRedirectT]): ... class _BaseExitStack(Generic[_ExitT_co]): def enter_context(self, cm: AbstractContextManager[_T, _ExitT_co]) -> _T: ... diff --git a/mypy/typeshed/stdlib/enum.pyi b/mypy/typeshed/stdlib/enum.pyi index 4ac860f5e611d..c131c93923937 100644 --- a/mypy/typeshed/stdlib/enum.pyi +++ b/mypy/typeshed/stdlib/enum.pyi @@ -309,6 +309,7 @@ if sys.version_info >= (3, 11): def global_enum(cls: _EnumerationT, update_str: bool = False) -> _EnumerationT: ... def global_enum_repr(self: Enum) -> str: ... def global_flag_repr(self: Flag) -> str: ... + def show_flag_values(value: int) -> list[int]: ... if sys.version_info >= (3, 12): # The body of the class is the same, but the base classes are different. diff --git a/mypy/typeshed/stdlib/os/__init__.pyi b/mypy/typeshed/stdlib/os/__init__.pyi index 71c79dfac399f..580452739f7f0 100644 --- a/mypy/typeshed/stdlib/os/__init__.pyi +++ b/mypy/typeshed/stdlib/os/__init__.pyi @@ -729,6 +729,9 @@ environ: _Environ[str] if sys.platform != "win32": environb: _Environ[bytes] +if sys.version_info >= (3, 14): + def reload_environ() -> None: ... + if sys.version_info >= (3, 11) or sys.platform != "win32": EX_OK: Final[int] diff --git a/mypy/typeshed/stdlib/sys/__init__.pyi b/mypy/typeshed/stdlib/sys/__init__.pyi index 7807b0eab01f6..97e65d3094aae 100644 --- a/mypy/typeshed/stdlib/sys/__init__.pyi +++ b/mypy/typeshed/stdlib/sys/__init__.pyi @@ -354,6 +354,13 @@ else: def _current_frames() -> dict[int, FrameType]: ... def _getframe(depth: int = 0, /) -> FrameType: ... +# documented -- see https://docs.python.org/3/library/sys.html#sys._current_exceptions +if sys.version_info >= (3, 12): + def _current_exceptions() -> dict[int, BaseException | None]: ... + +else: + def _current_exceptions() -> dict[int, OptExcInfo]: ... + if sys.version_info >= (3, 12): def _getframemodulename(depth: int = 0) -> str | None: ... @@ -366,6 +373,10 @@ if sys.version_info >= (3, 11): def exception() -> BaseException | None: ... def exit(status: _ExitCode = None, /) -> NoReturn: ... + +if sys.platform == "android": # noqa: Y008 + def getandroidapilevel() -> int: ... + def getallocatedblocks() -> int: ... def getdefaultencoding() -> str: ... @@ -501,3 +512,4 @@ if sys.version_info >= (3, 12): if sys.version_info >= (3, 14): def is_remote_debug_enabled() -> bool: ... def remote_exec(pid: int, script: StrOrBytesPath) -> None: ... + def _is_immortal(op: object, /) -> bool: ... diff --git a/mypy/typeshed/stdlib/sysconfig.pyi b/mypy/typeshed/stdlib/sysconfig.pyi index 807a979050e80..c6419222df970 100644 --- a/mypy/typeshed/stdlib/sysconfig.pyi +++ b/mypy/typeshed/stdlib/sysconfig.pyi @@ -1,6 +1,6 @@ import sys from typing import IO, Any, Literal, overload -from typing_extensions import deprecated +from typing_extensions import LiteralString, deprecated __all__ = [ "get_config_h_filename", @@ -28,8 +28,10 @@ def get_config_vars(arg: str, /, *args: str) -> list[Any]: ... def get_scheme_names() -> tuple[str, ...]: ... if sys.version_info >= (3, 10): - def get_default_scheme() -> str: ... - def get_preferred_scheme(key: Literal["prefix", "home", "user"]) -> str: ... + def get_default_scheme() -> LiteralString: ... + def get_preferred_scheme(key: Literal["prefix", "home", "user"]) -> LiteralString: ... + # Documented -- see https://docs.python.org/3/library/sysconfig.html#sysconfig._get_preferred_schemes + def _get_preferred_schemes() -> dict[Literal["prefix", "home", "user"], LiteralString]: ... def get_path_names() -> tuple[str, ...]: ... def get_path(name: str, scheme: str = ..., vars: dict[str, Any] | None = None, expand: bool = True) -> str: ... diff --git a/mypy/typeshed/stdlib/turtle.pyi b/mypy/typeshed/stdlib/turtle.pyi index 9b9b329bd74bc..b5f536d0e28e5 100644 --- a/mypy/typeshed/stdlib/turtle.pyi +++ b/mypy/typeshed/stdlib/turtle.pyi @@ -266,7 +266,7 @@ class TurtleScreen(TurtleScreenBase): def window_height(self) -> int: ... def getcanvas(self) -> Canvas: ... def getshapes(self) -> list[str]: ... - def onclick(self, fun: Callable[[float, float], object], btn: int = 1, add: Any | None = None) -> None: ... + def onclick(self, fun: Callable[[float, float], object], btn: int = 1, add: bool | None = None) -> None: ... def onkey(self, fun: Callable[[], object], key: str) -> None: ... def listen(self, xdummy: float | None = None, ydummy: float | None = None) -> None: ... def ontimer(self, fun: Callable[[], object], t: int = 0) -> None: ... @@ -561,7 +561,7 @@ def window_width() -> int: ... def window_height() -> int: ... def getcanvas() -> Canvas: ... def getshapes() -> list[str]: ... -def onclick(fun: Callable[[float, float], object], btn: int = 1, add: Any | None = None) -> None: ... +def onclick(fun: Callable[[float, float], object], btn: int = 1, add: bool | None = None) -> None: ... def onkey(fun: Callable[[], object], key: str) -> None: ... def listen(xdummy: float | None = None, ydummy: float | None = None) -> None: ... def ontimer(fun: Callable[[], object], t: int = 0) -> None: ... @@ -776,8 +776,8 @@ def getturtle() -> Turtle: ... getpen = getturtle -def onrelease(fun: Callable[[float, float], object], btn: int = 1, add: Any | None = None) -> None: ... -def ondrag(fun: Callable[[float, float], object], btn: int = 1, add: Any | None = None) -> None: ... +def onrelease(fun: Callable[[float, float], object], btn: int = 1, add: bool | None = None) -> None: ... +def ondrag(fun: Callable[[float, float], object], btn: int = 1, add: bool | None = None) -> None: ... def undo() -> None: ... turtlesize = shapesize diff --git a/mypy/typeshed/stdlib/zlib.pyi b/mypy/typeshed/stdlib/zlib.pyi index 4e410fdd18ad9..d5998cab90fef 100644 --- a/mypy/typeshed/stdlib/zlib.pyi +++ b/mypy/typeshed/stdlib/zlib.pyi @@ -26,8 +26,8 @@ Z_RLE: Final = 3 Z_SYNC_FLUSH: Final = 2 Z_TREES: Final = 6 -if sys.version_info >= (3, 14) and sys.platform == "win32": - # Available when zlib was built with zlib-ng, usually only on Windows +if sys.version_info >= (3, 14): + # Available when zlib was built with zlib-ng ZLIBNG_VERSION: Final[str] class error(Exception): ... From 92101f3ec406534cb6be14bef7eb99b7e8fc5348 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 1 Nov 2025 00:57:29 +0000 Subject: [PATCH 363/424] Force-discard cache if cache format changed (#20152) If either low-level (i.e. `librt`) or high-level cache format changes, discard the cache. Note I intentionally don't use `librt` to read/write the first two bytes of cache meta, se we are 100% sure we can always read them. --- mypy-requirements.txt | 2 +- mypy/build.py | 18 ++++++++++++++---- mypy/cache.py | 6 ++++++ pyproject.toml | 4 ++-- test-requirements.txt | 2 +- 5 files changed, 24 insertions(+), 8 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index 622a8c3f36135..7c83178ae1eb8 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -4,4 +4,4 @@ typing_extensions>=4.6.0 mypy_extensions>=1.0.0 pathspec>=0.9.0 tomli>=1.1.0; python_version<'3.11' -librt>=0.3.0 +librt>=0.4.0 diff --git a/mypy/build.py b/mypy/build.py index 0058fb7eaaa06..0b78f879c547e 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -28,8 +28,10 @@ from typing import TYPE_CHECKING, Any, Callable, ClassVar, Final, NoReturn, TextIO, TypedDict from typing_extensions import TypeAlias as _TypeAlias +from librt.internal import cache_version + import mypy.semanal_main -from mypy.cache import Buffer, CacheMeta +from mypy.cache import CACHE_VERSION, Buffer, CacheMeta from mypy.checker import TypeChecker from mypy.error_formatter import OUTPUT_CHOICES, ErrorFormatter from mypy.errors import CompileError, ErrorInfo, Errors, report_internal_error @@ -1334,12 +1336,18 @@ def find_cache_meta(id: str, path: str, manager: BuildManager) -> CacheMeta | No return None t1 = time.time() if isinstance(meta, bytes): - data_io = Buffer(meta) + # If either low-level buffer format or high-level cache layout changed, we + # cannot use the cache files, even with --skip-version-check. + # TODO: switch to something like librt.internal.read_byte() if this is slow. + if meta[0] != cache_version() or meta[1] != CACHE_VERSION: + manager.log(f"Metadata abandoned for {id}: incompatible cache format") + return None + data_io = Buffer(meta[2:]) m = CacheMeta.read(data_io, data_file) else: m = CacheMeta.deserialize(meta, data_file) if m is None: - manager.log(f"Metadata abandoned for {id}: attributes are missing") + manager.log(f"Metadata abandoned for {id}: cannot deserialize data") return None t2 = time.time() manager.add_stats( @@ -1671,7 +1679,9 @@ def write_cache_meta(meta: CacheMeta, manager: BuildManager, meta_file: str) -> if manager.options.fixed_format_cache: data_io = Buffer() meta.write(data_io) - meta_bytes = data_io.getvalue() + # Prefix with both low- and high-level cache format versions for future validation. + # TODO: switch to something like librt.internal.write_byte() if this is slow. + meta_bytes = bytes([cache_version(), CACHE_VERSION]) + data_io.getvalue() else: meta_dict = meta.serialize() meta_bytes = json_dumps(meta_dict, manager.options.debug_cache) diff --git a/mypy/cache.py b/mypy/cache.py index 0d2db67fac948..900815b9f7e73 100644 --- a/mypy/cache.py +++ b/mypy/cache.py @@ -40,6 +40,9 @@ serialization. The write method should write both class tag and end tag. The read method conventionally *does not* read the start tag (to simplify logic for unions). Known exceptions are MypyFile.read() and SymbolTableNode.read(), since those two never appear in a union. + +If any of these details change, or if the structure of CacheMeta changes please +bump CACHE_VERSION below. """ from __future__ import annotations @@ -65,6 +68,9 @@ ) from mypy_extensions import u8 +# High-level cache layout format +CACHE_VERSION: Final = 0 + class CacheMeta: """Class representing cache metadata for a module.""" diff --git a/pyproject.toml b/pyproject.toml index 42e10967cba29..0de739be9b55d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ requires = [ "mypy_extensions>=1.0.0", "pathspec>=0.9.0", "tomli>=1.1.0; python_version<'3.11'", - "librt>=0.3.0", + "librt>=0.4.0", # the following is from build-requirements.txt "types-psutil", "types-setuptools", @@ -54,7 +54,7 @@ dependencies = [ "mypy_extensions>=1.0.0", "pathspec>=0.9.0", "tomli>=1.1.0; python_version<'3.11'", - "librt>=0.3.0", + "librt>=0.4.0", ] dynamic = ["version"] diff --git a/test-requirements.txt b/test-requirements.txt index b9ff4ffe085b7..126abd7149e62 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -22,7 +22,7 @@ identify==2.6.15 # via pre-commit iniconfig==2.1.0 # via pytest -librt==0.3.0 +librt==0.4.0 # via -r mypy-requirements.txt lxml==6.0.2 ; python_version < "3.15" # via -r test-requirements.in From d31d5260a33f9a748986baab6ef56187d85f953d Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Sat, 1 Nov 2025 02:53:40 +0100 Subject: [PATCH 364/424] Cleanup fastparse (#20159) Remove some unused code in mypy/fastparse.py. --- mypy/fastparse.py | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 276e183a6bf0e..c5e4589ec0256 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -144,9 +144,6 @@ def ast3_parse( ) -NamedExpr = ast3.NamedExpr -Constant = ast3.Constant - if sys.version_info >= (3, 10): Match = ast3.Match MatchValue = ast3.MatchValue @@ -957,7 +954,7 @@ def do_func_def( # for ellipsis arg if ( len(func_type_ast.argtypes) == 1 - and isinstance(func_type_ast.argtypes[0], Constant) + and isinstance(func_type_ast.argtypes[0], ast3.Constant) and func_type_ast.argtypes[0].value is Ellipsis ): if n.returns: @@ -1492,7 +1489,7 @@ def visit_Continue(self, n: ast3.Continue) -> ContinueStmt: # --- expr --- - def visit_NamedExpr(self, n: NamedExpr) -> AssignmentExpr: + def visit_NamedExpr(self, n: ast3.NamedExpr) -> AssignmentExpr: s = AssignmentExpr(self.visit(n.target), self.visit(n.value)) return self.set_line(s, n) @@ -1648,8 +1645,8 @@ def visit_Call(self, n: Call) -> CallExpr: ) return self.set_line(e, n) - # Constant(object value) -- a constant, in Python 3.8. - def visit_Constant(self, n: Constant) -> Any: + # Constant(object value) + def visit_Constant(self, n: ast3.Constant) -> Any: val = n.value e: Any = None if val is None: @@ -1773,8 +1770,6 @@ def visit_Tuple(self, n: ast3.Tuple) -> TupleExpr: e = TupleExpr(self.translate_expr_list(n.elts)) return self.set_line(e, n) - # --- slice --- - # Slice(expr? lower, expr? upper, expr? step) def visit_Slice(self, n: ast3.Slice) -> SliceExpr: e = SliceExpr(self.visit(n.lower), self.visit(n.upper), self.visit(n.step)) @@ -2030,9 +2025,9 @@ def translate_argument_list(self, l: Sequence[ast3.expr]) -> TypeList: return TypeList([self.visit(e) for e in l], line=self.line) def _extract_argument_name(self, n: ast3.expr) -> str | None: - if isinstance(n, Constant) and isinstance(n.value, str): + if isinstance(n, ast3.Constant) and isinstance(n.value, str): return n.value.strip() - elif isinstance(n, Constant) and n.value is None: + elif isinstance(n, ast3.Constant) and n.value is None: return None self.fail( message_registry.ARG_NAME_EXPECTED_STRING_LITERAL.format(type(n).__name__), @@ -2058,7 +2053,7 @@ def visit_BinOp(self, n: ast3.BinOp) -> Type: uses_pep604_syntax=True, ) - def visit_Constant(self, n: Constant) -> Type: + def visit_Constant(self, n: ast3.Constant) -> Type: val = n.value if val is None: # None is a type. @@ -2114,16 +2109,10 @@ def numeric_type(self, value: object, n: AST) -> Type: numeric_value, type_name, line=self.line, column=getattr(n, "col_offset", -1) ) - def visit_Index(self, n: ast3.Index) -> Type: - # cast for mypyc's benefit on Python 3.9 - value = self.visit(cast(Any, n).value) - assert isinstance(value, Type) - return value - def visit_Slice(self, n: ast3.Slice) -> Type: return self.invalid_type(n, note="did you mean to use ',' instead of ':' ?") - # Subscript(expr value, expr slice, expr_context ctx) # Python 3.9 and later + # Subscript(expr value, expr slice, expr_context ctx) def visit_Subscript(self, n: ast3.Subscript) -> Type: empty_tuple_index = False if isinstance(n.slice, ast3.Tuple): From 698e910c29c485419c5d5a1e6d99be0630b9d496 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Sat, 1 Nov 2025 19:46:18 -0600 Subject: [PATCH 365/424] [nit] command_line.rst: the standard format for multiple choice has no space (#19868) Removing this spaces causes this documentation to match the other documentation on the page, and also in `--help` There are no tests for this change. I manually verified that the link to the option currently is https://mypy.readthedocs.io/en/stable/command_line.html#cmdoption-mypy-enable-incomplete-feature, so changing the arguments text does not necessitate adding a new anchor in to preserve old inbound links to this section. --- docs/source/command_line.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index d667fa0ff7277..79dd68a84b28d 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -1159,7 +1159,7 @@ format into the specified directory. Enabling incomplete/experimental features ***************************************** -.. option:: --enable-incomplete-feature {PreciseTupleTypes, InlineTypedDict} +.. option:: --enable-incomplete-feature {PreciseTupleTypes,InlineTypedDict} Some features may require several mypy releases to implement, for example due to their complexity, potential for backwards incompatibility, or From 3618369b1263116804255a6af4cbd93abb7c69aa Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Sat, 1 Nov 2025 19:49:10 -0600 Subject: [PATCH 366/424] Update kinds_of_types.rst: keep old anchor for #no-strict-optional (#19828) Addresses https://github.com/python/mypy/pull/19252#issuecomment-2953310853 > support this old anchor, for people on older mypy versions This way, when people get the old documentation link, it will continue to go to the right place. I have looked at the documentation that gets generated locally to see if everything still works. It does. In our vast panoply of documentation, we now have two very similar refs, `no-strict-optional` and `no_strict_optional` (used in https://mypy.readthedocs.io/en/stable/common_issues.html#no-errors-reported-for-obviously-wrong-code to link to https://mypy.readthedocs.io/en/stable/command_line.html#no-strict-optional) but the software handles them fine, without getting confused. --- docs/source/kinds_of_types.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/kinds_of_types.rst b/docs/source/kinds_of_types.rst index 8e721c0fb3218..23ebc14e8670e 100644 --- a/docs/source/kinds_of_types.rst +++ b/docs/source/kinds_of_types.rst @@ -294,6 +294,7 @@ isn't supported by the runtime with some limitations, if you use def f(x: int | str) -> None: # OK on Python 3.7 and later ... +.. _no-strict-optional: .. _strict_optional: Optional types and the None type From 7aed6962621a53e4c31301a5cb07a8aa3547da55 Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Mon, 3 Nov 2025 08:40:54 +0100 Subject: [PATCH 367/424] Stubtest: check `_value_` for ellipsis-valued stub enum members (#19760) Currently stubtest allows unsound definitions: ```python # a.pyi from enum import Enum class E(Enum): _value_: str FOO = ... ``` ```python # a.py from enum import Enum class E(Enum): FOO = 0 ``` This PR teaches `stubtest` that `_value_` attribute ([spec](https://typing.python.org/en/latest/spec/enums.html#member-values)) should be used as a fallback in such case. --- mypy/stubtest.py | 18 +++++++++++++++--- mypy/test/teststubtest.py | 24 ++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index 99404dbe52abd..ada56a2489fe1 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -1274,6 +1274,7 @@ def verify_var( yield Error(object_path, "is read-only at runtime but not in the stub", stub, runtime) runtime_type = get_mypy_type_of_runtime_value(runtime, type_context=stub.type) + note = "" if ( runtime_type is not None and stub.type is not None @@ -1286,17 +1287,28 @@ def verify_var( runtime_type = get_mypy_type_of_runtime_value(runtime.value) if runtime_type is not None and is_subtype_helper(runtime_type, stub.type): should_error = False - # We always allow setting the stub value to ... + # We always allow setting the stub value to Ellipsis (...), but use + # _value_ type as a fallback if given. If a member is ... and _value_ + # type is given, all runtime types should be assignable to _value_. proper_type = mypy.types.get_proper_type(stub.type) if ( isinstance(proper_type, mypy.types.Instance) and proper_type.type.fullname in mypy.types.ELLIPSIS_TYPE_NAMES ): - should_error = False + value_t = stub.info.get("_value_") + if value_t is None or value_t.type is None or runtime_type is None: + should_error = False + elif is_subtype_helper(runtime_type, value_t.type): + should_error = False + else: + note = " (incompatible '_value_')" if should_error: yield Error( - object_path, f"variable differs from runtime type {runtime_type}", stub, runtime + object_path, + f"variable differs from runtime type {runtime_type}{note}", + stub, + runtime, ) diff --git a/mypy/test/teststubtest.py b/mypy/test/teststubtest.py index dfbde217e82f5..4bec5daf3ffbe 100644 --- a/mypy/test/teststubtest.py +++ b/mypy/test/teststubtest.py @@ -1483,6 +1483,30 @@ class HasEmptySlots: """, error=None, ) + yield Case( + stub=""" + class HasCompatibleValue(enum.Enum): + _value_: str + FOO = ... + """, + runtime=""" + class HasCompatibleValue(enum.Enum): + FOO = "foo" + """, + error=None, + ) + yield Case( + stub=""" + class HasIncompatibleValue(enum.Enum): + _value_: int + FOO = ... + """, + runtime=""" + class HasIncompatibleValue(enum.Enum): + FOO = "foo" + """, + error="HasIncompatibleValue.FOO", + ) @collect_cases def test_decorator(self) -> Iterator[Case]: From 3174d3f1d4e04d101384f1632e65ecc7911f20a0 Mon Sep 17 00:00:00 2001 From: Theodore Ando Date: Mon, 3 Nov 2025 01:54:26 -0600 Subject: [PATCH 368/424] Use pretty_callable more often for callable expressions (#20128) Fixes #5490 Uses pretty_callable for formatting Callable expressions that would otherwise be formatted with complex Args/VarArgs. Avoids pretty_callable for things that would be formatted with only positional args such as `Callable[[X, ..., Y], Z]` --- mypy/messages.py | 26 ++++++++++++- test-data/unit/check-assert-type-fail.test | 2 +- test-data/unit/check-callable.test | 6 +-- test-data/unit/check-functions.test | 38 +++++++++---------- test-data/unit/check-functools.test | 16 ++++---- test-data/unit/check-incremental.test | 2 +- test-data/unit/check-inference.test | 14 +++---- .../unit/check-parameter-specification.test | 8 ++-- test-data/unit/check-protocols.test | 34 ++++++++--------- test-data/unit/check-python311.test | 2 +- test-data/unit/check-statements.test | 2 +- test-data/unit/check-typevar-tuple.test | 26 ++++++------- test-data/unit/check-varargs.test | 2 +- test-data/unit/pythoneval.test | 2 +- 14 files changed, 102 insertions(+), 78 deletions(-) diff --git a/mypy/messages.py b/mypy/messages.py index c6378c2647578..a9e8ee2e43abf 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -2530,6 +2530,15 @@ def quote_type_string(type_string: str) -> str: return f'"{type_string}"' +def should_format_arg_as_type(arg_kind: ArgKind, arg_name: str | None, verbosity: int) -> bool: + """ + Determine whether a function argument should be formatted as its Type or with name. + """ + return (arg_kind == ARG_POS and arg_name is None) or ( + verbosity == 0 and arg_kind.is_positional() + ) + + def format_callable_args( arg_types: list[Type], arg_kinds: list[ArgKind], @@ -2540,7 +2549,7 @@ def format_callable_args( """Format a bunch of Callable arguments into a string""" arg_strings = [] for arg_name, arg_type, arg_kind in zip(arg_names, arg_types, arg_kinds): - if arg_kind == ARG_POS and arg_name is None or verbosity == 0 and arg_kind.is_positional(): + if should_format_arg_as_type(arg_kind, arg_name, verbosity): arg_strings.append(format(arg_type)) else: constructor = ARG_CONSTRUCTOR_NAMES[arg_kind] @@ -2558,13 +2567,18 @@ def format_type_inner( options: Options, fullnames: set[str] | None, module_names: bool = False, + use_pretty_callable: bool = True, ) -> str: """ Convert a type to a relatively short string suitable for error messages. Args: + typ: type to be formatted verbosity: a coarse grained control on the verbosity of the type + options: Options object controlling formatting fullnames: a set of names that should be printed in full + module_names: whether to show module names for module types + use_pretty_callable: use pretty_callable to format Callable types. """ def format(typ: Type) -> str: @@ -2761,6 +2775,16 @@ def format_literal_value(typ: LiteralType) -> str: param_spec = func.param_spec() if param_spec is not None: return f"Callable[{format(param_spec)}, {return_type}]" + + # Use pretty format (def-style) for complex signatures with named, optional, or star args. + # Use compact Callable[[...], ...] only for signatures with all simple positional args. + if use_pretty_callable: + if any( + not should_format_arg_as_type(kind, name, verbosity) + for kind, name in zip(func.arg_kinds, func.arg_names) + ): + return pretty_callable(func, options) + args = format_callable_args( func.arg_types, func.arg_kinds, func.arg_names, format, verbosity ) diff --git a/test-data/unit/check-assert-type-fail.test b/test-data/unit/check-assert-type-fail.test index 5146506496418..98aae0ba6b329 100644 --- a/test-data/unit/check-assert-type-fail.test +++ b/test-data/unit/check-assert-type-fail.test @@ -30,7 +30,7 @@ def f(si: arr.array[int]): [case testAssertTypeFailCallableArgKind] from typing import assert_type, Callable def myfunc(arg: int) -> None: pass -assert_type(myfunc, Callable[[int], None]) # E: Expression is of type "Callable[[Arg(int, 'arg')], None]", not "Callable[[int], None]" +assert_type(myfunc, Callable[[int], None]) # E: Expression is of type "def myfunc(arg: int) -> None", not "Callable[[int], None]" [case testAssertTypeOverload] from typing import assert_type, overload diff --git a/test-data/unit/check-callable.test b/test-data/unit/check-callable.test index 23db0bf50a4ec..0157ff3d2c536 100644 --- a/test-data/unit/check-callable.test +++ b/test-data/unit/check-callable.test @@ -654,13 +654,13 @@ class Call(Protocol): def f1() -> None: ... a1: Call = f1 # E: Incompatible types in assignment (expression has type "Callable[[], None]", variable has type "Call") \ - # N: "Call.__call__" has type "Callable[[Arg(int, 'x'), VarArg(Any), KwArg(Any)], None]" + # N: "Call.__call__" has type "def __call__(self, x: int, *args: Any, **kwargs: Any) -> None" def f2(x: str) -> None: ... a2: Call = f2 # E: Incompatible types in assignment (expression has type "Callable[[str], None]", variable has type "Call") \ - # N: "Call.__call__" has type "Callable[[Arg(int, 'x'), VarArg(Any), KwArg(Any)], None]" + # N: "Call.__call__" has type "def __call__(self, x: int, *args: Any, **kwargs: Any) -> None" def f3(y: int) -> None: ... a3: Call = f3 # E: Incompatible types in assignment (expression has type "Callable[[int], None]", variable has type "Call") \ - # N: "Call.__call__" has type "Callable[[Arg(int, 'x'), VarArg(Any), KwArg(Any)], None]" + # N: "Call.__call__" has type "def __call__(self, x: int, *args: Any, **kwargs: Any) -> None" def f4(x: int) -> None: ... a4: Call = f4 diff --git a/test-data/unit/check-functions.test b/test-data/unit/check-functions.test index 7fa34a398ea05..1882f235f7e30 100644 --- a/test-data/unit/check-functions.test +++ b/test-data/unit/check-functions.test @@ -107,30 +107,30 @@ if int(): [case testSubtypingFunctionsDoubleCorrespondence] def l(x) -> None: ... def r(__x, *, x) -> None: ... -r = l # E: Incompatible types in assignment (expression has type "Callable[[Any], None]", variable has type "Callable[[Any, NamedArg(Any, 'x')], None]") +r = l # E: Incompatible types in assignment (expression has type "Callable[[Any], None]", variable has type "def r(Any, /, *, x: Any) -> None") [case testSubtypingFunctionsDoubleCorrespondenceNamedOptional] def l(x) -> None: ... def r(__x, *, x = 1) -> None: ... -r = l # E: Incompatible types in assignment (expression has type "Callable[[Any], None]", variable has type "Callable[[Any, DefaultNamedArg(Any, 'x')], None]") +r = l # E: Incompatible types in assignment (expression has type "Callable[[Any], None]", variable has type "def r(Any, /, *, x: Any = ...) -> None") [case testSubtypingFunctionsDoubleCorrespondenceBothNamedOptional] def l(x = 1) -> None: ... def r(__x, *, x = 1) -> None: ... -r = l # E: Incompatible types in assignment (expression has type "Callable[[Any], None]", variable has type "Callable[[Any, DefaultNamedArg(Any, 'x')], None]") +r = l # E: Incompatible types in assignment (expression has type "Callable[[Any], None]", variable has type "def r(Any, /, *, x: Any = ...) -> None") [case testSubtypingFunctionsTrivialSuffixRequired] def l(__x) -> None: ... def r(x, *args, **kwargs) -> None: ... -r = l # E: Incompatible types in assignment (expression has type "Callable[[Any], None]", variable has type "Callable[[Arg(Any, 'x'), VarArg(Any), KwArg(Any)], None]") +r = l # E: Incompatible types in assignment (expression has type "Callable[[Any], None]", variable has type "def r(x: Any, *args: Any, **kwargs: Any) -> None") [builtins fixtures/dict.pyi] [case testSubtypingFunctionsTrivialSuffixOptional] def l(__x = 1) -> None: ... def r(x = 1, *args, **kwargs) -> None: ... -r = l # E: Incompatible types in assignment (expression has type "Callable[[DefaultArg(Any)], None]", variable has type "Callable[[DefaultArg(Any, 'x'), VarArg(Any), KwArg(Any)], None]") +r = l # E: Incompatible types in assignment (expression has type "def l(Any = ..., /) -> None", variable has type "def r(x: Any = ..., *args: Any, **kwargs: Any) -> None") [builtins fixtures/dict.pyi] [case testSubtypingFunctionsRequiredLeftArgNotPresent] @@ -170,13 +170,13 @@ if int(): if int(): ff_nonames = f_nonames # reset if int(): - ff = ff_nonames # E: Incompatible types in assignment (expression has type "Callable[[int, str], None]", variable has type "Callable[[Arg(int, 'a'), Arg(str, 'b')], None]") + ff = ff_nonames # E: Incompatible types in assignment (expression has type "Callable[[int, str], None]", variable has type "def f(a: int, b: str) -> None") if int(): ff = f # reset if int(): - gg = ff # E: Incompatible types in assignment (expression has type "Callable[[Arg(int, 'a'), Arg(str, 'b')], None]", variable has type "Callable[[Arg(int, 'a'), DefaultArg(str, 'b')], None]") + gg = ff # E: Incompatible types in assignment (expression has type "def f(a: int, b: str) -> None", variable has type "def g(a: int, b: str = ...) -> None") if int(): - gg = hh # E: Incompatible types in assignment (expression has type "Callable[[Arg(int, 'aa'), DefaultArg(str, 'b')], None]", variable has type "Callable[[Arg(int, 'a'), DefaultArg(str, 'b')], None]") + gg = hh # E: Incompatible types in assignment (expression has type "def h(aa: int, b: str = ...) -> None", variable has type "def g(a: int, b: str = ...) -> None") [case testSubtypingFunctionsArgsKwargs] from typing import Any, Callable @@ -245,7 +245,7 @@ gg = g if int(): ff = g if int(): - gg = f # E: Incompatible types in assignment (expression has type "Callable[[int, str], None]", variable has type "Callable[[Arg(int, 'a'), Arg(str, 'b')], None]") + gg = f # E: Incompatible types in assignment (expression has type "Callable[[int, str], None]", variable has type "def g(a: int, b: str) -> None") [case testLackOfNamesFastparse] def f(__a: int, __b: str) -> None: pass @@ -257,7 +257,7 @@ gg = g if int(): ff = g if int(): - gg = f # E: Incompatible types in assignment (expression has type "Callable[[int, str], None]", variable has type "Callable[[Arg(int, 'a'), Arg(str, 'b')], None]") + gg = f # E: Incompatible types in assignment (expression has type "Callable[[int, str], None]", variable has type "def g(a: int, b: str) -> None") [case testFunctionTypeCompatibilityWithOtherTypes] # flags: --no-strict-optional @@ -2016,12 +2016,12 @@ def isf_unnamed(__i: int, __s: str) -> str: int_str_fun = isf int_str_fun = isf_unnamed -int_named_str_fun = isf_unnamed # E: Incompatible types in assignment (expression has type "Callable[[int, str], str]", variable has type "Callable[[int, Arg(str, 's')], str]") +int_named_str_fun = isf_unnamed # E: Incompatible types in assignment (expression has type "Callable[[int, str], str]", variable has type "def (int, /, s: str) -> str") int_opt_str_fun = iosf int_str_fun = iosf -int_opt_str_fun = isf # E: Incompatible types in assignment (expression has type "Callable[[Arg(int, 'ii'), Arg(str, 'ss')], str]", variable has type "Callable[[int, DefaultArg(str)], str]") +int_opt_str_fun = isf # E: Incompatible types in assignment (expression has type "def isf(ii: int, ss: str) -> str", variable has type "def (int, str = ..., /) -> str") -int_named_str_fun = isf # E: Incompatible types in assignment (expression has type "Callable[[Arg(int, 'ii'), Arg(str, 'ss')], str]", variable has type "Callable[[int, Arg(str, 's')], str]") +int_named_str_fun = isf # E: Incompatible types in assignment (expression has type "def isf(ii: int, ss: str) -> str", variable has type "def (int, /, s: str) -> str") int_named_str_fun = iosf [builtins fixtures/dict.pyi] @@ -2076,7 +2076,7 @@ def g4(*, y: int) -> str: pass f(g1) f(g2) f(g3) -f(g4) # E: Argument 1 to "f" has incompatible type "Callable[[NamedArg(int, 'y')], str]"; expected "Callable[..., int]" +f(g4) # E: Argument 1 to "f" has incompatible type "def g4(*, y: int) -> str"; expected "Callable[..., int]" [case testCallableWithArbitraryArgsSubtypingWithGenericFunc] from typing import Callable, TypeVar @@ -2238,7 +2238,7 @@ def g(x, y): pass def h(x): pass def j(y) -> Any: pass f = h -f = j # E: Incompatible types in assignment (expression has type "Callable[[Arg(Any, 'y')], Any]", variable has type "Callable[[Arg(Any, 'x')], Any]") +f = j # E: Incompatible types in assignment (expression has type "def j(y: Any) -> Any", variable has type "def f(x: Any) -> Any") f = g # E: Incompatible types in assignment (expression has type "Callable[[Any, Any], Any]", variable has type "Callable[[Any], Any]") [case testRedefineFunction2] @@ -3531,7 +3531,7 @@ def decorator(f: Callable[P, None]) -> Callable[[Callable[P, A]], None]: def key(x: int) -> None: ... def fn_b(b: int) -> B: ... -decorator(key)(fn_b) # E: Argument 1 has incompatible type "Callable[[Arg(int, 'b')], B]"; expected "Callable[[Arg(int, 'x')], A]" +decorator(key)(fn_b) # E: Argument 1 has incompatible type "def fn_b(b: int) -> B"; expected "def (x: int) -> A" def decorator2(f: Callable[P, None]) -> Callable[ [Callable[P, Awaitable[None]]], @@ -3542,7 +3542,7 @@ def decorator2(f: Callable[P, None]) -> Callable[ def key2(x: int) -> None: ... -@decorator2(key2) # E: Argument 1 has incompatible type "Callable[[Arg(int, 'y')], Coroutine[Any, Any, None]]"; expected "Callable[[Arg(int, 'x')], Awaitable[None]]" +@decorator2(key2) # E: Argument 1 has incompatible type "def foo2(y: int) -> Coroutine[Any, Any, None]"; expected "def (x: int) -> Awaitable[None]" async def foo2(y: int) -> None: ... @@ -3552,7 +3552,7 @@ class Parent: class Child(Parent): method_without: Callable[[], "Child"] - method_with: Callable[[str], "Child"] # E: Incompatible types in assignment (expression has type "Callable[[str], Child]", base class "Parent" defined the type as "Callable[[Arg(str, 'param')], Parent]") + method_with: Callable[[str], "Child"] # E: Incompatible types in assignment (expression has type "Callable[[str], Child]", base class "Parent" defined the type as "def method_with(self, param: str) -> Parent") [builtins fixtures/tuple.pyi] [case testDistinctFormattingUnion] @@ -3562,7 +3562,7 @@ from mypy_extensions import Arg def f(x: Callable[[Arg(int, 'x')], None]) -> None: pass y: Callable[[Union[int, str]], None] -f(y) # E: Argument 1 to "f" has incompatible type "Callable[[Union[int, str]], None]"; expected "Callable[[Arg(int, 'x')], None]" +f(y) # E: Argument 1 to "f" has incompatible type "Callable[[Union[int, str]], None]"; expected "def (x: int) -> None" [builtins fixtures/tuple.pyi] [case testAbstractOverloadsWithoutImplementationAllowed] diff --git a/test-data/unit/check-functools.test b/test-data/unit/check-functools.test index fa2cacda275db..650928b1a5edb 100644 --- a/test-data/unit/check-functools.test +++ b/test-data/unit/check-functools.test @@ -162,7 +162,7 @@ def takes_callable_int(f: Callable[..., int]) -> None: ... def takes_callable_str(f: Callable[..., str]) -> None: ... takes_callable_int(p1) takes_callable_str(p1) # E: Argument 1 to "takes_callable_str" has incompatible type "partial[int]"; expected "Callable[..., str]" \ - # N: "partial[int].__call__" has type "Callable[[VarArg(Any), KwArg(Any)], int]" + # N: "partial[int].__call__" has type "def __call__(__self, *args: Any, **kwargs: Any) -> int" p2 = functools.partial(foo, 1) p2("a") # OK @@ -386,7 +386,7 @@ q: partial[bool] = partial(generic, resulting_type=str) # E: Argument "resultin pc: Callable[..., str] = partial(generic, resulting_type=str) qc: Callable[..., bool] = partial(generic, resulting_type=str) # E: Incompatible types in assignment (expression has type "partial[str]", variable has type "Callable[..., bool]") \ - # N: "partial[str].__call__" has type "Callable[[VarArg(Any), KwArg(Any)], str]" + # N: "partial[str].__call__" has type "def __call__(__self, *args: Any, **kwargs: Any) -> str" [builtins fixtures/tuple.pyi] [case testFunctoolsPartialNestedPartial] @@ -697,11 +697,11 @@ use_int_callable(partial(func_b, b="")) use_func_callable(partial(func_b, b="")) use_int_callable(partial(func_c, b="")) use_func_callable(partial(func_c, b="")) -use_int_callable(partial(func_fn, b="")) # E: Argument 1 to "use_int_callable" has incompatible type "partial[Callable[[VarArg(Any), KwArg(Any)], Any]]"; expected "Callable[[int], int]" \ - # N: "partial[Callable[[VarArg(Any), KwArg(Any)], Any]].__call__" has type "Callable[[VarArg(Any), KwArg(Any)], Callable[[VarArg(Any), KwArg(Any)], Any]]" +use_int_callable(partial(func_fn, b="")) # E: Argument 1 to "use_int_callable" has incompatible type "partial[def (*Any, **Any) -> Any]"; expected "Callable[[int], int]" \ + # N: "partial[def (*Any, **Any) -> Any].__call__" has type "def __call__(__self, *args: Any, **kwargs: Any) -> def (*Any, **Any) -> Any" use_func_callable(partial(func_fn, b="")) -use_int_callable(partial(func_fn_unpack, b="")) # E: Argument 1 to "use_int_callable" has incompatible type "partial[Callable[[VarArg(Any)], Any]]"; expected "Callable[[int], int]" \ - # N: "partial[Callable[[VarArg(Any)], Any]].__call__" has type "Callable[[VarArg(Any), KwArg(Any)], Callable[[VarArg(Any)], Any]]" +use_int_callable(partial(func_fn_unpack, b="")) # E: Argument 1 to "use_int_callable" has incompatible type "partial[def (*Any) -> Any]"; expected "Callable[[int], int]" \ + # N: "partial[def (*Any) -> Any].__call__" has type "def __call__(__self, *args: Any, **kwargs: Any) -> def (*Any) -> Any" use_func_callable(partial(func_fn_unpack, b="")) # But we should not erase typevars that aren't bound by function @@ -714,7 +714,7 @@ def outer_b(arg: Tb) -> None: reveal_type(partial(inner, b="")) # N: Revealed type is "functools.partial[Tb`-1]" use_int_callable(partial(inner, b="")) # E: Argument 1 to "use_int_callable" has incompatible type "partial[Tb]"; expected "Callable[[int], int]" \ - # N: "partial[Tb].__call__" has type "Callable[[VarArg(Any), KwArg(Any)], Tb]" + # N: "partial[Tb].__call__" has type "def __call__(__self, *args: Any, **kwargs: Any) -> Tb" def outer_c(arg: Tc) -> None: @@ -724,5 +724,5 @@ def outer_c(arg: Tc) -> None: reveal_type(partial(inner, b="")) # N: Revealed type is "functools.partial[builtins.int]" \ # N: Revealed type is "functools.partial[builtins.str]" use_int_callable(partial(inner, b="")) # E: Argument 1 to "use_int_callable" has incompatible type "partial[str]"; expected "Callable[[int], int]" \ - # N: "partial[str].__call__" has type "Callable[[VarArg(Any), KwArg(Any)], str]" + # N: "partial[str].__call__" has type "def __call__(__self, *args: Any, **kwargs: Any) -> str" [builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 94f65a950062c..5fbaa4f2c9044 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -6950,7 +6950,7 @@ p3 = functools.partial(foo, b="a") [out] tmp/a.py:8: note: Revealed type is "functools.partial[builtins.int]" tmp/a.py:13: error: Argument 1 to "takes_callable_str" has incompatible type "partial[int]"; expected "Callable[..., str]" -tmp/a.py:13: note: "partial[int].__call__" has type "Callable[[VarArg(Any), KwArg(Any)], int]" +tmp/a.py:13: note: "partial[int].__call__" has type "def __call__(__self, *args: Any, **kwargs: Any) -> int" tmp/a.py:18: error: Argument 1 to "foo" has incompatible type "int"; expected "str" tmp/a.py:19: error: Too many arguments for "foo" tmp/a.py:19: error: Argument 1 to "foo" has incompatible type "int"; expected "str" diff --git a/test-data/unit/check-inference.test b/test-data/unit/check-inference.test index 9ed9c5e9ec787..17f79dbcb663a 100644 --- a/test-data/unit/check-inference.test +++ b/test-data/unit/check-inference.test @@ -1109,7 +1109,7 @@ def f(*, x: int) -> int: ... def g(*, y: int) -> int: ... def h(*, x: int) -> int: ... -list_1 = [f, g] # E: List item 0 has incompatible type "Callable[[NamedArg(int, 'x')], int]"; expected "Callable[[NamedArg(int, 'y')], int]" +list_1 = [f, g] # E: List item 0 has incompatible type "def f(*, x: int) -> int"; expected "def g(*, y: int) -> int" list_2 = [f, h] [builtins fixtures/list.pyi] @@ -1434,7 +1434,7 @@ from typing import Callable def f(a: Callable[..., None] = lambda *a, **k: None): pass -def g(a: Callable[..., None] = lambda *a, **k: 1): # E: Incompatible default for argument "a" (default has type "Callable[[VarArg(Any), KwArg(Any)], int]", argument has type "Callable[..., None]") +def g(a: Callable[..., None] = lambda *a, **k: 1): # E: Incompatible default for argument "a" (default has type "def (*a: Any, **k: Any) -> int", argument has type "Callable[..., None]") pass [builtins fixtures/dict.pyi] @@ -3769,7 +3769,7 @@ def f(x: Call[T]) -> Tuple[T, T]: ... def g(__x: str) -> None: pass reveal_type(f(g)) # N: Revealed type is "tuple[Never, Never]" \ # E: Argument 1 to "f" has incompatible type "Callable[[str], None]"; expected "Call[Never]" \ - # N: "Call[Never].__call__" has type "Callable[[NamedArg(Never, 'x')], None]" + # N: "Call[Never].__call__" has type "def __call__(self, *, x: Never) -> None" [builtins fixtures/list.pyi] [case testCallableInferenceAgainstCallableNamedVsPosOnly] @@ -3785,7 +3785,7 @@ def f(x: Call[T]) -> Tuple[T, T]: ... def g(*, x: str) -> None: pass reveal_type(f(g)) # N: Revealed type is "tuple[Never, Never]" \ - # E: Argument 1 to "f" has incompatible type "Callable[[NamedArg(str, 'x')], None]"; expected "Call[Never]" \ + # E: Argument 1 to "f" has incompatible type "def g(*, x: str) -> None"; expected "Call[Never]" \ # N: "Call[Never].__call__" has type "Callable[[Never], None]" [builtins fixtures/list.pyi] @@ -3802,7 +3802,7 @@ def f(x: Call[T]) -> Tuple[T, T]: ... def g(**x: str) -> None: pass reveal_type(f(g)) # N: Revealed type is "tuple[Never, Never]" \ - # E: Argument 1 to "f" has incompatible type "Callable[[KwArg(str)], None]"; expected "Call[Never]" \ + # E: Argument 1 to "f" has incompatible type "def g(**x: str) -> None"; expected "Call[Never]" \ # N: "Call[Never].__call__" has type "Callable[[Never], None]" [builtins fixtures/list.pyi] @@ -3819,8 +3819,8 @@ def f(x: Call[T]) -> Tuple[T, T]: ... def g(*args: str) -> None: pass reveal_type(f(g)) # N: Revealed type is "tuple[Never, Never]" \ - # E: Argument 1 to "f" has incompatible type "Callable[[VarArg(str)], None]"; expected "Call[Never]" \ - # N: "Call[Never].__call__" has type "Callable[[NamedArg(Never, 'x')], None]" + # E: Argument 1 to "f" has incompatible type "def g(*args: str) -> None"; expected "Call[Never]" \ + # N: "Call[Never].__call__" has type "def __call__(self, *, x: Never) -> None" [builtins fixtures/list.pyi] [case testInferenceAgainstTypeVarActualBound] diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index 2b4f92c7c8195..bffd34782f517 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -443,7 +443,7 @@ reveal_type(register(lambda: f(1))) # N: Revealed type is "def ()" reveal_type(register(lambda x: f(x), x=1)) # N: Revealed type is "def (x: Literal[1]?)" register(lambda x: f(x)) # E: Cannot infer type of lambda \ # E: Argument 1 to "register" has incompatible type "Callable[[Any], None]"; expected "Callable[[], None]" -register(lambda x: f(x), y=1) # E: Argument 1 to "register" has incompatible type "Callable[[Arg(int, 'x')], None]"; expected "Callable[[Arg(int, 'y')], None]" +register(lambda x: f(x), y=1) # E: Argument 1 to "register" has incompatible type "def (x: int) -> None"; expected "def (y: int) -> None" reveal_type(register(lambda x: f(x), 1)) # N: Revealed type is "def (Literal[1]?)" reveal_type(register(lambda x, y: g(x, y), 1, "a")) # N: Revealed type is "def (Literal[1]?, Literal['a']?)" reveal_type(register(lambda x, y: g(x, y), 1, y="a")) # N: Revealed type is "def (Literal[1]?, y: Literal['a']?)" @@ -623,10 +623,10 @@ def expects_int_first(x: Callable[Concatenate[int, P], int]) -> None: ... # N: This is likely because "one" has named arguments: "x". Consider marking them positional-only def one(x: str) -> int: ... -@expects_int_first # E: Argument 1 to "expects_int_first" has incompatible type "Callable[[NamedArg(int, 'x')], int]"; expected "Callable[[int, NamedArg(int, 'x')], int]" +@expects_int_first # E: Argument 1 to "expects_int_first" has incompatible type "def two(*, x: int) -> int"; expected "def (int, /, *, x: int) -> int" def two(*, x: int) -> int: ... -@expects_int_first # E: Argument 1 to "expects_int_first" has incompatible type "Callable[[KwArg(int)], int]"; expected "Callable[[int, KwArg(int)], int]" +@expects_int_first # E: Argument 1 to "expects_int_first" has incompatible type "def three(**kwargs: int) -> int"; expected "def (int, /, **kwargs: int) -> int" def three(**kwargs: int) -> int: ... @expects_int_first # Accepted @@ -2154,7 +2154,7 @@ reveal_type(submit( # N: Revealed type is "__main__.Result" backend="asyncio", )) submit( - run, # E: Argument 1 to "submit" has incompatible type "Callable[[Callable[[], R], VarArg(object), DefaultNamedArg(str, 'backend')], R]"; expected "Callable[[Callable[[], Result], int], Result]" + run, # E: Argument 1 to "submit" has incompatible type "def [R] run(func: Callable[[], R], *args: object, backend: str = ...) -> R"; expected "Callable[[Callable[[], Result], int], Result]" run_portal, backend=int(), ) diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index e7971cd5b5d83..fd7f0c3449daf 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -1907,7 +1907,7 @@ reveal_type(apply_gen(Add5())) # N: Revealed type is "builtins.int" def apply_str(f: Callable[[str], int], x: str) -> int: return f(x) apply_str(Add5(), 'a') # E: Argument 1 to "apply_str" has incompatible type "Add5"; expected "Callable[[str], int]" \ - # N: "Add5.__call__" has type "Callable[[Arg(int, 'x')], int]" + # N: "Add5.__call__" has type "def __call__(self, x: int) -> int" [builtins fixtures/isinstancelist.pyi] [case testMoreComplexCallableStructuralSubtyping] @@ -1923,10 +1923,10 @@ class Bad1: class Bad2: def __call__(self, y: int, *rest: str) -> int: pass call_soon(Good()) -call_soon(Bad1()) # E: Argument 1 to "call_soon" has incompatible type "Bad1"; expected "Callable[[int, VarArg(str)], int]" \ - # N: "Bad1.__call__" has type "Callable[[Arg(int, 'x'), VarArg(int)], int]" -call_soon(Bad2()) # E: Argument 1 to "call_soon" has incompatible type "Bad2"; expected "Callable[[int, VarArg(str)], int]" \ - # N: "Bad2.__call__" has type "Callable[[Arg(int, 'y'), VarArg(str)], int]" +call_soon(Bad1()) # E: Argument 1 to "call_soon" has incompatible type "Bad1"; expected "def (x: int, *str) -> int" \ + # N: "Bad1.__call__" has type "def __call__(self, x: int, *rest: int) -> int" +call_soon(Bad2()) # E: Argument 1 to "call_soon" has incompatible type "Bad2"; expected "def (x: int, *str) -> int" \ + # N: "Bad2.__call__" has type "def __call__(self, y: int, *rest: str) -> int" [builtins fixtures/isinstancelist.pyi] [case testStructuralSupportForPartial] @@ -2486,8 +2486,8 @@ def func(caller: Caller) -> None: pass func(call) -func(bad) # E: Argument 1 to "func" has incompatible type "Callable[[int, VarArg(str)], None]"; expected "Caller" \ - # N: "Caller.__call__" has type "Callable[[Arg(str, 'x'), VarArg(int)], None]" +func(bad) # E: Argument 1 to "func" has incompatible type "def bad(x: int, *args: str) -> None"; expected "Caller" \ + # N: "Caller.__call__" has type "def __call__(self, x: str, *args: int) -> None" [builtins fixtures/tuple.pyi] [out] @@ -2525,7 +2525,7 @@ def func(caller: Caller) -> None: func(call) func(bad) # E: Argument 1 to "func" has incompatible type "Callable[[int], int]"; expected "Caller" \ - # N: "Caller.__call__" has type "Callable[[Arg(T, 'x')], T]" + # N: "Caller.__call__" has type "def [T] __call__(self, x: T) -> T" [builtins fixtures/tuple.pyi] [out] @@ -2546,7 +2546,7 @@ def func(caller: Caller) -> None: func(call) func(bad) # E: Argument 1 to "func" has incompatible type "Callable[[T], tuple[T, T]]"; expected "Caller" \ - # N: "Caller.__call__" has type "Callable[[Arg(int, 'x')], int]" + # N: "Caller.__call__" has type "def __call__(self, x: int) -> int" [builtins fixtures/tuple.pyi] [out] @@ -2586,8 +2586,8 @@ class Caller(Protocol): def bad(x: int, *args: str) -> None: pass -cb: Caller = bad # E: Incompatible types in assignment (expression has type "Callable[[int, VarArg(str)], None]", variable has type "Caller") \ - # N: "Caller.__call__" has type "Callable[[Arg(str, 'x'), VarArg(int)], None]" +cb: Caller = bad # E: Incompatible types in assignment (expression has type "def bad(x: int, *args: str) -> None", variable has type "Caller") \ + # N: "Caller.__call__" has type "def __call__(self, x: str, *args: int) -> None" [builtins fixtures/tuple.pyi] [out] @@ -2614,7 +2614,7 @@ def anon(caller: CallerAnon) -> None: func(call) func(bad) # E: Argument 1 to "func" has incompatible type "Callable[[str], None]"; expected "Caller" \ - # N: "Caller.__call__" has type "Callable[[Arg(str, 'x')], None]" + # N: "Caller.__call__" has type "def __call__(self, x: str) -> None" anon(bad) [out] @@ -2638,7 +2638,7 @@ b: Bad func(a) func(b) # E: Argument 1 to "func" has incompatible type "Bad"; expected "One" \ - # N: "One.__call__" has type "Callable[[Arg(str, 'x')], None]" + # N: "One.__call__" has type "def __call__(self, x: str) -> None" [out] [case testJoinProtocolCallback] @@ -3610,7 +3610,7 @@ test(C) # E: Argument 1 to "test" has incompatible type "type[C]"; expected "P" # N: def __call__(x: int, y: int) -> Any \ # N: Got: \ # N: def __init__(x: int, y: str) -> C \ - # N: "P.__call__" has type "Callable[[Arg(int, 'x'), Arg(int, 'y')], Any]" + # N: "P.__call__" has type "def __call__(self, x: int, y: int) -> Any" [case testProtocolClassObjectPureCallback] from typing import Any, ClassVar, Protocol @@ -3632,7 +3632,7 @@ test(C) # E: Argument 1 to "test" has incompatible type "type[C]"; expected "P" # N: def __call__(x: int, y: int) -> Any \ # N: Got: \ # N: def __init__(x: int, y: str) -> C \ - # N: "P.__call__" has type "Callable[[Arg(int, 'x'), Arg(int, 'y')], Any]" + # N: "P.__call__" has type "def __call__(self, x: int, y: int) -> Any" [builtins fixtures/type.pyi] [case testProtocolClassObjectCallableError] @@ -3655,7 +3655,7 @@ p: P = C # E: Incompatible types in assignment (expression has type "type[C]", # N: def __call__(app: int) -> Callable[[str], None] \ # N: Got: \ # N: def __init__(app: str) -> C \ - # N: "P.__call__" has type "Callable[[Arg(int, 'app')], Callable[[str], None]]" + # N: "P.__call__" has type "def __call__(self, app: int) -> Callable[[str], None]" [builtins fixtures/type.pyi] @@ -3814,7 +3814,7 @@ def f_good(t: S) -> S: return t g: C = f_bad # E: Incompatible types in assignment (expression has type "Callable[[int], int]", variable has type "C") \ - # N: "C.__call__" has type "Callable[[Arg(T, 't')], T]" + # N: "C.__call__" has type "def [T] __call__(self, t: T) -> T" g = f_good # OK [case testModuleAsProtocolImplementation] diff --git a/test-data/unit/check-python311.test b/test-data/unit/check-python311.test index 09c8d6082365c..c2a0bb09810aa 100644 --- a/test-data/unit/check-python311.test +++ b/test-data/unit/check-python311.test @@ -157,7 +157,7 @@ Alias1 = Callable[[*Ts], int] # E: Variable "__main__.Ts" is not valid as a typ x1: Alias1[int] # E: Bad number of arguments for type alias, expected 0, given 1 reveal_type(x1) # N: Revealed type is "def (*Any) -> builtins.int" x1 = good -x1 = bad # E: Incompatible types in assignment (expression has type "Callable[[VarArg(int), NamedArg(int, 'y')], int]", variable has type "Callable[[VarArg(Any)], int]") +x1 = bad # E: Incompatible types in assignment (expression has type "def bad(*x: int, y: int) -> int", variable has type "def (*Any) -> int") Alias2 = Callable[[*T], int] # E: "T" cannot be unpacked (must be tuple or TypeVarTuple) x2: Alias2[int] diff --git a/test-data/unit/check-statements.test b/test-data/unit/check-statements.test index 9ab68b32472d1..658bee76ef0de 100644 --- a/test-data/unit/check-statements.test +++ b/test-data/unit/check-statements.test @@ -2358,6 +2358,6 @@ describe(CAny()) describe(C()) describe(CNone()) describe(CWrong()) # E: Argument 1 to "describe" has incompatible type "CWrong"; expected "Callable[[], None]" \ - # N: "CWrong.__call__" has type "Callable[[Arg(int, 'x')], None]" + # N: "CWrong.__call__" has type "def __call__(self, x: int) -> None" describe(f) [builtins fixtures/isinstancelist.pyi] diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index 927a4f037a4a0..cb5029ee4e6d2 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -518,15 +518,15 @@ call(target=func, args=(0, 'foo')) call(target=func, args=('bar', 'foo')) # E: Argument "target" to "call" has incompatible type "Callable[[int, str], None]"; expected "Callable[[str, str], None]" call(target=func, args=(True, 'foo', 0)) # E: Argument "target" to "call" has incompatible type "Callable[[int, str], None]"; expected "Callable[[bool, str, int], None]" call(target=func, args=(0, 0, 'foo')) # E: Argument "target" to "call" has incompatible type "Callable[[int, str], None]"; expected "Callable[[int, int, str], None]" -call(target=func, args=vargs) # E: Argument "target" to "call" has incompatible type "Callable[[int, str], None]"; expected "Callable[[VarArg(int)], None]" +call(target=func, args=vargs) # E: Argument "target" to "call" has incompatible type "Callable[[int, str], None]"; expected "def (*int) -> None" # NOTE: This behavior may be a bit contentious, it is maybe inconsistent with our handling of # PEP646 but consistent with our handling of callable constraints. -call(target=func2, args=vargs) # E: Argument "target" to "call" has incompatible type "Callable[[int, int], None]"; expected "Callable[[VarArg(int)], None]" +call(target=func2, args=vargs) # E: Argument "target" to "call" has incompatible type "Callable[[int, int], None]"; expected "def (*int) -> None" call(target=func3, args=vargs) call(target=func3, args=(0,1)) -call(target=func3, args=(0,'foo')) # E: Argument "target" to "call" has incompatible type "Callable[[VarArg(int)], None]"; expected "Callable[[int, str], None]" -call(target=func3, args=vargs_str) # E: Argument "target" to "call" has incompatible type "Callable[[VarArg(int)], None]"; expected "Callable[[VarArg(str)], None]" +call(target=func3, args=(0,'foo')) # E: Argument "target" to "call" has incompatible type "def func3(*args: int) -> None"; expected "Callable[[int, str], None]" +call(target=func3, args=vargs_str) # E: Argument "target" to "call" has incompatible type "def func3(*args: int) -> None"; expected "def (*str) -> None" [builtins fixtures/tuple.pyi] [case testTypeVarTuplePep646CallableWithPrefixSuffix] @@ -1903,7 +1903,7 @@ def foo3(func: Callable[[int, Unpack[Args2]], T], *args: Unpack[Args2]) -> T: return submit2(func, 1, *args) def foo_bad(func: Callable[[Unpack[Args2]], T], *args: Unpack[Args2]) -> T: - return submit2(func, 1, *args) # E: Argument 1 to "submit2" has incompatible type "Callable[[VarArg(Unpack[Args2])], T]"; expected "Callable[[int, VarArg(Unpack[Args2])], T]" + return submit2(func, 1, *args) # E: Argument 1 to "submit2" has incompatible type "def (*Unpack[Args2]) -> T"; expected "def (int, /, *Unpack[Args2]) -> T" [builtins fixtures/tuple.pyi] [case testTypeVarTupleParamSpecInteraction] @@ -2321,8 +2321,8 @@ higher_order(good2) higher_order(ok1) higher_order(ok2) -higher_order(bad1) # E: Argument 1 to "higher_order" has incompatible type "Callable[[NamedArg(str, 'd')], int]"; expected "Callable[[VarArg(Any)], Any]" -higher_order(bad2) # E: Argument 1 to "higher_order" has incompatible type "Callable[[KwArg(None)], None]"; expected "Callable[[VarArg(Any)], Any]" +higher_order(bad1) # E: Argument 1 to "higher_order" has incompatible type "def bad1(*, d: str) -> int"; expected "def (*Any) -> Any" +higher_order(bad2) # E: Argument 1 to "higher_order" has incompatible type "def bad2(**kwargs: None) -> None"; expected "def (*Any) -> Any" [builtins fixtures/tuple.pyi] [case testAliasToCallableWithUnpack2] @@ -2338,10 +2338,10 @@ def bad3(*, d: str) -> int: ... def bad4(**kwargs: None) -> None: ... higher_order(good) -higher_order(bad1) # E: Argument 1 to "higher_order" has incompatible type "Callable[[str, int], None]"; expected "Callable[[int, str, VarArg(Unpack[tuple[Unpack[tuple[Any, ...]], int]])], Any]" -higher_order(bad2) # E: Argument 1 to "higher_order" has incompatible type "Callable[[bytes, VarArg(int)], str]"; expected "Callable[[int, str, VarArg(Unpack[tuple[Unpack[tuple[Any, ...]], int]])], Any]" -higher_order(bad3) # E: Argument 1 to "higher_order" has incompatible type "Callable[[NamedArg(str, 'd')], int]"; expected "Callable[[int, str, VarArg(Unpack[tuple[Unpack[tuple[Any, ...]], int]])], Any]" -higher_order(bad4) # E: Argument 1 to "higher_order" has incompatible type "Callable[[KwArg(None)], None]"; expected "Callable[[int, str, VarArg(Unpack[tuple[Unpack[tuple[Any, ...]], int]])], Any]" +higher_order(bad1) # E: Argument 1 to "higher_order" has incompatible type "Callable[[str, int], None]"; expected "def (int, str, /, *Unpack[tuple[Unpack[tuple[Any, ...]], int]]) -> Any" +higher_order(bad2) # E: Argument 1 to "higher_order" has incompatible type "def bad2(c: bytes, *args: int) -> str"; expected "def (int, str, /, *Unpack[tuple[Unpack[tuple[Any, ...]], int]]) -> Any" +higher_order(bad3) # E: Argument 1 to "higher_order" has incompatible type "def bad3(*, d: str) -> int"; expected "def (int, str, /, *Unpack[tuple[Unpack[tuple[Any, ...]], int]]) -> Any" +higher_order(bad4) # E: Argument 1 to "higher_order" has incompatible type "def bad4(**kwargs: None) -> None"; expected "def (int, str, /, *Unpack[tuple[Unpack[tuple[Any, ...]], int]]) -> Any" [builtins fixtures/tuple.pyi] [case testAliasToCallableWithUnpackInvalid] @@ -2357,7 +2357,7 @@ Alias = Callable[[Unpack[T]], int] # E: "T" cannot be unpacked (must be tuple o x: Alias[int] reveal_type(x) # N: Revealed type is "def (*Any) -> builtins.int" x = good -x = bad # E: Incompatible types in assignment (expression has type "Callable[[VarArg(int), NamedArg(int, 'y')], int]", variable has type "Callable[[VarArg(Any)], int]") +x = bad # E: Incompatible types in assignment (expression has type "def bad(*x: int, y: int) -> int", variable has type "def (*Any) -> int") [builtins fixtures/tuple.pyi] [case testTypeVarTupleInvariant] @@ -2443,7 +2443,7 @@ class CM(Generic[R]): ... def cm(fn: Callable[P, List[R]]) -> Callable[P, CM[R]]: ... Ts = TypeVarTuple("Ts") -@cm # E: Argument 1 to "cm" has incompatible type "Callable[[VarArg(Unpack[Ts])], tuple[Unpack[Ts]]]"; expected "Callable[[VarArg(Never)], list[Never]]" +@cm # E: Argument 1 to "cm" has incompatible type "def [Ts`-1] test(*args: Unpack[Ts]) -> tuple[Unpack[Ts]]"; expected "def (*args: Never) -> list[Never]" def test(*args: Unpack[Ts]) -> Tuple[Unpack[Ts]]: ... reveal_type(test) # N: Revealed type is "def (*args: Never) -> __main__.CM[Never]" diff --git a/test-data/unit/check-varargs.test b/test-data/unit/check-varargs.test index 680021a166f2b..3b80b9e8829aa 100644 --- a/test-data/unit/check-varargs.test +++ b/test-data/unit/check-varargs.test @@ -660,7 +660,7 @@ x: Callable[[int], None] def f(*x: int) -> None: pass def g(*x: str) -> None: pass x = f -x = g # E: Incompatible types in assignment (expression has type "Callable[[VarArg(str)], None]", variable has type "Callable[[int], None]") +x = g # E: Incompatible types in assignment (expression has type "def g(*x: str) -> None", variable has type "Callable[[int], None]") [builtins fixtures/list.pyi] [out] diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index 2069d082df178..7dfcf7447b619 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -616,7 +616,7 @@ def f(*args: str) -> str: return args[0] map(f, ['x']) map(f, [1]) [out] -_program.py:4: error: Argument 1 to "map" has incompatible type "Callable[[VarArg(str)], str]"; expected "Callable[[int], str]" +_program.py:4: error: Argument 1 to "map" has incompatible type "def f(*args: str) -> str"; expected "Callable[[int], str]" [case testMapStr] import typing From 630a1439f2f3226f20ef744c32ba3f03bda58ecb Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 3 Nov 2025 01:54:05 -0800 Subject: [PATCH 369/424] Run ubuntu mypyc tests on 3.10 (#20169) from Emma https://github.com/python/mypy/pull/20155#issuecomment-3475902175 --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 07b4b3f030204..6fe8257480733 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -85,8 +85,8 @@ jobs: toxenv: py tox_extra_args: "-n 3 mypyc/test/test_run.py mypyc/test/test_external.py" - - name: mypyc runtime tests with py313-ubuntu - python: '3.13' + - name: mypyc runtime tests with py310-ubuntu + python: '3.10' os: ubuntu-latest toxenv: py tox_extra_args: "-n 3 mypyc/test/test_run.py mypyc/test/test_external.py" From 2809328d9f86f4d3c434998d0a2338931362bec0 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 3 Nov 2025 10:25:14 +0000 Subject: [PATCH 370/424] Consistently raise ValueError on corrupted cache data and test more (#20153) Test random and arbitrary cache data. Deserialization should fail in a predictable manner. --- mypyc/lib-rt/librt_internal.c | 14 +++++++ mypyc/test-data/run-classes.test | 68 ++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+) diff --git a/mypyc/lib-rt/librt_internal.c b/mypyc/lib-rt/librt_internal.c index 6cae63cfadcb1..eaf451eff22ba 100644 --- a/mypyc/lib-rt/librt_internal.c +++ b/mypyc/lib-rt/librt_internal.c @@ -319,6 +319,11 @@ read_str_internal(PyObject *data) { CPyTagged tagged_size = _read_short_int(data, first); if (tagged_size == CPY_INT_TAG) return NULL; + if ((Py_ssize_t)tagged_size < 0) { + // Fail fast for invalid/tampered data. + PyErr_SetString(PyExc_ValueError, "invalid str size"); + return NULL; + } Py_ssize_t size = tagged_size >> 1; // Read string content. char *buf = ((BufferObject *)data)->buf; @@ -437,6 +442,11 @@ read_bytes_internal(PyObject *data) { CPyTagged tagged_size = _read_short_int(data, first); if (tagged_size == CPY_INT_TAG) return NULL; + if ((Py_ssize_t)tagged_size < 0) { + // Fail fast for invalid/tampered data. + PyErr_SetString(PyExc_ValueError, "invalid bytes size"); + return NULL; + } Py_ssize_t size = tagged_size >> 1; // Read bytes content. char *buf = ((BufferObject *)data)->buf; @@ -601,6 +611,10 @@ read_int_internal(PyObject *data) { Py_ssize_t size_and_sign = _read_short_int(data, first); if (size_and_sign == CPY_INT_TAG) return CPY_INT_TAG; + if ((Py_ssize_t)size_and_sign < 0) { + PyErr_SetString(PyExc_ValueError, "invalid int data"); + return CPY_INT_TAG; + } bool sign = (size_and_sign >> 1) & 1; Py_ssize_t size = size_and_sign >> 2; diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index b02d10446800c..0805da184e1a8 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -5359,3 +5359,71 @@ def test_deletable_attr() -> None: assert i.del_counter == 1 test_deletable_attr() + +[case testBufferCorruptedData_librt_internal] +from librt.internal import ( + Buffer, read_bool, read_str, read_float, read_int, read_tag, read_bytes +) +from random import randbytes + +def check(data: bytes) -> None: + b = Buffer(data) + try: + while True: + read_bool(b) + except ValueError: + pass + b = Buffer(data) + read_tag(b) # Always succeeds + try: + while True: + read_int(b) + except ValueError: + pass + b = Buffer(data) + try: + while True: + read_str(b) + except ValueError: + pass + b = Buffer(data) + try: + while True: + read_bytes(b) + except ValueError: + pass + b = Buffer(data) + try: + while True: + read_float(b) + except ValueError: + pass + +import time + +def test_read_corrupted_data() -> None: + # Test various deterministic byte sequences (1 to 4 bytes). + t0 = time.time() + for a in range(256): + check(bytes([a])) + for a in range(256): + for b in range(256): + check(bytes([a, b])) + for a in range(32): + for b in range(48): + for c in range(48): + check(bytes([a, b, c])) + for a in range(32): + for b in (0, 5, 17, 34): + for c in (0, 5, 17, 34): + for d in (0, 5, 17, 34): + check(bytes([a, b, c, d])) + # Also test some random data. + for i in range(20000): + data = randbytes(16) + try: + check(data) + except BaseException as e: + print("RANDOMIZED TEST FAILURE -- please open an issue with the following context:") + print(">>>", e, data) + raise From 1b7e717ecc56cd13d76bc110a1db2796e8b3c918 Mon Sep 17 00:00:00 2001 From: David Foster Date: Mon, 3 Nov 2025 02:53:50 -0800 Subject: [PATCH 371/424] [PEP 747] Recognize TypeForm[T] type and values (#9773) (#19596) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _(This PR replaces an earlier draft of the same feature: https://github.com/python/mypy/pull/18690 )_ Feedback from @JukkaL integrated since the last PR, by commit title: * Apply feedback: Change MAYBE_UNRECOGNIZED_STR_TYPEFORM from unaccompanied note to standalone error * Apply feedback: Refactor extract save/restore of SemanticAnalyzer state to a new context manager * Apply feedback: Suppress SyntaxWarnings when parsing strings as types at the _most-targeted_ location * Apply feedback: Add TypeForm profiling counters to SemanticAnalyzer and the --dump-build-stats option * Increase efficiency of quick rejection heuristic from 85.8% -> 99.6% in SemanticAnalyzer.try_parse_as_type_expression() * Apply feedback: Recognize assignment to union of TypeForm with non-TypeForm * Apply feedback: Alter primitives.pyi fixture rather than tuple.pyi and dict.pyi Feedback NOT integrated, with rationale: * ✖️ Add tests related to recursive types * Recursive cases are already well-covered by tests related to TypeType (is_type_form=False). * I _did_ find an infinite recursion bug affecting garden-variety `Type[...]`, which I can fix in a separate PR. * ✖️ Define `TypeForm(...)` in value contexts as a regular function like `Callable[[TypeForm[T]], TypeForm[T]]` rather than as a special expression node (TypeFormExpr). * The special expression node allows mypy to print out _better error messages_ when a user puts an invalid type expression inside `TypeForm(...)`. See case 4 of testTypeFormExpression in check-typeform.test There is one commit unrelated to the core function of this PR that could be split to a separate PR: * Allow TypeAlias and PlaceholderNode to be stringified/printed Closes #9773 --- _(Most of the following description is copied from the original PR, **except for the text in bold**)_ Implements the [TypeForm PEP 747](https://peps.python.org/pep-0747/), as an opt-in feature enabled by the CLI flag `--enable-incomplete-feature=TypeForm`. Implementation approach: * The `TypeForm[T]` is represented as a type using the existing `TypeType` class, with an `is_type_form=True` constructor parameter. `Type[C]` continues to be represented using `TypeType`, but with `is_type_form=False` (the default). * Recognizing a type expression literal such as `int | str` requires parsing an `Expression` as a type expression. Only the SemanticAnalyzer pass has the ability to parse **arbitrary** type expressions **(including stringified annotations)**, using `SemanticAnalyzer.expr_to_analyzed_type()`. **(I've extended the `TypeChecker` pass to parse all kinds of type expressions except stringified annotations, using the new `TypeCheckerAsSemanticAnalyzer` adapter.)** * Therefore during the SemanticAnalyzer pass, at certain syntactic locations (i.e. assignment r-values, callable arguments, returned expressions), the analyzer tries to parse the `Expression` it is looking at using `try_parse_as_type_expression()` - a new function - and stores the result (a `Type`) in `{IndexExpr, OpExpr, StrExpr}.as_type` - a new attribute. * During the later TypeChecker pass, when looking at an `Expression` to determine its type, if the expression is in a type context that expects some kind of `TypeForm[...]` and the expression was successfully parsed as a type expression by the earlier SemanticAnalyzer pass **(or can be parsed as a type expression immediately during the type checker pass)**, the expression will be given the type `TypeForm[expr.as_type]` rather than using the regular type inference rules for a value expression. * Key relationships between `TypeForm[T]`, `Type[C]`, and `object` types are defined in the visitors powering `is_subtype`, `join_types`, and `meet_types`. * The `TypeForm(T)` expression is recognized as a `TypeFormExpr` and has the return type `TypeForm[T]`. * The new test suite in `check-typeform.test` is a good reference to the expected behaviors for operations that interact with `TypeForm` in some way. Controversial parts of this PR, in @davidfstr 's opinion: * Type form literals **containing stringified annotations** are only recognized in certain syntactic locations (and not ALL possible locations). Namely they are recognized as (1) assignment r-values, (2) callable expression arguments, and (3) as returned expressions, but nowhere else. For example they aren't recognized in expressions like `dict_with_typx_keys[int | str]`. **Attempting to use stringified annotations in other locations will emit a MAYBE_UNRECOGNIZED_STR_TYPEFORM error.** * The existing `TypeType` class is now used to represent BOTH the `Type[T]` and `TypeForm[T]` types, rather than introducing a distinct subclass of `Type` to represent the `TypeForm[T]` type. This was done to simplify logic that manipulates both `Type[T]` and `TypeForm[T]` values, since they are both manipulated in very similar ways. * The "normalized" form of `TypeForm[X | Y]` - as returned by `TypeType.make_normalized()` - is just `TypeForm[X | Y]` rather than `TypeForm[X] | TypeForm[Y]`, differing from the normalization behavior of `Type[X | Y]`. --- docs/source/error_code_list.rst | 59 ++ misc/analyze_typeform_stats.py | 91 ++ mypy/checker.py | 114 ++- mypy/checkexpr.py | 124 ++- mypy/copytype.py | 2 +- mypy/erasetype.py | 4 +- mypy/errorcodes.py | 5 + mypy/evalexpr.py | 3 + mypy/expandtype.py | 2 +- mypy/fastparse.py | 19 +- mypy/join.py | 6 +- mypy/literals.py | 4 + mypy/meet.py | 18 +- mypy/messages.py | 6 +- mypy/mixedtraverser.py | 5 + mypy/nodes.py | 49 +- mypy/options.py | 3 +- mypy/semanal.py | 257 +++++- mypy/semanal_main.py | 11 + mypy/server/astdiff.py | 2 +- mypy/server/astmerge.py | 5 + mypy/server/deps.py | 5 + mypy/server/subexpr.py | 5 + mypy/strconv.py | 9 + mypy/subtypes.py | 72 +- mypy/traverser.py | 44 + mypy/treetransform.py | 4 + mypy/type_visitor.py | 4 +- mypy/typeanal.py | 23 +- mypy/typeops.py | 2 +- mypy/types.py | 55 +- mypy/visitor.py | 7 + mypyc/irbuild/visitor.py | 4 + test-data/unit/check-fastparse.test | 15 + test-data/unit/check-typeform.test | 842 ++++++++++++++++++ test-data/unit/fixtures/primitives.pyi | 4 + test-data/unit/fixtures/typing-full.pyi | 1 + test-data/unit/lib-stub/typing_extensions.pyi | 2 + 38 files changed, 1814 insertions(+), 73 deletions(-) create mode 100644 misc/analyze_typeform_stats.py create mode 100644 test-data/unit/check-typeform.test diff --git a/docs/source/error_code_list.rst b/docs/source/error_code_list.rst index 229230eda4caa..d4e2c83323ac6 100644 --- a/docs/source/error_code_list.rst +++ b/docs/source/error_code_list.rst @@ -1286,6 +1286,65 @@ type must be a subtype of the original type:: def g(x: object) -> TypeIs[str]: # OK ... +.. _code-maybe-unrecognized-str-typeform: + +String appears in a context which expects a TypeForm [maybe-unrecognized-str-typeform] +-------------------------------------------------------------------------------------- + +TypeForm literals may contain string annotations: + +.. code-block:: python + + typx1: TypeForm = str | None + typx2: TypeForm = 'str | None' # OK + typx3: TypeForm = 'str' | None # OK + +However TypeForm literals containing a string annotation can only be recognized +by mypy in the following locations: + +.. code-block:: python + + typx_var: TypeForm = 'str | None' # assignment r-value + + def func(typx_param: TypeForm) -> TypeForm: + return 'str | None' # returned expression + + func('str | None') # callable's argument + +If you try to use a string annotation in some other location +which expects a TypeForm, the string value will always be treated as a ``str`` +even if a ``TypeForm`` would be more appropriate and this error code +will be generated: + +.. code-block:: python + + # Error: TypeForm containing a string annotation cannot be recognized here. Surround with TypeForm(...) to recognize. [maybe-unrecognized-str-typeform] + # Error: List item 0 has incompatible type "str"; expected "TypeForm[Any]" [list-item] + list_of_typx: list[TypeForm] = ['str | None', float] + +Fix the error by surrounding the entire type with ``TypeForm(...)``: + +.. code-block:: python + + list_of_typx: list[TypeForm] = [TypeForm('str | None'), float] # OK + +Similarly, if you try to use a string literal in a location which expects a +TypeForm, this error code will be generated: + +.. code-block:: python + + dict_of_typx = {'str_or_none': TypeForm(str | None)} + # Error: TypeForm containing a string annotation cannot be recognized here. Surround with TypeForm(...) to recognize. [maybe-unrecognized-str-typeform] + list_of_typx: list[TypeForm] = [dict_of_typx['str_or_none']] + +Fix the error by adding ``# type: ignore[maybe-unrecognized-str-typeform]`` +to the line with the string literal: + +.. code-block:: python + + dict_of_typx = {'str_or_none': TypeForm(str | None)} + list_of_typx: list[TypeForm] = [dict_of_typx['str_or_none']] # type: ignore[maybe-unrecognized-str-typeform] + .. _code-misc: Miscellaneous checks [misc] diff --git a/misc/analyze_typeform_stats.py b/misc/analyze_typeform_stats.py new file mode 100644 index 0000000000000..0a540610bc620 --- /dev/null +++ b/misc/analyze_typeform_stats.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +""" +Analyze TypeForm parsing efficiency from mypy build stats. + +Usage: + python3 analyze_typeform_stats.py '' + python3 -m mypy --dump-build-stats file.py 2>&1 | python3 analyze_typeform_stats.py + +Example output: + TypeForm Expression Parsing Statistics: + ================================================== + Total calls to SA.try_parse_as_type_expression: 14,555 + Quick rejections (no full parse): 14,255 + Full parses attempted: 300 + - Successful: 248 + - Failed: 52 + + Efficiency Metrics: + - Quick rejection rate: 97.9% + - Full parse rate: 2.1% + - Full parse success rate: 82.7% + - Overall success rate: 1.7% + + Performance Implications: + - Expensive failed full parses: 52 (0.4% of all calls) + +See also: + - mypy/semanal.py: SemanticAnalyzer.try_parse_as_type_expression() + - mypy/semanal.py: DEBUG_TYPE_EXPRESSION_FULL_PARSE_FAILURES +""" + +import re +import sys + + +def analyze_stats(output: str) -> None: + """Parse mypy stats output and calculate TypeForm parsing efficiency.""" + + # Extract the three counters + total_match = re.search(r"type_expression_parse_count:\s*(\d+)", output) + success_match = re.search(r"type_expression_full_parse_success_count:\s*(\d+)", output) + failure_match = re.search(r"type_expression_full_parse_failure_count:\s*(\d+)", output) + + if not (total_match and success_match and failure_match): + print("Error: Could not find all required counters in output") + return + + total = int(total_match.group(1)) + successes = int(success_match.group(1)) + failures = int(failure_match.group(1)) + + full_parses = successes + failures + + print("TypeForm Expression Parsing Statistics:") + print("=" * 50) + print(f"Total calls to SA.try_parse_as_type_expression: {total:,}") + print(f"Quick rejections (no full parse): {total - full_parses:,}") + print(f"Full parses attempted: {full_parses:,}") + print(f" - Successful: {successes:,}") + print(f" - Failed: {failures:,}") + if total > 0: + print() + print("Efficiency Metrics:") + print(f" - Quick rejection rate: {((total - full_parses) / total * 100):.1f}%") + print(f" - Full parse rate: {(full_parses / total * 100):.1f}%") + print(f" - Full parse success rate: {(successes / full_parses * 100):.1f}%") + print(f" - Overall success rate: {(successes / total * 100):.1f}%") + print() + print("Performance Implications:") + print( + f" - Expensive failed full parses: {failures:,} ({(failures / total * 100):.1f}% of all calls)" + ) + + +if __name__ == "__main__": + if len(sys.argv) == 1: + # Read from stdin + output = sys.stdin.read() + elif len(sys.argv) == 2: + # Read from command line argument + output = sys.argv[1] + else: + print("Usage: python3 analyze_typeform_stats.py [mypy_output_with_stats]") + print("Examples:") + print( + " python3 -m mypy --dump-build-stats file.py 2>&1 | python3 analyze_typeform_stats.py" + ) + print(" python3 analyze_typeform_stats.py 'output_string'") + sys.exit(1) + + analyze_stats(output) diff --git a/mypy/checker.py b/mypy/checker.py index 63e128f78310a..f4746bc0c8862 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -155,6 +155,7 @@ from mypy.scope import Scope from mypy.semanal import is_trivial_body, refers_to_fullname, set_callable_name from mypy.semanal_enum import ENUM_BASES, ENUM_SPECIAL_PROPS +from mypy.semanal_shared import SemanticAnalyzerCoreInterface from mypy.sharedparse import BINARY_MAGIC_METHODS from mypy.state import state from mypy.subtypes import ( @@ -346,6 +347,8 @@ class TypeChecker(NodeVisitor[None], TypeCheckerSharedApi): tscope: Scope scope: CheckerScope + # Innermost enclosing type + type: TypeInfo | None # Stack of function return types return_types: list[Type] # Flags; true for dynamically typed functions @@ -423,6 +426,7 @@ def __init__( self.scope = CheckerScope(tree) self.binder = ConditionalTypeBinder(options) self.globals = tree.names + self.type = None self.return_types = [] self.dynamic_funcs = [] self.partial_types = [] @@ -2661,7 +2665,11 @@ def visit_class_def(self, defn: ClassDef) -> None: self.fail(message_registry.CANNOT_INHERIT_FROM_FINAL.format(base.name), defn) if not can_have_shared_disjoint_base(typ.bases): self.fail(message_registry.INCOMPATIBLE_DISJOINT_BASES.format(typ.name), defn) - with self.tscope.class_scope(defn.info), self.enter_partial_types(is_class=True): + with ( + self.tscope.class_scope(defn.info), + self.enter_partial_types(is_class=True), + self.enter_class(defn.info), + ): old_binder = self.binder self.binder = ConditionalTypeBinder(self.options) with self.binder.top_frame_context(): @@ -2729,6 +2737,15 @@ def visit_class_def(self, defn: ClassDef) -> None: self.check_enum(defn) infer_class_variances(defn.info) + @contextmanager + def enter_class(self, type: TypeInfo) -> Iterator[None]: + original_type = self.type + self.type = type + try: + yield + finally: + self.type = original_type + def check_final_deletable(self, typ: TypeInfo) -> None: # These checks are only for mypyc. Only perform some checks that are easier # to implement here than in mypyc. @@ -8023,7 +8040,9 @@ def add_any_attribute_to_type(self, typ: Type, name: str) -> Type: fallback = typ.fallback.copy_with_extra_attr(name, any_type) return typ.copy_modified(fallback=fallback) if isinstance(typ, TypeType) and isinstance(typ.item, Instance): - return TypeType.make_normalized(self.add_any_attribute_to_type(typ.item, name)) + return TypeType.make_normalized( + self.add_any_attribute_to_type(typ.item, name), is_type_form=typ.is_type_form + ) if isinstance(typ, TypeVarType): return typ.copy_modified( upper_bound=self.add_any_attribute_to_type(typ.upper_bound, name), @@ -8151,6 +8170,97 @@ def visit_global_decl(self, o: GlobalDecl, /) -> None: return None +class TypeCheckerAsSemanticAnalyzer(SemanticAnalyzerCoreInterface): + """ + Adapts TypeChecker to the SemanticAnalyzerCoreInterface, + allowing most type expressions to be parsed during the TypeChecker pass. + + See ExpressionChecker.try_parse_as_type_expression() to understand how this + class is used. + """ + + _chk: TypeChecker + _names: dict[str, SymbolTableNode] + did_fail: bool + + def __init__(self, chk: TypeChecker, names: dict[str, SymbolTableNode]) -> None: + self._chk = chk + self._names = names + self.did_fail = False + + def lookup_qualified( + self, name: str, ctx: Context, suppress_errors: bool = False + ) -> SymbolTableNode | None: + sym = self._names.get(name) + # All names being looked up should have been previously gathered, + # even if the related SymbolTableNode does not refer to a valid SymbolNode + assert sym is not None, name + return sym + + def lookup_fully_qualified(self, fullname: str, /) -> SymbolTableNode: + ret = self.lookup_fully_qualified_or_none(fullname) + assert ret is not None, fullname + return ret + + def lookup_fully_qualified_or_none(self, fullname: str, /) -> SymbolTableNode | None: + try: + return self._chk.lookup_qualified(fullname) + except KeyError: + return None + + def fail( + self, + msg: str, + ctx: Context, + serious: bool = False, + *, + blocker: bool = False, + code: ErrorCode | None = None, + ) -> None: + self.did_fail = True + + def note(self, msg: str, ctx: Context, *, code: ErrorCode | None = None) -> None: + pass + + def incomplete_feature_enabled(self, feature: str, ctx: Context) -> bool: + if feature not in self._chk.options.enable_incomplete_feature: + self.fail("__ignored__", ctx) + return False + return True + + def record_incomplete_ref(self) -> None: + pass + + def defer(self, debug_context: Context | None = None, force_progress: bool = False) -> None: + pass + + def is_incomplete_namespace(self, fullname: str) -> bool: + return False + + @property + def final_iteration(self) -> bool: + return True + + def is_future_flag_set(self, flag: str) -> bool: + return self._chk.tree.is_future_flag_set(flag) + + @property + def is_stub_file(self) -> bool: + return self._chk.tree.is_stub + + def is_func_scope(self) -> bool: + # Return arbitrary value. + # + # This method is currently only used to decide whether to pair + # a fail() message with a note() message or not. Both of those + # message types are ignored. + return False + + @property + def type(self) -> TypeInfo | None: + return self._chk.type + + class CollectArgTypeVarTypes(TypeTraverserVisitor): """Collects the non-nested argument types in a set.""" diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 3eb54579a0500..a06af690f8dbf 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -27,6 +27,7 @@ freshen_all_functions_type_vars, freshen_function_type_vars, ) +from mypy.exprtotype import TypeTranslationError, expr_to_unanalyzed_type from mypy.infer import ArgumentInferContext, infer_function_type_arguments, infer_type_arguments from mypy.literals import literal from mypy.maptype import map_instance_to_supertype @@ -43,6 +44,7 @@ LITERAL_TYPE, REVEAL_LOCALS, REVEAL_TYPE, + UNBOUND_IMPORTED, ArgKind, AssertTypeExpr, AssignmentExpr, @@ -68,11 +70,13 @@ LambdaExpr, ListComprehension, ListExpr, + MaybeTypeExpression, MemberExpr, MypyFile, NamedTupleExpr, NameExpr, NewTypeExpr, + NotParsed, OpExpr, OverloadedFuncDef, ParamSpecExpr, @@ -87,12 +91,14 @@ StrExpr, SuperExpr, SymbolNode, + SymbolTableNode, TempNode, TupleExpr, TypeAlias, TypeAliasExpr, TypeApplication, TypedDictExpr, + TypeFormExpr, TypeInfo, TypeVarExpr, TypeVarLikeExpr, @@ -101,6 +107,7 @@ Var, YieldExpr, YieldFromExpr, + get_member_expr_fullname, ) from mypy.options import PRECISE_TUPLE_TYPES from mypy.plugin import ( @@ -119,8 +126,14 @@ is_subtype, non_method_protocol_members, ) -from mypy.traverser import has_await_expression +from mypy.traverser import ( + all_name_and_member_expressions, + has_await_expression, + has_str_expression, +) +from mypy.tvar_scope import TypeVarLikeScope from mypy.typeanal import ( + TypeAnalyser, check_for_explicit_any, fix_instance, has_any_from_unimported_type, @@ -4692,6 +4705,10 @@ def visit_cast_expr(self, expr: CastExpr) -> Type: ) return target_type + def visit_type_form_expr(self, expr: TypeFormExpr) -> Type: + typ = expr.type + return TypeType.make_normalized(typ, line=typ.line, column=typ.column, is_type_form=True) + def visit_assert_type_expr(self, expr: AssertTypeExpr) -> Type: source_type = self.accept( expr.expr, @@ -6018,6 +6035,7 @@ def accept( old_is_callee = self.is_callee self.is_callee = is_callee try: + p_type_context = get_proper_type(type_context) if allow_none_return and isinstance(node, CallExpr): typ = self.visit_call_expr(node, allow_none_return=True) elif allow_none_return and isinstance(node, YieldFromExpr): @@ -6026,6 +6044,37 @@ def accept( typ = self.visit_conditional_expr(node, allow_none_return=True) elif allow_none_return and isinstance(node, AwaitExpr): typ = self.visit_await_expr(node, allow_none_return=True) + + elif ( + isinstance(p_type_context, TypeType) + and p_type_context.is_type_form + and (node_as_type := self.try_parse_as_type_expression(node)) is not None + ): + typ = TypeType.make_normalized( + node_as_type, + line=node_as_type.line, + column=node_as_type.column, + is_type_form=True, + ) # r-value type, when interpreted as a type expression + elif ( + isinstance(p_type_context, UnionType) + and any( + isinstance(p_item := get_proper_type(item), TypeType) and p_item.is_type_form + for item in p_type_context.items + ) + and (node_as_type := self.try_parse_as_type_expression(node)) is not None + ): + typ1 = TypeType.make_normalized( + node_as_type, + line=node_as_type.line, + column=node_as_type.column, + is_type_form=True, + ) + if is_subtype(typ1, p_type_context): + typ = typ1 # r-value type, when interpreted as a type expression + else: + typ2 = node.accept(self) + typ = typ2 # r-value type, when interpreted as a value expression # Deeply nested generic calls can deteriorate performance dramatically. # Although in most cases caching makes little difference, in worst case # it avoids exponential complexity. @@ -6047,7 +6096,7 @@ def accept( else: typ = self.accept_maybe_cache(node, type_context=type_context) else: - typ = node.accept(self) + typ = node.accept(self) # r-value type, when interpreted as a value expression except Exception as err: report_internal_error( err, self.chk.errors.file, node.line, self.chk.errors, self.chk.options @@ -6379,6 +6428,77 @@ def has_abstract_type(self, caller_type: ProperType, callee_type: ProperType) -> and not self.chk.allow_abstract_call ) + def try_parse_as_type_expression(self, maybe_type_expr: Expression) -> Type | None: + """Try to parse a value Expression as a type expression. + If success then return the type that it spells. + If fails then return None. + + A value expression that is parsable as a type expression may be used + where a TypeForm is expected to represent the spelled type. + + Unlike SemanticAnalyzer.try_parse_as_type_expression() + (used in the earlier SemanticAnalyzer pass), this function can only + recognize type expressions which contain no string annotations.""" + if not isinstance(maybe_type_expr, MaybeTypeExpression): + return None + + # Check whether has already been parsed as a type expression + # by SemanticAnalyzer.try_parse_as_type_expression(), + # perhaps containing a string annotation + if ( + isinstance(maybe_type_expr, (StrExpr, IndexExpr, OpExpr)) + and maybe_type_expr.as_type != NotParsed.VALUE + ): + return maybe_type_expr.as_type + + # If is potentially a type expression containing a string annotation, + # don't try to parse it because there isn't enough information + # available to the TypeChecker pass to resolve string annotations + if has_str_expression(maybe_type_expr): + self.chk.fail( + "TypeForm containing a string annotation cannot be recognized here. " + "Surround with TypeForm(...) to recognize.", + maybe_type_expr, + code=codes.MAYBE_UNRECOGNIZED_STR_TYPEFORM, + ) + return None + + # Collect symbols targeted by NameExprs and MemberExprs, + # to be looked up by TypeAnalyser when binding the + # UnboundTypes corresponding to those expressions. + (name_exprs, member_exprs) = all_name_and_member_expressions(maybe_type_expr) + sym_for_name = {e.name: SymbolTableNode(UNBOUND_IMPORTED, e.node) for e in name_exprs} | { + e_name: SymbolTableNode(UNBOUND_IMPORTED, e.node) + for e in member_exprs + if (e_name := get_member_expr_fullname(e)) is not None + } + + chk_sem = mypy.checker.TypeCheckerAsSemanticAnalyzer(self.chk, sym_for_name) + tpan = TypeAnalyser( + chk_sem, + # NOTE: Will never need to lookup type vars in this scope because + # SemanticAnalyzer.try_parse_as_type_expression() will have + # already recognized any type var referenced in a NameExpr. + # String annotations (which may also reference type vars) + # can't be resolved in the TypeChecker pass anyway. + TypeVarLikeScope(), # empty scope + self.plugin, + self.chk.options, + self.chk.tree, + self.chk.is_typeshed_stub, + ) + + try: + typ1 = expr_to_unanalyzed_type( + maybe_type_expr, self.chk.options, self.chk.is_typeshed_stub + ) + typ2 = typ1.accept(tpan) + if chk_sem.did_fail: + return None + return typ2 + except TypeTranslationError: + return None + def has_any_type(t: Type, ignore_in_type_obj: bool = False) -> bool: """Whether t contains an Any type""" diff --git a/mypy/copytype.py b/mypy/copytype.py index ecb1a89759b67..a890431a1772b 100644 --- a/mypy/copytype.py +++ b/mypy/copytype.py @@ -122,7 +122,7 @@ def visit_overloaded(self, t: Overloaded) -> ProperType: def visit_type_type(self, t: TypeType) -> ProperType: # Use cast since the type annotations in TypeType are imprecise. - return self.copy_common(t, TypeType(cast(Any, t.item))) + return self.copy_common(t, TypeType(cast(Any, t.item), is_type_form=t.is_type_form)) def visit_type_alias_type(self, t: TypeAliasType) -> ProperType: assert False, "only ProperTypes supported" diff --git a/mypy/erasetype.py b/mypy/erasetype.py index 6645bcf916d90..500d8fd5ae087 100644 --- a/mypy/erasetype.py +++ b/mypy/erasetype.py @@ -134,7 +134,9 @@ def visit_union_type(self, t: UnionType) -> ProperType: return make_simplified_union(erased_items) def visit_type_type(self, t: TypeType) -> ProperType: - return TypeType.make_normalized(t.item.accept(self), line=t.line) + return TypeType.make_normalized( + t.item.accept(self), line=t.line, is_type_form=t.is_type_form + ) def visit_type_alias_type(self, t: TypeAliasType) -> ProperType: raise RuntimeError("Type aliases should be expanded before accepting this visitor") diff --git a/mypy/errorcodes.py b/mypy/errorcodes.py index fbfa572b94397..785b6166b18bf 100644 --- a/mypy/errorcodes.py +++ b/mypy/errorcodes.py @@ -269,6 +269,11 @@ def __hash__(self) -> int: default_enabled=False, ) METACLASS: Final = ErrorCode("metaclass", "Ensure that metaclass is valid", "General") +MAYBE_UNRECOGNIZED_STR_TYPEFORM: Final = ErrorCode( + "maybe-unrecognized-str-typeform", + "Error when a string is used where a TypeForm is expected but a string annotation cannot be recognized", + "General", +) # Syntax errors are often blocking. SYNTAX: Final = ErrorCode("syntax", "Report syntax errors", "General") diff --git a/mypy/evalexpr.py b/mypy/evalexpr.py index e39c5840d47a8..218d50e37ec36 100644 --- a/mypy/evalexpr.py +++ b/mypy/evalexpr.py @@ -75,6 +75,9 @@ def visit_comparison_expr(self, o: mypy.nodes.ComparisonExpr) -> object: def visit_cast_expr(self, o: mypy.nodes.CastExpr) -> object: return o.expr.accept(self) + def visit_type_form_expr(self, o: mypy.nodes.TypeFormExpr) -> object: + return UNKNOWN + def visit_assert_type_expr(self, o: mypy.nodes.AssertTypeExpr) -> object: return o.expr.accept(self) diff --git a/mypy/expandtype.py b/mypy/expandtype.py index be1c2dd91e77c..891ea4d89a806 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -521,7 +521,7 @@ def visit_type_type(self, t: TypeType) -> Type: # union of instances or Any). Sadly we can't report errors # here yet. item = t.item.accept(self) - return TypeType.make_normalized(item) + return TypeType.make_normalized(item, is_type_form=t.is_type_form) def visit_type_alias_type(self, t: TypeAliasType) -> Type: # Target of the type alias cannot contain type variables (not bound by the type diff --git a/mypy/fastparse.py b/mypy/fastparse.py index c5e4589ec0256..0e7b418d03752 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -135,13 +135,18 @@ def ast3_parse( source: str | bytes, filename: str, mode: str, feature_version: int = PY_MINOR_VERSION ) -> AST: - return ast3.parse( - source, - filename, - mode, - type_comments=True, # This works the magic - feature_version=feature_version, - ) + # Ignore warnings that look like: + # :1: SyntaxWarning: invalid escape sequence '\.' + # because `source` could be anything, including literals like r'(re\.match)' + with warnings.catch_warnings(): + warnings.simplefilter("ignore", SyntaxWarning) + return ast3.parse( + source, + filename, + mode, + type_comments=True, # This works the magic + feature_version=feature_version, + ) if sys.version_info >= (3, 10): diff --git a/mypy/join.py b/mypy/join.py index 099df02680f06..0822ddbfd89aa 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -637,7 +637,11 @@ def visit_partial_type(self, t: PartialType) -> ProperType: def visit_type_type(self, t: TypeType) -> ProperType: if isinstance(self.s, TypeType): - return TypeType.make_normalized(join_types(t.item, self.s.item), line=t.line) + return TypeType.make_normalized( + join_types(t.item, self.s.item), + line=t.line, + is_type_form=self.s.is_type_form or t.is_type_form, + ) elif isinstance(self.s, Instance) and self.s.type.fullname == "builtins.type": return self.s else: diff --git a/mypy/literals.py b/mypy/literals.py index 5b0c46f4bee88..fd17e04714403 100644 --- a/mypy/literals.py +++ b/mypy/literals.py @@ -48,6 +48,7 @@ TypeAliasExpr, TypeApplication, TypedDictExpr, + TypeFormExpr, TypeVarExpr, TypeVarTupleExpr, UnaryExpr, @@ -244,6 +245,9 @@ def visit_slice_expr(self, e: SliceExpr) -> None: def visit_cast_expr(self, e: CastExpr) -> None: return None + def visit_type_form_expr(self, e: TypeFormExpr) -> None: + return None + def visit_assert_type_expr(self, e: AssertTypeExpr) -> None: return None diff --git a/mypy/meet.py b/mypy/meet.py index 1cb291ff90d50..42229f9b23c15 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -187,12 +187,24 @@ def narrow_declared_type(declared: Type, narrowed: Type) -> Type: elif isinstance(narrowed, TypeVarType) and is_subtype(narrowed.upper_bound, declared): return narrowed elif isinstance(declared, TypeType) and isinstance(narrowed, TypeType): - return TypeType.make_normalized(narrow_declared_type(declared.item, narrowed.item)) + return TypeType.make_normalized( + narrow_declared_type(declared.item, narrowed.item), + is_type_form=declared.is_type_form and narrowed.is_type_form, + ) elif ( isinstance(declared, TypeType) and isinstance(narrowed, Instance) and narrowed.type.is_metaclass() ): + if declared.is_type_form: + # The declared TypeForm[T] after narrowing must be a kind of + # type object at least as narrow as Type[T] + return narrow_declared_type( + TypeType.make_normalized( + declared.item, line=declared.line, column=declared.column, is_type_form=False + ), + original_narrowed, + ) # We'd need intersection types, so give up. return original_declared elif isinstance(declared, Instance): @@ -1115,7 +1127,9 @@ def visit_type_type(self, t: TypeType) -> ProperType: if isinstance(self.s, TypeType): typ = self.meet(t.item, self.s.item) if not isinstance(typ, NoneType): - typ = TypeType.make_normalized(typ, line=t.line) + typ = TypeType.make_normalized( + typ, line=t.line, is_type_form=self.s.is_type_form and t.is_type_form + ) return typ elif isinstance(self.s, Instance) and self.s.type.fullname == "builtins.type": return t diff --git a/mypy/messages.py b/mypy/messages.py index a9e8ee2e43abf..68b9b83572f12 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -2756,7 +2756,11 @@ def format_literal_value(typ: LiteralType) -> str: elif isinstance(typ, UninhabitedType): return "Never" elif isinstance(typ, TypeType): - return f"type[{format(typ.item)}]" + if typ.is_type_form: + type_name = "TypeForm" + else: + type_name = "type" + return f"{type_name}[{format(typ.item)}]" elif isinstance(typ, FunctionLike): func = typ if func.is_type_obj(): diff --git a/mypy/mixedtraverser.py b/mypy/mixedtraverser.py index f47d762934bc7..39fba49cf3c7a 100644 --- a/mypy/mixedtraverser.py +++ b/mypy/mixedtraverser.py @@ -15,6 +15,7 @@ TypeAliasStmt, TypeApplication, TypedDictExpr, + TypeFormExpr, TypeVarExpr, Var, WithStmt, @@ -109,6 +110,10 @@ def visit_cast_expr(self, o: CastExpr, /) -> None: super().visit_cast_expr(o) o.type.accept(self) + def visit_type_form_expr(self, o: TypeFormExpr, /) -> None: + super().visit_type_form_expr(o) + o.type.accept(self) + def visit_assert_type_expr(self, o: AssertTypeExpr, /) -> None: super().visit_assert_type_expr(o) o.type.accept(self) diff --git a/mypy/nodes.py b/mypy/nodes.py index 6cf984e5a2185..539995ce92295 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -61,6 +61,11 @@ from mypy.patterns import Pattern +@unique +class NotParsed(Enum): + VALUE = "NotParsed" + + class Context: """Base type for objects that are valid as error message locations.""" @@ -2005,15 +2010,20 @@ def accept(self, visitor: ExpressionVisitor[T]) -> T: class StrExpr(Expression): """String literal""" - __slots__ = ("value",) + __slots__ = ("value", "as_type") __match_args__ = ("value",) value: str # '' by default + # If this value expression can also be parsed as a valid type expression, + # represents the type denoted by the type expression. + # None means "is not a type expression". + as_type: NotParsed | mypy.types.Type | None def __init__(self, value: str) -> None: super().__init__() self.value = value + self.as_type = NotParsed.VALUE def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_str_expr(self) @@ -2314,7 +2324,7 @@ class IndexExpr(Expression): Also wraps type application such as List[int] as a special form. """ - __slots__ = ("base", "index", "method_type", "analyzed") + __slots__ = ("base", "index", "method_type", "analyzed", "as_type") __match_args__ = ("base", "index") @@ -2325,6 +2335,10 @@ class IndexExpr(Expression): # If not None, this is actually semantically a type application # Class[type, ...] or a type alias initializer. analyzed: TypeApplication | TypeAliasExpr | None + # If this value expression can also be parsed as a valid type expression, + # represents the type denoted by the type expression. + # None means "is not a type expression". + as_type: NotParsed | mypy.types.Type | None def __init__(self, base: Expression, index: Expression) -> None: super().__init__() @@ -2332,6 +2346,7 @@ def __init__(self, base: Expression, index: Expression) -> None: self.index = index self.method_type = None self.analyzed = None + self.as_type = NotParsed.VALUE def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_index_expr(self) @@ -2389,6 +2404,7 @@ class OpExpr(Expression): "right_always", "right_unreachable", "analyzed", + "as_type", ) __match_args__ = ("left", "op", "right") @@ -2404,6 +2420,10 @@ class OpExpr(Expression): right_unreachable: bool # Used for expressions that represent a type "X | Y" in some contexts analyzed: TypeAliasExpr | None + # If this value expression can also be parsed as a valid type expression, + # represents the type denoted by the type expression. + # None means "is not a type expression". + as_type: NotParsed | mypy.types.Type | None def __init__( self, op: str, left: Expression, right: Expression, analyzed: TypeAliasExpr | None = None @@ -2416,11 +2436,19 @@ def __init__( self.right_always = False self.right_unreachable = False self.analyzed = analyzed + self.as_type = NotParsed.VALUE def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_op_expr(self) +# Expression subtypes that could represent the root of a valid type expression. +# +# May have an "as_type" attribute to hold the type for a type expression parsed +# during the SemanticAnalyzer pass. +MaybeTypeExpression = (IndexExpr, MemberExpr, NameExpr, OpExpr, StrExpr) + + class ComparisonExpr(Expression): """Comparison expression (e.g. a < b > c < d).""" @@ -2498,6 +2526,23 @@ def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_cast_expr(self) +class TypeFormExpr(Expression): + """TypeForm(type) expression.""" + + __slots__ = ("type",) + + __match_args__ = ("type",) + + type: mypy.types.Type + + def __init__(self, typ: mypy.types.Type) -> None: + super().__init__() + self.type = typ + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_type_form_expr(self) + + class AssertTypeExpr(Expression): """Represents a typing.assert_type(expr, type) call.""" diff --git a/mypy/options.py b/mypy/options.py index 209759763a5ac..39490c9f0beeb 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -83,7 +83,8 @@ class BuildType: PRECISE_TUPLE_TYPES: Final = "PreciseTupleTypes" NEW_GENERIC_SYNTAX: Final = "NewGenericSyntax" INLINE_TYPEDDICT: Final = "InlineTypedDict" -INCOMPLETE_FEATURES: Final = frozenset((PRECISE_TUPLE_TYPES, INLINE_TYPEDDICT)) +TYPE_FORM: Final = "TypeForm" +INCOMPLETE_FEATURES: Final = frozenset((PRECISE_TUPLE_TYPES, INLINE_TYPEDDICT, TYPE_FORM)) COMPLETE_FEATURES: Final = frozenset((TYPE_VAR_TUPLE, UNPACK, NEW_GENERIC_SYNTAX)) diff --git a/mypy/semanal.py b/mypy/semanal.py index d7b50bd09496e..1fdea22e8962d 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -50,10 +50,11 @@ from __future__ import annotations +import re from collections.abc import Collection, Iterable, Iterator from contextlib import contextmanager from typing import Any, Callable, Final, TypeVar, cast -from typing_extensions import TypeAlias as _TypeAlias, TypeGuard +from typing_extensions import TypeAlias as _TypeAlias, TypeGuard, assert_never from mypy import errorcodes as codes, message_registry from mypy.constant_fold import constant_fold_expr @@ -136,6 +137,7 @@ ListExpr, Lvalue, MatchStmt, + MaybeTypeExpression, MemberExpr, MypyFile, NamedTupleExpr, @@ -172,6 +174,7 @@ TypeAliasStmt, TypeApplication, TypedDictExpr, + TypeFormExpr, TypeInfo, TypeParam, TypeVarExpr, @@ -190,7 +193,7 @@ type_aliases_source_versions, typing_extensions_aliases, ) -from mypy.options import Options +from mypy.options import TYPE_FORM, Options from mypy.patterns import ( AsPattern, ClassPattern, @@ -311,6 +314,13 @@ T = TypeVar("T") +# Whether to print diagnostic information for failed full parses +# in SemanticAnalyzer.try_parse_as_type_expression(). +# +# See also: misc/analyze_typeform_stats.py +DEBUG_TYPE_EXPRESSION_FULL_PARSE_FAILURES: Final = False + + FUTURE_IMPORTS: Final = { "__future__.nested_scopes": "nested_scopes", "__future__.generators": "generators", @@ -342,6 +352,22 @@ Tag: _TypeAlias = int +# Matches two words separated by whitespace, where each word lacks +# any symbols which have special meaning in a type expression. +# +# Any string literal matching this common pattern cannot be a valid +# type expression and can be ignored quickly when attempting to parse a +# string literal as a type expression. +_MULTIPLE_WORDS_NONTYPE_RE = re.compile(r'\s*[^\s.\'"|\[]+\s+[^\s.\'"|\[]') + +# Matches any valid Python identifier, including identifiers with Unicode characters. +# +# [^\d\W] = word character that is not a digit +# \w = word character +# \Z = match end of string; does not allow a trailing \n, unlike $ +_IDENTIFIER_RE = re.compile(r"^[^\d\W]\w*\Z", re.UNICODE) + + class SemanticAnalyzer( NodeVisitor[None], SemanticAnalyzerInterface, SemanticAnalyzerPluginInterface ): @@ -503,6 +529,11 @@ def __init__( self._function_type: Instance | None = None self._object_type: Instance | None = None + # TypeForm profiling counters + self.type_expression_parse_count: int = 0 # Total try_parse_as_type_expression calls + self.type_expression_full_parse_success_count: int = 0 # Successful full parses + self.type_expression_full_parse_failure_count: int = 0 # Failed full parses + # mypyc doesn't properly handle implementing an abstractproperty # with a regular attribute so we make them properties @property @@ -3272,6 +3303,7 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None: self.store_final_status(s) self.check_classvar(s) self.process_type_annotation(s) + self.analyze_rvalue_as_type_form(s) self.apply_dynamic_class_hook(s) if not s.type: self.process_module_assignment(s.lvalues, s.rvalue, s) @@ -3607,6 +3639,10 @@ def analyze_lvalues(self, s: AssignmentStmt) -> None: has_explicit_value=has_explicit_value, ) + def analyze_rvalue_as_type_form(self, s: AssignmentStmt) -> None: + if TYPE_FORM in self.options.enable_incomplete_feature: + self.try_parse_as_type_expression(s.rvalue) + def apply_dynamic_class_hook(self, s: AssignmentStmt) -> None: if not isinstance(s.rvalue, CallExpr): return @@ -5337,6 +5373,8 @@ def visit_return_stmt(self, s: ReturnStmt) -> None: self.fail('"return" not allowed in except* block', s, serious=True) if s.expr: s.expr.accept(self) + if TYPE_FORM in self.options.enable_incomplete_feature: + self.try_parse_as_type_expression(s.expr) self.statement = old def visit_raise_stmt(self, s: RaiseStmt) -> None: @@ -5856,10 +5894,31 @@ def visit_call_expr(self, expr: CallExpr) -> None: with self.allow_unbound_tvars_set(): for a in expr.args: a.accept(self) + elif refers_to_fullname(expr.callee, ("typing.TypeForm", "typing_extensions.TypeForm")): + # Special form TypeForm(...). + if not self.check_fixed_args(expr, 1, "TypeForm"): + return + # Translate first argument to an unanalyzed type. + try: + typ = self.expr_to_unanalyzed_type(expr.args[0]) + except TypeTranslationError: + self.fail("TypeForm argument is not a type", expr) + # Suppress future error: "" not callable + expr.analyzed = CastExpr(expr.args[0], AnyType(TypeOfAny.from_error)) + return + # Piggyback TypeFormExpr object to the CallExpr object; it takes + # precedence over the CallExpr semantics. + expr.analyzed = TypeFormExpr(typ) + expr.analyzed.line = expr.line + expr.analyzed.column = expr.column + expr.analyzed.accept(self) else: # Normal call expression. + calculate_type_forms = TYPE_FORM in self.options.enable_incomplete_feature for a in expr.args: a.accept(self) + if calculate_type_forms: + self.try_parse_as_type_expression(a) if ( isinstance(expr.callee, MemberExpr) @@ -6108,6 +6167,11 @@ def visit_cast_expr(self, expr: CastExpr) -> None: if analyzed is not None: expr.type = analyzed + def visit_type_form_expr(self, expr: TypeFormExpr) -> None: + analyzed = self.anal_type(expr.type) + if analyzed is not None: + expr.type = analyzed + def visit_assert_type_expr(self, expr: AssertTypeExpr) -> None: expr.expr.accept(self) analyzed = self.anal_type(expr.type) @@ -7699,6 +7763,195 @@ def visit_pass_stmt(self, o: PassStmt, /) -> None: def visit_singleton_pattern(self, o: SingletonPattern, /) -> None: return None + def try_parse_as_type_expression(self, maybe_type_expr: Expression) -> None: + """Try to parse a value Expression as a type expression. + If success then annotate the Expression with the type that it spells. + If fails then emit no errors and take no further action. + + A value expression that is parsable as a type expression may be used + where a TypeForm is expected to represent the spelled type. + + Unlike ExpressionChecker.try_parse_as_type_expression() + (used in the later TypeChecker pass), this function can recognize + ALL kinds of type expressions, including type expressions containing + string annotations. + + If the provided Expression will be parsable later in + ExpressionChecker.try_parse_as_type_expression(), this function will + skip parsing the Expression to improve performance, because the later + function is called many fewer times (i.e. only lazily in a rare TypeForm + type context) than this function is called (i.e. eagerly for EVERY + expression in certain syntactic positions). + """ + # Count every call to this method for profiling + self.type_expression_parse_count += 1 + + # Bail ASAP if the Expression matches a common pattern that cannot possibly + # be a valid type expression, because this function is called very frequently + if not isinstance(maybe_type_expr, MaybeTypeExpression): + return + # Check types in order from most common to least common, for best performance + if isinstance(maybe_type_expr, (NameExpr, MemberExpr)): + # Defer parsing to the later TypeChecker pass, + # and only lazily in contexts where a TypeForm is expected + return + elif isinstance(maybe_type_expr, StrExpr): + str_value = maybe_type_expr.value # cache + # Filter out string literals with common patterns that could not + # possibly be in a type expression + if _MULTIPLE_WORDS_NONTYPE_RE.match(str_value): + # A common pattern in string literals containing a sentence. + # But cannot be a type expression. + maybe_type_expr.as_type = None + return + # Filter out string literals which look like an identifier but + # cannot be a type expression, for a few common reasons + if _IDENTIFIER_RE.fullmatch(str_value): + sym = self.lookup(str_value, UnboundType(str_value), suppress_errors=True) + if sym is None: + # Does not refer to anything in the local symbol table + maybe_type_expr.as_type = None + return + else: # sym is not None + node = sym.node # cache + if isinstance(node, PlaceholderNode) and not node.becomes_typeinfo: + # Either: + # 1. f'Cannot resolve name "{t.name}" (possible cyclic definition)' + # 2. Reference to an unknown placeholder node. + maybe_type_expr.as_type = None + return + unbound_tvar_or_paramspec = ( + isinstance(node, (TypeVarExpr, TypeVarTupleExpr, ParamSpecExpr)) + and self.tvar_scope.get_binding(sym) is None + ) + if unbound_tvar_or_paramspec: + # Either: + # 1. unbound_tvar: 'Type variable "{}" is unbound' [codes.VALID_TYPE] + # 2. unbound_paramspec: f'ParamSpec "{name}" is unbound' [codes.VALID_TYPE] + maybe_type_expr.as_type = None + return + else: # does not look like an identifier + if '"' in str_value or "'" in str_value: + # Only valid inside a Literal[...] type + if "[" not in str_value: + # Cannot be a Literal[...] type + maybe_type_expr.as_type = None + return + elif str_value == "": + # Empty string is not a valid type + maybe_type_expr.as_type = None + return + elif isinstance(maybe_type_expr, IndexExpr): + if isinstance(maybe_type_expr.base, NameExpr): + if isinstance( + maybe_type_expr.base.node, Var + ) and not self.var_is_typing_special_form(maybe_type_expr.base.node): + # Leftmost part of IndexExpr refers to a Var. Not a valid type. + maybe_type_expr.as_type = None + return + elif isinstance(maybe_type_expr.base, MemberExpr): + next_leftmost = maybe_type_expr.base + while True: + leftmost = next_leftmost.expr + if not isinstance(leftmost, MemberExpr): + break + next_leftmost = leftmost + if isinstance(leftmost, NameExpr): + if isinstance(leftmost.node, Var) and not self.var_is_typing_special_form( + leftmost.node + ): + # Leftmost part of IndexExpr refers to a Var. Not a valid type. + maybe_type_expr.as_type = None + return + else: + # Leftmost part of IndexExpr is not a NameExpr. Not a valid type. + maybe_type_expr.as_type = None + return + else: + # IndexExpr base is neither a NameExpr nor MemberExpr. Not a valid type. + maybe_type_expr.as_type = None + return + elif isinstance(maybe_type_expr, OpExpr): + if maybe_type_expr.op != "|": + # Binary operators other than '|' never spell a valid type + maybe_type_expr.as_type = None + return + else: + assert_never(maybe_type_expr) + + with self.isolated_error_analysis(): + try: + t = self.expr_to_analyzed_type(maybe_type_expr) + if self.errors.is_errors(): + t = None + except TypeTranslationError: + # Not a type expression + t = None + + if DEBUG_TYPE_EXPRESSION_FULL_PARSE_FAILURES and t is None: + original_flushed_files = set(self.errors.flushed_files) # save + try: + errors = self.errors.new_messages() # capture + finally: + self.errors.flushed_files = original_flushed_files # restore + + print( + f"SA.try_parse_as_type_expression: Full parse failure: {maybe_type_expr}, errors={errors!r}" + ) + + # Count full parse attempts for profiling + if t is not None: + self.type_expression_full_parse_success_count += 1 + else: + self.type_expression_full_parse_failure_count += 1 + + maybe_type_expr.as_type = t + + @staticmethod + def var_is_typing_special_form(var: Var) -> bool: + return var.fullname.startswith("typing") and var.fullname in [ + "typing.Annotated", + "typing_extensions.Annotated", + "typing.Callable", + "typing.Literal", + "typing_extensions.Literal", + "typing.Optional", + "typing.TypeGuard", + "typing_extensions.TypeGuard", + "typing.TypeIs", + "typing_extensions.TypeIs", + "typing.Union", + ] + + @contextmanager + def isolated_error_analysis(self) -> Iterator[None]: + """ + Context manager for performing error analysis that should not + affect the main SemanticAnalyzer state. + + Upon entering this context, `self.errors` will start empty. + Within this context, you can analyze expressions for errors. + Upon exiting this context, the original `self.errors` will be restored, + and any errors collected during the analysis will be discarded. + """ + # Save state + original_errors = self.errors + original_num_incomplete_refs = self.num_incomplete_refs + original_progress = self.progress + original_deferred = self.deferred + original_deferral_debug_context_len = len(self.deferral_debug_context) + + self.errors = Errors(Options()) + try: + yield + finally: + # Restore state + self.errors = original_errors + self.num_incomplete_refs = original_num_incomplete_refs + self.progress = original_progress + self.deferred = original_deferred + del self.deferral_debug_context[original_deferral_debug_context_len:] + def replace_implicit_first_type(sig: FunctionLike, new: Type) -> FunctionLike: if isinstance(sig, CallableType): diff --git a/mypy/semanal_main.py b/mypy/semanal_main.py index 7301e9f9b9b3e..b2c43e6becb8b 100644 --- a/mypy/semanal_main.py +++ b/mypy/semanal_main.py @@ -107,6 +107,17 @@ def semantic_analysis_for_scc(graph: Graph, scc: list[str], errors: Errors) -> N if "builtins" in scc: cleanup_builtin_scc(graph["builtins"]) + # Report TypeForm profiling stats + if len(scc) >= 1: + # Get manager from any state in the SCC (they all share the same manager) + manager = graph[scc[0]].manager + analyzer = manager.semantic_analyzer + manager.add_stats( + type_expression_parse_count=analyzer.type_expression_parse_count, + type_expression_full_parse_success_count=analyzer.type_expression_full_parse_success_count, + type_expression_full_parse_failure_count=analyzer.type_expression_full_parse_failure_count, + ) + def cleanup_builtin_scc(state: State) -> None: """Remove imported names from builtins namespace. diff --git a/mypy/server/astdiff.py b/mypy/server/astdiff.py index 25542ce37588f..15d472b648866 100644 --- a/mypy/server/astdiff.py +++ b/mypy/server/astdiff.py @@ -519,7 +519,7 @@ def visit_partial_type(self, typ: PartialType) -> SnapshotItem: raise RuntimeError def visit_type_type(self, typ: TypeType) -> SnapshotItem: - return ("TypeType", snapshot_type(typ.item)) + return ("TypeType", snapshot_type(typ.item), typ.is_type_form) def visit_type_alias_type(self, typ: TypeAliasType) -> SnapshotItem: assert typ.alias is not None diff --git a/mypy/server/astmerge.py b/mypy/server/astmerge.py index cda1d20fb8e4f..56f2f935481c5 100644 --- a/mypy/server/astmerge.py +++ b/mypy/server/astmerge.py @@ -75,6 +75,7 @@ SymbolTable, TypeAlias, TypedDictExpr, + TypeFormExpr, TypeInfo, Var, ) @@ -291,6 +292,10 @@ def visit_cast_expr(self, node: CastExpr) -> None: super().visit_cast_expr(node) self.fixup_type(node.type) + def visit_type_form_expr(self, node: TypeFormExpr) -> None: + super().visit_type_form_expr(node) + self.fixup_type(node.type) + def visit_assert_type_expr(self, node: AssertTypeExpr) -> None: super().visit_assert_type_expr(node) self.fixup_type(node.type) diff --git a/mypy/server/deps.py b/mypy/server/deps.py index 076d95e2baf98..ba622329665ea 100644 --- a/mypy/server/deps.py +++ b/mypy/server/deps.py @@ -125,6 +125,7 @@ class 'mod.Cls'. This can also refer to an attribute inherited from a TypeAliasExpr, TypeApplication, TypedDictExpr, + TypeFormExpr, TypeInfo, TypeVarExpr, UnaryExpr, @@ -766,6 +767,10 @@ def visit_cast_expr(self, e: CastExpr) -> None: super().visit_cast_expr(e) self.add_type_dependencies(e.type) + def visit_type_form_expr(self, e: TypeFormExpr) -> None: + super().visit_type_form_expr(e) + self.add_type_dependencies(e.type) + def visit_assert_type_expr(self, e: AssertTypeExpr) -> None: super().visit_assert_type_expr(e) self.add_type_dependencies(e.type) diff --git a/mypy/server/subexpr.py b/mypy/server/subexpr.py index c94db44445dc3..013b936e8b7c9 100644 --- a/mypy/server/subexpr.py +++ b/mypy/server/subexpr.py @@ -28,6 +28,7 @@ StarExpr, TupleExpr, TypeApplication, + TypeFormExpr, UnaryExpr, YieldExpr, YieldFromExpr, @@ -122,6 +123,10 @@ def visit_cast_expr(self, e: CastExpr) -> None: self.add(e) super().visit_cast_expr(e) + def visit_type_form_expr(self, e: TypeFormExpr) -> None: + self.add(e) + super().visit_type_form_expr(e) + def visit_assert_type_expr(self, e: AssertTypeExpr) -> None: self.add(e) super().visit_assert_type_expr(e) diff --git a/mypy/strconv.py b/mypy/strconv.py index d1595139572ac..168a8bcffdc7e 100644 --- a/mypy/strconv.py +++ b/mypy/strconv.py @@ -213,6 +213,12 @@ def visit_nonlocal_decl(self, o: mypy.nodes.NonlocalDecl) -> str: def visit_decorator(self, o: mypy.nodes.Decorator) -> str: return self.dump([o.var, o.decorators, o.func], o) + def visit_type_alias(self, o: mypy.nodes.TypeAlias, /) -> str: + return self.dump([o.name, o.target, o.alias_tvars, o.no_args], o) + + def visit_placeholder_node(self, o: mypy.nodes.PlaceholderNode, /) -> str: + return self.dump([o.fullname], o) + # Statements def visit_block(self, o: mypy.nodes.Block) -> str: @@ -466,6 +472,9 @@ def visit_comparison_expr(self, o: mypy.nodes.ComparisonExpr) -> str: def visit_cast_expr(self, o: mypy.nodes.CastExpr) -> str: return self.dump([o.expr, o.type], o) + def visit_type_form_expr(self, o: mypy.nodes.TypeFormExpr) -> str: + return self.dump([o.type], o) + def visit_assert_type_expr(self, o: mypy.nodes.AssertTypeExpr) -> str: return self.dump([o.expr, o.type], o) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 7da258a827f33..c02ff068560b4 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -1108,38 +1108,50 @@ def visit_partial_type(self, left: PartialType) -> bool: def visit_type_type(self, left: TypeType) -> bool: right = self.right - if isinstance(right, TypeType): - return self._is_subtype(left.item, right.item) - if isinstance(right, Overloaded) and right.is_type_obj(): - # Same as in other direction: if it's a constructor callable, all - # items should belong to the same class' constructor, so it's enough - # to check one of them. - return self._is_subtype(left, right.items[0]) - if isinstance(right, CallableType): - if self.proper_subtype and not right.is_type_obj(): - # We can't accept `Type[X]` as a *proper* subtype of Callable[P, X] - # since this will break transitivity of subtyping. + if left.is_type_form: + if isinstance(right, TypeType): + if not right.is_type_form: + return False + return self._is_subtype(left.item, right.item) + if isinstance(right, Instance): + if right.type.fullname == "builtins.object": + return True return False - # This is unsound, we don't check the __init__ signature. - return self._is_subtype(left.item, right.ret_type) - if isinstance(right, Instance): - if right.type.fullname in ["builtins.object", "builtins.type"]: - # TODO: Strictly speaking, the type builtins.type is considered equivalent to - # Type[Any]. However, this would break the is_proper_subtype check in - # conditional_types for cases like isinstance(x, type) when the type - # of x is Type[int]. It's unclear what's the right way to address this. - return True - item = left.item - if isinstance(item, TypeVarType): - item = get_proper_type(item.upper_bound) - if isinstance(item, Instance): - if right.type.is_protocol and is_protocol_implementation( - item, right, proper_subtype=self.proper_subtype, class_obj=True - ): + return False + else: # not left.is_type_form + if isinstance(right, TypeType): + return self._is_subtype(left.item, right.item) + if isinstance(right, Overloaded) and right.is_type_obj(): + # Same as in other direction: if it's a constructor callable, all + # items should belong to the same class' constructor, so it's enough + # to check one of them. + return self._is_subtype(left, right.items[0]) + if isinstance(right, CallableType): + if self.proper_subtype and not right.is_type_obj(): + # We can't accept `Type[X]` as a *proper* subtype of Callable[P, X] + # since this will break transitivity of subtyping. + return False + # This is unsound, we don't check the __init__ signature. + return self._is_subtype(left.item, right.ret_type) + + if isinstance(right, Instance): + if right.type.fullname in ["builtins.object", "builtins.type"]: + # TODO: Strictly speaking, the type builtins.type is considered equivalent to + # Type[Any]. However, this would break the is_proper_subtype check in + # conditional_types for cases like isinstance(x, type) when the type + # of x is Type[int]. It's unclear what's the right way to address this. return True - metaclass = item.type.metaclass_type - return metaclass is not None and self._is_subtype(metaclass, right) - return False + item = left.item + if isinstance(item, TypeVarType): + item = get_proper_type(item.upper_bound) + if isinstance(item, Instance): + if right.type.is_protocol and is_protocol_implementation( + item, right, proper_subtype=self.proper_subtype, class_obj=True + ): + return True + metaclass = item.type.metaclass_type + return metaclass is not None and self._is_subtype(metaclass, right) + return False def visit_type_alias_type(self, left: TypeAliasType) -> bool: assert False, f"This should be never called, got {left}" diff --git a/mypy/traverser.py b/mypy/traverser.py index 3c249391bdf87..baf234cc1b251 100644 --- a/mypy/traverser.py +++ b/mypy/traverser.py @@ -76,6 +76,7 @@ TypeAliasStmt, TypeApplication, TypedDictExpr, + TypeFormExpr, TypeVarExpr, TypeVarTupleExpr, UnaryExpr, @@ -289,6 +290,9 @@ def visit_slice_expr(self, o: SliceExpr, /) -> None: def visit_cast_expr(self, o: CastExpr, /) -> None: o.expr.accept(self) + def visit_type_form_expr(self, o: TypeFormExpr, /) -> None: + pass + def visit_assert_type_expr(self, o: AssertTypeExpr, /) -> None: o.expr.accept(self) @@ -737,6 +741,11 @@ def visit_cast_expr(self, o: CastExpr, /) -> None: return super().visit_cast_expr(o) + def visit_type_form_expr(self, o: TypeFormExpr, /) -> None: + if not self.visit(o): + return + super().visit_type_form_expr(o) + def visit_assert_type_expr(self, o: AssertTypeExpr, /) -> None: if not self.visit(o): return @@ -935,6 +944,41 @@ def has_return_statement(fdef: FuncBase) -> bool: return seeker.found +class NameAndMemberCollector(TraverserVisitor): + def __init__(self) -> None: + super().__init__() + self.name_exprs: list[NameExpr] = [] + self.member_exprs: list[MemberExpr] = [] + + def visit_name_expr(self, o: NameExpr, /) -> None: + self.name_exprs.append(o) + super().visit_name_expr(o) + + def visit_member_expr(self, o: MemberExpr, /) -> None: + self.member_exprs.append(o) + super().visit_member_expr(o) + + +def all_name_and_member_expressions(node: Expression) -> tuple[list[NameExpr], list[MemberExpr]]: + v = NameAndMemberCollector() + node.accept(v) + return (v.name_exprs, v.member_exprs) + + +class StringSeeker(TraverserVisitor): + def __init__(self) -> None: + self.found = False + + def visit_str_expr(self, o: StrExpr, /) -> None: + self.found = True + + +def has_str_expression(node: Expression) -> bool: + v = StringSeeker() + node.accept(v) + return v.found + + class FuncCollectorBase(TraverserVisitor): def __init__(self) -> None: self.inside_func = False diff --git a/mypy/treetransform.py b/mypy/treetransform.py index 0abf98a52336b..f5af5fb777b58 100644 --- a/mypy/treetransform.py +++ b/mypy/treetransform.py @@ -83,6 +83,7 @@ TypeAliasExpr, TypeApplication, TypedDictExpr, + TypeFormExpr, TypeVarExpr, TypeVarTupleExpr, UnaryExpr, @@ -540,6 +541,9 @@ def visit_comparison_expr(self, node: ComparisonExpr) -> ComparisonExpr: def visit_cast_expr(self, node: CastExpr) -> CastExpr: return CastExpr(self.expr(node.expr), self.type(node.type)) + def visit_type_form_expr(self, node: TypeFormExpr) -> TypeFormExpr: + return TypeFormExpr(self.type(node.type)) + def visit_assert_type_expr(self, node: AssertTypeExpr) -> AssertTypeExpr: return AssertTypeExpr(self.expr(node.expr), self.type(node.type)) diff --git a/mypy/type_visitor.py b/mypy/type_visitor.py index 35846e7c3ddd3..1b38481ba0004 100644 --- a/mypy/type_visitor.py +++ b/mypy/type_visitor.py @@ -328,7 +328,9 @@ def visit_overloaded(self, t: Overloaded, /) -> Type: return Overloaded(items=items) def visit_type_type(self, t: TypeType, /) -> Type: - return TypeType.make_normalized(t.item.accept(self), line=t.line, column=t.column) + return TypeType.make_normalized( + t.item.accept(self), line=t.line, column=t.column, is_type_form=t.is_type_form + ) @abstractmethod def visit_type_alias_type(self, t: TypeAliasType, /) -> Type: diff --git a/mypy/typeanal.py b/mypy/typeanal.py index d7a07c9f48e32..06fa847c54345 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -49,7 +49,7 @@ check_arg_kinds, check_arg_names, ) -from mypy.options import INLINE_TYPEDDICT, Options +from mypy.options import INLINE_TYPEDDICT, TYPE_FORM, Options from mypy.plugin import AnalyzeTypeContext, Plugin, TypeAnalyzerPluginInterface from mypy.semanal_shared import ( SemanticAnalyzerCoreInterface, @@ -665,6 +665,23 @@ def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Typ self.fail(f'{type_str} can\'t contain "{bad_item_name}"', t, code=codes.VALID_TYPE) item = AnyType(TypeOfAny.from_error) return TypeType.make_normalized(item, line=t.line, column=t.column) + elif fullname in ("typing_extensions.TypeForm", "typing.TypeForm"): + if TYPE_FORM not in self.options.enable_incomplete_feature: + self.fail( + "TypeForm is experimental," + " must be enabled with --enable-incomplete-feature=TypeForm", + t, + ) + if len(t.args) == 0: + any_type = self.get_omitted_any(t) + return TypeType(any_type, line=t.line, column=t.column, is_type_form=True) + if len(t.args) != 1: + type_str = "TypeForm[...]" + self.fail( + type_str + " must have exactly one type argument", t, code=codes.VALID_TYPE + ) + item = self.anal_type(t.args[0]) + return TypeType.make_normalized(item, line=t.line, column=t.column, is_type_form=True) elif fullname == "typing.ClassVar": if self.nesting_level > 0: self.fail( @@ -1400,7 +1417,9 @@ def visit_ellipsis_type(self, t: EllipsisType) -> Type: return AnyType(TypeOfAny.from_error) def visit_type_type(self, t: TypeType) -> Type: - return TypeType.make_normalized(self.anal_type(t.item), line=t.line) + return TypeType.make_normalized( + self.anal_type(t.item), line=t.line, is_type_form=t.is_type_form + ) def visit_placeholder_type(self, t: PlaceholderType) -> Type: n = ( diff --git a/mypy/typeops.py b/mypy/typeops.py index 341c96c089315..d2f9f4da44e40 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -501,7 +501,7 @@ def erase_to_bound(t: Type) -> Type: return t.upper_bound if isinstance(t, TypeType): if isinstance(t.item, TypeVarType): - return TypeType.make_normalized(t.item.upper_bound) + return TypeType.make_normalized(t.item.upper_bound, is_type_form=t.is_type_form) return t diff --git a/mypy/types.py b/mypy/types.py index 6013f4c252988..1c43310224965 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -3495,11 +3495,14 @@ def serialize(self) -> JsonDict: class TypeType(ProperType): - """For types like Type[User]. + """For types like Type[User] or TypeForm[User | None]. - This annotates variables that are class objects, constrained by + Type[C] annotates variables that are class objects, constrained by the type argument. See PEP 484 for more details. + TypeForm[T] annotates variables that hold the result of evaluating + a type expression. See PEP 747 for more details. + We may encounter expressions whose values are specific classes; those are represented as callables (possibly overloaded) corresponding to the class's constructor's signature and returning @@ -3522,35 +3525,47 @@ class TypeType(ProperType): assumption). """ - __slots__ = ("item",) + __slots__ = ("item", "is_type_form") # This can't be everything, but it can be a class reference, # a generic class instance, a union, Any, a type variable... item: ProperType + # If True then this TypeType represents a TypeForm[T]. + # If False then this TypeType represents a Type[C]. + is_type_form: bool + def __init__( self, item: Bogus[Instance | AnyType | TypeVarType | TupleType | NoneType | CallableType], *, line: int = -1, column: int = -1, + is_type_form: bool = False, ) -> None: """To ensure Type[Union[A, B]] is always represented as Union[Type[A], Type[B]], item of type UnionType must be handled through make_normalized static method. """ super().__init__(line, column) self.item = item + self.is_type_form = is_type_form @staticmethod - def make_normalized(item: Type, *, line: int = -1, column: int = -1) -> ProperType: + def make_normalized( + item: Type, *, line: int = -1, column: int = -1, is_type_form: bool = False + ) -> ProperType: item = get_proper_type(item) - if isinstance(item, UnionType): - return UnionType.make_union( - [TypeType.make_normalized(union_item) for union_item in item.items], - line=line, - column=column, - ) - return TypeType(item, line=line, column=column) # type: ignore[arg-type] + if is_type_form: + # Don't convert TypeForm[X | Y] to (TypeForm[X] | TypeForm[Y]) + pass + else: + if isinstance(item, UnionType): + return UnionType.make_union( + [TypeType.make_normalized(union_item) for union_item in item.items], + line=line, + column=column, + ) + return TypeType(item, line=line, column=column, is_type_form=is_type_form) # type: ignore[arg-type] def accept(self, visitor: TypeVisitor[T]) -> T: return visitor.visit_type_type(self) @@ -3561,15 +3576,21 @@ def __hash__(self) -> int: def __eq__(self, other: object) -> bool: if not isinstance(other, TypeType): return NotImplemented - return self.item == other.item + return self.item == other.item and self.is_type_form == other.is_type_form def serialize(self) -> JsonDict: - return {".class": "TypeType", "item": self.item.serialize()} + return { + ".class": "TypeType", + "item": self.item.serialize(), + "is_type_form": self.is_type_form, + } @classmethod def deserialize(cls, data: JsonDict) -> Type: assert data[".class"] == "TypeType" - return TypeType.make_normalized(deserialize_type(data["item"])) + return TypeType.make_normalized( + deserialize_type(data["item"]), is_type_form=data["is_type_form"] + ) def write(self, data: Buffer) -> None: write_tag(data, TYPE_TYPE) @@ -3945,7 +3966,11 @@ def visit_ellipsis_type(self, t: EllipsisType, /) -> str: return "..." def visit_type_type(self, t: TypeType, /) -> str: - return f"type[{t.item.accept(self)}]" + if t.is_type_form: + type_name = "TypeForm" + else: + type_name = "type" + return f"{type_name}[{t.item.accept(self)}]" def visit_placeholder_type(self, t: PlaceholderType, /) -> str: return f"" diff --git a/mypy/visitor.py b/mypy/visitor.py index d1b2ca4164101..e150788ec3c16 100644 --- a/mypy/visitor.py +++ b/mypy/visitor.py @@ -79,6 +79,10 @@ def visit_comparison_expr(self, o: mypy.nodes.ComparisonExpr, /) -> T: def visit_cast_expr(self, o: mypy.nodes.CastExpr, /) -> T: pass + @abstractmethod + def visit_type_form_expr(self, o: mypy.nodes.TypeFormExpr, /) -> T: + pass + @abstractmethod def visit_assert_type_expr(self, o: mypy.nodes.AssertTypeExpr, /) -> T: pass @@ -511,6 +515,9 @@ def visit_comparison_expr(self, o: mypy.nodes.ComparisonExpr, /) -> T: def visit_cast_expr(self, o: mypy.nodes.CastExpr, /) -> T: raise NotImplementedError() + def visit_type_form_expr(self, o: mypy.nodes.TypeFormExpr, /) -> T: + raise NotImplementedError() + def visit_assert_type_expr(self, o: mypy.nodes.AssertTypeExpr, /) -> T: raise NotImplementedError() diff --git a/mypyc/irbuild/visitor.py b/mypyc/irbuild/visitor.py index 05a033c3e6ad4..dc81e95a2980e 100644 --- a/mypyc/irbuild/visitor.py +++ b/mypyc/irbuild/visitor.py @@ -73,6 +73,7 @@ TypeAliasStmt, TypeApplication, TypedDictExpr, + TypeFormExpr, TypeVarExpr, TypeVarTupleExpr, UnaryExpr, @@ -387,6 +388,9 @@ def visit_var(self, o: Var) -> None: def visit_cast_expr(self, o: CastExpr) -> Value: assert False, "CastExpr should have been handled in CallExpr" + def visit_type_form_expr(self, o: TypeFormExpr) -> Value: + assert False, "TypeFormExpr should have been handled in CallExpr" + def visit_assert_type_expr(self, o: AssertTypeExpr) -> Value: assert False, "AssertTypeExpr should have been handled in CallExpr" diff --git a/test-data/unit/check-fastparse.test b/test-data/unit/check-fastparse.test index 80d314333ddce..e7417d049442e 100644 --- a/test-data/unit/check-fastparse.test +++ b/test-data/unit/check-fastparse.test @@ -324,3 +324,18 @@ class Bla: def call() -> str: pass [builtins fixtures/module.pyi] + +[case testInvalidEscapeSequenceWarningsSuppressed] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +# Test that SyntaxWarnings for invalid escape sequences are suppressed +# when parsing potential type expressions containing regex patterns or +# similar strings. Callable arguments are always potential type expressions. +from typing import TypeForm + +def identity(typx: TypeForm) -> TypeForm: + return typx + +# This should not generate SyntaxWarning despite invalid escape sequence +identity(r"re\.match") # E: Argument 1 to "identity" has incompatible type "str"; expected "TypeForm[Any]" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] diff --git a/test-data/unit/check-typeform.test b/test-data/unit/check-typeform.test new file mode 100644 index 0000000000000..627067903a92d --- /dev/null +++ b/test-data/unit/check-typeform.test @@ -0,0 +1,842 @@ +-- TypeForm Type + +[case testRecognizesUnparameterizedTypeFormInAnnotation] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +typx: TypeForm = str +reveal_type(typx) # N: Revealed type is "TypeForm[Any]" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testRecognizesParameterizedTypeFormInAnnotation] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +typx: TypeForm[str] = str +reveal_type(typx) # N: Revealed type is "TypeForm[builtins.str]" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + + +-- Type Expression Location: Assignment + +[case testCanAssignTypeExpressionToTypeFormVariable] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +typx: TypeForm[str] = str +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testCanAssignTypeExpressionToUnionTypeFormVariable] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +typx: TypeForm[str | None] = str | None +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testCannotAssignTypeExpressionToTypeFormVariableWithIncompatibleItemType] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +typx: TypeForm[str] = int # E: Incompatible types in assignment (expression has type "TypeForm[int]", variable has type "TypeForm[str]") +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testCanAssignValueExpressionToTypeFormVariableIfValueIsATypeForm1] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +typx1: TypeForm = str +typx2: TypeForm = typx1 # looks like a type expression: name +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testCanAssignValueExpressionToTypeFormVariableIfValueIsATypeForm2] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +def identity_tf(x: TypeForm) -> TypeForm: + return x +typx1: TypeForm = str +typx2: TypeForm = identity_tf(typx1) # does not look like a type expression +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testCannotAssignValueExpressionToTypeFormVariableIfValueIsNotATypeForm] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +val: int = 42 +typx: TypeForm = val # E: Incompatible types in assignment (expression has type "int", variable has type "TypeForm[Any]") +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testCanAssignNoneTypeExpressionToTypeFormVariable] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +typx: TypeForm = None +reveal_type(typx) # N: Revealed type is "TypeForm[Any]" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testCanAssignTypeExpressionToTypeFormVariableDeclaredEarlier] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing import Type, TypeForm +typ: Type +typ = int | None # E: Incompatible types in assignment (expression has type "object", variable has type "type[Any]") +typx: TypeForm +typx = int | None +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testCanAssignTypeExpressionWithStringAnnotationToTypeFormVariable] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +typx: TypeForm[str | None] = 'str | None' +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + + +-- Type Expression Location: Function Parameter + +[case testCanPassTypeExpressionToTypeFormParameterInFunction] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +def is_type(typx: TypeForm) -> bool: + return isinstance(typx, type) +is_type(int | None) +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testCannotPassTypeExpressionToTypeParameter] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +def is_type(typ: type) -> bool: + return isinstance(typ, type) +is_type(int | None) # E: Argument 1 to "is_type" has incompatible type "object"; expected "type" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testCanPassTypeExpressionToTypeFormParameterInMethod] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +class C: + def is_type(self, typx: TypeForm) -> bool: + return isinstance(typx, type) +C().is_type(int | None) +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testCanPassTypeExpressionToTypeFormParameterInOverload] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing import overload, TypeForm +@overload +def is_type(typx: TypeForm) -> bool: ... +@overload +def is_type(typx: type) -> bool: ... +def is_type(typx): + return isinstance(typx, type) +is_type(int | None) +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testCanPassTypeExpressionToTypeFormParameterInDecorator] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing import Callable, TypeForm, TypeVar +P = TypeVar('P') +R = TypeVar('R') +def expects_type(typx: TypeForm) -> Callable[[Callable[[P], R]], Callable[[P], R]]: + def wrap(func: Callable[[P], R]) -> Callable[[P], R]: + func.expected_type = typx # type: ignore[attr-defined] + return func + return wrap +@expects_type(int | None) +def sum_ints(x: int | None) -> int: + return (x or 0) +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testCanPassTypeExpressionToTypeFormVarargsParameter] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing import Callable, ParamSpec, TypeForm, TypeVar +P = ParamSpec('P') +R = TypeVar('R') +def expects_types(*typxs: TypeForm) -> Callable[[Callable[P, R]], Callable[P, R]]: + def wrap(func: Callable[P, R]) -> Callable[P, R]: + func.expected_types = typxs # type: ignore[attr-defined] + return func + return wrap +@expects_types(int | None, int) +def sum_ints(x: int | None, y: int) -> tuple[int, int]: + return ((x or 0), y) +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testCanPassTypeExpressionWithStringAnnotationToTypeFormParameter] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +def is_type(typx: TypeForm) -> bool: + return isinstance(typx, type) +is_type('int | None') +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + + +-- Type Expression Location: Return Statement + +[case testCanReturnTypeExpressionInFunctionWithTypeFormReturnType] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +def maybe_int_type() -> TypeForm: + return int | None +reveal_type(maybe_int_type()) # N: Revealed type is "TypeForm[Any]" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testCanReturnTypeExpressionWithStringAnnotationInFunctionWithTypeFormReturnType] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +def maybe_int_type() -> TypeForm: + return 'int | None' +reveal_type(maybe_int_type()) # N: Revealed type is "TypeForm[Any]" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + + +-- Type Expression Location: Other + +-- In particular ensure that ExpressionChecker.try_parse_as_type_expression() in +-- the TypeChecker pass is able to parse types correctly even though it doesn't +-- have the same rich context as SemanticAnalyzer.try_parse_as_type_expression(). + +[case testTypeExpressionWithoutStringAnnotationRecognizedInOtherSyntacticLocations] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing import Dict, List, TypeForm +list_of_typx: List[TypeForm] = [int | str] +dict_with_typx_keys: Dict[TypeForm, int] = { + int | str: 1, + str | None: 2, +} +dict_with_typx_keys[int | str] + 1 +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testTypeExpressionWithStringAnnotationNotRecognizedInOtherSyntacticLocations] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing import Dict, List, TypeForm +list_of_typx: List[TypeForm] = ['int | str'] # E: TypeForm containing a string annotation cannot be recognized here. Surround with TypeForm(...) to recognize. \ + # E: List item 0 has incompatible type "str"; expected "TypeForm[Any]" +dict_with_typx_keys: Dict[TypeForm, int] = { + 'int | str': 1, # E: TypeForm containing a string annotation cannot be recognized here. Surround with TypeForm(...) to recognize. \ + # E: Dict entry 0 has incompatible type "str": "int"; expected "TypeForm[Any]": "int" + 'str | None': 2, # E: TypeForm containing a string annotation cannot be recognized here. Surround with TypeForm(...) to recognize. \ + # E: Dict entry 1 has incompatible type "str": "int"; expected "TypeForm[Any]": "int" +} +dict_with_typx_keys['int | str'] += 1 # E: TypeForm containing a string annotation cannot be recognized here. Surround with TypeForm(...) to recognize. \ + # E: Invalid index type "str" for "dict[TypeForm[Any], int]"; expected type "TypeForm[Any]" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-full.pyi] + +[case testValueExpressionWithStringInTypeFormContextEmitsConservativeWarning] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing import Any, Dict, List, TypeForm +types: Dict[str, TypeForm] = {'any': Any} +# Ensure warning can be ignored if does not apply. +list_of_typx1: List[TypeForm] = [types['any']] # E: TypeForm containing a string annotation cannot be recognized here. Surround with TypeForm(...) to recognize. +list_of_typx2: List[TypeForm] = [types['any']] # type: ignore[maybe-unrecognized-str-typeform] +# Ensure warning can be fixed using the suggested fix in the warning message. +list_of_typx3: List[TypeForm] = ['Any'] # E: TypeForm containing a string annotation cannot be recognized here. Surround with TypeForm(...) to recognize. \ + # E: List item 0 has incompatible type "str"; expected "TypeForm[Any]" +list_of_typx4: List[TypeForm] = [TypeForm('Any')] +[builtins fixtures/dict.pyi] +[typing fixtures/typing-full.pyi] + +[case testSelfRecognizedInOtherSyntacticLocations] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing import List, Self, TypeForm +class C: + def foo(self) -> None: + list_of_typx1: List[TypeForm] = [Self] + typx1: TypeForm = Self + typx2: TypeForm = 'Self' +list_of_typx2: List[TypeForm] = [Self] # E: List item 0 has incompatible type "int"; expected "TypeForm[Any]" +typx3: TypeForm = Self # E: Incompatible types in assignment (expression has type "int", variable has type "TypeForm[Any]") +typx4: TypeForm = 'Self' # E: Incompatible types in assignment (expression has type "str", variable has type "TypeForm[Any]") +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testNameOrDottedNameRecognizedInOtherSyntacticLocations] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +import typing +from typing import List, TypeForm +list_of_typx: List[TypeForm] = [List | typing.Optional[str]] +typx: TypeForm = List | typing.Optional[str] +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testInvalidNameOrDottedNameRecognizedInOtherSyntacticLocations] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing import List, TypeForm +list_of_typx1: List[TypeForm] = [NoSuchType] # E: Name "NoSuchType" is not defined +list_of_typx2: List[TypeForm] = [no_such_module.NoSuchType] # E: Name "no_such_module" is not defined +typx1: TypeForm = NoSuchType # E: Name "NoSuchType" is not defined +typx2: TypeForm = no_such_module.NoSuchType # E: Name "no_such_module" is not defined +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + + +-- Type Expression Context: Union[TypeForm, ] + +[case testAcceptsTypeFormLiteralAssignedToUnionOfTypeFormAndNonStr] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +typx_or_int1: TypeForm[int | None] | int = int | None # No error; interpret as TypeForm +typx_or_int2: TypeForm[int | None] | int = str | None # E: Incompatible types in assignment (expression has type "object", variable has type "Union[TypeForm[Optional[int]], int]") +typx_or_int3: TypeForm[int | None] | int = 1 +typx_or_int4: TypeForm[int | None] | int = object() # E: Incompatible types in assignment (expression has type "object", variable has type "Union[TypeForm[Optional[int]], int]") +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testAcceptsTypeFormLiteralAssignedToUnionOfTypeFormAndStr] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +typx_or_str1: TypeForm[int | None] | str = 'int | None' +typx_or_str2: TypeForm[int | None] | str = 'str | None' # No error; interpret as str +typx_or_str3: TypeForm[int | None] | str = 'hello' +typx_or_str4: TypeForm[int | None] | str = object() # E: Incompatible types in assignment (expression has type "object", variable has type "Union[TypeForm[Optional[int]], str]") +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testValueExpressionWithStringInTypeFormUnionContextEmitsConservativeWarning1] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing import List, TypeForm +list_of_typx1: List[TypeForm[int | None] | str] = ['int | None'] # E: TypeForm containing a string annotation cannot be recognized here. Surround with TypeForm(...) to recognize. +list_of_typx2: List[TypeForm[int | None] | str] = ['str | None'] # E: TypeForm containing a string annotation cannot be recognized here. Surround with TypeForm(...) to recognize. +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testValueExpressionWithStringInTypeFormUnionContextEmitsConservativeWarning2] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing import List, TypeForm +list_of_typx3: List[TypeForm[int | None] | int] = ['int | None'] # E: TypeForm containing a string annotation cannot be recognized here. Surround with TypeForm(...) to recognize. \ + # E: List item 0 has incompatible type "str"; expected "Union[TypeForm[Optional[int]], int]" +list_of_typx4: List[TypeForm[str | None] | int] = ['str | None'] # E: TypeForm containing a string annotation cannot be recognized here. Surround with TypeForm(...) to recognize. \ + # E: List item 0 has incompatible type "str"; expected "Union[TypeForm[Optional[str]], int]" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + + +-- Assignability (is_subtype) + +[case testTypeFormToTypeFormAssignability] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +# - TypeForm[T1] is assignable to TypeForm[T2] iff T1 is assignable to T2. +# - In particular TypeForm[Any] is assignable to TypeForm[Any]. +from typing_extensions import TypeForm +INT_OR_STR_TF: TypeForm[int | str] = int | str +INT_TF: TypeForm[int] = int +STR_TF: TypeForm[str] = str +OBJECT_TF: TypeForm[object] = object +ANY_TF: TypeForm = object +reveal_type(ANY_TF) # N: Revealed type is "TypeForm[Any]" +typx1: TypeForm[int | str] = INT_OR_STR_TF +typx2: TypeForm[int | str] = INT_TF +typx3: TypeForm[int | str] = STR_TF +typx4: TypeForm[int | str] = OBJECT_TF # E: Incompatible types in assignment (expression has type "TypeForm[object]", variable has type "TypeForm[Union[int, str]]") +typx5: TypeForm[int | str] = ANY_TF # no error +typx6: TypeForm[int] = INT_OR_STR_TF # E: Incompatible types in assignment (expression has type "TypeForm[Union[int, str]]", variable has type "TypeForm[int]") +typx7: TypeForm[int] = INT_TF +typx8: TypeForm[int] = STR_TF # E: Incompatible types in assignment (expression has type "TypeForm[str]", variable has type "TypeForm[int]") +typx9: TypeForm[int] = OBJECT_TF # E: Incompatible types in assignment (expression has type "TypeForm[object]", variable has type "TypeForm[int]") +typx10: TypeForm[int] = ANY_TF # no error +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testTypeToTypeFormAssignability] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +# - Type[C] is assignable to TypeForm[T] iff C is assignable to T. +# - In particular Type[Any] is assignable to TypeForm[Any]. +from typing import Type, TypeForm +INT_T: Type[int] = int +STR_T: Type[str] = str +OBJECT_T: Type[object] = object +ANY_T: Type = object +reveal_type(ANY_T) # N: Revealed type is "type[Any]" +typx1: TypeForm[int | str] = INT_T +typx2: TypeForm[int | str] = STR_T +typx3: TypeForm[int | str] = OBJECT_T # E: Incompatible types in assignment (expression has type "type[object]", variable has type "TypeForm[Union[int, str]]") +typx4: TypeForm[int | str] = ANY_T # no error +typx5: TypeForm[int] = INT_T +typx6: TypeForm[int] = STR_T # E: Incompatible types in assignment (expression has type "type[str]", variable has type "TypeForm[int]") +typx7: TypeForm[int] = OBJECT_T # E: Incompatible types in assignment (expression has type "type[object]", variable has type "TypeForm[int]") +typx8: TypeForm[int] = ANY_T # no error +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testTypeFormToTypeAssignability] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +# - TypeForm[T] is NOT assignable to Type[C]. +# - In particular TypeForm[Any] is NOT assignable to Type[Any]. +from typing import Type, TypeForm +INT_OR_STR_TF: TypeForm[int | str] = int | str +INT_TF: TypeForm[int] = int +STR_TF: TypeForm[str] = str +OBJECT_TF: TypeForm[object] = object +ANY_TF: TypeForm = object +reveal_type(ANY_TF) # N: Revealed type is "TypeForm[Any]" +typ1: Type[int] = INT_OR_STR_TF # E: Incompatible types in assignment (expression has type "TypeForm[Union[int, str]]", variable has type "type[int]") +typ2: Type[int] = INT_TF # E: Incompatible types in assignment (expression has type "TypeForm[int]", variable has type "type[int]") +typ3: Type[int] = STR_TF # E: Incompatible types in assignment (expression has type "TypeForm[str]", variable has type "type[int]") +typ4: Type[int] = OBJECT_TF # E: Incompatible types in assignment (expression has type "TypeForm[object]", variable has type "type[int]") +typ5: Type[int] = ANY_TF # E: Incompatible types in assignment (expression has type "TypeForm[Any]", variable has type "type[int]") +typ6: Type[object] = INT_OR_STR_TF # E: Incompatible types in assignment (expression has type "TypeForm[Union[int, str]]", variable has type "type[object]") +typ7: Type[object] = INT_TF # E: Incompatible types in assignment (expression has type "TypeForm[int]", variable has type "type[object]") +typ8: Type[object] = STR_TF # E: Incompatible types in assignment (expression has type "TypeForm[str]", variable has type "type[object]") +typ9: Type[object] = OBJECT_TF # E: Incompatible types in assignment (expression has type "TypeForm[object]", variable has type "type[object]") +typ10: Type[object] = ANY_TF # E: Incompatible types in assignment (expression has type "TypeForm[Any]", variable has type "type[object]") +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +# NOTE: This test doesn't involve TypeForm at all, but is still illustrative +# when compared with similarly structured TypeForm-related tests above. +[case testTypeToTypeAssignability] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +# - Type[C1] is assignable to Type[C2] iff C1 is assignable to C2. +# - In particular Type[Any] is assignable to Type[Any]. +from typing import Type +INT_T: Type[int] = int +STR_T: Type[str] = str +OBJECT_T: Type[object] = object +ANY_T: Type = object +reveal_type(ANY_T) # N: Revealed type is "type[Any]" +typ1: Type[int] = INT_T +typ2: Type[int] = STR_T # E: Incompatible types in assignment (expression has type "type[str]", variable has type "type[int]") +typ3: Type[int] = OBJECT_T # E: Incompatible types in assignment (expression has type "type[object]", variable has type "type[int]") +typ4: Type[int] = ANY_T # no error +typ5: Type[object] = INT_T +typ6: Type[object] = STR_T +typ7: Type[object] = OBJECT_T +typ8: Type[object] = ANY_T # no error +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testTypeFormToObjectAssignability] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +# - TypeForm[T] is assignable to object and Any. +from typing import Any, TypeForm +INT_TF: TypeForm[int] = int +OBJECT_TF: TypeForm[object] = object +ANY_TF: TypeForm = object +reveal_type(ANY_TF) # N: Revealed type is "TypeForm[Any]" +obj1: object = INT_TF +obj2: object = OBJECT_TF +obj3: object = ANY_TF +any1: Any = INT_TF +any2: Any = OBJECT_TF +any3: Any = ANY_TF +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + + +-- Join (join_types) + +[case testTypeFormToTypeFormJoin] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +# - TypeForm[T1] join TypeForm[T2] == TypeForm[T1 join T2] +from typing_extensions import TypeForm +class AB: + pass +class A(AB): + pass +class B(AB): + pass +A_TF: TypeForm[A] = A +B_TF: TypeForm[B] = B +reveal_type([A_TF, B_TF][0]) # N: Revealed type is "TypeForm[__main__.AB]" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testTypeToTypeFormJoin] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +# - TypeForm[T1] join Type[T2] == TypeForm[T1 join T2] +from typing import Type, TypeForm +class AB: + pass +class A(AB): + pass +class B(AB): + pass +A_T: Type[A] = A +B_TF: TypeForm[B] = B +reveal_type([A_T, B_TF][0]) # N: Revealed type is "TypeForm[__main__.AB]" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testTypeFormToTypeJoin] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +# - TypeForm[T1] join Type[T2] == TypeForm[T1 join T2] +from typing import Type, TypeForm +class AB: + pass +class A(AB): + pass +class B(AB): + pass +A_TF: TypeForm[A] = A +B_T: Type[B] = B +reveal_type([A_TF, B_T][0]) # N: Revealed type is "TypeForm[__main__.AB]" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +# NOTE: This test doesn't involve TypeForm at all, but is still illustrative +# when compared with similarly structured TypeForm-related tests above. +[case testTypeToTypeJoin] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +# - Type[T1] join Type[T2] == Type[T1 join T2] +from typing import Type, TypeForm +class AB: + pass +class A(AB): + pass +class B(AB): + pass +A_T: Type[A] = A +B_T: Type[B] = B +reveal_type([A_T, B_T][0]) # N: Revealed type is "type[__main__.AB]" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + + +-- Meet (meet_types) + +[case testTypeFormToTypeFormMeet] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +# - TypeForm[T1] meet TypeForm[T2] == TypeForm[T1 meet T2] +from typing import Callable, TypeForm, TypeVar +class AB: + pass +class A(AB): + pass +class B(AB): + pass +class C(AB): + pass +T = TypeVar('T') +def f(x: Callable[[T, T], None]) -> T: pass # type: ignore[empty-body] +def g(x: TypeForm[A | B], y: TypeForm[B | C]) -> None: pass +reveal_type(f(g)) # N: Revealed type is "TypeForm[__main__.B]" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testTypeToTypeFormMeet] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +# - TypeForm[T1] meet Type[T2] == Type[T1 meet T2] +from typing import Callable, Type, TypeForm, TypeVar +class AB: + pass +class A(AB): + pass +class B(AB): + pass +class C(AB): + pass +T = TypeVar('T') +def f(x: Callable[[T, T], None]) -> T: pass # type: ignore[empty-body] +def g(x: Type[B], y: TypeForm[B | C]) -> None: pass +reveal_type(f(g)) # N: Revealed type is "type[__main__.B]" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testTypeFormToTypeMeet] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +# - TypeForm[T1] meet Type[T2] == Type[T1 meet T2] +from typing import Callable, Type, TypeForm, TypeVar +class AB: + pass +class A(AB): + pass +class B(AB): + pass +class C(AB): + pass +T = TypeVar('T') +def f(x: Callable[[T, T], None]) -> T: pass # type: ignore[empty-body] +def g(x: TypeForm[A | B], y: Type[B]) -> None: pass +reveal_type(f(g)) # N: Revealed type is "type[__main__.B]" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +# NOTE: This test doesn't involve TypeForm at all, but is still illustrative +# when compared with similarly structured TypeForm-related tests above. +[case testTypeToTypeMeet] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +# - Type[T1] meet Type[T2] == Type[T1 meet T2] +from typing import Callable, Type, TypedDict, TypeForm, TypeVar +class AB(TypedDict): + a: str + b: str +class BC(TypedDict): + b: str + c: str +T = TypeVar('T') +def f(x: Callable[[T, T], None]) -> T: pass # type: ignore[empty-body] +def g(x: Type[AB], y: Type[BC]) -> None: pass +reveal_type(f(g)) # N: Revealed type is "type[TypedDict({'b': builtins.str, 'c': builtins.str, 'a': builtins.str})]" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + + +-- TypeForm(...) Expression + +[case testTypeFormExpression] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +tf1 = TypeForm(int | str) +reveal_type(tf1) # N: Revealed type is "TypeForm[Union[builtins.int, builtins.str]]" +tf2 = TypeForm('int | str') +reveal_type(tf2) # N: Revealed type is "TypeForm[Union[builtins.int, builtins.str]]" +tf3: TypeForm = TypeForm(int | str) +reveal_type(tf3) # N: Revealed type is "TypeForm[Any]" +tf4: TypeForm = TypeForm(1) # E: Invalid type: try using Literal[1] instead? +tf5: TypeForm = TypeForm(int) | TypeForm(str) # E: Incompatible types in assignment (expression has type "object", variable has type "TypeForm[Any]") +tf6: TypeForm = TypeForm(TypeForm(int) | TypeForm(str)) # E: TypeForm argument is not a type +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + + +-- isinstance + +[case testTypeFormAndTypeIsinstance] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +typx: TypeForm[str] = str +if isinstance(typx, type): + reveal_type(typx) # N: Revealed type is "type[builtins.str]" +else: + reveal_type(typx) # N: Revealed type is "TypeForm[builtins.str]" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + + +-- Type Variables + +[case testLinkTypeFormToTypeFormWithTypeVariable] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm, TypeVar +T = TypeVar('T') +def as_typeform(typx: TypeForm[T]) -> TypeForm[T]: + return typx +reveal_type(as_typeform(int | str)) # N: Revealed type is "TypeForm[Union[builtins.int, builtins.str]]" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testLinkTypeFormToTypeWithTypeVariable] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing import Type, TypeForm, TypeVar +T = TypeVar('T') +def as_type(typx: TypeForm[T]) -> Type[T] | None: + if isinstance(typx, type): + return typx + else: + return None +reveal_type(as_type(int | str)) # N: Revealed type is "Union[type[builtins.int], type[builtins.str], None]" +reveal_type(as_type(int)) # N: Revealed type is "Union[type[builtins.int], None]" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testLinkTypeFormToInstanceWithTypeVariable] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm, TypeVar +T = TypeVar('T') +def as_instance(typx: TypeForm[T]) -> T | None: + if isinstance(typx, type): + return typx() + else: + return None +reveal_type(as_instance(int | str)) # N: Revealed type is "Union[builtins.int, builtins.str, None]" +reveal_type(as_instance(int)) # N: Revealed type is "Union[builtins.int, None]" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testLinkTypeFormToTypeIsWithTypeVariable] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm, TypeVar +from typing_extensions import TypeIs +T = TypeVar('T') +def isassignable(value: object, typx: TypeForm[T]) -> TypeIs[T]: + raise BaseException() +count: int | str = 1 +if isassignable(count, int): + reveal_type(count) # N: Revealed type is "builtins.int" +else: + reveal_type(count) # N: Revealed type is "builtins.str" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testLinkTypeFormToTypeGuardWithTypeVariable] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm, TypeVar +from typing_extensions import TypeGuard +T = TypeVar('T') +def isassignable(value: object, typx: TypeForm[T]) -> TypeGuard[T]: + raise BaseException() +count: int | str = 1 +if isassignable(count, int): + reveal_type(count) # N: Revealed type is "builtins.int" +else: + reveal_type(count) # N: Revealed type is "Union[builtins.int, builtins.str]" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + + +-- Type Expressions Assignable To TypeForm Variable + +[case testEveryKindOfTypeExpressionIsAssignableToATypeFormVariable] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +# NOTE: Importing Callable from collections.abc also works OK +from typing import ( + Any, Callable, Dict, List, Literal, LiteralString, NoReturn, + Optional, ParamSpec, Self, Type, TypeGuard, TypeVar, Union, +) +from typing_extensions import ( + Annotated, Concatenate, Never, TypeAlias, TypeForm, TypeIs, + TypeVarTuple, Unpack, +) +# +class SomeClass: + pass +SomeTypeAlias: TypeAlias = SomeClass +SomeTypeVar = TypeVar('SomeTypeVar') +Ts = TypeVarTuple('Ts') +IntTuple: TypeAlias = tuple[int, Unpack[Ts]] +P = ParamSpec('P') +R = TypeVar('R') +# +typx: TypeForm +# Begin rules taken from: https://typing.python.org/en/latest/spec/annotations.html#grammar-token-expression-grammar-type_expression +# +typx = Any +class SelfBinder: + def bind_self(self) -> Self: + typx: TypeForm + # (valid only in some contexts) + typx = Self + return self +# +typx = LiteralString +# +typx = NoReturn +# +typx = Never +# +typx = None +# name (where name must refer to a valid in-scope class) +typx = SomeClass +# name (where name must refer to a valid in-scope type alias) +typx = SomeTypeAlias +# name (where name must refer to a valid in-scope TypeVar) +# NOTE: Unbound TypeVar isn't currently accepted as a TypeForm. Is that OK? +typx = SomeTypeVar # E: Incompatible types in assignment (expression has type "TypeVar", variable has type "TypeForm[Any]") +# name '[' type_expression (',' type_expression)* ']' +typx = Dict[str, int] +# (TODO: Add: name '[' unpacked ']') +# (TODO: Add: name '[' type_expression_list (',' type_expression_list)* ']') +# name '[' '(' ')' ']' (denoting specialization with an empty TypeVarTuple) +typx = IntTuple[()] +# '[' expression (',' expression) ']' +typx = Literal[1] +# type_expression '|' type_expression +typx = int | str +# '[' type_expression ']' +typx = Optional[str] +# '[' type_expression (',' type_expression)* ']' +typx = Union[int, str] +# '[' ']' +typx = type[Any] +# '[' name ']' (where name must refer to a valid in-scope class) +typx = type[int] +# (TODO: Add: '[' name ']' (where name must refer to a valid in-scope TypeVar)) +# '[' '...' ',' type_expression ']' +typx = Callable[..., str] +def bind_R(input: R) -> R: + typx: TypeForm + # '[' name ',' type_expression ']' (where name must be a valid in-scope ParamSpec) + typx = Callable[P, R] + # '[' '[' (type_expression ',')+ + # (name | '...') ']' ',' type_expression ']' + # (where name must be a valid in-scope ParamSpec) + typx = Callable[Concatenate[int, P], R] + return input +# '[' '[' maybe_unpacked (',' maybe_unpacked)* ']' ',' type_expression ']' +typx = Callable[[int, str], None] +# '[' '(' ')' ']' (representing an empty tuple) +typx = tuple[()] +# '[' type_expression ',' '...' ']' (representing an arbitrary-length tuple) +typx = tuple[int, ...] +# '[' maybe_unpacked (',' maybe_unpacked)* ']' +typx = tuple[int, str] +# '[' type_expression ',' expression (',' expression)* ']' +typx = Annotated[str, 'uppercase'] +# '[' type_expression ']' (valid only in some contexts) +typx = TypeGuard[List[str]] +# '[' type_expression ']' (valid only in some contexts) +typx = TypeIs[List[str]] +# string_annotation (must evaluate to a valid type_expression) +typx = 'int | str' +# End rules +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + + +-- Misc + +[case testTypeFormHasAllObjectAttributesAndMethods] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +typx: TypeForm[int | str] = int | str +print(typx.__class__) # OK +print(typx.__hash__()) # OK +obj: object = typx +[file builtins.py] +class object: + def __init__(self) -> None: pass + __class__: None + def __hash__(self) -> int: pass +def print(x): + raise BaseException() +class int: pass +class dict: pass +class str: pass +class type: pass +class tuple: pass +class ellipsis: pass +class BaseException: pass +class float: pass +[typing fixtures/typing-full.pyi] + +[case testDottedTypeFormsAreRecognized] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +import typing +class C1: + class C2: + pass +typx1: TypeForm[C1.C2] = C1.C2 # OK +typx2: TypeForm[typing.Any] = typing.Any # OK +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +-- mypy already refused to recognize TypeVars in value expressions before +-- the TypeForm feature was introduced. +[case testTypeVarTypeFormsAreOnlyRecognizedInStringAnnotation] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing import Generic, List, TypeForm, TypeVar +E = TypeVar('E') +class Box(Generic[E]): + def foo(self, e: E) -> None: + list_of_typx: List[TypeForm] = [E] # E: "E" is a type variable and only valid in type context + typx1: TypeForm = E # E: "E" is a type variable and only valid in type context + typx2: TypeForm = 'E' +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testIncompleteTypeFormsAreNotRecognized] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing import Optional, TypeForm +typx: TypeForm = Optional # E: Incompatible types in assignment (expression has type "int", variable has type "TypeForm[Any]") +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] diff --git a/test-data/unit/fixtures/primitives.pyi b/test-data/unit/fixtures/primitives.pyi index 2f8623c79b9ff..98e604e9e81ef 100644 --- a/test-data/unit/fixtures/primitives.pyi +++ b/test-data/unit/fixtures/primitives.pyi @@ -13,6 +13,8 @@ class object: class type: def __init__(self, x: object) -> None: pass + # Real implementation returns UnionType + def __or__(self, value: object, /) -> object: pass class int: # Note: this is a simplification of the actual signature @@ -72,3 +74,5 @@ class range(Sequence[int]): def __contains__(self, other: object) -> bool: pass def isinstance(x: object, t: Union[type, Tuple]) -> bool: pass + +class BaseException: pass diff --git a/test-data/unit/fixtures/typing-full.pyi b/test-data/unit/fixtures/typing-full.pyi index 1a63deaa727d0..87b66c0cd857e 100644 --- a/test-data/unit/fixtures/typing-full.pyi +++ b/test-data/unit/fixtures/typing-full.pyi @@ -30,6 +30,7 @@ Protocol = 0 Tuple = 0 _promote = 0 Type = 0 +TypeForm = 0 no_type_check = 0 ClassVar = 0 Final = 0 diff --git a/test-data/unit/lib-stub/typing_extensions.pyi b/test-data/unit/lib-stub/typing_extensions.pyi index 6158a0c9ebbc3..71a17a939d41a 100644 --- a/test-data/unit/lib-stub/typing_extensions.pyi +++ b/test-data/unit/lib-stub/typing_extensions.pyi @@ -33,6 +33,8 @@ Concatenate: _SpecialForm TypeAlias: _SpecialForm +TypeForm: _SpecialForm + TypeGuard: _SpecialForm TypeIs: _SpecialForm Never: _SpecialForm From 40821bfaac5e42a5f569758b2fc4ea25e563162a Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Mon, 3 Nov 2025 13:46:02 +0100 Subject: [PATCH 372/424] Use the fallback for `ModuleSpec` early if it can never be resolved (#20167) Fixes #18237. --- mypy/semanal.py | 10 +++++++++- test-data/unit/cmdline.test | 18 ++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 1fdea22e8962d..a21995fdafa30 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -750,7 +750,15 @@ def add_implicit_module_attrs(self, file_node: MypyFile) -> None: else: inst = self.named_type_or_none("importlib.machinery.ModuleSpec") if inst is None: - if self.final_iteration: + if ( + self.final_iteration + or self.options.clone_for_module("importlib.machinery").follow_imports + == "skip" + ): + # If we are not allowed to resolve imports from `importlib.machinery`, + # ModuleSpec will not be available at any iteration. + # Use the fallback earlier. + # (see https://github.com/python/mypy/issues/18237) inst = self.named_type_or_none("builtins.object") assert inst is not None, "Cannot find builtins.object" else: diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test index 35d7b700b1618..bed4134bff727 100644 --- a/test-data/unit/cmdline.test +++ b/test-data/unit/cmdline.test @@ -1393,3 +1393,21 @@ Ts = TypeVarTuple("Ts") Warning: TypeVarTuple is already enabled by default Warning: Unpack is already enabled by default == Return code: 0 + +[case testImportlibImportCannotBeResolved] +# cmd: mypy a.py +# This test is here because it needs use_builtins_fixtures off. + +[file a.py] +from typing import NamedTuple + +class CodecKey(NamedTuple): + def foo(self) -> "CodecKey": + ... + +[file mypy.ini] +\[mypy] + +\[mypy-importlib.*] +follow_imports = skip +follow_imports_for_stubs = True From 62770f484007a8f7a3ea4dbab0959772b3cf2267 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 3 Nov 2025 04:54:17 -0800 Subject: [PATCH 373/424] Fix type checking of dict type aliases (#20170) Resolves a bad false negative and a false positive Previously, for `D = dict[str, str]` we would require a type annotation for `d = D()` and we would fail to error on `D(x=1)` Fixes #11093, fixes #11246, fixes #13320, fixes #16047 --- mypy/semanal.py | 6 +++++- test-data/unit/check-type-aliases.test | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index a21995fdafa30..e55819b898e96 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -5888,7 +5888,11 @@ def visit_call_expr(self, expr: CallExpr) -> None: expr.analyzed = PromoteExpr(target) expr.analyzed.line = expr.line expr.analyzed.accept(self) - elif refers_to_fullname(expr.callee, "builtins.dict"): + elif refers_to_fullname(expr.callee, "builtins.dict") and not ( + isinstance(expr.callee, RefExpr) + and isinstance(expr.callee.node, TypeAlias) + and not expr.callee.node.no_args + ): expr.analyzed = self.translate_dict_call(expr) elif refers_to_fullname(expr.callee, "builtins.divmod"): if not self.check_fixed_args(expr, 2, "divmod"): diff --git a/test-data/unit/check-type-aliases.test b/test-data/unit/check-type-aliases.test index 5bbb503a578a1..06e2237161895 100644 --- a/test-data/unit/check-type-aliases.test +++ b/test-data/unit/check-type-aliases.test @@ -1318,3 +1318,17 @@ from typing_extensions import TypeAlias Foo: TypeAlias = ClassVar[int] # E: ClassVar[...] can't be used inside a type alias [builtins fixtures/tuple.pyi] + + +[case testTypeAliasDict] +D = dict[str, int] +d = D() +reveal_type(d) # N: Revealed type is "builtins.dict[builtins.str, builtins.int]" +reveal_type(D()) # N: Revealed type is "builtins.dict[builtins.str, builtins.int]" +reveal_type(D(x=1)) # N: Revealed type is "builtins.dict[builtins.str, builtins.int]" +reveal_type(D(x="asdf")) # E: No overload variant of "dict" matches argument type "str" \ + # N: Possible overload variants: \ + # N: def __init__(self, **kwargs: int) -> dict[str, int] \ + # N: def __init__(self, arg: Iterable[tuple[str, int]], **kwargs: int) -> dict[str, int] \ + # N: Revealed type is "Any" +[builtins fixtures/dict.pyi] From 904fc779118a7fe9538286de704dc563d20b69b6 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Mon, 3 Nov 2025 05:58:26 -0700 Subject: [PATCH 374/424] [docs] Change the InlineTypedDict example (#20172) This way it will be clear that there is no necessary connection between the name of the field and its type. --- docs/source/command_line.rst | 4 ++-- docs/source/typed_dict.rst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index 79dd68a84b28d..d6b70c4976dca 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -1211,8 +1211,8 @@ List of currently incomplete/experimental features: .. code-block:: python - def test_values() -> {"int": int, "str": str}: - return {"int": 42, "str": "test"} + def test_values() -> {"width": int, "description": str}: + return {"width": 42, "description": "test"} Miscellaneous diff --git a/docs/source/typed_dict.rst b/docs/source/typed_dict.rst index bbb10a12abe8f..d42b434b05b2d 100644 --- a/docs/source/typed_dict.rst +++ b/docs/source/typed_dict.rst @@ -303,8 +303,8 @@ to use inline TypedDict syntax. For example: .. code-block:: python - def test_values() -> {"int": int, "str": str}: - return {"int": 42, "str": "test"} + def test_values() -> {"width": int, "description": str}: + return {"width": 42, "description": "test"} class Response(TypedDict): status: int From 5bdfe7ef86ea45dc7da0d618104ae93537bc7c6c Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Mon, 3 Nov 2025 18:30:11 +0100 Subject: [PATCH 375/424] Respect force-union-syntax flag in error hint (#20165) The error hint for `Need type annotations for ...` should defer to `options.use_union_syntax` when evaluating if `Union` or `|` should be used. This will also make it easier to eventually upgrade all tests to use the PEP 604 syntax. --- mypy/checker.py | 14 +++++--------- mypy/messages.py | 7 +++---- mypy/suggestions.py | 3 ++- mypy/test/testcheck.py | 2 +- test-data/unit/check-inference.test | 2 +- 5 files changed, 12 insertions(+), 16 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index f4746bc0c8862..fedb4e745909c 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -4449,7 +4449,7 @@ def infer_variable_type( # partial type which will be made more specific later. A partial type # gets generated in assignment like 'x = []' where item type is not known. if name.name != "_" and not self.infer_partial_type(name, lvalue, init_type): - self.msg.need_annotation_for_var(name, context, self.options.python_version) + self.msg.need_annotation_for_var(name, context, self.options) self.set_inference_error_fallback_type(name, lvalue, init_type) elif ( isinstance(lvalue, MemberExpr) @@ -4459,7 +4459,7 @@ def infer_variable_type( and not is_same_type(self.inferred_attribute_types[lvalue.def_var], init_type) ): # Multiple, inconsistent types inferred for an attribute. - self.msg.need_annotation_for_var(name, context, self.options.python_version) + self.msg.need_annotation_for_var(name, context, self.options) name.type = AnyType(TypeOfAny.from_error) else: # Infer type of the target. @@ -4656,9 +4656,7 @@ def check_simple_assignment( rvalue, type_context=lvalue_type, always_allow_any=always_allow_any ) if not is_valid_inferred_type(rvalue_type, self.options) and inferred is not None: - self.msg.need_annotation_for_var( - inferred, context, self.options.python_version - ) + self.msg.need_annotation_for_var(inferred, context, self.options) rvalue_type = rvalue_type.accept(SetNothingToAny()) if ( @@ -7680,7 +7678,7 @@ def enter_partial_types( var.type = NoneType() else: if var not in self.partial_reported and not permissive: - self.msg.need_annotation_for_var(var, context, self.options.python_version) + self.msg.need_annotation_for_var(var, context, self.options) self.partial_reported.add(var) if var.type: fixed = fixup_partial_type(var.type) @@ -7707,9 +7705,7 @@ def handle_partial_var_type( if in_scope: context = partial_types[node] if is_local or not self.options.allow_untyped_globals: - self.msg.need_annotation_for_var( - node, context, self.options.python_version - ) + self.msg.need_annotation_for_var(node, context, self.options) self.partial_reported.add(node) else: # Defer the node -- we might get a better type in the outer scope diff --git a/mypy/messages.py b/mypy/messages.py index 68b9b83572f12..9fdfb748b2882 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -1803,18 +1803,17 @@ def unimported_type_becomes_any(self, prefix: str, typ: Type, ctx: Context) -> N ) def need_annotation_for_var( - self, node: SymbolNode, context: Context, python_version: tuple[int, int] | None = None + self, node: SymbolNode, context: Context, options: Options | None = None ) -> None: hint = "" - pep604_supported = not python_version or python_version >= (3, 10) # type to recommend the user adds recommended_type = None # Only gives hint if it's a variable declaration and the partial type is a builtin type - if python_version and isinstance(node, Var) and isinstance(node.type, PartialType): + if options and isinstance(node, Var) and isinstance(node.type, PartialType): type_dec = "" if not node.type.type: # partial None - if pep604_supported: + if options.use_or_syntax(): recommended_type = f"{type_dec} | None" else: recommended_type = f"Optional[{type_dec}]" diff --git a/mypy/suggestions.py b/mypy/suggestions.py index 45aa5ade47a4d..756cf6ae2ccdc 100644 --- a/mypy/suggestions.py +++ b/mypy/suggestions.py @@ -890,7 +890,8 @@ def visit_typeddict_type(self, t: TypedDictType) -> str: def visit_union_type(self, t: UnionType) -> str: if len(t.items) == 2 and is_overlapping_none(t): - return f"Optional[{remove_optional(t).accept(self)}]" + s = remove_optional(t).accept(self) + return f"{s} | None" if self.options.use_or_syntax() else f"Optional[{s}]" else: return super().visit_union_type(t) diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index f59cce701ea6c..f2b7057d9f20d 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -140,7 +140,7 @@ def run_case_once( options.hide_error_codes = False if "abstract" not in testcase.file: options.allow_empty_bodies = not testcase.name.endswith("_no_empty") - if "union-error" not in testcase.file: + if "union-error" not in testcase.file and "Pep604" not in testcase.name: options.force_union_syntax = True if incremental_step and options.incremental: diff --git a/test-data/unit/check-inference.test b/test-data/unit/check-inference.test index 17f79dbcb663a..bc4b56e49622c 100644 --- a/test-data/unit/check-inference.test +++ b/test-data/unit/check-inference.test @@ -3548,7 +3548,7 @@ if x: [builtins fixtures/dict.pyi] [case testSuggestPep604AnnotationForPartialNone] -# flags: --local-partial-types --python-version 3.10 +# flags: --local-partial-types --python-version 3.10 --no-force-union-syntax x = None # E: Need type annotation for "x" (hint: "x: | None = ...") [case testTupleContextFromIterable] From 08a0b0742d75120fe948da7dd3c475135cf9ea2e Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Mon, 3 Nov 2025 22:45:02 +0100 Subject: [PATCH 376/424] Do not cache fast container types inside lambdas (#20166) Fixes #20163. `fast_container_type` uses another layer of expression cache, it also has to be bypassed from within lambdas. --- mypy/checkexpr.py | 10 ++++++++-- test-data/unit/check-inference-context.test | 14 ++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index a06af690f8dbf..03ebc5058cee9 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -5099,7 +5099,10 @@ def fast_container_type( self.resolved_type[e] = NoneType() return None ct = self.chk.named_generic_type(container_fullname, [vt]) - self.resolved_type[e] = ct + if not self.in_lambda_expr: + # We cannot cache results in lambdas - their bodies can be accepted in + # error-suppressing watchers too early + self.resolved_type[e] = ct return ct def _first_or_join_fast_item(self, items: list[Type]) -> Type | None: @@ -5334,7 +5337,10 @@ def fast_dict_type(self, e: DictExpr) -> Type | None: self.resolved_type[e] = NoneType() return None dt = self.chk.named_generic_type("builtins.dict", [kt, vt]) - self.resolved_type[e] = dt + if not self.in_lambda_expr: + # We cannot cache results in lambdas - their bodies can be accepted in + # error-suppressing watchers too early + self.resolved_type[e] = dt return dt def check_typeddict_literal_in_context( diff --git a/test-data/unit/check-inference-context.test b/test-data/unit/check-inference-context.test index a41ee5f59670e..b5b5d778d90fb 100644 --- a/test-data/unit/check-inference-context.test +++ b/test-data/unit/check-inference-context.test @@ -709,6 +709,20 @@ f( A(), r=B()) [builtins fixtures/isinstance.pyi] +[case testLambdaWithFastContainerType] +from collections.abc import Callable +from typing import Never, TypeVar + +T = TypeVar("T") + +def f(a: Callable[[], T]) -> None: ... + +def foo(x: str) -> Never: ... + +f(lambda: [foo(0)]) # E: Argument 1 to "foo" has incompatible type "int"; expected "str" +f(lambda: {"x": foo(0)}) # E: Argument 1 to "foo" has incompatible type "int"; expected "str" +[builtins fixtures/tuple.pyi] + -- Overloads + generic functions -- ----------------------------- From 902acd153b0fa04e98ac8bf074c74f0514f9041b Mon Sep 17 00:00:00 2001 From: Frank Dana Date: Mon, 3 Nov 2025 16:47:16 -0500 Subject: [PATCH 377/424] Don't let help formatter line-wrap URLs (#19825) Fixes #19816 This PR adjusts the `AugmentedHelpFormatter` for the command-line `--help` output so that it will no longer wrap/break long URLs in the output. It also (in a separate commit) adds `https://` in front of two URLs that lacked it, so they can be recognized as links by anything parsing the help text. --- mypy/main.py | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index a44632ed96f94..7d5721851c3d6 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -256,10 +256,19 @@ def _fill_text(self, text: str, width: int, indent: str) -> str: if "\n" in text: # Assume we want to manually format the text return super()._fill_text(text, width, indent) - else: - # Assume we want argparse to manage wrapping, indenting, and - # formatting the text for us. - return argparse.HelpFormatter._fill_text(self, text, width, indent) + # Format the text like argparse, but overflow rather than + # breaking long words (like URLs) + text = self._whitespace_matcher.sub(" ", text).strip() + import textwrap + + return textwrap.fill( + text, + width, + initial_indent=indent, + subsequent_indent=indent, + break_on_hyphens=False, + break_long_words=False, + ) # Define pairs of flag prefixes with inverse meaning. @@ -544,10 +553,15 @@ def add_invertible_flag( # Feel free to add subsequent sentences that add additional details. # 3. If you cannot think of a meaningful description for a new group, omit it entirely. # (E.g. see the "miscellaneous" sections). - # 4. The group description should end with a period (unless the last line is a link). If you - # do end the group description with a link, omit the 'http://' prefix. (Some links are too - # long and will break up into multiple lines if we include that prefix, so for consistency - # we omit the prefix on all links.) + # 4. The text of the group description should end with a period, optionally followed + # by a documentation reference (URL). + # 5. If you want to include a documentation reference, place it at the end of the + # description. Feel free to open with a brief reference ("See also:", "For more + # information:", etc.), followed by a space, then the entire URL including + # "https://" scheme identifier and fragment ("#some-target-heading"), if any. + # Do not end with a period (or any other characters not part of the URL). + # URLs longer than the available terminal width will overflow without being + # broken apart. This facilitates both URL detection, and manual copy-pasting. general_group = parser.add_argument_group(title="Optional arguments") general_group.add_argument( @@ -1034,7 +1048,7 @@ def add_invertible_flag( "Mypy caches type information about modules into a cache to " "let you speed up future invocations of mypy. Also see " "mypy's daemon mode: " - "mypy.readthedocs.io/en/stable/mypy_daemon.html#mypy-daemon", + "https://mypy.readthedocs.io/en/stable/mypy_daemon.html#mypy-daemon", ) incremental_group.add_argument( "-i", "--incremental", action="store_true", help=argparse.SUPPRESS @@ -1278,7 +1292,7 @@ def add_invertible_flag( code_group = parser.add_argument_group( title="Running code", description="Specify the code you want to type check. For more details, see " - "mypy.readthedocs.io/en/stable/running_mypy.html#running-mypy", + "https://mypy.readthedocs.io/en/stable/running_mypy.html#running-mypy", ) add_invertible_flag( "--explicit-package-bases", From 3c30736847141e6212701647de95bc0da2728a7f Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 3 Nov 2025 18:56:57 -0800 Subject: [PATCH 378/424] Fix errors for raise NotImplemented (#20168) See also discussion on https://github.com/python/typeshed/pull/14966 --- mypy/checker.py | 15 ++++++++++----- mypy/types.py | 2 ++ test-data/unit/check-statements.test | 2 +- test-data/unit/fixtures/notimplemented.pyi | 13 ++++++++++--- 4 files changed, 23 insertions(+), 9 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index fedb4e745909c..f3a93d1eeda18 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -198,6 +198,7 @@ from mypy.types import ( ANY_STRATEGY, MYPYC_NATIVE_INT_NAMES, + NOT_IMPLEMENTED_TYPE_NAMES, OVERLOAD_NAMES, AnyType, BoolTypeQuery, @@ -4974,10 +4975,7 @@ def check_return_stmt(self, s: ReturnStmt) -> None: ) # Treat NotImplemented as having type Any, consistent with its # definition in typeshed prior to python/typeshed#4222. - if ( - isinstance(typ, Instance) - and typ.type.fullname == "builtins._NotImplementedType" - ): + if isinstance(typ, Instance) and typ.type.fullname in NOT_IMPLEMENTED_TYPE_NAMES: typ = AnyType(TypeOfAny.special_form) if defn.is_async_generator: @@ -5136,7 +5134,14 @@ def type_check_raise(self, e: Expression, s: RaiseStmt, optional: bool = False) # https://github.com/python/mypy/issues/11089 self.expr_checker.check_call(typ, [], [], e) - if isinstance(typ, Instance) and typ.type.fullname == "builtins._NotImplementedType": + if ( + isinstance(typ, Instance) + and typ.type.fullname in {"builtins._NotImplementedType", "types.NotImplementedType"} + ) or ( + isinstance(e, CallExpr) + and isinstance(e.callee, RefExpr) + and e.callee.fullname in {"builtins.NotImplemented"} + ): self.fail( message_registry.INVALID_EXCEPTION.with_additional_msg( '; did you mean "NotImplementedError"?' diff --git a/mypy/types.py b/mypy/types.py index 1c43310224965..7a8343097204f 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -200,6 +200,8 @@ ELLIPSIS_TYPE_NAMES: Final = ("builtins.ellipsis", "types.EllipsisType") +NOT_IMPLEMENTED_TYPE_NAMES: Final = ("builtins._NotImplementedType", "types.NotImplementedType") + # A placeholder used for Bogus[...] parameters _dummy: Final[Any] = object() diff --git a/test-data/unit/check-statements.test b/test-data/unit/check-statements.test index 658bee76ef0de..87d015f3de0f8 100644 --- a/test-data/unit/check-statements.test +++ b/test-data/unit/check-statements.test @@ -523,7 +523,7 @@ if object(): if object(): raise NotImplemented # E: Exception must be derived from BaseException; did you mean "NotImplementedError"? if object(): - raise NotImplemented() # E: NotImplemented? not callable + raise NotImplemented() # E: Exception must be derived from BaseException; did you mean "NotImplementedError"? [builtins fixtures/notimplemented.pyi] [case testTryFinallyStatement] diff --git a/test-data/unit/fixtures/notimplemented.pyi b/test-data/unit/fixtures/notimplemented.pyi index 92edf84a7fd11..c9e58f0994774 100644 --- a/test-data/unit/fixtures/notimplemented.pyi +++ b/test-data/unit/fixtures/notimplemented.pyi @@ -10,9 +10,16 @@ class bool: pass class int: pass class str: pass class dict: pass +class tuple: pass +class ellipsis: pass -class _NotImplementedType(Any): - __call__: NotImplemented # type: ignore -NotImplemented: _NotImplementedType +import sys + +if sys.version_info >= (3, 10): # type: ignore + from types import NotImplementedType + NotImplemented: NotImplementedType +else: + class _NotImplementedType(Any): ... + NotImplemented: _NotImplementedType class BaseException: pass From 08dde9fdd70297054efcc58f450ce4390bab9e44 Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Tue, 4 Nov 2025 17:11:08 +0100 Subject: [PATCH 379/424] refactor: reuse NOT_IMPLEMENTED_TYPE_NAMES (#20178) --- mypy/checker.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index f3a93d1eeda18..9f8299e6805de 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -5134,13 +5134,10 @@ def type_check_raise(self, e: Expression, s: RaiseStmt, optional: bool = False) # https://github.com/python/mypy/issues/11089 self.expr_checker.check_call(typ, [], [], e) - if ( - isinstance(typ, Instance) - and typ.type.fullname in {"builtins._NotImplementedType", "types.NotImplementedType"} - ) or ( + if (isinstance(typ, Instance) and typ.type.fullname in NOT_IMPLEMENTED_TYPE_NAMES) or ( isinstance(e, CallExpr) and isinstance(e.callee, RefExpr) - and e.callee.fullname in {"builtins.NotImplemented"} + and e.callee.fullname == "builtins.NotImplemented" ): self.fail( message_registry.INVALID_EXCEPTION.with_additional_msg( From bed188f7d159da3089bc538b6f8ba31d3e12f3ee Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Tue, 4 Nov 2025 09:31:13 -0700 Subject: [PATCH 380/424] [docs] document --enable-incomplete-feature TypeForm, minimally but sufficiently (#20173) --- docs/source/command_line.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index d6b70c4976dca..5efec68555931 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -1159,7 +1159,7 @@ format into the specified directory. Enabling incomplete/experimental features ***************************************** -.. option:: --enable-incomplete-feature {PreciseTupleTypes,InlineTypedDict} +.. option:: --enable-incomplete-feature {PreciseTupleTypes,InlineTypedDict,TypeForm} Some features may require several mypy releases to implement, for example due to their complexity, potential for backwards incompatibility, or @@ -1214,6 +1214,9 @@ List of currently incomplete/experimental features: def test_values() -> {"width": int, "description": str}: return {"width": 42, "description": "test"} +* ``TypeForm``: this feature enables ``TypeForm``, as described in + `PEP 747 – Annotating Type Forms _`. + Miscellaneous ************* From 9ed7bb48050950d670f27190658f46e669272e18 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Wed, 5 Nov 2025 09:37:19 -0700 Subject: [PATCH 381/424] Update duck_type_compatibility.rst: mention strict-bytes & mypy 2.0 (#20121) Seems like a good idea to mention this, while we're on the subject. --- docs/source/duck_type_compatibility.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/duck_type_compatibility.rst b/docs/source/duck_type_compatibility.rst index e801f9251db5e..7f4b67503ebef 100644 --- a/docs/source/duck_type_compatibility.rst +++ b/docs/source/duck_type_compatibility.rst @@ -9,6 +9,8 @@ supported for a small set of built-in types: * ``int`` is duck type compatible with ``float`` and ``complex``. * ``float`` is duck type compatible with ``complex``. * ``bytearray`` and ``memoryview`` are duck type compatible with ``bytes``. + (this will be disabled by default in **mypy 2.0**, and currently can be + disabled with :option:`--strict-bytes `.) For example, mypy considers an ``int`` object to be valid whenever a ``float`` object is expected. Thus code like this is nice and clean From 45aa599633e16b714205445ec67b4bcf80739359 Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Wed, 5 Nov 2025 17:44:54 +0100 Subject: [PATCH 382/424] Do not abort constructing TypeAlias if only type parameters hold us back. (#20162) Fixes #20135. I am not fully certain that this is the right way. If the type alias can be reasonably constructed and only type parameters prevent that, we can create an "almost" equivalent alias with all placeholder-bearing typevars replaced with their "simple" equivalents with default values/bound/default values. This would cause current iteration to not emit some parameterization errors later, but, since we already defer the alias as a whole, we will recheck all of them again. This obviously adds some pointless work (we check parameterization that will not be used later), but probably is not that big of a deal, recursive aliases are relatively rare in wild. If this turns out to be a bottleneck, we can add a `parameters_ready` flag to aliases and skip the heavy parts if it is False, but I think it isn't necessary now. --- mypy/semanal.py | 56 +++++++++++++++++++++++++++-- mypy/semanal_typeargs.py | 2 +- test-data/unit/check-python312.test | 24 +++++++++++++ 3 files changed, 78 insertions(+), 4 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index e55819b898e96..fe6bd71c1ab93 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -5637,13 +5637,20 @@ def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None: else: incomplete_target = has_placeholder(res) - incomplete_tv = any(has_placeholder(tv) for tv in alias_tvars) - if self.found_incomplete_ref(tag) or incomplete_target or incomplete_tv: + if self.found_incomplete_ref(tag) or incomplete_target: # Since we have got here, we know this must be a type alias (incomplete refs # may appear in nested positions), therefore use becomes_typeinfo=True. self.mark_incomplete(s.name.name, s.value, becomes_typeinfo=True) return + # Now go through all new variables and temporary replace all tvars that still + # refer to some placeholders. We defer the whole alias and will revisit it again, + # as well as all its dependents. + for i, tv in enumerate(alias_tvars): + if has_placeholder(tv): + self.mark_incomplete(s.name.name, s.value, becomes_typeinfo=True) + alias_tvars[i] = self._trivial_typevarlike_like(tv) + self.add_type_alias_deps(depends_on) check_for_explicit_any( res, self.options, self.is_typeshed_stub_file, self.msg, context=s @@ -5677,7 +5684,10 @@ def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None: ): updated = False if isinstance(existing.node, TypeAlias): - if existing.node.target != res: + if ( + existing.node.target != res + or existing.node.alias_tvars != alias_node.alias_tvars + ): # Copy expansion to the existing alias, this matches how we update base classes # for a TypeInfo _in place_ if there are nested placeholders. existing.node.target = res @@ -5707,6 +5717,46 @@ def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None: finally: self.pop_type_args(s.type_args) + def _trivial_typevarlike_like(self, tv: TypeVarLikeType) -> TypeVarLikeType: + object_type = self.named_type("builtins.object") + if isinstance(tv, TypeVarType): + return TypeVarType( + tv.name, + tv.fullname, + tv.id, + values=[], + upper_bound=object_type, + default=AnyType(TypeOfAny.from_omitted_generics), + variance=tv.variance, + line=tv.line, + column=tv.column, + ) + elif isinstance(tv, TypeVarTupleType): + tuple_type = self.named_type("builtins.tuple", [object_type]) + return TypeVarTupleType( + tv.name, + tv.fullname, + tv.id, + upper_bound=tuple_type, + tuple_fallback=tuple_type, + default=AnyType(TypeOfAny.from_omitted_generics), + line=tv.line, + column=tv.column, + ) + elif isinstance(tv, ParamSpecType): + return ParamSpecType( + tv.name, + tv.fullname, + tv.id, + flavor=tv.flavor, + upper_bound=object_type, + default=AnyType(TypeOfAny.from_omitted_generics), + line=tv.line, + column=tv.column, + ) + else: + assert False, f"Unknown TypeVarLike: {tv!r}" + # # Expressions # diff --git a/mypy/semanal_typeargs.py b/mypy/semanal_typeargs.py index 686e7a57042d9..86f8a8700def6 100644 --- a/mypy/semanal_typeargs.py +++ b/mypy/semanal_typeargs.py @@ -151,7 +151,7 @@ def validate_args( is_error = False is_invalid = False - for (i, arg), tvar in zip(enumerate(args), type_vars): + for arg, tvar in zip(args, type_vars): context = ctx if arg.line < 0 else arg if isinstance(tvar, TypeVarType): if isinstance(arg, ParamSpecType): diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index be46ff6ee5c0b..840a708fecf33 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -2057,6 +2057,7 @@ reveal_type(AA.XX) # N: Revealed type is "def () -> __main__.XX" y: B.Y reveal_type(y) # N: Revealed type is "__main__.Y" [builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] [case testPEP695TypeAliasRecursiveInvalid] type X = X # E: Cannot resolve name "X" (possible cyclic definition) @@ -2168,3 +2169,26 @@ x: MyTuple[int, str] reveal_type(x[0]) # N: Revealed type is "Any" [builtins fixtures/tuple.pyi] [typing fixtures/typing-full.pyi] + +[case testPEP695TypeAliasRecursiveInParameterBound] +from typing import Any + +type A1[T: B1] = list[int] +type B1 = None | A1[B1] +x1: A1[B1] +y1: A1[int] # E: Type argument "int" of "A1" must be a subtype of "B1" +z1: A1[None] + +type A2[T: B2] = list[T] +type B2 = None | A2[Any] +x2: A2[B2] +y2: A2[int] # E: Type argument "int" of "A2" must be a subtype of "B2" +z2: A2[None] + +type A3[T: B3] = list[T] +type B3 = None | A3[B3] +x3: A3[B3] +y3: A3[int] # E: Type argument "int" of "A3" must be a subtype of "B3" +z3: A3[None] +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] From f10b462447900ebf9675cd81034340719438cc44 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 6 Nov 2025 12:37:30 +0000 Subject: [PATCH 383/424] Stricter handling of submodules as attributes (#20179) Fixes https://github.com/python/mypy/issues/20174 The idea is quite straightforward: we only allow `foo.bar` without explicit re-export if `foo.bar` was imported in any transitive dependency (and not in some unrelated module). Note: only `import foo.bar` takes effect, effect of using `from foo.bar import ...` is not propagated for two reasons: * It will cost large performance penalty * It is relatively obscure Python feature that may be considered by some as "implementation detail" --- mypy/build.py | 50 ++++++++-------- mypy/nodes.py | 20 +++---- mypy/semanal.py | 48 ++++++++++++++- test-data/unit/check-incremental.test | 84 +++++++++++++++++++++++++++ 4 files changed, 168 insertions(+), 34 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 0b78f879c547e..e9c50ce6b2244 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -603,6 +603,7 @@ def __init__( self.options = options self.version_id = version_id self.modules: dict[str, MypyFile] = {} + self.import_map: dict[str, set[str]] = {} self.missing_modules: set[str] = set() self.fg_deps_meta: dict[str, FgDepMeta] = {} # fg_deps holds the dependencies of every module that has been @@ -623,6 +624,7 @@ def __init__( self.incomplete_namespaces, self.errors, self.plugin, + self.import_map, ) self.all_types: dict[Expression, Type] = {} # Enabled by export_types self.indirection_detector = TypeIndirectionVisitor() @@ -742,6 +744,26 @@ def getmtime(self, path: str) -> int: else: return int(self.metastore.getmtime(path)) + def correct_rel_imp(self, file: MypyFile, imp: ImportFrom | ImportAll) -> str: + """Function to correct for relative imports.""" + file_id = file.fullname + rel = imp.relative + if rel == 0: + return imp.id + if os.path.basename(file.path).startswith("__init__."): + rel -= 1 + if rel != 0: + file_id = ".".join(file_id.split(".")[:-rel]) + new_id = file_id + "." + imp.id if imp.id else file_id + + if not new_id: + self.errors.set_file(file.path, file.name, self.options) + self.errors.report( + imp.line, 0, "No parent module -- cannot perform relative import", blocker=True + ) + + return new_id + def all_imported_modules_in_file(self, file: MypyFile) -> list[tuple[int, str, int]]: """Find all reachable import statements in a file. @@ -750,27 +772,6 @@ def all_imported_modules_in_file(self, file: MypyFile) -> list[tuple[int, str, i Can generate blocking errors on bogus relative imports. """ - - def correct_rel_imp(imp: ImportFrom | ImportAll) -> str: - """Function to correct for relative imports.""" - file_id = file.fullname - rel = imp.relative - if rel == 0: - return imp.id - if os.path.basename(file.path).startswith("__init__."): - rel -= 1 - if rel != 0: - file_id = ".".join(file_id.split(".")[:-rel]) - new_id = file_id + "." + imp.id if imp.id else file_id - - if not new_id: - self.errors.set_file(file.path, file.name, self.options) - self.errors.report( - imp.line, 0, "No parent module -- cannot perform relative import", blocker=True - ) - - return new_id - res: list[tuple[int, str, int]] = [] for imp in file.imports: if not imp.is_unreachable: @@ -785,7 +786,7 @@ def correct_rel_imp(imp: ImportFrom | ImportAll) -> str: ancestors.append(part) res.append((ancestor_pri, ".".join(ancestors), imp.line)) elif isinstance(imp, ImportFrom): - cur_id = correct_rel_imp(imp) + cur_id = self.correct_rel_imp(file, imp) all_are_submodules = True # Also add any imported names that are submodules. pri = import_priority(imp, PRI_MED) @@ -805,7 +806,7 @@ def correct_rel_imp(imp: ImportFrom | ImportAll) -> str: res.append((pri, cur_id, imp.line)) elif isinstance(imp, ImportAll): pri = import_priority(imp, PRI_HIGH) - res.append((pri, correct_rel_imp(imp), imp.line)) + res.append((pri, self.correct_rel_imp(file, imp), imp.line)) # Sort such that module (e.g. foo.bar.baz) comes before its ancestors (e.g. foo # and foo.bar) so that, if FindModuleCache finds the target module in a @@ -2898,6 +2899,9 @@ def dispatch(sources: list[BuildSource], manager: BuildManager, stdout: TextIO) manager.cache_enabled = False graph = load_graph(sources, manager) + for id in graph: + manager.import_map[id] = set(graph[id].dependencies + graph[id].suppressed) + t1 = time.time() manager.add_stats( graph_size=len(graph), diff --git a/mypy/nodes.py b/mypy/nodes.py index 539995ce92295..13ba011eebc0b 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5002,27 +5002,27 @@ def local_definitions( SYMBOL_TABLE_NODE: Final[Tag] = 61 -def read_symbol(data: Buffer) -> mypy.nodes.SymbolNode: +def read_symbol(data: Buffer) -> SymbolNode: tag = read_tag(data) # The branches here are ordered manually by type "popularity". if tag == VAR: - return mypy.nodes.Var.read(data) + return Var.read(data) if tag == FUNC_DEF: - return mypy.nodes.FuncDef.read(data) + return FuncDef.read(data) if tag == DECORATOR: - return mypy.nodes.Decorator.read(data) + return Decorator.read(data) if tag == TYPE_INFO: - return mypy.nodes.TypeInfo.read(data) + return TypeInfo.read(data) if tag == OVERLOADED_FUNC_DEF: - return mypy.nodes.OverloadedFuncDef.read(data) + return OverloadedFuncDef.read(data) if tag == TYPE_VAR_EXPR: - return mypy.nodes.TypeVarExpr.read(data) + return TypeVarExpr.read(data) if tag == TYPE_ALIAS: - return mypy.nodes.TypeAlias.read(data) + return TypeAlias.read(data) if tag == PARAM_SPEC_EXPR: - return mypy.nodes.ParamSpecExpr.read(data) + return ParamSpecExpr.read(data) if tag == TYPE_VAR_TUPLE_EXPR: - return mypy.nodes.TypeVarTupleExpr.read(data) + return TypeVarTupleExpr.read(data) assert False, f"Unknown symbol tag {tag}" diff --git a/mypy/semanal.py b/mypy/semanal.py index fe6bd71c1ab93..973a28db0588b 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -451,6 +451,7 @@ def __init__( incomplete_namespaces: set[str], errors: Errors, plugin: Plugin, + import_map: dict[str, set[str]], ) -> None: """Construct semantic analyzer. @@ -483,6 +484,7 @@ def __init__( self.loop_depth = [0] self.errors = errors self.modules = modules + self.import_map = import_map self.msg = MessageBuilder(errors, modules) self.missing_modules = missing_modules self.missing_names = [set()] @@ -534,6 +536,16 @@ def __init__( self.type_expression_full_parse_success_count: int = 0 # Successful full parses self.type_expression_full_parse_failure_count: int = 0 # Failed full parses + # Imports of submodules transitively visible from given module. + # This is needed to support patterns like this + # [a.py] + # import b + # import foo + # foo.bar # <- this should work even if bar is not re-exported in foo + # [b.py] + # import foo.bar + self.transitive_submodule_imports: dict[str, set[str]] = {} + # mypyc doesn't properly handle implementing an abstractproperty # with a regular attribute so we make them properties @property @@ -6687,7 +6699,7 @@ def get_module_symbol(self, node: MypyFile, name: str) -> SymbolTableNode | None sym = names.get(name) if not sym: fullname = module + "." + name - if fullname in self.modules: + if fullname in self.modules and self.is_visible_import(module, fullname): sym = SymbolTableNode(GDEF, self.modules[fullname]) elif self.is_incomplete_namespace(module): self.record_incomplete_ref() @@ -6706,6 +6718,40 @@ def get_module_symbol(self, node: MypyFile, name: str) -> SymbolTableNode | None sym = None return sym + def is_visible_import(self, base_id: str, id: str) -> bool: + if id in self.import_map[self.cur_mod_id]: + # Fast path: module is imported locally. + return True + if base_id not in self.transitive_submodule_imports: + # This is a performance optimization for a common pattern. If one module + # in a codebase uses import numpy as np; np.foo.bar, then it is likely that + # other modules use similar pattern as well. So we pre-compute transitive + # dependencies for np, to avoid possible duplicate work in the future. + self.add_transitive_submodule_imports(base_id) + if self.cur_mod_id not in self.transitive_submodule_imports: + self.add_transitive_submodule_imports(self.cur_mod_id) + return id in self.transitive_submodule_imports[self.cur_mod_id] + + def add_transitive_submodule_imports(self, mod_id: str) -> None: + if mod_id not in self.import_map: + return + todo = self.import_map[mod_id] + seen = {mod_id} + result = {mod_id} + while todo: + dep = todo.pop() + if dep in seen: + continue + seen.add(dep) + if "." in dep: + result.add(dep) + if dep in self.transitive_submodule_imports: + result |= self.transitive_submodule_imports[dep] + continue + if dep in self.import_map: + todo |= self.import_map[dep] + self.transitive_submodule_imports[mod_id] = result + def is_missing_module(self, module: str) -> bool: return module in self.missing_modules diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 5fbaa4f2c9044..56c9cef80f342 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -7512,3 +7512,87 @@ tmp/impl.py:31: note: Revealed type is "builtins.object" tmp/impl.py:32: note: Revealed type is "Union[builtins.int, builtins.str, lib.Unrelated]" tmp/impl.py:33: note: Revealed type is "builtins.object" tmp/impl.py:34: note: Revealed type is "builtins.object" + +[case testIncrementalAccessSubmoduleWithoutExplicitImport] +import b +import a + +[file a.py] +import pkg + +pkg.submod.foo() + +[file a.py.2] +import pkg + +pkg.submod.foo() +x = 1 + +[file b.py] +import c + +[file c.py] +from pkg import submod + +[file pkg/__init__.pyi] +[file pkg/submod.pyi] +def foo() -> None: pass +[out] +tmp/a.py:3: error: "object" has no attribute "submod" +[out2] +tmp/a.py:3: error: "object" has no attribute "submod" + +[case testIncrementalAccessSubmoduleWithoutExplicitImportNested] +import a + +[file a.py] +import pandas +pandas.core.dtypes + +[file a.py.2] +import pandas +pandas.core.dtypes +# touch + +[file pandas/__init__.py] +import pandas.core.api + +[file pandas/core/__init__.py] +[file pandas/core/api.py] +import pandas.core.dtypes.dtypes + +[file pandas/core/dtypes/__init__.py] +[file pandas/core/dtypes/dtypes.py] +X = 0 +[out] +[out2] + +[case testIncrementalAccessSubmoduleWithoutExplicitImportNestedFrom] +import a + +[file a.py] +import pandas + +# Although this actually works at runtime, we do not support this, since +# this would cause major slowdown for a rare edge case. This test verifies +# that we fail consistently on cold and warm runs. +pandas.core.dtypes + +[file a.py.2] +import pandas +pandas.core.dtypes + +[file pandas/__init__.py] +import pandas.core.api + +[file pandas/core/__init__.py] +[file pandas/core/api.py] +from pandas.core.dtypes.dtypes import X + +[file pandas/core/dtypes/__init__.py] +[file pandas/core/dtypes/dtypes.py] +X = 0 +[out] +tmp/a.py:6: error: "object" has no attribute "dtypes" +[out2] +tmp/a.py:2: error: "object" has no attribute "dtypes" From 66dd2c146be9835902b88c7c9155b9eb82fe6f79 Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Fri, 7 Nov 2025 08:44:39 +0100 Subject: [PATCH 384/424] Adjust HTML reports in tests to support newer libxml (#20199) Fixes #20070. I tested this manually in debian:sid container with `libxml2-16` installed via `apt` and `lxml==6.0.2` built against it (`--no-binary`). HTML reports testcases fail on master as reported and pass on my branch. --- mypy/test/helpers.py | 9 +++++++++ test-data/unit/reports.test | 12 ++++++------ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/mypy/test/helpers.py b/mypy/test/helpers.py index 36ad5ad4ec1a2..8ff6874e746a6 100644 --- a/mypy/test/helpers.py +++ b/mypy/test/helpers.py @@ -444,6 +444,8 @@ def check_test_output_files( if testcase.suite.native_sep and os.path.sep == "\\": normalized_output = [fix_cobertura_filename(line) for line in normalized_output] normalized_output = normalize_error_messages(normalized_output) + if os.path.basename(testcase.file) == "reports.test": + normalized_output = normalize_report_meta(normalized_output) assert_string_arrays_equal( expected_content.splitlines(), normalized_output, @@ -467,6 +469,13 @@ def normalize_file_output(content: list[str], current_abs_path: str) -> list[str return result +def normalize_report_meta(content: list[str]) -> list[str]: + # libxml 2.15 and newer emits the "modern" version of this element. + # Normalize the old style to look the same. + html_meta = '' + return ['' if x == html_meta else x for x in content] + + def find_test_files(pattern: str, exclude: list[str] | None = None) -> list[str]: return [ path.name diff --git a/test-data/unit/reports.test b/test-data/unit/reports.test index cce2f7295e3bf..6e80683ad957b 100644 --- a/test-data/unit/reports.test +++ b/test-data/unit/reports.test @@ -118,7 +118,7 @@ class A(object): [outfile report/html/n.py.html] - + @@ -172,7 +172,7 @@ T = TypeVar('T') [outfile report/html/n.py.html] - + @@ -214,7 +214,7 @@ def bar(x): [outfile report/html/n.py.html] - + @@ -255,7 +255,7 @@ old_stdout = sys.stdout [outfile report/html/n.py.html] - + @@ -487,7 +487,7 @@ DisplayToSource = Callable[[int], int] [outfile report/html/n.py.html] - + @@ -529,7 +529,7 @@ namespace_packages = True [outfile report/html/folder/subfolder/something.py.html] - + From 6986993532d84c3033dc1f28cda1c0c14ee1a55a Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Sun, 9 Nov 2025 00:13:28 +0100 Subject: [PATCH 385/424] Do not assume that args of decorated functions can be cleanly mapped to their nodes (#20203) Fixes #20059. If any non-trivial decorator is present, avoid trying to pick the parameter corresponding to i-th parameter of the callable type. --- mypy/checker.py | 13 +++++++- test-data/unit/check-classes.test | 52 +++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/mypy/checker.py b/mypy/checker.py index 9f8299e6805de..07f5c520de957 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -2578,7 +2578,18 @@ def erase_override(t: Type) -> Type: continue if not is_subtype(original_arg_type, erase_override(override_arg_type)): context: Context = node - if isinstance(node, FuncDef) and not node.is_property: + if ( + isinstance(node, FuncDef) + and not node.is_property + and ( + not node.is_decorated # fast path + # allow trivial decorators like @classmethod and @override + or not (sym := node.info.get(node.name)) + or not isinstance(sym.node, Decorator) + or not sym.node.decorators + ) + ): + # If there's any decorator, we can no longer map arguments 1:1 reliably. arg_node = node.arguments[i + override.bound()] if arg_node.line != -1: context = arg_node diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index c0b1114db5120..0e9d6357af1ac 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -585,6 +585,58 @@ class B(A): @dec def f(self) -> int: pass +[case testOverrideWithDecoratorReturningCallable] +from typing import Any, Callable, TypeVar + +class Base: + def get(self, a: str) -> None: ... + +def dec(fn: Any) -> Callable[[Any, int], None]: ... + +class Derived(Base): + @dec + def get(self) -> None: ... # E: Argument 1 of "get" is incompatible with supertype "Base"; supertype defines the argument type as "str" \ + # N: This violates the Liskov substitution principle \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides +[builtins fixtures/tuple.pyi] + +[case testOverrideWithDecoratorReturningCallable2] +# flags: --pretty +from typing import Any, Callable, TypeVar + +_C = TypeVar("_C", bound=Callable[..., Any]) + +def infer_signature(f: _C) -> Callable[[Any], _C]: ... + +class Base: + def get(self, a: str, b: str, c: str) -> None: ... + def post(self, a: str, b: str) -> None: ... + +# Third argument incompatible +def get(self, a: str, b: str, c: int) -> None: ... + +# Second argument incompatible - still should not map to **kwargs +def post(self, a: str, b: int) -> None: ... + +class Derived(Base): + @infer_signature(get) + def get(self, *args: Any, **kwargs: Any) -> None: ... + + @infer_signature(post) + def post(self, *args: Any, **kwargs: Any) -> None: ... +[builtins fixtures/tuple.pyi] +[out] +main:20: error: Argument 3 of "get" is incompatible with supertype "Base"; supertype defines the argument type as "str" + def get(self, *args: Any, **kwargs: Any) -> None: ... + ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +main:20: note: This violates the Liskov substitution principle +main:20: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides +main:23: error: Argument 2 of "post" is incompatible with supertype "Base"; supertype defines the argument type as "str" + def post(self, *args: Any, **kwargs: Any) -> None: ... + ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +main:23: note: This violates the Liskov substitution principle +main:23: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides + [case testOverrideWithDecoratorReturningInstance] def dec(f) -> str: pass From 566ba1ed27ccde4d85e736c8fe03d4ac8f744186 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Sun, 9 Nov 2025 20:12:21 -0500 Subject: [PATCH 386/424] chore: fix typo in comment (#20210) Title says it all --- mypyc/transform/flag_elimination.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mypyc/transform/flag_elimination.py b/mypyc/transform/flag_elimination.py index 605e5bc46ae4b..c78e60d47cbdc 100644 --- a/mypyc/transform/flag_elimination.py +++ b/mypyc/transform/flag_elimination.py @@ -78,10 +78,9 @@ def __init__(self, builder: LowLevelIRBuilder, branch_map: dict[Register, Branch self.branches = set(branch_map.values()) def visit_assign(self, op: Assign) -> None: - old_branch = self.branch_map.get(op.dest) - if old_branch: + if old_branch := self.branch_map.get(op.dest): # Replace assignment with a copy of the old branch, which is in a - # separate basic block. The old branch will be deletecd in visit_branch. + # separate basic block. The old branch will be deleted in visit_branch. new_branch = Branch( op.src, old_branch.true, From ad2b72be21462c98f8b877fe0859123d92d876c0 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 10 Nov 2025 16:13:46 +0000 Subject: [PATCH 387/424] Micro-optimize cache primitives in librt.internal (#20194) This includes the following optimizations: * Split Buffer into ReadBuffer and WriteBuffer which are more specialized * Only check buffer types in wrapper functions, not in C primitives * Use pointers instead of integer indexes in the buffer objects This improves the performance of a micro-benchmark that reads integers in a loop by 5%, but this could help more if we'd inline some of the smaller functions (in the future). By making the functions simpler, inlining is more feasible. I'm not sure what's the best way to merge this -- maybe we'll need to have broken master for a while, and then we ca publish a new version of `librt`, and then update mypy to work using the new `librt` version. --- mypy/typeshed/stubs/librt/librt/internal.pyi | 32 +- mypyc/codegen/emit.py | 14 +- mypyc/ir/rtypes.py | 2 +- mypyc/lib-rt/librt_internal.c | 446 ++++++++++++------- mypyc/lib-rt/librt_internal.h | 37 +- mypyc/primitives/misc_ops.py | 23 +- mypyc/test-data/irbuild-classes.test | 42 +- mypyc/test-data/run-classes.test | 196 +++++--- 8 files changed, 517 insertions(+), 275 deletions(-) diff --git a/mypy/typeshed/stubs/librt/librt/internal.pyi b/mypy/typeshed/stubs/librt/librt/internal.pyi index 8654e31c100e1..78e7f9caa1170 100644 --- a/mypy/typeshed/stubs/librt/librt/internal.pyi +++ b/mypy/typeshed/stubs/librt/librt/internal.pyi @@ -1,19 +1,27 @@ from mypy_extensions import u8 +# TODO: Remove Buffer -- right now we have hacky support for BOTH the old and new APIs + class Buffer: def __init__(self, source: bytes = ...) -> None: ... def getvalue(self) -> bytes: ... -def write_bool(data: Buffer, value: bool) -> None: ... -def read_bool(data: Buffer) -> bool: ... -def write_str(data: Buffer, value: str) -> None: ... -def read_str(data: Buffer) -> str: ... -def write_bytes(data: Buffer, value: bytes) -> None: ... -def read_bytes(data: Buffer) -> bytes: ... -def write_float(data: Buffer, value: float) -> None: ... -def read_float(data: Buffer) -> float: ... -def write_int(data: Buffer, value: int) -> None: ... -def read_int(data: Buffer) -> int: ... -def write_tag(data: Buffer, value: u8) -> None: ... -def read_tag(data: Buffer) -> u8: ... +class ReadBuffer: + def __init__(self, source: bytes) -> None: ... + +class WriteBuffer: + def getvalue(self) -> bytes: ... + +def write_bool(data: WriteBuffer | Buffer, value: bool) -> None: ... +def read_bool(data: ReadBuffer | Buffer) -> bool: ... +def write_str(data: WriteBuffer | Buffer, value: str) -> None: ... +def read_str(data: ReadBuffer | Buffer) -> str: ... +def write_bytes(data: WriteBuffer | Buffer, value: bytes) -> None: ... +def read_bytes(data: ReadBuffer | Buffer) -> bytes: ... +def write_float(data: WriteBuffer | Buffer, value: float) -> None: ... +def read_float(data: ReadBuffer | Buffer) -> float: ... +def write_int(data: WriteBuffer | Buffer, value: int) -> None: ... +def read_int(data: ReadBuffer | Buffer) -> int: ... +def write_tag(data: WriteBuffer | Buffer, value: u8) -> None: ... +def read_tag(data: ReadBuffer | Buffer) -> u8: ... def cache_version() -> u8: ... diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index 4ef53296ef0d1..f2a2271e020ea 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -705,13 +705,25 @@ def emit_cast( self.emit_lines(f" {dest} = {src};", "else {") self.emit_cast_error_handler(error, src, dest, typ, raise_exception) self.emit_line("}") - elif is_object_rprimitive(typ) or is_native_rprimitive(typ): + elif is_object_rprimitive(typ): if declare_dest: self.emit_line(f"PyObject *{dest};") self.emit_arg_check(src, dest, typ, "", optional) self.emit_line(f"{dest} = {src};") if optional: self.emit_line("}") + elif is_native_rprimitive(typ): + # Native primitive types have type check functions of form "CPy_Check(...)". + if declare_dest: + self.emit_line(f"PyObject *{dest};") + short_name = typ.name.rsplit(".", 1)[-1] + check = f"(CPy{short_name}_Check({src}))" + if likely: + check = f"(likely{check})" + self.emit_arg_check(src, dest, typ, check, optional) + self.emit_lines(f" {dest} = {src};", "else {") + self.emit_cast_error_handler(error, src, dest, typ, raise_exception) + self.emit_line("}") elif isinstance(typ, RUnion): self.emit_union_cast( src, dest, typ, declare_dest, error, optional, src_type, raise_exception diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 941670ab230dd..66b98e5d63983 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -514,7 +514,7 @@ def __hash__(self) -> int: KNOWN_NATIVE_TYPES: Final = { name: RPrimitive(name, is_unboxed=False, is_refcounted=True) - for name in ["librt.internal.Buffer"] + for name in ["librt.internal.WriteBuffer", "librt.internal.ReadBuffer"] } diff --git a/mypyc/lib-rt/librt_internal.c b/mypyc/lib-rt/librt_internal.c index eaf451eff22ba..ada2dfeb39a52 100644 --- a/mypyc/lib-rt/librt_internal.c +++ b/mypyc/lib-rt/librt_internal.c @@ -30,18 +30,30 @@ #define CPY_NONE_ERROR 2 #define CPY_NONE 1 -#define _CHECK_BUFFER(data, err) if (unlikely(_check_buffer(data) == CPY_NONE_ERROR)) \ - return err; -#define _CHECK_SIZE(data, need) if (unlikely(_check_size((BufferObject *)data, need) == CPY_NONE_ERROR)) \ - return CPY_NONE_ERROR; -#define _CHECK_READ(data, size, err) if (unlikely(_check_read((BufferObject *)data, size) == CPY_NONE_ERROR)) \ - return err; - -#define _READ(data, type) *(type *)(((BufferObject *)data)->buf + ((BufferObject *)data)->pos); \ - ((BufferObject *)data)->pos += sizeof(type); - -#define _WRITE(data, type, v) *(type *)(((BufferObject *)data)->buf + ((BufferObject *)data)->pos) = v; \ - ((BufferObject *)data)->pos += sizeof(type); +#define _CHECK_READ_BUFFER(data, err) if (unlikely(_check_read_buffer(data) == CPY_NONE_ERROR)) \ + return err; +#define _CHECK_WRITE_BUFFER(data, err) if (unlikely(_check_write_buffer(data) == CPY_NONE_ERROR)) \ + return err; +#define _CHECK_WRITE(data, need) if (unlikely(_check_size((WriteBufferObject *)data, need) == CPY_NONE_ERROR)) \ + return CPY_NONE_ERROR; +#define _CHECK_READ(data, size, err) if (unlikely(_check_read((ReadBufferObject *)data, size) == CPY_NONE_ERROR)) \ + return err; + +#define _READ(result, data, type) \ + do { \ + *(result) = *(type *)(((ReadBufferObject *)data)->ptr); \ + ((ReadBufferObject *)data)->ptr += sizeof(type); \ + } while (0) + +#define _WRITE(data, type, v) \ + do { \ + *(type *)(((WriteBufferObject *)data)->ptr) = v; \ + ((WriteBufferObject *)data)->ptr += sizeof(type); \ + } while (0) + +// +// ReadBuffer +// #if PY_BIG_ENDIAN uint16_t reverse_16(uint16_t number) { @@ -55,78 +67,59 @@ uint32_t reverse_32(uint32_t number) { typedef struct { PyObject_HEAD - Py_ssize_t pos; - Py_ssize_t end; - Py_ssize_t size; - char *buf; -} BufferObject; + char *ptr; // Current read location in the buffer + char *end; // End of the buffer + PyObject *source; // The object that contains the buffer +} ReadBufferObject; -static PyTypeObject BufferType; +static PyTypeObject ReadBufferType; static PyObject* -Buffer_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +ReadBuffer_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - if (type != &BufferType) { - PyErr_SetString(PyExc_TypeError, "Buffer should not be subclassed"); + if (type != &ReadBufferType) { + PyErr_SetString(PyExc_TypeError, "ReadBuffer should not be subclassed"); return NULL; } - BufferObject *self = (BufferObject *)type->tp_alloc(type, 0); + ReadBufferObject *self = (ReadBufferObject *)type->tp_alloc(type, 0); if (self != NULL) { - self->pos = 0; - self->end = 0; - self->size = 0; - self->buf = NULL; + self->source = NULL; + self->ptr = NULL; + self->end = NULL; } return (PyObject *) self; } - static int -Buffer_init_internal(BufferObject *self, PyObject *source) { - if (source) { - if (!PyBytes_Check(source)) { - PyErr_SetString(PyExc_TypeError, "source must be a bytes object"); - return -1; - } - self->end = PyBytes_GET_SIZE(source); - // Allocate at least one byte to simplify resizing logic. - // The original bytes buffer has last null byte, so this is safe. - self->size = self->end + 1; - // This returns a pointer to internal bytes data, so make our own copy. - char *buf = PyBytes_AsString(source); - self->buf = PyMem_Malloc(self->size); - memcpy(self->buf, buf, self->end); - } else { - self->buf = PyMem_Malloc(START_SIZE); - self->size = START_SIZE; +ReadBuffer_init_internal(ReadBufferObject *self, PyObject *source) { + if (!PyBytes_CheckExact(source)) { + PyErr_SetString(PyExc_TypeError, "source must be a bytes object"); + return -1; } + self->source = Py_NewRef(source); + self->ptr = PyBytes_AS_STRING(source); + self->end = self->ptr + PyBytes_GET_SIZE(source); return 0; } static PyObject* -Buffer_internal(PyObject *source) { - BufferObject *self = (BufferObject *)BufferType.tp_alloc(&BufferType, 0); +ReadBuffer_internal(PyObject *source) { + ReadBufferObject *self = (ReadBufferObject *)ReadBufferType.tp_alloc(&ReadBufferType, 0); if (self == NULL) return NULL; - self->pos = 0; - self->end = 0; - self->size = 0; - self->buf = NULL; - if (Buffer_init_internal(self, source) == -1) { + self->ptr = NULL; + self->end = NULL; + self->source = NULL; + if (ReadBuffer_init_internal(self, source) == -1) { Py_DECREF(self); return NULL; } return (PyObject *)self; } -static PyObject* -Buffer_internal_empty(void) { - return Buffer_internal(NULL); -} - static int -Buffer_init(BufferObject *self, PyObject *args, PyObject *kwds) +ReadBuffer_init(ReadBufferObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"source", NULL}; PyObject *source = NULL; @@ -134,53 +127,166 @@ Buffer_init(BufferObject *self, PyObject *args, PyObject *kwds) if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O", kwlist, &source)) return -1; - return Buffer_init_internal(self, source); + return ReadBuffer_init_internal(self, source); +} + +static void +ReadBuffer_dealloc(ReadBufferObject *self) +{ + Py_CLEAR(self->source); + Py_TYPE(self)->tp_free((PyObject *)self); +} + +static PyMethodDef ReadBuffer_methods[] = { + {NULL} /* Sentinel */ +}; + +static PyTypeObject ReadBufferType = { + .ob_base = PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "ReadBuffer", + .tp_doc = PyDoc_STR("Mypy cache buffer objects"), + .tp_basicsize = sizeof(ReadBufferObject), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_new = ReadBuffer_new, + .tp_init = (initproc) ReadBuffer_init, + .tp_dealloc = (destructor) ReadBuffer_dealloc, + .tp_methods = ReadBuffer_methods, +}; + +// +// WriteBuffer +// + +typedef struct { + PyObject_HEAD + char *buf; // Beginning of the buffer + char *ptr; // Current write location in the buffer + char *end; // End of the buffer +} WriteBufferObject; + +static PyTypeObject WriteBufferType; + +static PyObject* +WriteBuffer_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + if (type != &WriteBufferType) { + PyErr_SetString(PyExc_TypeError, "WriteBuffer cannot be subclassed"); + return NULL; + } + + WriteBufferObject *self = (WriteBufferObject *)type->tp_alloc(type, 0); + if (self != NULL) { + self->buf = NULL; + self->ptr = NULL; + self->end = NULL; + } + return (PyObject *)self; +} + +static int +WriteBuffer_init_internal(WriteBufferObject *self) { + Py_ssize_t size = START_SIZE; + self->buf = PyMem_Malloc(size + 1); + if (self->buf == NULL) { + PyErr_NoMemory(); + return -1; + } + self->ptr = self->buf; + self->end = self->buf + size; + return 0; +} + +static PyObject* +WriteBuffer_internal(void) { + WriteBufferObject *self = (WriteBufferObject *)WriteBufferType.tp_alloc(&WriteBufferType, 0); + if (self == NULL) + return NULL; + self->buf = NULL; + self->ptr = NULL; + self->end = NULL; + if (WriteBuffer_init_internal(self) == -1) { + Py_DECREF(self); + return NULL; + } + return (PyObject *)self; +} + +static int +WriteBuffer_init(WriteBufferObject *self, PyObject *args, PyObject *kwds) +{ + if (!PyArg_ParseTuple(args, "")) { + return -1; + } + + if (kwds != NULL && PyDict_Size(kwds) > 0) { + PyErr_SetString(PyExc_TypeError, + "WriteBuffer() takes no keyword arguments"); + return -1; + } + + return WriteBuffer_init_internal(self); } static void -Buffer_dealloc(BufferObject *self) +WriteBuffer_dealloc(WriteBufferObject *self) { PyMem_Free(self->buf); + self->buf = NULL; Py_TYPE(self)->tp_free((PyObject *)self); } static PyObject* -Buffer_getvalue_internal(PyObject *self) +WriteBuffer_getvalue_internal(PyObject *self) { - return PyBytes_FromStringAndSize(((BufferObject *)self)->buf, ((BufferObject *)self)->end); + WriteBufferObject *obj = (WriteBufferObject *)self; + return PyBytes_FromStringAndSize(obj->buf, obj->ptr - obj->buf); } static PyObject* -Buffer_getvalue(BufferObject *self, PyObject *Py_UNUSED(ignored)) +WriteBuffer_getvalue(WriteBufferObject *self, PyObject *Py_UNUSED(ignored)) { - return PyBytes_FromStringAndSize(self->buf, self->end); + return PyBytes_FromStringAndSize(self->buf, self->ptr - self->buf); } -static PyMethodDef Buffer_methods[] = { - {"getvalue", (PyCFunction) Buffer_getvalue, METH_NOARGS, +static PyMethodDef WriteBuffer_methods[] = { + {"getvalue", (PyCFunction) WriteBuffer_getvalue, METH_NOARGS, "Return the buffer content as bytes object" }, {NULL} /* Sentinel */ }; -static PyTypeObject BufferType = { +static PyTypeObject WriteBufferType = { .ob_base = PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "Buffer", + .tp_name = "WriteBuffer", .tp_doc = PyDoc_STR("Mypy cache buffer objects"), - .tp_basicsize = sizeof(BufferObject), + .tp_basicsize = sizeof(WriteBufferObject), .tp_itemsize = 0, .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_new = Buffer_new, - .tp_init = (initproc) Buffer_init, - .tp_dealloc = (destructor) Buffer_dealloc, - .tp_methods = Buffer_methods, + .tp_new = WriteBuffer_new, + .tp_init = (initproc) WriteBuffer_init, + .tp_dealloc = (destructor) WriteBuffer_dealloc, + .tp_methods = WriteBuffer_methods, }; +// ---------- + +static inline char +_check_read_buffer(PyObject *data) { + if (unlikely(Py_TYPE(data) != &ReadBufferType)) { + PyErr_Format( + PyExc_TypeError, "data must be a ReadBuffer object, got %s", Py_TYPE(data)->tp_name + ); + return CPY_NONE_ERROR; + } + return CPY_NONE; +} + static inline char -_check_buffer(PyObject *data) { - if (unlikely(Py_TYPE(data) != &BufferType)) { +_check_write_buffer(PyObject *data) { + if (unlikely(Py_TYPE(data) != &WriteBufferType)) { PyErr_Format( - PyExc_TypeError, "data must be a Buffer object, got %s", Py_TYPE(data)->tp_name + PyExc_TypeError, "data must be a WriteBuffer object, got %s", Py_TYPE(data)->tp_name ); return CPY_NONE_ERROR; } @@ -188,24 +294,28 @@ _check_buffer(PyObject *data) { } static inline char -_check_size(BufferObject *data, Py_ssize_t need) { - Py_ssize_t target = data->pos + need; - if (target <= data->size) +_check_size(WriteBufferObject *data, Py_ssize_t need) { + if (data->end - data->ptr >= need) return CPY_NONE; - do - data->size *= 2; - while (target >= data->size); - data->buf = PyMem_Realloc(data->buf, data->size); + Py_ssize_t index = data->ptr - data->buf; + Py_ssize_t target = index + need; + Py_ssize_t size = data->end - data->buf; + do { + size *= 2; + } while (target >= size); + data->buf = PyMem_Realloc(data->buf, size); if (unlikely(data->buf == NULL)) { PyErr_NoMemory(); return CPY_NONE_ERROR; } + data->ptr = data->buf + index; + data->end = data->buf + size; return CPY_NONE; } static inline char -_check_read(BufferObject *data, Py_ssize_t need) { - if (unlikely(data->pos + need > data->end)) { +_check_read(ReadBufferObject *data, Py_ssize_t need) { + if (unlikely((data->end - data->ptr) < need)) { PyErr_SetString(PyExc_ValueError, "reading past the buffer end"); return CPY_NONE_ERROR; } @@ -220,9 +330,9 @@ bool format: single byte static char read_bool_internal(PyObject *data) { - _CHECK_BUFFER(data, CPY_BOOL_ERROR) _CHECK_READ(data, 1, CPY_BOOL_ERROR) - char res = _READ(data, char) + char res; + _READ(&res, data, char); if (unlikely((res != 0) & (res != 1))) { PyErr_SetString(PyExc_ValueError, "invalid bool value"); return CPY_BOOL_ERROR; @@ -238,6 +348,7 @@ read_bool(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames if (unlikely(!CPyArg_ParseStackAndKeywordsOneArg(args, nargs, kwnames, &parser, &data))) { return NULL; } + _CHECK_READ_BUFFER(data, NULL) char res = read_bool_internal(data); if (unlikely(res == CPY_BOOL_ERROR)) return NULL; @@ -248,10 +359,8 @@ read_bool(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames static char write_bool_internal(PyObject *data, char value) { - _CHECK_BUFFER(data, CPY_NONE_ERROR) - _CHECK_SIZE(data, 1) - _WRITE(data, char, value) - ((BufferObject *)data)->end += 1; + _CHECK_WRITE(data, 1) + _WRITE(data, char, value); return CPY_NONE; } @@ -264,6 +373,7 @@ write_bool(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwname if (unlikely(!CPyArg_ParseStackAndKeywordsSimple(args, nargs, kwnames, &parser, &data, &value))) { return NULL; } + _CHECK_WRITE_BUFFER(data, NULL) if (unlikely(!PyBool_Check(value))) { PyErr_SetString(PyExc_TypeError, "value must be a bool"); return NULL; @@ -289,14 +399,14 @@ _read_short_int(PyObject *data, uint8_t first) { } if ((first & FOUR_BYTES_INT_BIT) == 0) { _CHECK_READ(data, 1, CPY_INT_TAG) - second = _READ(data, uint8_t) + _READ(&second, data, uint8_t); return ((((Py_ssize_t)second) << 6) + (Py_ssize_t)(first >> 2) + MIN_TWO_BYTES_INT) << 1; } // The caller is responsible to verify this is called only for short ints. _CHECK_READ(data, 3, CPY_INT_TAG) // TODO: check if compilers emit optimal code for these two reads, and tweak if needed. - second = _READ(data, uint8_t) - two_more = _READ(data, uint16_t) + _READ(&second, data, uint8_t); + _READ(&two_more, data, uint16_t); #if PY_BIG_ENDIAN two_more = reverse_16(two_more); #endif @@ -306,11 +416,10 @@ _read_short_int(PyObject *data, uint8_t first) { static PyObject* read_str_internal(PyObject *data) { - _CHECK_BUFFER(data, NULL) - // Read string length. _CHECK_READ(data, 1, NULL) - uint8_t first = _READ(data, uint8_t) + uint8_t first; + _READ(&first, data, uint8_t); if (unlikely(first == LONG_INT_TRAILER)) { // Fail fast for invalid/tampered data. PyErr_SetString(PyExc_ValueError, "invalid str size"); @@ -326,14 +435,12 @@ read_str_internal(PyObject *data) { } Py_ssize_t size = tagged_size >> 1; // Read string content. - char *buf = ((BufferObject *)data)->buf; + char *ptr = ((ReadBufferObject *)data)->ptr; _CHECK_READ(data, size, NULL) - PyObject *res = PyUnicode_FromStringAndSize( - buf + ((BufferObject *)data)->pos, (Py_ssize_t)size - ); + PyObject *res = PyUnicode_FromStringAndSize(ptr, (Py_ssize_t)size); if (unlikely(res == NULL)) return NULL; - ((BufferObject *)data)->pos += size; + ((ReadBufferObject *)data)->ptr += size; return res; } @@ -345,6 +452,7 @@ read_str(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames) if (unlikely(!CPyArg_ParseStackAndKeywordsOneArg(args, nargs, kwnames, &parser, &data))) { return NULL; } + _CHECK_READ_BUFFER(data, NULL) return read_str_internal(data); } @@ -352,35 +460,30 @@ read_str(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames) static inline char _write_short_int(PyObject *data, Py_ssize_t real_value) { if (real_value >= MIN_ONE_BYTE_INT && real_value <= MAX_ONE_BYTE_INT) { - _CHECK_SIZE(data, 1) - _WRITE(data, uint8_t, (uint8_t)(real_value - MIN_ONE_BYTE_INT) << 1) - ((BufferObject *)data)->end += 1; + _CHECK_WRITE(data, 1) + _WRITE(data, uint8_t, (uint8_t)(real_value - MIN_ONE_BYTE_INT) << 1); } else if (real_value >= MIN_TWO_BYTES_INT && real_value <= MAX_TWO_BYTES_INT) { - _CHECK_SIZE(data, 2) + _CHECK_WRITE(data, 2) #if PY_BIG_ENDIAN uint16_t to_write = ((uint16_t)(real_value - MIN_TWO_BYTES_INT) << 2) | TWO_BYTES_INT_BIT; _WRITE(data, uint16_t, reverse_16(to_write)) #else - _WRITE(data, uint16_t, ((uint16_t)(real_value - MIN_TWO_BYTES_INT) << 2) | TWO_BYTES_INT_BIT) + _WRITE(data, uint16_t, ((uint16_t)(real_value - MIN_TWO_BYTES_INT) << 2) | TWO_BYTES_INT_BIT); #endif - ((BufferObject *)data)->end += 2; } else { - _CHECK_SIZE(data, 4) + _CHECK_WRITE(data, 4) #if PY_BIG_ENDIAN uint32_t to_write = ((uint32_t)(real_value - MIN_FOUR_BYTES_INT) << 3) | FOUR_BYTES_INT_TRAILER; _WRITE(data, uint32_t, reverse_32(to_write)) #else - _WRITE(data, uint32_t, ((uint32_t)(real_value - MIN_FOUR_BYTES_INT) << 3) | FOUR_BYTES_INT_TRAILER) + _WRITE(data, uint32_t, ((uint32_t)(real_value - MIN_FOUR_BYTES_INT) << 3) | FOUR_BYTES_INT_TRAILER); #endif - ((BufferObject *)data)->end += 4; } return CPY_NONE; } static char write_str_internal(PyObject *data, PyObject *value) { - _CHECK_BUFFER(data, CPY_NONE_ERROR) - Py_ssize_t size; const char *chunk = PyUnicode_AsUTF8AndSize(value, &size); if (unlikely(chunk == NULL)) @@ -395,11 +498,10 @@ write_str_internal(PyObject *data, PyObject *value) { return CPY_NONE_ERROR; } // Write string content. - _CHECK_SIZE(data, size) - char *buf = ((BufferObject *)data)->buf; - memcpy(buf + ((BufferObject *)data)->pos, chunk, size); - ((BufferObject *)data)->pos += size; - ((BufferObject *)data)->end += size; + _CHECK_WRITE(data, size) + char *ptr = ((WriteBufferObject *)data)->ptr; + memcpy(ptr, chunk, size); + ((WriteBufferObject *)data)->ptr += size; return CPY_NONE; } @@ -412,6 +514,7 @@ write_str(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames if (unlikely(!CPyArg_ParseStackAndKeywordsSimple(args, nargs, kwnames, &parser, &data, &value))) { return NULL; } + _CHECK_WRITE_BUFFER(data, NULL) if (unlikely(!PyUnicode_Check(value))) { PyErr_SetString(PyExc_TypeError, "value must be a str"); return NULL; @@ -429,11 +532,10 @@ bytes format: size as int (see below) followed by bytes static PyObject* read_bytes_internal(PyObject *data) { - _CHECK_BUFFER(data, NULL) - // Read length. _CHECK_READ(data, 1, NULL) - uint8_t first = _READ(data, uint8_t) + uint8_t first; + _READ(&first, data, uint8_t); if (unlikely(first == LONG_INT_TRAILER)) { // Fail fast for invalid/tampered data. PyErr_SetString(PyExc_ValueError, "invalid bytes size"); @@ -449,14 +551,12 @@ read_bytes_internal(PyObject *data) { } Py_ssize_t size = tagged_size >> 1; // Read bytes content. - char *buf = ((BufferObject *)data)->buf; + char *ptr = ((ReadBufferObject *)data)->ptr; _CHECK_READ(data, size, NULL) - PyObject *res = PyBytes_FromStringAndSize( - buf + ((BufferObject *)data)->pos, (Py_ssize_t)size - ); + PyObject *res = PyBytes_FromStringAndSize(ptr, (Py_ssize_t)size); if (unlikely(res == NULL)) return NULL; - ((BufferObject *)data)->pos += size; + ((ReadBufferObject *)data)->ptr += size; return res; } @@ -468,13 +568,12 @@ read_bytes(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwname if (unlikely(!CPyArg_ParseStackAndKeywordsOneArg(args, nargs, kwnames, &parser, &data))) { return NULL; } + _CHECK_READ_BUFFER(data, NULL) return read_bytes_internal(data); } static char write_bytes_internal(PyObject *data, PyObject *value) { - _CHECK_BUFFER(data, CPY_NONE_ERROR) - const char *chunk = PyBytes_AsString(value); if (unlikely(chunk == NULL)) return CPY_NONE_ERROR; @@ -489,11 +588,10 @@ write_bytes_internal(PyObject *data, PyObject *value) { return CPY_NONE_ERROR; } // Write bytes content. - _CHECK_SIZE(data, size) - char *buf = ((BufferObject *)data)->buf; - memcpy(buf + ((BufferObject *)data)->pos, chunk, size); - ((BufferObject *)data)->pos += size; - ((BufferObject *)data)->end += size; + _CHECK_WRITE(data, size) + char *ptr = ((WriteBufferObject *)data)->ptr; + memcpy(ptr, chunk, size); + ((WriteBufferObject *)data)->ptr += size; return CPY_NONE; } @@ -506,6 +604,7 @@ write_bytes(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnam if (unlikely(!CPyArg_ParseStackAndKeywordsSimple(args, nargs, kwnames, &parser, &data, &value))) { return NULL; } + _CHECK_WRITE_BUFFER(data, NULL) if (unlikely(!PyBytes_Check(value))) { PyErr_SetString(PyExc_TypeError, "value must be a bytes object"); return NULL; @@ -524,13 +623,12 @@ float format: static double read_float_internal(PyObject *data) { - _CHECK_BUFFER(data, CPY_FLOAT_ERROR) _CHECK_READ(data, 8, CPY_FLOAT_ERROR) - char *buf = ((BufferObject *)data)->buf; - double res = PyFloat_Unpack8(buf + ((BufferObject *)data)->pos, 1); + char *ptr = ((ReadBufferObject *)data)->ptr; + double res = PyFloat_Unpack8(ptr, 1); if (unlikely((res == -1.0) && PyErr_Occurred())) return CPY_FLOAT_ERROR; - ((BufferObject *)data)->pos += 8; + ((ReadBufferObject *)data)->ptr += 8; return res; } @@ -542,6 +640,7 @@ read_float(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwname if (unlikely(!CPyArg_ParseStackAndKeywordsOneArg(args, nargs, kwnames, &parser, &data))) { return NULL; } + _CHECK_READ_BUFFER(data, NULL) double retval = read_float_internal(data); if (unlikely(retval == CPY_FLOAT_ERROR && PyErr_Occurred())) { return NULL; @@ -551,14 +650,12 @@ read_float(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwname static char write_float_internal(PyObject *data, double value) { - _CHECK_BUFFER(data, CPY_NONE_ERROR) - _CHECK_SIZE(data, 8) - char *buf = ((BufferObject *)data)->buf; - int res = PyFloat_Pack8(value, buf + ((BufferObject *)data)->pos, 1); + _CHECK_WRITE(data, 8) + char *ptr = ((WriteBufferObject *)data)->ptr; + int res = PyFloat_Pack8(value, ptr, 1); if (unlikely(res == -1)) return CPY_NONE_ERROR; - ((BufferObject *)data)->pos += 8; - ((BufferObject *)data)->end += 8; + ((WriteBufferObject *)data)->ptr += 8; return CPY_NONE; } @@ -571,6 +668,7 @@ write_float(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnam if (unlikely(!CPyArg_ParseStackAndKeywordsSimple(args, nargs, kwnames, &parser, &data, &value))) { return NULL; } + _CHECK_WRITE_BUFFER(data, NULL) if (unlikely(!PyFloat_Check(value))) { PyErr_SetString(PyExc_TypeError, "value must be a float"); return NULL; @@ -595,10 +693,10 @@ since negative integers are much more rare. static CPyTagged read_int_internal(PyObject *data) { - _CHECK_BUFFER(data, CPY_INT_TAG) _CHECK_READ(data, 1, CPY_INT_TAG) - uint8_t first = _READ(data, uint8_t) + uint8_t first; + _READ(&first, data, uint8_t); if (likely(first != LONG_INT_TRAILER)) { return _read_short_int(data, first); } @@ -607,7 +705,7 @@ read_int_internal(PyObject *data) { // Read byte length and sign. _CHECK_READ(data, 1, CPY_INT_TAG) - first = _READ(data, uint8_t) + _READ(&first, data, uint8_t); Py_ssize_t size_and_sign = _read_short_int(data, first); if (size_and_sign == CPY_INT_TAG) return CPY_INT_TAG; @@ -620,12 +718,11 @@ read_int_internal(PyObject *data) { // Construct an int object from the byte array. _CHECK_READ(data, size, CPY_INT_TAG) - char *buf = ((BufferObject *)data)->buf; - PyObject *num = _PyLong_FromByteArray( - (unsigned char *)(buf + ((BufferObject *)data)->pos), size, 1, 0); + char *ptr = ((ReadBufferObject *)data)->ptr; + PyObject *num = _PyLong_FromByteArray((unsigned char *)ptr, size, 1, 0); if (num == NULL) return CPY_INT_TAG; - ((BufferObject *)data)->pos += size; + ((ReadBufferObject *)data)->ptr += size; if (sign) { PyObject *old = num; num = PyNumber_Negative(old); @@ -645,6 +742,7 @@ read_int(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames) if (unlikely(!CPyArg_ParseStackAndKeywordsOneArg(args, nargs, kwnames, &parser, &data))) { return NULL; } + _CHECK_READ_BUFFER(data, NULL) CPyTagged retval = read_int_internal(data); if (unlikely(retval == CPY_INT_TAG)) { return NULL; @@ -664,9 +762,8 @@ static inline int hex_to_int(char c) { static inline char _write_long_int(PyObject *data, CPyTagged value) { - _CHECK_SIZE(data, 1) - _WRITE(data, uint8_t, LONG_INT_TRAILER) - ((BufferObject *)data)->end += 1; + _CHECK_WRITE(data, 1) + _WRITE(data, uint8_t, LONG_INT_TRAILER); PyObject *hex_str = NULL; PyObject* int_value = CPyTagged_AsObject(value); @@ -731,8 +828,6 @@ _write_long_int(PyObject *data, CPyTagged value) { static char write_int_internal(PyObject *data, CPyTagged value) { - _CHECK_BUFFER(data, CPY_NONE_ERROR) - if (likely((value & CPY_INT_TAG) == 0)) { Py_ssize_t real_value = CPyTagged_ShortAsSsize_t(value); if (likely(real_value >= MIN_FOUR_BYTES_INT && real_value <= MAX_FOUR_BYTES_INT)) { @@ -754,6 +849,7 @@ write_int(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames if (unlikely(!CPyArg_ParseStackAndKeywordsSimple(args, nargs, kwnames, &parser, &data, &value))) { return NULL; } + _CHECK_WRITE_BUFFER(data, NULL) if (unlikely(!PyLong_Check(value))) { PyErr_SetString(PyExc_TypeError, "value must be an int"); return NULL; @@ -773,9 +869,9 @@ integer tag format (0 <= t <= 255): static uint8_t read_tag_internal(PyObject *data) { - _CHECK_BUFFER(data, CPY_LL_UINT_ERROR) _CHECK_READ(data, 1, CPY_LL_UINT_ERROR) - uint8_t ret = _READ(data, uint8_t) + uint8_t ret; + _READ(&ret, data, uint8_t); return ret; } @@ -787,6 +883,7 @@ read_tag(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames) if (unlikely(!CPyArg_ParseStackAndKeywordsOneArg(args, nargs, kwnames, &parser, &data))) { return NULL; } + _CHECK_READ_BUFFER(data, NULL) uint8_t retval = read_tag_internal(data); if (unlikely(retval == CPY_LL_UINT_ERROR && PyErr_Occurred())) { return NULL; @@ -796,10 +893,8 @@ read_tag(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames) static char write_tag_internal(PyObject *data, uint8_t value) { - _CHECK_BUFFER(data, CPY_NONE_ERROR) - _CHECK_SIZE(data, 1) - _WRITE(data, uint8_t, value) - ((BufferObject *)data)->end += 1; + _CHECK_WRITE(data, 1) + _WRITE(data, uint8_t, value); return CPY_NONE; } @@ -812,6 +907,7 @@ write_tag(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames if (unlikely(!CPyArg_ParseStackAndKeywordsSimple(args, nargs, kwnames, &parser, &data, &value))) { return NULL; } + _CHECK_WRITE_BUFFER(data, NULL) uint8_t unboxed = CPyLong_AsUInt8(value); if (unlikely(unboxed == CPY_LL_UINT_ERROR && PyErr_Occurred())) { CPy_TypeError("u8", value); @@ -834,6 +930,16 @@ cache_version(PyObject *self, PyObject *Py_UNUSED(ignored)) { return PyLong_FromLong(cache_version_internal()); } +static PyTypeObject * +ReadBuffer_type_internal(void) { + return &ReadBufferType; // Return borrowed reference +} + +static PyTypeObject * +WriteBuffer_type_internal(void) { + return &WriteBufferType; // Return borrowed reference +}; + static PyMethodDef librt_internal_module_methods[] = { {"write_bool", (PyCFunction)write_bool, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("write a bool")}, {"read_bool", (PyCFunction)read_bool, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("read a bool")}, @@ -859,18 +965,24 @@ NativeInternal_ABI_Version(void) { static int librt_internal_module_exec(PyObject *m) { - if (PyType_Ready(&BufferType) < 0) { + if (PyType_Ready(&ReadBufferType) < 0) { + return -1; + } + if (PyType_Ready(&WriteBufferType) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "ReadBuffer", (PyObject *) &ReadBufferType) < 0) { return -1; } - if (PyModule_AddObjectRef(m, "Buffer", (PyObject *) &BufferType) < 0) { + if (PyModule_AddObjectRef(m, "WriteBuffer", (PyObject *) &WriteBufferType) < 0) { return -1; } // Export mypy internal C API, be careful with the order! - static void *NativeInternal_API[17] = { - (void *)Buffer_internal, - (void *)Buffer_internal_empty, - (void *)Buffer_getvalue_internal, + static void *NativeInternal_API[LIBRT_INTERNAL_API_LEN] = { + (void *)ReadBuffer_internal, + (void *)WriteBuffer_internal, + (void *)WriteBuffer_getvalue_internal, (void *)write_bool_internal, (void *)read_bool_internal, (void *)write_str_internal, @@ -885,6 +997,8 @@ librt_internal_module_exec(PyObject *m) (void *)write_bytes_internal, (void *)read_bytes_internal, (void *)cache_version_internal, + (void *)ReadBuffer_type_internal, + (void *)WriteBuffer_type_internal, }; PyObject *c_api_object = PyCapsule_New((void *)NativeInternal_API, "librt.internal._C_API", NULL); if (PyModule_Add(m, "_C_API", c_api_object) < 0) { diff --git a/mypyc/lib-rt/librt_internal.h b/mypyc/lib-rt/librt_internal.h index 1d16e1cb127f5..329a0fd68c111 100644 --- a/mypyc/lib-rt/librt_internal.h +++ b/mypyc/lib-rt/librt_internal.h @@ -1,13 +1,16 @@ #ifndef LIBRT_INTERNAL_H #define LIBRT_INTERNAL_H -#define LIBRT_INTERNAL_ABI_VERSION 0 +#define LIBRT_INTERNAL_ABI_VERSION 1 +#define LIBRT_INTERNAL_API_LEN 19 #ifdef LIBRT_INTERNAL_MODULE -static PyObject *Buffer_internal(PyObject *source); -static PyObject *Buffer_internal_empty(void); -static PyObject *Buffer_getvalue_internal(PyObject *self); +static PyObject *ReadBuffer_internal(PyObject *source); +static PyObject *WriteBuffer_internal(void); +static PyObject *WriteBuffer_getvalue_internal(PyObject *self); +static PyObject *ReadBuffer_internal(PyObject *source); +static PyObject *ReadBuffer_internal_empty(void); static char write_bool_internal(PyObject *data, char value); static char read_bool_internal(PyObject *data); static char write_str_internal(PyObject *data, PyObject *value); @@ -22,14 +25,16 @@ static int NativeInternal_ABI_Version(void); static char write_bytes_internal(PyObject *data, PyObject *value); static PyObject *read_bytes_internal(PyObject *data); static uint8_t cache_version_internal(void); +static PyTypeObject *ReadBuffer_type_internal(void); +static PyTypeObject *WriteBuffer_type_internal(void); #else -static void **NativeInternal_API; +static void *NativeInternal_API[LIBRT_INTERNAL_API_LEN]; -#define Buffer_internal (*(PyObject* (*)(PyObject *source)) NativeInternal_API[0]) -#define Buffer_internal_empty (*(PyObject* (*)(void)) NativeInternal_API[1]) -#define Buffer_getvalue_internal (*(PyObject* (*)(PyObject *source)) NativeInternal_API[2]) +#define ReadBuffer_internal (*(PyObject* (*)(PyObject *source)) NativeInternal_API[0]) +#define WriteBuffer_internal (*(PyObject* (*)(void)) NativeInternal_API[1]) +#define WriteBuffer_getvalue_internal (*(PyObject* (*)(PyObject *source)) NativeInternal_API[2]) #define write_bool_internal (*(char (*)(PyObject *source, char value)) NativeInternal_API[3]) #define read_bool_internal (*(char (*)(PyObject *source)) NativeInternal_API[4]) #define write_str_internal (*(char (*)(PyObject *source, PyObject *value)) NativeInternal_API[5]) @@ -44,6 +49,8 @@ static void **NativeInternal_API; #define write_bytes_internal (*(char (*)(PyObject *source, PyObject *value)) NativeInternal_API[14]) #define read_bytes_internal (*(PyObject* (*)(PyObject *source)) NativeInternal_API[15]) #define cache_version_internal (*(uint8_t (*)(void)) NativeInternal_API[16]) +#define ReadBuffer_type_internal (*(PyTypeObject* (*)(void)) NativeInternal_API[17]) +#define WriteBuffer_type_internal (*(PyTypeObject* (*)(void)) NativeInternal_API[18]) static int import_librt_internal(void) @@ -52,9 +59,10 @@ import_librt_internal(void) if (mod == NULL) return -1; Py_DECREF(mod); // we import just for the side effect of making the below work. - NativeInternal_API = (void **)PyCapsule_Import("librt.internal._C_API", 0); - if (NativeInternal_API == NULL) + void *capsule = PyCapsule_Import("librt.internal._C_API", 0); + if (capsule == NULL) return -1; + memcpy(NativeInternal_API, capsule, sizeof(NativeInternal_API)); if (NativeInternal_ABI_Version() != LIBRT_INTERNAL_ABI_VERSION) { PyErr_SetString(PyExc_ValueError, "ABI version conflict for librt.internal"); return -1; @@ -63,4 +71,13 @@ import_librt_internal(void) } #endif + +static inline bool CPyReadBuffer_Check(PyObject *obj) { + return Py_TYPE(obj) == ReadBuffer_type_internal(); +} + +static inline bool CPyWriteBuffer_Check(PyObject *obj) { + return Py_TYPE(obj) == WriteBuffer_type_internal(); +} + #endif // LIBRT_INTERNAL_H diff --git a/mypyc/primitives/misc_ops.py b/mypyc/primitives/misc_ops.py index 10f4bc001e293..f685b1cfbcf53 100644 --- a/mypyc/primitives/misc_ops.py +++ b/mypyc/primitives/misc_ops.py @@ -333,31 +333,32 @@ error_kind=ERR_NEVER, ) -buffer_rprimitive = KNOWN_NATIVE_TYPES["librt.internal.Buffer"] +write_buffer_rprimitive = KNOWN_NATIVE_TYPES["librt.internal.WriteBuffer"] +read_buffer_rprimitive = KNOWN_NATIVE_TYPES["librt.internal.ReadBuffer"] -# Buffer(source) +# ReadBuffer(source) function_op( - name="librt.internal.Buffer", + name="librt.internal.ReadBuffer", arg_types=[bytes_rprimitive], - return_type=buffer_rprimitive, - c_function_name="Buffer_internal", + return_type=read_buffer_rprimitive, + c_function_name="ReadBuffer_internal", error_kind=ERR_MAGIC, ) -# Buffer() +# WriteBuffer() function_op( - name="librt.internal.Buffer", + name="librt.internal.WriteBuffer", arg_types=[], - return_type=buffer_rprimitive, - c_function_name="Buffer_internal_empty", + return_type=write_buffer_rprimitive, + c_function_name="WriteBuffer_internal", error_kind=ERR_MAGIC, ) method_op( name="getvalue", - arg_types=[buffer_rprimitive], + arg_types=[write_buffer_rprimitive], return_type=bytes_rprimitive, - c_function_name="Buffer_getvalue_internal", + c_function_name="WriteBuffer_getvalue_internal", error_kind=ERR_MAGIC, ) diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index a8ee7213ef965..0f8ec2b094f05 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -1453,7 +1453,7 @@ class TestOverload: from typing import Final from mypy_extensions import u8 from librt.internal import ( - Buffer, write_bool, read_bool, write_str, read_str, write_float, read_float, + WriteBuffer, ReadBuffer, write_bool, read_bool, write_str, read_str, write_float, read_float, write_int, read_int, write_tag, read_tag, write_bytes, read_bytes, cache_version, ) @@ -1462,7 +1462,7 @@ Tag = u8 TAG: Final[Tag] = 1 def foo() -> None: - b = Buffer() + b = WriteBuffer() write_str(b, "foo") write_bytes(b, b"bar") write_bool(b, True) @@ -1470,23 +1470,23 @@ def foo() -> None: write_int(b, 1) write_tag(b, TAG) - b = Buffer(b.getvalue()) - x = read_str(b) - xb = read_bytes(b) - y = read_bool(b) - z = read_float(b) - t = read_int(b) - u = read_tag(b) + rb = ReadBuffer(b.getvalue()) + x = read_str(rb) + xb = read_bytes(rb) + y = read_bool(rb) + z = read_float(rb) + t = read_int(rb) + u = read_tag(rb) v = cache_version() [out] def foo(): - r0, b :: librt.internal.Buffer + r0, b :: librt.internal.WriteBuffer r1 :: str r2 :: None r3 :: bytes r4, r5, r6, r7, r8 :: None r9 :: bytes - r10 :: librt.internal.Buffer + r10, rb :: librt.internal.ReadBuffer r11, x :: str r12, xb :: bytes r13, y :: bool @@ -1494,7 +1494,7 @@ def foo(): r15, t :: int r16, u, r17, v :: u8 L0: - r0 = Buffer_internal_empty() + r0 = WriteBuffer_internal() b = r0 r1 = 'foo' r2 = write_str_internal(b, r1) @@ -1504,20 +1504,20 @@ L0: r6 = write_float_internal(b, 0.1) r7 = write_int_internal(b, 2) r8 = write_tag_internal(b, 1) - r9 = Buffer_getvalue_internal(b) - r10 = Buffer_internal(r9) - b = r10 - r11 = read_str_internal(b) + r9 = WriteBuffer_getvalue_internal(b) + r10 = ReadBuffer_internal(r9) + rb = r10 + r11 = read_str_internal(rb) x = r11 - r12 = read_bytes_internal(b) + r12 = read_bytes_internal(rb) xb = r12 - r13 = read_bool_internal(b) + r13 = read_bool_internal(rb) y = r13 - r14 = read_float_internal(b) + r14 = read_float_internal(rb) z = r14 - r15 = read_int_internal(b) + r15 = read_int_internal(rb) t = r15 - r16 = read_tag_internal(b) + r16 = read_tag_internal(rb) u = r16 r17 = cache_version_internal() v = r17 diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 0805da184e1a8..2c2eac5057971 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -2711,14 +2711,18 @@ from native import Player Player.MIN = [case testBufferRoundTrip_librt_internal] -from typing import Final +from __future__ import annotations + +from typing import Final, Any from mypy_extensions import u8 from librt.internal import ( - Buffer, write_bool, read_bool, write_str, read_str, write_float, read_float, + ReadBuffer, WriteBuffer, write_bool, read_bool, write_str, read_str, write_float, read_float, write_int, read_int, write_tag, read_tag, write_bytes, read_bytes, cache_version, ) +from testutil import assertRaises + Tag = u8 TAG_A: Final[Tag] = 33 TAG_B: Final[Tag] = 255 @@ -2726,11 +2730,89 @@ TAG_SPECIAL: Final[Tag] = 239 def test_buffer_basic() -> None: assert cache_version() == 0 - b = Buffer(b"foo") - assert b.getvalue() == b"foo" + w = WriteBuffer() + write_str(w, "foo") + r = ReadBuffer(w.getvalue()) + assert read_str(r) == "foo" + +def test_buffer_grow() -> None: + w = WriteBuffer() + n = 100 * 1000 + for i in range(n): + write_int(w, i & 63) + r = ReadBuffer(w.getvalue()) + for i in range(n): + assert read_int(r) == (i & 63) + with assertRaises(ValueError): + read_int(r) + +def test_buffer_primitive_types() -> None: + a1: Any = WriteBuffer() + w: WriteBuffer = a1 + write_str(w, "foo") + data = w.getvalue() + assert read_str(ReadBuffer(data)) == "foo" + a2: Any = ReadBuffer(b"foo") + with assertRaises(TypeError): + w2: WriteBuffer = a2 + + a3: Any = ReadBuffer(data) + r: ReadBuffer = a3 + assert read_str(r) == "foo" + a4: Any = WriteBuffer() + with assertRaises(TypeError): + r2: ReadBuffer = a4 + +def test_type_check_args_in_write_functions() -> None: + # Test calling wrapper functions with invalid arg types + from librt import internal + alias: Any = internal + w = WriteBuffer() + with assertRaises(TypeError): + alias.write_str(None, "foo") + with assertRaises(TypeError): + alias.write_str(w, None) + with assertRaises(TypeError): + alias.write_bool(None, True) + with assertRaises(TypeError): + alias.write_bool(w, None) + with assertRaises(TypeError): + alias.write_bytes(None, b"foo") + with assertRaises(TypeError): + alias.write_bytes(w, None) + with assertRaises(TypeError): + alias.write_float(None, 1.5) + with assertRaises(TypeError): + alias.write_float(w, None) + with assertRaises(TypeError): + alias.write_int(None, 15) + with assertRaises(TypeError): + alias.write_int(w, None) + with assertRaises(TypeError): + alias.write_tag(None, 15) + with assertRaises(TypeError): + alias.write_tag(w, None) + +def test_type_check_buffer_in_read_functions() -> None: + # Test calling wrapper functions with invalid arg types + from librt import internal + alias: Any = internal + with assertRaises(TypeError): + alias.read_str(None) + with assertRaises(TypeError): + alias.read_bool(None) + with assertRaises(TypeError): + alias.read_bytes(None) + with assertRaises(TypeError): + alias.read_float(None) + with assertRaises(TypeError): + alias.read_int(None) + with assertRaises(TypeError): + alias.read_tag(None) def test_buffer_roundtrip() -> None: - b = Buffer() + b: WriteBuffer | ReadBuffer + b = WriteBuffer() write_str(b, "foo") write_bool(b, True) write_str(b, "bar" * 1000) @@ -2757,7 +2839,7 @@ def test_buffer_roundtrip() -> None: write_int(b, 536860912) write_int(b, 1234567891) - b = Buffer(b.getvalue()) + b = ReadBuffer(b.getvalue()) assert read_str(b) == "foo" assert read_bool(b) is True assert read_str(b) == "bar" * 1000 @@ -2785,77 +2867,83 @@ def test_buffer_roundtrip() -> None: assert read_int(b) == 1234567891 def test_buffer_int_size() -> None: + b: WriteBuffer | ReadBuffer for i in (-10, -9, 0, 116, 117): - b = Buffer() + b = WriteBuffer() write_int(b, i) assert len(b.getvalue()) == 1 - b = Buffer(b.getvalue()) + b = ReadBuffer(b.getvalue()) assert read_int(b) == i for i in (-100, -11, 118, 12344, 16283): - b = Buffer() + b = WriteBuffer() write_int(b, i) assert len(b.getvalue()) == 2 - b = Buffer(b.getvalue()) + b = ReadBuffer(b.getvalue()) assert read_int(b) == i for i in (-10000, 16284, 123456789): - b = Buffer() + b = WriteBuffer() write_int(b, i) assert len(b.getvalue()) == 4 - b = Buffer(b.getvalue()) + b = ReadBuffer(b.getvalue()) assert read_int(b) == i def test_buffer_int_powers() -> None: # 0, 1, 2 are tested above for p in range(2, 200): - b = Buffer() + b = WriteBuffer() write_int(b, 1 << p) write_int(b, (1 << p) - 1) write_int(b, -1 << p) write_int(b, (-1 << p) + 1) - b = Buffer(b.getvalue()) - assert read_int(b) == 1 << p - assert read_int(b) == (1 << p) - 1 - assert read_int(b) == -1 << p - assert read_int(b) == (-1 << p) + 1 + rb = ReadBuffer(b.getvalue()) + assert read_int(rb) == 1 << p + assert read_int(rb) == (1 << p) - 1 + assert read_int(rb) == -1 << p + assert read_int(rb) == (-1 << p) + 1 def test_positive_long_int_serialized_bytes() -> None: - b = Buffer() + b = WriteBuffer() n = 0x123456789ab write_int(b, n) x = b.getvalue() # Two prefix bytes, followed by little endian encoded integer in variable-length format assert x == b"\x0f\x2c\xab\x89\x67\x45\x23\x01" - b = Buffer(x) - assert read_int(b) == n + rb = ReadBuffer(x) + assert read_int(rb) == n def test_negative_long_int_serialized_bytes() -> None: - b = Buffer() + b = WriteBuffer() n = -0x123456789abcde write_int(b, n) x = b.getvalue() assert x == b"\x0f\x32\xde\xbc\x9a\x78\x56\x34\x12" - b = Buffer(x) - assert read_int(b) == n + rb = ReadBuffer(x) + assert read_int(rb) == n def test_buffer_str_size() -> None: + b: WriteBuffer | ReadBuffer for s in ("", "a", "a" * 117): - b = Buffer() + b = WriteBuffer() write_str(b, s) assert len(b.getvalue()) == len(s) + 1 - b = Buffer(b.getvalue()) + b = ReadBuffer(b.getvalue()) assert read_str(b) == s for s in ("a" * 118, "a" * 16283): - b = Buffer() + b = WriteBuffer() write_str(b, s) assert len(b.getvalue()) == len(s) + 2 - b = Buffer(b.getvalue()) + b = ReadBuffer(b.getvalue()) assert read_str(b) == s [file driver.py] from native import * test_buffer_basic() +test_buffer_grow() +test_buffer_primitive_types() +test_type_check_args_in_write_functions() +test_type_check_buffer_in_read_functions() test_buffer_roundtrip() test_buffer_int_size() test_buffer_str_size() @@ -2864,11 +2952,13 @@ test_positive_long_int_serialized_bytes() test_negative_long_int_serialized_bytes() def test_buffer_basic_interpreted() -> None: - b = Buffer(b"foo") - assert b.getvalue() == b"foo" + b = WriteBuffer() + write_str(b, "foo") + b = ReadBuffer(b.getvalue()) + assert read_str(b) == "foo" def test_buffer_roundtrip_interpreted() -> None: - b = Buffer() + b = WriteBuffer() write_str(b, "foo") write_bool(b, True) write_str(b, "bar" * 1000) @@ -2893,7 +2983,7 @@ def test_buffer_roundtrip_interpreted() -> None: write_int(b, 536860912) write_int(b, 1234567891) - b = Buffer(b.getvalue()) + b = ReadBuffer(b.getvalue()) assert read_str(b) == "foo" assert read_bool(b) is True assert read_str(b) == "bar" * 1000 @@ -2920,47 +3010,47 @@ def test_buffer_roundtrip_interpreted() -> None: def test_buffer_int_size_interpreted() -> None: for i in (-10, -9, 0, 116, 117): - b = Buffer() + b = WriteBuffer() write_int(b, i) assert len(b.getvalue()) == 1 - b = Buffer(b.getvalue()) + b = ReadBuffer(b.getvalue()) assert read_int(b) == i for i in (-100, -11, 118, 12344, 16283): - b = Buffer() + b = WriteBuffer() write_int(b, i) assert len(b.getvalue()) == 2 - b = Buffer(b.getvalue()) + b = ReadBuffer(b.getvalue()) assert read_int(b) == i for i in (-10000, 16284, 123456789): - b = Buffer() + b = WriteBuffer() write_int(b, i) assert len(b.getvalue()) == 4 - b = Buffer(b.getvalue()) + b = ReadBuffer(b.getvalue()) assert read_int(b) == i def test_buffer_int_powers_interpreted() -> None: # 0, 1, 2 are tested above for p in range(2, 9): - b = Buffer() + b = WriteBuffer() write_int(b, 1 << p) write_int(b, -1 << p) - b = Buffer(b.getvalue()) + b = ReadBuffer(b.getvalue()) assert read_int(b) == 1 << p assert read_int(b) == -1 << p def test_buffer_str_size_interpreted() -> None: for s in ("", "a", "a" * 117): - b = Buffer() + b = WriteBuffer() write_str(b, s) assert len(b.getvalue()) == len(s) + 1 - b = Buffer(b.getvalue()) + b = ReadBuffer(b.getvalue()) assert read_str(b) == s for s in ("a" * 118, "a" * 16283): - b = Buffer() + b = WriteBuffer() write_str(b, s) assert len(b.getvalue()) == len(s) + 2 - b = Buffer(b.getvalue()) + b = ReadBuffer(b.getvalue()) assert read_str(b) == s test_buffer_basic_interpreted() @@ -2970,12 +3060,12 @@ test_buffer_str_size_interpreted() test_buffer_int_powers_interpreted() [case testBufferEmpty_librt_internal] -from librt.internal import Buffer, write_int, read_int +from librt.internal import WriteBuffer, ReadBuffer, write_int, read_int def test_empty() -> None: - b = Buffer(b"") + b = WriteBuffer() write_int(b, 42) - b1 = Buffer(b.getvalue()) + b1 = ReadBuffer(b.getvalue()) assert read_int(b1) == 42 [case testEnumMethodCalls] @@ -5362,37 +5452,37 @@ test_deletable_attr() [case testBufferCorruptedData_librt_internal] from librt.internal import ( - Buffer, read_bool, read_str, read_float, read_int, read_tag, read_bytes + ReadBuffer, read_bool, read_str, read_float, read_int, read_tag, read_bytes ) from random import randbytes def check(data: bytes) -> None: - b = Buffer(data) + b = ReadBuffer(data) try: while True: read_bool(b) except ValueError: pass - b = Buffer(data) + b = ReadBuffer(data) read_tag(b) # Always succeeds try: while True: read_int(b) except ValueError: pass - b = Buffer(data) + b = ReadBuffer(data) try: while True: read_str(b) except ValueError: pass - b = Buffer(data) + b = ReadBuffer(data) try: while True: read_bytes(b) except ValueError: pass - b = Buffer(data) + b = ReadBuffer(data) try: while True: read_float(b) From 68d8f9d626f29b20d72cdcb31bb9cf22a80d035d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 10 Nov 2025 17:37:01 +0000 Subject: [PATCH 388/424] Update librt dependency to 0.5.0 (#20214) This new version has breaking API and ABI changes (see https://github.com/python/mypy/pull/20194 for context). This is okay since we haven't made a public release with librt as a dependency yet. (Note that librt is part of the mypy project but distributed separately.) --- mypy-requirements.txt | 2 +- mypy/build.py | 12 +-- mypy/cache.py | 55 +++++----- mypy/exportjson.py | 4 +- mypy/nodes.py | 71 ++++++------- mypy/types.py | 105 ++++++++++--------- mypy/typeshed/stubs/librt/librt/internal.pyi | 30 +++--- pyproject.toml | 4 +- test-requirements.txt | 2 +- 9 files changed, 141 insertions(+), 144 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index 7c83178ae1eb8..a69d31088e554 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -4,4 +4,4 @@ typing_extensions>=4.6.0 mypy_extensions>=1.0.0 pathspec>=0.9.0 tomli>=1.1.0; python_version<'3.11' -librt>=0.4.0 +librt>=0.5.0 diff --git a/mypy/build.py b/mypy/build.py index e9c50ce6b2244..853e54e445ac6 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -31,7 +31,7 @@ from librt.internal import cache_version import mypy.semanal_main -from mypy.cache import CACHE_VERSION, Buffer, CacheMeta +from mypy.cache import CACHE_VERSION, CacheMeta, ReadBuffer, WriteBuffer from mypy.checker import TypeChecker from mypy.error_formatter import OUTPUT_CHOICES, ErrorFormatter from mypy.errors import CompileError, ErrorInfo, Errors, report_internal_error @@ -1343,7 +1343,7 @@ def find_cache_meta(id: str, path: str, manager: BuildManager) -> CacheMeta | No if meta[0] != cache_version() or meta[1] != CACHE_VERSION: manager.log(f"Metadata abandoned for {id}: incompatible cache format") return None - data_io = Buffer(meta[2:]) + data_io = ReadBuffer(meta[2:]) m = CacheMeta.read(data_io, data_file) else: m = CacheMeta.deserialize(meta, data_file) @@ -1594,7 +1594,7 @@ def write_cache( # Serialize data and analyze interface if manager.options.fixed_format_cache: - data_io = Buffer() + data_io = WriteBuffer() tree.write(data_io) data_bytes = data_io.getvalue() else: @@ -1678,7 +1678,7 @@ def write_cache_meta(meta: CacheMeta, manager: BuildManager, meta_file: str) -> # Write meta cache file metastore = manager.metastore if manager.options.fixed_format_cache: - data_io = Buffer() + data_io = WriteBuffer() meta.write(data_io) # Prefix with both low- and high-level cache format versions for future validation. # TODO: switch to something like librt.internal.write_byte() if this is slow. @@ -2111,7 +2111,7 @@ def load_tree(self, temporary: bool = False) -> None: t0 = time.time() # TODO: Assert data file wasn't changed. if isinstance(data, bytes): - data_io = Buffer(data) + data_io = ReadBuffer(data) self.tree = MypyFile.read(data_io) else: self.tree = MypyFile.deserialize(data) @@ -2484,7 +2484,7 @@ def write_cache(self) -> tuple[CacheMeta, str] | None: if self.options.debug_serialize: try: if self.manager.options.fixed_format_cache: - data = Buffer() + data = WriteBuffer() self.tree.write(data) else: self.tree.serialize() diff --git a/mypy/cache.py b/mypy/cache.py index 900815b9f7e73..ad12fd96f1fa4 100644 --- a/mypy/cache.py +++ b/mypy/cache.py @@ -52,7 +52,8 @@ from typing_extensions import TypeAlias as _TypeAlias from librt.internal import ( - Buffer as Buffer, + ReadBuffer as ReadBuffer, + WriteBuffer as WriteBuffer, read_bool as read_bool, read_bytes as read_bytes_bare, read_float as read_float_bare, @@ -165,7 +166,7 @@ def deserialize(cls, meta: dict[str, Any], data_file: str) -> CacheMeta | None: except (KeyError, ValueError): return None - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_str(data, self.id) write_str(data, self.path) write_int(data, self.mtime) @@ -187,7 +188,7 @@ def write(self, data: Buffer) -> None: write_json_value(data, self.plugin_data) @classmethod - def read(cls, data: Buffer, data_file: str) -> CacheMeta | None: + def read(cls, data: ReadBuffer, data_file: str) -> CacheMeta | None: try: return CacheMeta( id=read_str(data), @@ -240,7 +241,7 @@ def read(cls, data: Buffer, data_file: str) -> CacheMeta | None: END_TAG: Final[Tag] = 255 -def read_literal(data: Buffer, tag: Tag) -> int | str | bool | float: +def read_literal(data: ReadBuffer, tag: Tag) -> int | str | bool | float: if tag == LITERAL_INT: return read_int_bare(data) elif tag == LITERAL_STR: @@ -256,7 +257,7 @@ def read_literal(data: Buffer, tag: Tag) -> int | str | bool | float: # There is an intentional asymmetry between read and write for literals because # None and/or complex values are only allowed in some contexts but not in others. -def write_literal(data: Buffer, value: int | str | bool | float | complex | None) -> None: +def write_literal(data: WriteBuffer, value: int | str | bool | float | complex | None) -> None: if isinstance(value, bool): write_bool(data, value) elif isinstance(value, int): @@ -276,37 +277,37 @@ def write_literal(data: Buffer, value: int | str | bool | float | complex | None write_tag(data, LITERAL_NONE) -def read_int(data: Buffer) -> int: +def read_int(data: ReadBuffer) -> int: assert read_tag(data) == LITERAL_INT return read_int_bare(data) -def write_int(data: Buffer, value: int) -> None: +def write_int(data: WriteBuffer, value: int) -> None: write_tag(data, LITERAL_INT) write_int_bare(data, value) -def read_str(data: Buffer) -> str: +def read_str(data: ReadBuffer) -> str: assert read_tag(data) == LITERAL_STR return read_str_bare(data) -def write_str(data: Buffer, value: str) -> None: +def write_str(data: WriteBuffer, value: str) -> None: write_tag(data, LITERAL_STR) write_str_bare(data, value) -def read_bytes(data: Buffer) -> bytes: +def read_bytes(data: ReadBuffer) -> bytes: assert read_tag(data) == LITERAL_BYTES return read_bytes_bare(data) -def write_bytes(data: Buffer, value: bytes) -> None: +def write_bytes(data: WriteBuffer, value: bytes) -> None: write_tag(data, LITERAL_BYTES) write_bytes_bare(data, value) -def read_int_opt(data: Buffer) -> int | None: +def read_int_opt(data: ReadBuffer) -> int | None: tag = read_tag(data) if tag == LITERAL_NONE: return None @@ -314,7 +315,7 @@ def read_int_opt(data: Buffer) -> int | None: return read_int_bare(data) -def write_int_opt(data: Buffer, value: int | None) -> None: +def write_int_opt(data: WriteBuffer, value: int | None) -> None: if value is not None: write_tag(data, LITERAL_INT) write_int_bare(data, value) @@ -322,7 +323,7 @@ def write_int_opt(data: Buffer, value: int | None) -> None: write_tag(data, LITERAL_NONE) -def read_str_opt(data: Buffer) -> str | None: +def read_str_opt(data: ReadBuffer) -> str | None: tag = read_tag(data) if tag == LITERAL_NONE: return None @@ -330,7 +331,7 @@ def read_str_opt(data: Buffer) -> str | None: return read_str_bare(data) -def write_str_opt(data: Buffer, value: str | None) -> None: +def write_str_opt(data: WriteBuffer, value: str | None) -> None: if value is not None: write_tag(data, LITERAL_STR) write_str_bare(data, value) @@ -338,52 +339,52 @@ def write_str_opt(data: Buffer, value: str | None) -> None: write_tag(data, LITERAL_NONE) -def read_int_list(data: Buffer) -> list[int]: +def read_int_list(data: ReadBuffer) -> list[int]: assert read_tag(data) == LIST_INT size = read_int_bare(data) return [read_int_bare(data) for _ in range(size)] -def write_int_list(data: Buffer, value: list[int]) -> None: +def write_int_list(data: WriteBuffer, value: list[int]) -> None: write_tag(data, LIST_INT) write_int_bare(data, len(value)) for item in value: write_int_bare(data, item) -def read_str_list(data: Buffer) -> list[str]: +def read_str_list(data: ReadBuffer) -> list[str]: assert read_tag(data) == LIST_STR size = read_int_bare(data) return [read_str_bare(data) for _ in range(size)] -def write_str_list(data: Buffer, value: Sequence[str]) -> None: +def write_str_list(data: WriteBuffer, value: Sequence[str]) -> None: write_tag(data, LIST_STR) write_int_bare(data, len(value)) for item in value: write_str_bare(data, item) -def read_bytes_list(data: Buffer) -> list[bytes]: +def read_bytes_list(data: ReadBuffer) -> list[bytes]: assert read_tag(data) == LIST_BYTES size = read_int_bare(data) return [read_bytes_bare(data) for _ in range(size)] -def write_bytes_list(data: Buffer, value: Sequence[bytes]) -> None: +def write_bytes_list(data: WriteBuffer, value: Sequence[bytes]) -> None: write_tag(data, LIST_BYTES) write_int_bare(data, len(value)) for item in value: write_bytes_bare(data, item) -def read_str_opt_list(data: Buffer) -> list[str | None]: +def read_str_opt_list(data: ReadBuffer) -> list[str | None]: assert read_tag(data) == LIST_GEN size = read_int_bare(data) return [read_str_opt(data) for _ in range(size)] -def write_str_opt_list(data: Buffer, value: list[str | None]) -> None: +def write_str_opt_list(data: WriteBuffer, value: list[str | None]) -> None: write_tag(data, LIST_GEN) write_int_bare(data, len(value)) for item in value: @@ -393,7 +394,7 @@ def write_str_opt_list(data: Buffer, value: list[str | None]) -> None: JsonValue: _TypeAlias = Union[None, int, str, bool, list["JsonValue"], dict[str, "JsonValue"]] -def read_json_value(data: Buffer) -> JsonValue: +def read_json_value(data: ReadBuffer) -> JsonValue: tag = read_tag(data) if tag == LITERAL_NONE: return None @@ -416,7 +417,7 @@ def read_json_value(data: Buffer) -> JsonValue: # Currently tuples are used by mypyc plugin. They will be normalized to # JSON lists after a roundtrip. -def write_json_value(data: Buffer, value: JsonValue | tuple[JsonValue, ...]) -> None: +def write_json_value(data: WriteBuffer, value: JsonValue | tuple[JsonValue, ...]) -> None: if value is None: write_tag(data, LITERAL_NONE) elif isinstance(value, bool): @@ -444,13 +445,13 @@ def write_json_value(data: Buffer, value: JsonValue | tuple[JsonValue, ...]) -> # These are functions for JSON *dictionaries* specifically. Unfortunately, we # must use imprecise types here, because the callers use imprecise types. -def read_json(data: Buffer) -> dict[str, Any]: +def read_json(data: ReadBuffer) -> dict[str, Any]: assert read_tag(data) == DICT_STR_GEN size = read_int_bare(data) return {read_str_bare(data): read_json_value(data) for _ in range(size)} -def write_json(data: Buffer, value: dict[str, Any]) -> None: +def write_json(data: WriteBuffer, value: dict[str, Any]) -> None: write_tag(data, DICT_STR_GEN) write_int_bare(data, len(value)) for key in sorted(value): diff --git a/mypy/exportjson.py b/mypy/exportjson.py index 09945f0ef28f0..dfc1cf5abbc6b 100644 --- a/mypy/exportjson.py +++ b/mypy/exportjson.py @@ -19,7 +19,7 @@ from typing import Any, Union from typing_extensions import TypeAlias as _TypeAlias -from librt.internal import Buffer +from librt.internal import ReadBuffer from mypy.nodes import ( FUNCBASE_FLAGS, @@ -78,7 +78,7 @@ def __init__(self, *, implicit_names: bool = True) -> None: def convert_binary_cache_to_json(data: bytes, *, implicit_names: bool = True) -> Json: - tree = MypyFile.read(Buffer(data)) + tree = MypyFile.read(ReadBuffer(data)) return convert_mypy_file_to_json(tree, Config(implicit_names=implicit_names)) diff --git a/mypy/nodes.py b/mypy/nodes.py index 13ba011eebc0b..e7d7e84d5ac2b 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -28,8 +28,9 @@ LIST_STR, LITERAL_COMPLEX, LITERAL_NONE, - Buffer, + ReadBuffer, Tag, + WriteBuffer, read_bool, read_int, read_int_list, @@ -285,11 +286,11 @@ def deserialize(cls, data: JsonDict) -> SymbolNode: return method(data) raise NotImplementedError(f"unexpected .class {classname}") - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: raise NotImplementedError(f"Cannot serialize {self.__class__.__name__} instance") @classmethod - def read(cls, data: Buffer) -> SymbolNode: + def read(cls, data: ReadBuffer) -> SymbolNode: raise NotImplementedError(f"Cannot deserialize {cls.__name__} instance") @@ -441,7 +442,7 @@ def deserialize(cls, data: JsonDict) -> MypyFile: tree.future_import_flags = set(data["future_import_flags"]) return tree - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, MYPY_FILE) write_str(data, self._fullname) self.names.write(data, self._fullname) @@ -452,7 +453,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> MypyFile: + def read(cls, data: ReadBuffer) -> MypyFile: assert read_tag(data) == MYPY_FILE tree = MypyFile([], []) tree._fullname = read_str(data) @@ -737,7 +738,7 @@ def deserialize(cls, data: JsonDict) -> OverloadedFuncDef: # NOTE: res.info will be set in the fixup phase. return res - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, OVERLOADED_FUNC_DEF) write_tag(data, LIST_GEN) write_int_bare(data, len(self.items)) @@ -755,7 +756,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> OverloadedFuncDef: + def read(cls, data: ReadBuffer) -> OverloadedFuncDef: assert read_tag(data) == LIST_GEN res = OverloadedFuncDef([read_overload_part(data) for _ in range(read_int_bare(data))]) typ = mypy.types.read_type_opt(data) @@ -1052,7 +1053,7 @@ def deserialize(cls, data: JsonDict) -> FuncDef: del ret.min_args return ret - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, FUNC_DEF) write_str(data, self._name) mypy.types.write_type_opt(data, self.type) @@ -1070,7 +1071,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> FuncDef: + def read(cls, data: ReadBuffer) -> FuncDef: name = read_str(data) typ: mypy.types.FunctionLike | None = None tag = read_tag(data) @@ -1168,7 +1169,7 @@ def deserialize(cls, data: JsonDict) -> Decorator: dec.is_overload = data["is_overload"] return dec - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, DECORATOR) self.func.write(data) self.var.write(data) @@ -1176,7 +1177,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> Decorator: + def read(cls, data: ReadBuffer) -> Decorator: assert read_tag(data) == FUNC_DEF func = FuncDef.read(data) assert read_tag(data) == VAR @@ -1362,7 +1363,7 @@ def deserialize(cls, data: JsonDict) -> Var: v.final_value = data.get("final_value") return v - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, VAR) write_str(data, self._name) mypy.types.write_type_opt(data, self.type) @@ -1373,7 +1374,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> Var: + def read(cls, data: ReadBuffer) -> Var: name = read_str(data) typ = mypy.types.read_type_opt(data) v = Var(name, typ) @@ -1504,7 +1505,7 @@ def deserialize(cls, data: JsonDict) -> ClassDef: res.fullname = data["fullname"] return res - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, CLASS_DEF) write_str(data, self.name) mypy.types.write_type_list(data, self.type_vars) @@ -1512,7 +1513,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> ClassDef: + def read(cls, data: ReadBuffer) -> ClassDef: res = ClassDef(read_str(data), Block([]), mypy.types.read_type_var_likes(data)) res.fullname = read_str(data) assert read_tag(data) == END_TAG @@ -2975,7 +2976,7 @@ def deserialize(cls, data: JsonDict) -> TypeVarExpr: data["variance"], ) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, TYPE_VAR_EXPR) write_str(data, self._name) write_str(data, self._fullname) @@ -2986,7 +2987,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> TypeVarExpr: + def read(cls, data: ReadBuffer) -> TypeVarExpr: ret = TypeVarExpr( read_str(data), read_str(data), @@ -3028,7 +3029,7 @@ def deserialize(cls, data: JsonDict) -> ParamSpecExpr: data["variance"], ) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, PARAM_SPEC_EXPR) write_str(data, self._name) write_str(data, self._fullname) @@ -3038,7 +3039,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> ParamSpecExpr: + def read(cls, data: ReadBuffer) -> ParamSpecExpr: ret = ParamSpecExpr( read_str(data), read_str(data), @@ -3099,7 +3100,7 @@ def deserialize(cls, data: JsonDict) -> TypeVarTupleExpr: data["variance"], ) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, TYPE_VAR_TUPLE_EXPR) self.tuple_fallback.write(data) write_str(data, self._name) @@ -3110,7 +3111,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> TypeVarTupleExpr: + def read(cls, data: ReadBuffer) -> TypeVarTupleExpr: assert read_tag(data) == mypy.types.INSTANCE fallback = mypy.types.Instance.read(data) ret = TypeVarTupleExpr( @@ -3994,7 +3995,7 @@ def deserialize(cls, data: JsonDict) -> TypeInfo: ti.deprecated = data.get("deprecated") return ti - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, TYPE_INFO) self.names.write(data, self.fullname) self.defn.write(data) @@ -4028,7 +4029,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> TypeInfo: + def read(cls, data: ReadBuffer) -> TypeInfo: names = SymbolTable.read(data) assert read_tag(data) == CLASS_DEF defn = ClassDef.read(data) @@ -4363,7 +4364,7 @@ def deserialize(cls, data: JsonDict) -> TypeAlias: python_3_12_type_alias=python_3_12_type_alias, ) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, TYPE_ALIAS) write_str(data, self._fullname) write_str(data, self.module) @@ -4375,7 +4376,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> TypeAlias: + def read(cls, data: ReadBuffer) -> TypeAlias: fullname = read_str(data) module = read_str(data) target = mypy.types.read_type(data) @@ -4652,7 +4653,7 @@ def deserialize(cls, data: JsonDict) -> SymbolTableNode: stnode.plugin_generated = data["plugin_generated"] return stnode - def write(self, data: Buffer, prefix: str, name: str) -> None: + def write(self, data: WriteBuffer, prefix: str, name: str) -> None: write_tag(data, SYMBOL_TABLE_NODE) write_int(data, self.kind) write_bool(data, self.module_hidden) @@ -4684,7 +4685,7 @@ def write(self, data: Buffer, prefix: str, name: str) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> SymbolTableNode: + def read(cls, data: ReadBuffer) -> SymbolTableNode: assert read_tag(data) == SYMBOL_TABLE_NODE sym = SymbolTableNode(read_int(data), None) sym.module_hidden = read_bool(data) @@ -4750,7 +4751,7 @@ def deserialize(cls, data: JsonDict) -> SymbolTable: st[key] = SymbolTableNode.deserialize(value) return st - def write(self, data: Buffer, fullname: str) -> None: + def write(self, data: WriteBuffer, fullname: str) -> None: size = 0 for key, value in self.items(): # Skip __builtins__: it's a reference to the builtins @@ -4771,7 +4772,7 @@ def write(self, data: Buffer, fullname: str) -> None: value.write(data, fullname, key) @classmethod - def read(cls, data: Buffer) -> SymbolTable: + def read(cls, data: ReadBuffer) -> SymbolTable: assert read_tag(data) == DICT_STR_GEN size = read_int_bare(data) return SymbolTable( @@ -4828,7 +4829,7 @@ def deserialize(cls, data: JsonDict) -> DataclassTransformSpec: field_specifiers=tuple(data.get("field_specifiers", [])), ) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, DT_SPEC) write_bool(data, self.eq_default) write_bool(data, self.order_default) @@ -4838,7 +4839,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> DataclassTransformSpec: + def read(cls, data: ReadBuffer) -> DataclassTransformSpec: ret = DataclassTransformSpec( eq_default=read_bool(data), order_default=read_bool(data), @@ -4859,12 +4860,12 @@ def set_flags(node: Node, flags: list[str]) -> None: setattr(node, name, True) -def write_flags(data: Buffer, node: SymbolNode, flags: list[str]) -> None: +def write_flags(data: WriteBuffer, node: SymbolNode, flags: list[str]) -> None: for flag in flags: write_bool(data, getattr(node, flag)) -def read_flags(data: Buffer, node: SymbolNode, flags: list[str]) -> None: +def read_flags(data: ReadBuffer, node: SymbolNode, flags: list[str]) -> None: for flag in flags: if read_bool(data): setattr(node, flag, True) @@ -5002,7 +5003,7 @@ def local_definitions( SYMBOL_TABLE_NODE: Final[Tag] = 61 -def read_symbol(data: Buffer) -> SymbolNode: +def read_symbol(data: ReadBuffer) -> SymbolNode: tag = read_tag(data) # The branches here are ordered manually by type "popularity". if tag == VAR: @@ -5026,7 +5027,7 @@ def read_symbol(data: Buffer) -> SymbolNode: assert False, f"Unknown symbol tag {tag}" -def read_overload_part(data: Buffer, tag: Tag | None = None) -> OverloadPart: +def read_overload_part(data: ReadBuffer, tag: Tag | None = None) -> OverloadPart: if tag is None: tag = read_tag(data) if tag == DECORATOR: diff --git a/mypy/types.py b/mypy/types.py index 7a8343097204f..056b99cc3f912 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -23,8 +23,9 @@ EXTRA_ATTRS, LIST_GEN, LITERAL_NONE, - Buffer, + ReadBuffer, Tag, + WriteBuffer, read_bool, read_int, read_int_list, @@ -312,11 +313,11 @@ def serialize(self) -> JsonDict | str: def deserialize(cls, data: JsonDict) -> Type: raise NotImplementedError(f"Cannot deserialize {cls.__name__} instance") - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: raise NotImplementedError(f"Cannot serialize {self.__class__.__name__} instance") @classmethod - def read(cls, data: Buffer) -> Type: + def read(cls, data: ReadBuffer) -> Type: raise NotImplementedError(f"Cannot deserialize {cls.__name__} instance") def is_singleton_type(self) -> bool: @@ -449,7 +450,7 @@ def deserialize(cls, data: JsonDict) -> TypeAliasType: alias.type_ref = data["type_ref"] return alias - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, TYPE_ALIAS_TYPE) write_type_list(data, self.args) assert self.alias is not None @@ -457,7 +458,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> TypeAliasType: + def read(cls, data: ReadBuffer) -> TypeAliasType: alias = TypeAliasType(None, read_type_list(data)) alias.type_ref = read_str(data) assert read_tag(data) == END_TAG @@ -730,7 +731,7 @@ def deserialize(cls, data: JsonDict) -> TypeVarType: variance=data["variance"], ) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, TYPE_VAR_TYPE) write_str(data, self.name) write_str(data, self.fullname) @@ -743,7 +744,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> TypeVarType: + def read(cls, data: ReadBuffer) -> TypeVarType: ret = TypeVarType( read_str(data), read_str(data), @@ -885,7 +886,7 @@ def deserialize(cls, data: JsonDict) -> ParamSpecType: prefix=Parameters.deserialize(data["prefix"]), ) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, PARAM_SPEC_TYPE) self.prefix.write(data) write_str(data, self.name) @@ -898,7 +899,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> ParamSpecType: + def read(cls, data: ReadBuffer) -> ParamSpecType: assert read_tag(data) == PARAMETERS prefix = Parameters.read(data) ret = ParamSpecType( @@ -968,7 +969,7 @@ def deserialize(cls, data: JsonDict) -> TypeVarTupleType: min_len=data["min_len"], ) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, TYPE_VAR_TUPLE_TYPE) self.tuple_fallback.write(data) write_str(data, self.name) @@ -981,7 +982,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> TypeVarTupleType: + def read(cls, data: ReadBuffer) -> TypeVarTupleType: assert read_tag(data) == INSTANCE fallback = Instance.read(data) ret = TypeVarTupleType( @@ -1127,7 +1128,7 @@ def deserialize(cls, data: JsonDict) -> UnboundType: original_str_fallback=data["expr_fallback"], ) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, UNBOUND_TYPE) write_str(data, self.name) write_type_list(data, self.args) @@ -1136,7 +1137,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> UnboundType: + def read(cls, data: ReadBuffer) -> UnboundType: ret = UnboundType( read_str(data), read_type_list(data), @@ -1240,13 +1241,13 @@ def accept(self, visitor: TypeVisitor[T]) -> T: def serialize(self) -> JsonDict: return {".class": "UnpackType", "type": self.type.serialize()} - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, UNPACK_TYPE) self.type.write(data) write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> UnpackType: + def read(cls, data: ReadBuffer) -> UnpackType: ret = UnpackType(read_type(data)) assert read_tag(data) == END_TAG return ret @@ -1352,7 +1353,7 @@ def deserialize(cls, data: JsonDict) -> AnyType: data["missing_import_name"], ) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, ANY_TYPE) write_type_opt(data, self.source_any) write_int(data, self.type_of_any) @@ -1360,7 +1361,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> AnyType: + def read(cls, data: ReadBuffer) -> AnyType: tag = read_tag(data) if tag != LITERAL_NONE: assert tag == ANY_TYPE @@ -1417,12 +1418,12 @@ def deserialize(cls, data: JsonDict) -> UninhabitedType: assert data[".class"] == "UninhabitedType" return UninhabitedType() - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, UNINHABITED_TYPE) write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> UninhabitedType: + def read(cls, data: ReadBuffer) -> UninhabitedType: assert read_tag(data) == END_TAG return UninhabitedType() @@ -1458,12 +1459,12 @@ def deserialize(cls, data: JsonDict) -> NoneType: assert data[".class"] == "NoneType" return NoneType() - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, NONE_TYPE) write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> NoneType: + def read(cls, data: ReadBuffer) -> NoneType: assert read_tag(data) == END_TAG return NoneType() @@ -1514,13 +1515,13 @@ def deserialize(cls, data: JsonDict) -> DeletedType: assert data[".class"] == "DeletedType" return DeletedType(data["source"]) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, DELETED_TYPE) write_str_opt(data, self.source) write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> DeletedType: + def read(cls, data: ReadBuffer) -> DeletedType: ret = DeletedType(read_str_opt(data)) assert read_tag(data) == END_TAG return ret @@ -1580,7 +1581,7 @@ def deserialize(cls, data: JsonDict) -> ExtraAttrs: data["mod_name"], ) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, EXTRA_ATTRS) write_type_map(data, self.attrs) write_str_list(data, sorted(self.immutable)) @@ -1588,7 +1589,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> ExtraAttrs: + def read(cls, data: ReadBuffer) -> ExtraAttrs: ret = ExtraAttrs(read_type_map(data), set(read_str_list(data)), read_str_opt(data)) assert read_tag(data) == END_TAG return ret @@ -1729,7 +1730,7 @@ def deserialize(cls, data: JsonDict | str) -> Instance: inst.extra_attrs = ExtraAttrs.deserialize(data["extra_attrs"]) return inst - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, INSTANCE) if not self.args and not self.last_known_value and not self.extra_attrs: type_ref = self.type.fullname @@ -1758,7 +1759,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> Instance: + def read(cls, data: ReadBuffer) -> Instance: tag = read_tag(data) # This is quite verbose, but this is very hot code, so we are not # using dictionary lookups here. @@ -2100,7 +2101,7 @@ def deserialize(cls, data: JsonDict) -> Parameters: imprecise_arg_kinds=data["imprecise_arg_kinds"], ) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, PARAMETERS) write_type_list(data, self.arg_types) write_int_list(data, [int(x.value) for x in self.arg_kinds]) @@ -2110,7 +2111,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> Parameters: + def read(cls, data: ReadBuffer) -> Parameters: ret = Parameters( read_type_list(data), # This is a micro-optimization until mypyc gets dedicated enum support. Otherwise, @@ -2629,7 +2630,7 @@ def deserialize(cls, data: JsonDict) -> CallableType: unpack_kwargs=data["unpack_kwargs"], ) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, CALLABLE_TYPE) self.fallback.write(data) write_type_list(data, self.arg_types) @@ -2649,7 +2650,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> CallableType: + def read(cls, data: ReadBuffer) -> CallableType: assert read_tag(data) == INSTANCE fallback = Instance.read(data) ret = CallableType( @@ -2747,13 +2748,13 @@ def deserialize(cls, data: JsonDict) -> Overloaded: assert data[".class"] == "Overloaded" return Overloaded([CallableType.deserialize(t) for t in data["items"]]) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, OVERLOADED) write_type_list(data, self.items) write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> Overloaded: + def read(cls, data: ReadBuffer) -> Overloaded: items = [] assert read_tag(data) == LIST_GEN for _ in range(read_int_bare(data)): @@ -2858,7 +2859,7 @@ def deserialize(cls, data: JsonDict) -> TupleType: implicit=data["implicit"], ) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, TUPLE_TYPE) self.partial_fallback.write(data) write_type_list(data, self.items) @@ -2866,7 +2867,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> TupleType: + def read(cls, data: ReadBuffer) -> TupleType: assert read_tag(data) == INSTANCE fallback = Instance.read(data) ret = TupleType(read_type_list(data), fallback, implicit=read_bool(data)) @@ -3043,7 +3044,7 @@ def deserialize(cls, data: JsonDict) -> TypedDictType: Instance.deserialize(data["fallback"]), ) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, TYPED_DICT_TYPE) self.fallback.write(data) write_type_map(data, self.items) @@ -3052,7 +3053,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> TypedDictType: + def read(cls, data: ReadBuffer) -> TypedDictType: assert read_tag(data) == INSTANCE fallback = Instance.read(data) ret = TypedDictType( @@ -3309,14 +3310,14 @@ def deserialize(cls, data: JsonDict) -> LiteralType: assert data[".class"] == "LiteralType" return LiteralType(value=data["value"], fallback=Instance.deserialize(data["fallback"])) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, LITERAL_TYPE) self.fallback.write(data) write_literal(data, self.value) write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> LiteralType: + def read(cls, data: ReadBuffer) -> LiteralType: assert read_tag(data) == INSTANCE fallback = Instance.read(data) tag = read_tag(data) @@ -3425,14 +3426,14 @@ def deserialize(cls, data: JsonDict) -> UnionType: uses_pep604_syntax=data["uses_pep604_syntax"], ) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, UNION_TYPE) write_type_list(data, self.items) write_bool(data, self.uses_pep604_syntax) write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> UnionType: + def read(cls, data: ReadBuffer) -> UnionType: ret = UnionType(read_type_list(data), uses_pep604_syntax=read_bool(data)) assert read_tag(data) == END_TAG return ret @@ -3594,13 +3595,13 @@ def deserialize(cls, data: JsonDict) -> Type: deserialize_type(data["item"]), is_type_form=data["is_type_form"] ) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, TYPE_TYPE) self.item.write(data) write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> Type: + def read(cls, data: ReadBuffer) -> Type: ret = TypeType.make_normalized(read_type(data)) assert read_tag(data) == END_TAG return ret @@ -4303,7 +4304,7 @@ def type_vars_as_args(type_vars: Sequence[TypeVarLikeType]) -> tuple[Type, ...]: PARAMETERS: Final[Tag] = 117 -def read_type(data: Buffer, tag: Tag | None = None) -> Type: +def read_type(data: ReadBuffer, tag: Tag | None = None) -> Type: if tag is None: tag = read_tag(data) # The branches here are ordered manually by type "popularity". @@ -4348,7 +4349,7 @@ def read_type(data: Buffer, tag: Tag | None = None) -> Type: assert False, f"Unknown type tag {tag}" -def read_function_like(data: Buffer, tag: Tag) -> FunctionLike: +def read_function_like(data: ReadBuffer, tag: Tag) -> FunctionLike: if tag == CALLABLE_TYPE: return CallableType.read(data) if tag == OVERLOADED: @@ -4356,7 +4357,7 @@ def read_function_like(data: Buffer, tag: Tag) -> FunctionLike: assert False, f"Invalid type tag for FunctionLike {tag}" -def read_type_var_likes(data: Buffer) -> list[TypeVarLikeType]: +def read_type_var_likes(data: ReadBuffer) -> list[TypeVarLikeType]: """Specialized version of read_type_list() for lists of type variables.""" assert read_tag(data) == LIST_GEN ret: list[TypeVarLikeType] = [] @@ -4373,40 +4374,40 @@ def read_type_var_likes(data: Buffer) -> list[TypeVarLikeType]: return ret -def read_type_opt(data: Buffer) -> Type | None: +def read_type_opt(data: ReadBuffer) -> Type | None: tag = read_tag(data) if tag == LITERAL_NONE: return None return read_type(data, tag) -def write_type_opt(data: Buffer, value: Type | None) -> None: +def write_type_opt(data: WriteBuffer, value: Type | None) -> None: if value is not None: value.write(data) else: write_tag(data, LITERAL_NONE) -def read_type_list(data: Buffer) -> list[Type]: +def read_type_list(data: ReadBuffer) -> list[Type]: assert read_tag(data) == LIST_GEN size = read_int_bare(data) return [read_type(data) for _ in range(size)] -def write_type_list(data: Buffer, value: Sequence[Type]) -> None: +def write_type_list(data: WriteBuffer, value: Sequence[Type]) -> None: write_tag(data, LIST_GEN) write_int_bare(data, len(value)) for item in value: item.write(data) -def read_type_map(data: Buffer) -> dict[str, Type]: +def read_type_map(data: ReadBuffer) -> dict[str, Type]: assert read_tag(data) == DICT_STR_GEN size = read_int_bare(data) return {read_str_bare(data): read_type(data) for _ in range(size)} -def write_type_map(data: Buffer, value: dict[str, Type]) -> None: +def write_type_map(data: WriteBuffer, value: dict[str, Type]) -> None: write_tag(data, DICT_STR_GEN) write_int_bare(data, len(value)) for key in sorted(value): diff --git a/mypy/typeshed/stubs/librt/librt/internal.pyi b/mypy/typeshed/stubs/librt/librt/internal.pyi index 78e7f9caa1170..2969ccfbadda7 100644 --- a/mypy/typeshed/stubs/librt/librt/internal.pyi +++ b/mypy/typeshed/stubs/librt/librt/internal.pyi @@ -1,27 +1,21 @@ from mypy_extensions import u8 -# TODO: Remove Buffer -- right now we have hacky support for BOTH the old and new APIs - -class Buffer: - def __init__(self, source: bytes = ...) -> None: ... - def getvalue(self) -> bytes: ... - class ReadBuffer: def __init__(self, source: bytes) -> None: ... class WriteBuffer: def getvalue(self) -> bytes: ... -def write_bool(data: WriteBuffer | Buffer, value: bool) -> None: ... -def read_bool(data: ReadBuffer | Buffer) -> bool: ... -def write_str(data: WriteBuffer | Buffer, value: str) -> None: ... -def read_str(data: ReadBuffer | Buffer) -> str: ... -def write_bytes(data: WriteBuffer | Buffer, value: bytes) -> None: ... -def read_bytes(data: ReadBuffer | Buffer) -> bytes: ... -def write_float(data: WriteBuffer | Buffer, value: float) -> None: ... -def read_float(data: ReadBuffer | Buffer) -> float: ... -def write_int(data: WriteBuffer | Buffer, value: int) -> None: ... -def read_int(data: ReadBuffer | Buffer) -> int: ... -def write_tag(data: WriteBuffer | Buffer, value: u8) -> None: ... -def read_tag(data: ReadBuffer | Buffer) -> u8: ... +def write_bool(data: WriteBuffer, value: bool) -> None: ... +def read_bool(data: ReadBuffer) -> bool: ... +def write_str(data: WriteBuffer, value: str) -> None: ... +def read_str(data: ReadBuffer) -> str: ... +def write_bytes(data: WriteBuffer, value: bytes) -> None: ... +def read_bytes(data: ReadBuffer) -> bytes: ... +def write_float(data: WriteBuffer, value: float) -> None: ... +def read_float(data: ReadBuffer) -> float: ... +def write_int(data: WriteBuffer, value: int) -> None: ... +def read_int(data: ReadBuffer) -> int: ... +def write_tag(data: WriteBuffer, value: u8) -> None: ... +def read_tag(data: ReadBuffer) -> u8: ... def cache_version() -> u8: ... diff --git a/pyproject.toml b/pyproject.toml index 0de739be9b55d..42ff3a6ca0199 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ requires = [ "mypy_extensions>=1.0.0", "pathspec>=0.9.0", "tomli>=1.1.0; python_version<'3.11'", - "librt>=0.4.0", + "librt>=0.5.0", # the following is from build-requirements.txt "types-psutil", "types-setuptools", @@ -54,7 +54,7 @@ dependencies = [ "mypy_extensions>=1.0.0", "pathspec>=0.9.0", "tomli>=1.1.0; python_version<'3.11'", - "librt>=0.4.0", + "librt>=0.5.0", ] dynamic = ["version"] diff --git a/test-requirements.txt b/test-requirements.txt index 126abd7149e62..b65b658844d2e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -22,7 +22,7 @@ identify==2.6.15 # via pre-commit iniconfig==2.1.0 # via pytest -librt==0.4.0 +librt==0.5.0 # via -r mypy-requirements.txt lxml==6.0.2 ; python_version < "3.15" # via -r test-requirements.in From 7c36b246363ff4f34e5d9130bc6d37bebdece5d4 Mon Sep 17 00:00:00 2001 From: jhance Date: Mon, 10 Nov 2025 15:31:41 -0800 Subject: [PATCH 389/424] Improve error message for librt abi mismatch (#20216) In the case where the abi does not match, it is easier to debug if we show what the expected and actual versions are. --- mypyc/lib-rt/librt_internal.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mypyc/lib-rt/librt_internal.h b/mypyc/lib-rt/librt_internal.h index 329a0fd68c111..bef9e196d2c1f 100644 --- a/mypyc/lib-rt/librt_internal.h +++ b/mypyc/lib-rt/librt_internal.h @@ -64,7 +64,12 @@ import_librt_internal(void) return -1; memcpy(NativeInternal_API, capsule, sizeof(NativeInternal_API)); if (NativeInternal_ABI_Version() != LIBRT_INTERNAL_ABI_VERSION) { - PyErr_SetString(PyExc_ValueError, "ABI version conflict for librt.internal"); + char err[128]; + snprintf(err, sizeof(err), "ABI version conflict for librt.internal, expected %d, found %d", + LIBRT_INTERNAL_ABI_VERSION, + NativeInternal_ABI_Version() + ); + PyErr_SetString(PyExc_ValueError, err); return -1; } return 0; From c2ee586e56fad3b51117d72912f6c7353114c422 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 12 Nov 2025 14:16:26 +0000 Subject: [PATCH 390/424] [librt] Add `librt.internal` API versioning with backward compat (#20221) This lets us add new API features by adding more functions to the capsule and the module namespace without breaking backward compatibility. We shouldn't use the capsule if the installed version is too old, since we could be calling functions via uninitialized pointers. Note that this is a breaking change in the `librt.internal` ABI. --- mypyc/lib-rt/librt_internal.c | 6 ++++++ mypyc/lib-rt/librt_internal.h | 27 +++++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/mypyc/lib-rt/librt_internal.c b/mypyc/lib-rt/librt_internal.c index ada2dfeb39a52..22d54de40f089 100644 --- a/mypyc/lib-rt/librt_internal.c +++ b/mypyc/lib-rt/librt_internal.c @@ -962,6 +962,11 @@ NativeInternal_ABI_Version(void) { return LIBRT_INTERNAL_ABI_VERSION; } +static int +NativeInternal_API_Version(void) { + return LIBRT_INTERNAL_API_VERSION; +} + static int librt_internal_module_exec(PyObject *m) { @@ -999,6 +1004,7 @@ librt_internal_module_exec(PyObject *m) (void *)cache_version_internal, (void *)ReadBuffer_type_internal, (void *)WriteBuffer_type_internal, + (void *)NativeInternal_API_Version, }; PyObject *c_api_object = PyCapsule_New((void *)NativeInternal_API, "librt.internal._C_API", NULL); if (PyModule_Add(m, "_C_API", c_api_object) < 0) { diff --git a/mypyc/lib-rt/librt_internal.h b/mypyc/lib-rt/librt_internal.h index bef9e196d2c1f..501162a279801 100644 --- a/mypyc/lib-rt/librt_internal.h +++ b/mypyc/lib-rt/librt_internal.h @@ -1,8 +1,19 @@ #ifndef LIBRT_INTERNAL_H #define LIBRT_INTERNAL_H -#define LIBRT_INTERNAL_ABI_VERSION 1 -#define LIBRT_INTERNAL_API_LEN 19 +// ABI version -- only an exact match is compatible. This will only be changed in +// very exceptional cases (likely never) due to strict backward compatibility +// requirements. +#define LIBRT_INTERNAL_ABI_VERSION 2 + +// API version -- more recent versions must maintain backward compatibility, i.e. +// we can add new features but not remove or change existing features (unless +// ABI version is changed, but see the comment above). + #define LIBRT_INTERNAL_API_VERSION 0 + +// Number of functions in the capsule API. If you add a new function, also increase +// LIBRT_INTERNAL_API_VERSION. +#define LIBRT_INTERNAL_API_LEN 20 #ifdef LIBRT_INTERNAL_MODULE @@ -27,6 +38,7 @@ static PyObject *read_bytes_internal(PyObject *data); static uint8_t cache_version_internal(void); static PyTypeObject *ReadBuffer_type_internal(void); static PyTypeObject *WriteBuffer_type_internal(void); +static int NativeInternal_API_Version(void); #else @@ -51,6 +63,7 @@ static void *NativeInternal_API[LIBRT_INTERNAL_API_LEN]; #define cache_version_internal (*(uint8_t (*)(void)) NativeInternal_API[16]) #define ReadBuffer_type_internal (*(PyTypeObject* (*)(void)) NativeInternal_API[17]) #define WriteBuffer_type_internal (*(PyTypeObject* (*)(void)) NativeInternal_API[18]) +#define NativeInternal_API_Version (*(int (*)(void)) NativeInternal_API[19]) static int import_librt_internal(void) @@ -72,6 +85,16 @@ import_librt_internal(void) PyErr_SetString(PyExc_ValueError, err); return -1; } + if (NativeInternal_API_Version() < LIBRT_INTERNAL_API_VERSION) { + char err[128]; + snprintf(err, sizeof(err), + "API version conflict for librt.internal, expected %d or newer, found %d (hint: upgrade librt)", + LIBRT_INTERNAL_API_VERSION, + NativeInternal_API_Version() + ); + PyErr_SetString(PyExc_ValueError, err); + return -1; + } return 0; } From ca2240c1b4e2dd95f016a7df01728c92bce3cb89 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 12 Nov 2025 15:28:50 +0000 Subject: [PATCH 391/424] Update librt dependency to 0.6.0 (#20225) Includes #20221 (which breaks backward compatibility). --- mypy-requirements.txt | 2 +- pyproject.toml | 4 ++-- test-requirements.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index a69d31088e554..06e0a9bffb1cb 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -4,4 +4,4 @@ typing_extensions>=4.6.0 mypy_extensions>=1.0.0 pathspec>=0.9.0 tomli>=1.1.0; python_version<'3.11' -librt>=0.5.0 +librt>=0.6.0 diff --git a/pyproject.toml b/pyproject.toml index 42ff3a6ca0199..336a16c489799 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ requires = [ "mypy_extensions>=1.0.0", "pathspec>=0.9.0", "tomli>=1.1.0; python_version<'3.11'", - "librt>=0.5.0", + "librt>=0.6.0", # the following is from build-requirements.txt "types-psutil", "types-setuptools", @@ -54,7 +54,7 @@ dependencies = [ "mypy_extensions>=1.0.0", "pathspec>=0.9.0", "tomli>=1.1.0; python_version<'3.11'", - "librt>=0.5.0", + "librt>=0.6.0", ] dynamic = ["version"] diff --git a/test-requirements.txt b/test-requirements.txt index b65b658844d2e..4d3f644d6bca3 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -22,7 +22,7 @@ identify==2.6.15 # via pre-commit iniconfig==2.1.0 # via pytest -librt==0.5.0 +librt==0.6.0 # via -r mypy-requirements.txt lxml==6.0.2 ; python_version < "3.15" # via -r test-requirements.in From 75592ff2866e3d911f7d9a0d1cdca65501dfba5d Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 13 Nov 2025 01:54:38 +0000 Subject: [PATCH 392/424] Fix compile on big-endian (#20231) --- mypyc/lib-rt/librt_internal.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypyc/lib-rt/librt_internal.c b/mypyc/lib-rt/librt_internal.c index 22d54de40f089..fe18c541c11fc 100644 --- a/mypyc/lib-rt/librt_internal.c +++ b/mypyc/lib-rt/librt_internal.c @@ -466,7 +466,7 @@ _write_short_int(PyObject *data, Py_ssize_t real_value) { _CHECK_WRITE(data, 2) #if PY_BIG_ENDIAN uint16_t to_write = ((uint16_t)(real_value - MIN_TWO_BYTES_INT) << 2) | TWO_BYTES_INT_BIT; - _WRITE(data, uint16_t, reverse_16(to_write)) + _WRITE(data, uint16_t, reverse_16(to_write)); #else _WRITE(data, uint16_t, ((uint16_t)(real_value - MIN_TWO_BYTES_INT) << 2) | TWO_BYTES_INT_BIT); #endif @@ -474,7 +474,7 @@ _write_short_int(PyObject *data, Py_ssize_t real_value) { _CHECK_WRITE(data, 4) #if PY_BIG_ENDIAN uint32_t to_write = ((uint32_t)(real_value - MIN_FOUR_BYTES_INT) << 3) | FOUR_BYTES_INT_TRAILER; - _WRITE(data, uint32_t, reverse_32(to_write)) + _WRITE(data, uint32_t, reverse_32(to_write)); #else _WRITE(data, uint32_t, ((uint32_t)(real_value - MIN_FOUR_BYTES_INT) << 3) | FOUR_BYTES_INT_TRAILER); #endif From 568b945c299cb921e937fa21c847004a39993c2e Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 13 Nov 2025 11:46:10 +0000 Subject: [PATCH 393/424] Fix crash on recursive tuple with Hashable (#20232) Fixes https://github.com/python/mypy/issues/20227 This is huge pain to reproduce, so I added just _some_ repro as a test. Note the change in `typeops.py` is not required for this fix, but it is a technically correct thing to do that I noticed while looking at this. --- mypy/typeops.py | 8 ++++---- mypy/types.py | 6 +++++- test-data/unit/check-python312.test | 12 ++++++++++++ test-data/unit/fixtures/tuple.pyi | 1 + 4 files changed, 22 insertions(+), 5 deletions(-) diff --git a/mypy/typeops.py b/mypy/typeops.py index d2f9f4da44e40..050252eb62050 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -666,15 +666,15 @@ def _remove_redundant_union_items(items: list[Type], keep_erased: bool) -> list[ else: # If not, check if we've seen a supertype of this type for j, tj in enumerate(new_items): - tj = get_proper_type(tj) + proper_tj = get_proper_type(tj) # If tj is an Instance with a last_known_value, do not remove proper_ti # (unless it's an instance with the same last_known_value) if ( - isinstance(tj, Instance) - and tj.last_known_value is not None + isinstance(proper_tj, Instance) + and proper_tj.last_known_value is not None and not ( isinstance(proper_ti, Instance) - and tj.last_known_value == proper_ti.last_known_value + and proper_tj.last_known_value == proper_ti.last_known_value ) ): continue diff --git a/mypy/types.py b/mypy/types.py index 056b99cc3f912..09e4b74bb821c 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -4159,7 +4159,11 @@ def flatten_nested_unions( tp = t if isinstance(tp, ProperType) and isinstance(tp, UnionType): flat_items.extend( - flatten_nested_unions(tp.items, handle_type_alias_type=handle_type_alias_type) + flatten_nested_unions( + tp.items, + handle_type_alias_type=handle_type_alias_type, + handle_recursive=handle_recursive, + ) ) else: # Must preserve original aliases when possible. diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 840a708fecf33..c8b306b6bd551 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -2192,3 +2192,15 @@ y3: A3[int] # E: Type argument "int" of "A3" must be a subtype of "B3" z3: A3[None] [builtins fixtures/tuple.pyi] [typing fixtures/typing-full.pyi] + +[case testPEP695TypeAliasRecursiveTupleUnionNoCrash] +from collections.abc import Hashable + +type HashableArg = int | tuple[Hashable | HashableArg] +x: HashableArg +reveal_type(x) # N: Revealed type is "Union[builtins.int, tuple[Union[typing.Hashable, ...]]]" +if isinstance(x, tuple): + y, = x + reveal_type(y) # N: Revealed type is "Union[typing.Hashable, Union[builtins.int, tuple[Union[typing.Hashable, ...]]]]" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] diff --git a/test-data/unit/fixtures/tuple.pyi b/test-data/unit/fixtures/tuple.pyi index d01cd0034d263..06dfcf5d0fbc1 100644 --- a/test-data/unit/fixtures/tuple.pyi +++ b/test-data/unit/fixtures/tuple.pyi @@ -14,6 +14,7 @@ class type: def __init__(self, *a: object) -> None: pass def __call__(self, *a: object) -> object: pass class tuple(Sequence[_Tco], Generic[_Tco]): + def __hash__(self) -> int: ... def __new__(cls: Type[_T], iterable: Iterable[_Tco] = ...) -> _T: ... def __iter__(self) -> Iterator[_Tco]: pass def __contains__(self, item: object) -> bool: pass From 3d237167e6277731395c34bf459c6003b071bc3c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 13 Nov 2025 13:14:25 +0000 Subject: [PATCH 394/424] [mypyc] Add minimal, experimental librt.base64 module (#20226) The module currently only has a `b64encode` function adapted from CPython. It's only enabled when librt is compiled with experimental features enabled, so that we are free to iterate on this and break backward compatibility until we are ready to declare the module as stable. This also adds a way to define experimental features in `librt` (and mypyc in general, but it's currently only used for `librt`). In follow-up PRs I'm planning to add a more efficient implementation of `b64encode` and add more features to the module, including decoding. I'm not planning to include every feature from the stdlib base64 module, since many of them aren't used very widely. --- mypy/typeshed/stubs/librt/librt/base64.pyi | 1 + mypyc/build.py | 13 +- mypyc/codegen/emitmodule.py | 6 + mypyc/ir/ops.py | 4 + mypyc/irbuild/ll_builder.py | 2 + mypyc/lib-rt/librt_base64.c | 148 +++++++++++++++++++++ mypyc/lib-rt/librt_base64.h | 60 +++++++++ mypyc/lib-rt/setup.py | 5 +- mypyc/options.py | 7 + mypyc/primitives/misc_ops.py | 9 ++ mypyc/primitives/registry.py | 6 + mypyc/test-data/run-base64.test | 52 ++++++++ mypyc/test/test_run.py | 19 ++- 13 files changed, 327 insertions(+), 5 deletions(-) create mode 100644 mypy/typeshed/stubs/librt/librt/base64.pyi create mode 100644 mypyc/lib-rt/librt_base64.c create mode 100644 mypyc/lib-rt/librt_base64.h create mode 100644 mypyc/test-data/run-base64.test diff --git a/mypy/typeshed/stubs/librt/librt/base64.pyi b/mypy/typeshed/stubs/librt/librt/base64.pyi new file mode 100644 index 0000000000000..36366f5754ce1 --- /dev/null +++ b/mypy/typeshed/stubs/librt/librt/base64.pyi @@ -0,0 +1 @@ +def b64encode(s: bytes) -> bytes: ... diff --git a/mypyc/build.py b/mypyc/build.py index 13648911c0b5a..2ff9d175947ff 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -42,7 +42,7 @@ from mypyc.namegen import exported_name from mypyc.options import CompilerOptions -LIBRT_MODULES = [("librt.internal", "librt_internal.c")] +LIBRT_MODULES = [("librt.internal", "librt_internal.c"), ("librt.base64", "librt_base64.c")] try: # Import setuptools so that it monkey-patch overrides distutils @@ -495,7 +495,9 @@ def mypycify( group_name: str | None = None, log_trace: bool = False, depends_on_librt_internal: bool = False, + depends_on_librt_base64: bool = False, install_librt: bool = False, + experimental_features: bool = False, ) -> list[Extension]: """Main entry point to building using mypyc. @@ -551,6 +553,9 @@ def mypycify( those are build and published on PyPI separately, but during tests, we want to use their development versions (i.e. from current commit). + experimental_features: Enable experimental features (install_librt=True is + also needed if using experimental librt features). These + have no backward compatibility guarantees! """ # Figure out our configuration @@ -565,6 +570,8 @@ def mypycify( group_name=group_name, log_trace=log_trace, depends_on_librt_internal=depends_on_librt_internal, + depends_on_librt_base64=depends_on_librt_base64, + experimental_features=experimental_features, ) # Generate all the actual important C code @@ -607,6 +614,8 @@ def mypycify( ] if log_trace: cflags.append("-DMYPYC_LOG_TRACE") + if experimental_features: + cflags.append("-DMYPYC_EXPERIMENTAL") elif compiler.compiler_type == "msvc": # msvc doesn't have levels, '/O2' is full and '/Od' is disable if opt_level == "0": @@ -633,6 +642,8 @@ def mypycify( cflags += ["/GL-", "/wd9025"] # warning about overriding /GL if log_trace: cflags.append("/DMYPYC_LOG_TRACE") + if experimental_features: + cflags.append("/DMYPYC_EXPERIMENTAL") # If configured to (defaults to yes in multi-file mode), copy the # runtime library in. Otherwise it just gets #included to save on diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index da60e145a7903..31fd23229253f 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -604,6 +604,8 @@ def generate_c_for_modules(self) -> list[tuple[str, str]]: ext_declarations.emit_line("#include ") if self.compiler_options.depends_on_librt_internal: ext_declarations.emit_line("#include ") + if self.compiler_options.depends_on_librt_base64: + ext_declarations.emit_line("#include ") declarations = Emitter(self.context) declarations.emit_line(f"#ifndef MYPYC_LIBRT_INTERNAL{self.group_suffix}_H") @@ -1034,6 +1036,10 @@ def emit_module_exec_func( emitter.emit_line("if (import_librt_internal() < 0) {") emitter.emit_line("return -1;") emitter.emit_line("}") + if self.compiler_options.depends_on_librt_base64: + emitter.emit_line("if (import_librt_base64() < 0) {") + emitter.emit_line("return -1;") + emitter.emit_line("}") emitter.emit_line("PyObject* modname = NULL;") if self.multi_phase_init: emitter.emit_line(f"{module_static} = module;") diff --git a/mypyc/ir/ops.py b/mypyc/ir/ops.py index ffce529f0756c..79a08dc2a9f5b 100644 --- a/mypyc/ir/ops.py +++ b/mypyc/ir/ops.py @@ -707,6 +707,7 @@ def __init__( extra_int_constants: list[tuple[int, RType]], priority: int, is_pure: bool, + experimental: bool, ) -> None: # Each primitive much have a distinct name, but otherwise they are arbitrary. self.name: Final = name @@ -729,6 +730,9 @@ def __init__( self.is_pure: Final = is_pure if is_pure: assert error_kind == ERR_NEVER + # Experimental primitives are not used unless mypyc experimental features are + # explicitly enabled + self.experimental = experimental def __repr__(self) -> str: return f"" diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index d33497d4987bd..e19bf54e07441 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -2212,6 +2212,8 @@ def matching_primitive_op( for desc in candidates: if len(desc.arg_types) != len(args): continue + if desc.experimental and not self.options.experimental_features: + continue if all( # formal is not None and # TODO is_subtype(actual.type, formal) diff --git a/mypyc/lib-rt/librt_base64.c b/mypyc/lib-rt/librt_base64.c new file mode 100644 index 0000000000000..1c3a6f8d01a59 --- /dev/null +++ b/mypyc/lib-rt/librt_base64.c @@ -0,0 +1,148 @@ +#define PY_SSIZE_T_CLEAN +#include +#include "librt_base64.h" +#include "pythoncapi_compat.h" + +#ifdef MYPYC_EXPERIMENTAL + +// b64encode_internal below is adapted from the CPython 3.14.0 binascii module + +static const unsigned char table_b2a_base64[] = +"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +#define BASE64_PAD '=' + +/* Max binary chunk size; limited only by available memory */ +#define BASE64_MAXBIN ((PY_SSIZE_T_MAX - 3) / 2) + +static PyObject * +b64encode_internal(PyObject *obj) { + unsigned char *ascii_data; + const unsigned char *bin_data; + int leftbits = 0; + unsigned char this_ch; + unsigned int leftchar = 0; + Py_ssize_t bin_len, out_len; + PyBytesWriter *writer; + int newline = 0; // TODO + + if (!PyBytes_Check(obj)) { + PyErr_SetString(PyExc_TypeError, "base64() expects a bytes object"); + return NULL; + } + + bin_data = (const unsigned char *)PyBytes_AS_STRING(obj); + bin_len = PyBytes_GET_SIZE(obj); + + assert(bin_len >= 0); + + if ( bin_len > BASE64_MAXBIN ) { + PyErr_SetString(PyExc_ValueError, "Too much data for base64 line"); + return NULL; + } + + /* We're lazy and allocate too much (fixed up later). + "+2" leaves room for up to two pad characters. + Note that 'b' gets encoded as 'Yg==\n' (1 in, 5 out). */ + out_len = bin_len*2 + 2; + if (newline) + out_len++; + writer = PyBytesWriter_Create(out_len); + ascii_data = PyBytesWriter_GetData(writer); + if (writer == NULL) + return NULL; + + for( ; bin_len > 0 ; bin_len--, bin_data++ ) { + /* Shift the data into our buffer */ + leftchar = (leftchar << 8) | *bin_data; + leftbits += 8; + + /* See if there are 6-bit groups ready */ + while ( leftbits >= 6 ) { + this_ch = (leftchar >> (leftbits-6)) & 0x3f; + leftbits -= 6; + *ascii_data++ = table_b2a_base64[this_ch]; + } + } + if ( leftbits == 2 ) { + *ascii_data++ = table_b2a_base64[(leftchar&3) << 4]; + *ascii_data++ = BASE64_PAD; + *ascii_data++ = BASE64_PAD; + } else if ( leftbits == 4 ) { + *ascii_data++ = table_b2a_base64[(leftchar&0xf) << 2]; + *ascii_data++ = BASE64_PAD; + } + if (newline) + *ascii_data++ = '\n'; /* Append a courtesy newline */ + + return PyBytesWriter_FinishWithSize(writer, ascii_data - (unsigned char *)PyBytesWriter_GetData(writer)); +} + +static PyObject* +b64encode(PyObject *self, PyObject *const *args, size_t nargs) { + if (nargs != 1) { + PyErr_SetString(PyExc_TypeError, "b64encode() takes exactly one argument"); + return 0; + } + return b64encode_internal(args[0]); +} + +#endif + +static PyMethodDef librt_base64_module_methods[] = { +#ifdef MYPYC_EXPERIMENTAL + {"b64encode", (PyCFunction)b64encode, METH_FASTCALL, PyDoc_STR("Encode bytes-like object using Base64.")}, +#endif + {NULL, NULL, 0, NULL} +}; + +static int +base64_abi_version(void) { + return 0; +} + +static int +base64_api_version(void) { + return 0; +} + +static int +librt_base64_module_exec(PyObject *m) +{ +#ifdef MYPYC_EXPERIMENTAL + // Export mypy internal C API, be careful with the order! + static void *base64_api[LIBRT_BASE64_API_LEN] = { + (void *)base64_abi_version, + (void *)base64_api_version, + (void *)b64encode_internal, + }; + PyObject *c_api_object = PyCapsule_New((void *)base64_api, "librt.base64._C_API", NULL); + if (PyModule_Add(m, "_C_API", c_api_object) < 0) { + return -1; + } +#endif + return 0; +} + +static PyModuleDef_Slot librt_base64_module_slots[] = { + {Py_mod_exec, librt_base64_module_exec}, +#ifdef Py_MOD_GIL_NOT_USED + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, +#endif + {0, NULL} +}; + +static PyModuleDef librt_base64_module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "base64", + .m_doc = "base64 encoding and decoding optimized for mypyc", + .m_size = 0, + .m_methods = librt_base64_module_methods, + .m_slots = librt_base64_module_slots, +}; + +PyMODINIT_FUNC +PyInit_base64(void) +{ + return PyModuleDef_Init(&librt_base64_module); +} diff --git a/mypyc/lib-rt/librt_base64.h b/mypyc/lib-rt/librt_base64.h new file mode 100644 index 0000000000000..cc97e54155fdb --- /dev/null +++ b/mypyc/lib-rt/librt_base64.h @@ -0,0 +1,60 @@ +#ifndef LIBRT_BASE64_H +#define LIBRT_BASE64_H + +#ifndef MYPYC_EXPERIMENTAL + +static int +import_librt_base64(void) +{ + // All librt.base64 features are experimental for now, so don't set up the API here + return 0; +} + +#else // MYPYC_EXPERIMENTAL + +#define LIBRT_BASE64_ABI_VERSION 0 +#define LIBRT_BASE64_API_VERSION 0 +#define LIBRT_BASE64_API_LEN 3 + +static void *LibRTBase64_API[LIBRT_BASE64_API_LEN]; + +#define LibRTBase64_ABIVersion (*(int (*)(void)) LibRTBase64_API[0]) +#define LibRTBase64_APIVersion (*(int (*)(void)) LibRTBase64_API[1]) +#define LibRTBase64_b64encode_internal (*(PyObject* (*)(PyObject *source)) LibRTBase64_API[2]) + +static int +import_librt_base64(void) +{ + PyObject *mod = PyImport_ImportModule("librt.base64"); + if (mod == NULL) + return -1; + Py_DECREF(mod); // we import just for the side effect of making the below work. + void *capsule = PyCapsule_Import("librt.base64._C_API", 0); + if (capsule == NULL) + return -1; + memcpy(LibRTBase64_API, capsule, sizeof(LibRTBase64_API)); + if (LibRTBase64_ABIVersion() != LIBRT_BASE64_ABI_VERSION) { + char err[128]; + snprintf(err, sizeof(err), "ABI version conflict for librt.base64, expected %d, found %d", + LIBRT_BASE64_ABI_VERSION, + LibRTBase64_ABIVersion() + ); + PyErr_SetString(PyExc_ValueError, err); + return -1; + } + if (LibRTBase64_APIVersion() < LIBRT_BASE64_API_VERSION) { + char err[128]; + snprintf(err, sizeof(err), + "API version conflict for librt.base64, expected %d or newer, found %d (hint: upgrade librt)", + LIBRT_BASE64_API_VERSION, + LibRTBase64_APIVersion() + ); + PyErr_SetString(PyExc_ValueError, err); + return -1; + } + return 0; +} + +#endif // MYPYC_EXPERIMENTAL + +#endif // LIBRT_BASE64_H diff --git a/mypyc/lib-rt/setup.py b/mypyc/lib-rt/setup.py index 299b0acd96e70..3c08c7dbb5ee0 100644 --- a/mypyc/lib-rt/setup.py +++ b/mypyc/lib-rt/setup.py @@ -96,6 +96,9 @@ def run(self) -> None: ], include_dirs=["."], extra_compile_args=cflags, - ) + ), + Extension( + "librt.base64", ["librt_base64.c"], include_dirs=["."], extra_compile_args=cflags + ), ] ) diff --git a/mypyc/options.py b/mypyc/options.py index e004a0a52c958..f0290cec4f06d 100644 --- a/mypyc/options.py +++ b/mypyc/options.py @@ -18,6 +18,8 @@ def __init__( group_name: str | None = None, log_trace: bool = False, depends_on_librt_internal: bool = False, + depends_on_librt_base64: bool = False, + experimental_features: bool = False, ) -> None: self.strip_asserts = strip_asserts self.multi_file = multi_file @@ -55,3 +57,8 @@ def __init__( # only for mypy itself, third-party code compiled with mypyc should not use # librt.internal. self.depends_on_librt_internal = depends_on_librt_internal + self.depends_on_librt_base64 = depends_on_librt_base64 + # Some experimental features are only available when building librt in + # experimental mode (e.g. use _experimental suffix in librt run test). + # These can't be used with a librt wheel installed from PyPI. + self.experimental_features = experimental_features diff --git a/mypyc/primitives/misc_ops.py b/mypyc/primitives/misc_ops.py index f685b1cfbcf53..b12ed3a2c523e 100644 --- a/mypyc/primitives/misc_ops.py +++ b/mypyc/primitives/misc_ops.py @@ -465,3 +465,12 @@ c_function_name="cache_version_internal", error_kind=ERR_NEVER, ) + +function_op( + name="librt.base64.b64encode", + arg_types=[bytes_rprimitive], + return_type=bytes_rprimitive, + c_function_name="LibRTBase64_b64encode_internal", + error_kind=ERR_MAGIC, + experimental=True, +) diff --git a/mypyc/primitives/registry.py b/mypyc/primitives/registry.py index 3188bc322809f..36b9e8d0835c7 100644 --- a/mypyc/primitives/registry.py +++ b/mypyc/primitives/registry.py @@ -144,6 +144,7 @@ def method_op( extra_int_constants, priority, is_pure=is_pure, + experimental=False, ) ops.append(desc) return desc @@ -162,6 +163,7 @@ def function_op( steals: StealsDescription = False, is_borrowed: bool = False, priority: int = 1, + experimental: bool = False, ) -> PrimitiveDescription: """Define a C function call op that replaces a function call. @@ -190,6 +192,7 @@ def function_op( extra_int_constants=extra_int_constants, priority=priority, is_pure=False, + experimental=experimental, ) ops.append(desc) return desc @@ -236,6 +239,7 @@ def binary_op( extra_int_constants=extra_int_constants, priority=priority, is_pure=False, + experimental=False, ) ops.append(desc) return desc @@ -314,6 +318,7 @@ def custom_primitive_op( extra_int_constants=extra_int_constants, priority=0, is_pure=is_pure, + experimental=False, ) @@ -355,6 +360,7 @@ def unary_op( extra_int_constants=extra_int_constants, priority=priority, is_pure=is_pure, + experimental=False, ) ops.append(desc) return desc diff --git a/mypyc/test-data/run-base64.test b/mypyc/test-data/run-base64.test new file mode 100644 index 0000000000000..31997d6305962 --- /dev/null +++ b/mypyc/test-data/run-base64.test @@ -0,0 +1,52 @@ +[case testAllBase64Features_librt_base64_experimental] +from typing import Any +import base64 + +from librt.base64 import b64encode + +from testutil import assertRaises + +def test_encode_basic() -> None: + assert b64encode(b"x") == b"eA==" + + with assertRaises(TypeError): + b64encode(bytearray(b"x")) + +def check_encode(b: bytes) -> None: + assert b64encode(b) == getattr(base64, "b64encode")(b) + +def test_encode_different_strings() -> None: + for i in range(256): + check_encode(bytes([i])) + check_encode(bytes([i]) + b"x") + check_encode(bytes([i]) + b"xy") + check_encode(bytes([i]) + b"xyz") + check_encode(bytes([i]) + b"xyza") + check_encode(b"x" + bytes([i])) + check_encode(b"xy" + bytes([i])) + check_encode(b"xyz" + bytes([i])) + check_encode(b"xyza" + bytes([i])) + + b = b"a\x00\xb7" * 1000 + for i in range(1000): + check_encode(b[:i]) + + for b in b"", b"ab", b"bac", b"1234", b"xyz88", b"abc" * 200: + check_encode(b) + +def test_encode_wrapper() -> None: + enc: Any = b64encode + assert enc(b"x") == b"eA==" + + with assertRaises(TypeError): + enc() + + with assertRaises(TypeError): + enc(b"x", b"y") + +[case testBase64FeaturesNotAvailableInNonExperimentalBuild_librt_base64] +# This also ensures librt.base64 can be built without experimental features +import librt.base64 + +def test_b64encode_not_available() -> None: + assert not hasattr(librt.base64, "b64encode") diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py index 953f613293953..26d8e44b784bd 100644 --- a/mypyc/test/test_run.py +++ b/mypyc/test/test_run.py @@ -73,6 +73,7 @@ "run-weakref.test", "run-python37.test", "run-python38.test", + "run-base64.test", ] if sys.version_info >= (3, 10): @@ -86,7 +87,8 @@ setup(name='test_run_output', ext_modules=mypycify({}, separate={}, skip_cgen_input={!r}, strip_asserts=False, - multi_file={}, opt_level='{}', install_librt={}), + multi_file={}, opt_level='{}', install_librt={}, + experimental_features={}), ) """ @@ -242,13 +244,18 @@ def run_case_step(self, testcase: DataDrivenTestCase, incremental_step: int) -> # Use _librt_internal to test mypy-specific parts of librt (they have # some special-casing in mypyc), for everything else use _librt suffix. librt_internal = testcase.name.endswith("_librt_internal") - librt = librt_internal or testcase.name.endswith("_librt") + librt_base64 = "_librt_base64" in testcase.name + librt = testcase.name.endswith("_librt") or "_librt_" in testcase.name + # Enable experimental features (local librt build also includes experimental features) + experimental_features = testcase.name.endswith("_experimental") try: compiler_options = CompilerOptions( multi_file=self.multi_file, separate=self.separate, strict_dunder_typing=self.strict_dunder_typing, depends_on_librt_internal=librt_internal, + depends_on_librt_base64=librt_base64, + experimental_features=experimental_features, ) result = emitmodule.parse_and_typecheck( sources=sources, @@ -281,7 +288,13 @@ def run_case_step(self, testcase: DataDrivenTestCase, incremental_step: int) -> with open(setup_file, "w", encoding="utf-8") as f: f.write( setup_format.format( - module_paths, separate, cfiles, self.multi_file, opt_level, librt + module_paths, + separate, + cfiles, + self.multi_file, + opt_level, + librt, + experimental_features, ) ) From 17f8b3133ace6581942becdd0792a06abc955cbf Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 13 Nov 2025 16:31:24 +0000 Subject: [PATCH 395/424] [mypyc] Import librt.base64 capsule automatically if needed (#20233) Allow primitives to specify the capsule they need via module name such as `librt.base64`. This way we can import the capsule automatically only when there are references to the contents of the capsule in the compiled code. Only make the change for `librt.base64`, but we can also do a similar thing for `librt.internal` in a follow-up PR. --- mypyc/analysis/capsule_deps.py | 27 +++++++++++++++++++++ mypyc/build.py | 2 -- mypyc/codegen/emitmodule.py | 9 +++++-- mypyc/ir/module_ir.py | 7 +++++- mypyc/ir/ops.py | 8 +++++++ mypyc/irbuild/ll_builder.py | 2 ++ mypyc/options.py | 2 -- mypyc/primitives/misc_ops.py | 1 + mypyc/primitives/registry.py | 12 ++++++++++ mypyc/test-data/irbuild-base64.test | 37 +++++++++++++++++++++++++++++ mypyc/test-data/run-base64.test | 11 ++++++++- mypyc/test/test_irbuild.py | 1 + mypyc/test/test_run.py | 2 -- mypyc/test/testutil.py | 2 ++ 14 files changed, 113 insertions(+), 10 deletions(-) create mode 100644 mypyc/analysis/capsule_deps.py create mode 100644 mypyc/test-data/irbuild-base64.test diff --git a/mypyc/analysis/capsule_deps.py b/mypyc/analysis/capsule_deps.py new file mode 100644 index 0000000000000..ada42ee03f289 --- /dev/null +++ b/mypyc/analysis/capsule_deps.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from mypyc.ir.func_ir import FuncIR +from mypyc.ir.ops import CallC, PrimitiveOp + + +def find_implicit_capsule_dependencies(fn: FuncIR) -> set[str] | None: + """Find implicit dependencies on capsules that need to be imported. + + Using primitives or types defined in librt submodules such as "librt.base64" + requires a capsule import. + + Note that a module can depend on a librt module even if it doesn't explicitly + import it, for example via re-exported names or via return types of functions + defined in other modules. + """ + deps: set[str] | None = None + for block in fn.blocks: + for op in block.ops: + # TODO: Also determine implicit type object dependencies (e.g. cast targets) + if isinstance(op, CallC) and op.capsule is not None: + if deps is None: + deps = set() + deps.add(op.capsule) + else: + assert not isinstance(op, PrimitiveOp), "Lowered IR is expected" + return deps diff --git a/mypyc/build.py b/mypyc/build.py index 2ff9d175947ff..351aa87b9264d 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -495,7 +495,6 @@ def mypycify( group_name: str | None = None, log_trace: bool = False, depends_on_librt_internal: bool = False, - depends_on_librt_base64: bool = False, install_librt: bool = False, experimental_features: bool = False, ) -> list[Extension]: @@ -570,7 +569,6 @@ def mypycify( group_name=group_name, log_trace=log_trace, depends_on_librt_internal=depends_on_librt_internal, - depends_on_librt_base64=depends_on_librt_base64, experimental_features=experimental_features, ) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 31fd23229253f..8dd9f750a920d 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -27,6 +27,7 @@ from mypy.options import Options from mypy.plugin import Plugin, ReportConfigContext from mypy.util import hash_digest, json_dumps +from mypyc.analysis.capsule_deps import find_implicit_capsule_dependencies from mypyc.codegen.cstring import c_string_initializer from mypyc.codegen.emit import Emitter, EmitterContext, HeaderDeclaration, c_array_initializer from mypyc.codegen.emitclass import generate_class, generate_class_reuse, generate_class_type_decl @@ -259,6 +260,10 @@ def compile_scc_to_ir( # Switch to lower abstraction level IR. lower_ir(fn, compiler_options) + # Calculate implicit module dependencies (needed for librt) + capsules = find_implicit_capsule_dependencies(fn) + if capsules is not None: + module.capsules.update(capsules) # Perform optimizations. do_copy_propagation(fn, compiler_options) do_flag_elimination(fn, compiler_options) @@ -604,7 +609,7 @@ def generate_c_for_modules(self) -> list[tuple[str, str]]: ext_declarations.emit_line("#include ") if self.compiler_options.depends_on_librt_internal: ext_declarations.emit_line("#include ") - if self.compiler_options.depends_on_librt_base64: + if any("librt.base64" in mod.capsules for mod in self.modules.values()): ext_declarations.emit_line("#include ") declarations = Emitter(self.context) @@ -1036,7 +1041,7 @@ def emit_module_exec_func( emitter.emit_line("if (import_librt_internal() < 0) {") emitter.emit_line("return -1;") emitter.emit_line("}") - if self.compiler_options.depends_on_librt_base64: + if "librt.base64" in module.capsules: emitter.emit_line("if (import_librt_base64() < 0) {") emitter.emit_line("return -1;") emitter.emit_line("}") diff --git a/mypyc/ir/module_ir.py b/mypyc/ir/module_ir.py index 7d95b48e197e7..5aef414490f9e 100644 --- a/mypyc/ir/module_ir.py +++ b/mypyc/ir/module_ir.py @@ -30,6 +30,8 @@ def __init__( # These are only visible in the module that defined them, so no need # to serialize. self.type_var_names = type_var_names + # Capsules needed by the module, specified via module names such as "librt.base64" + self.capsules: set[str] = set() def serialize(self) -> JsonDict: return { @@ -38,11 +40,12 @@ def serialize(self) -> JsonDict: "functions": [f.serialize() for f in self.functions], "classes": [c.serialize() for c in self.classes], "final_names": [(k, t.serialize()) for k, t in self.final_names], + "capsules": sorted(self.capsules), } @classmethod def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> ModuleIR: - return ModuleIR( + module = ModuleIR( data["fullname"], data["imports"], [ctx.functions[FuncDecl.get_id_from_json(f)] for f in data["functions"]], @@ -50,6 +53,8 @@ def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> ModuleIR: [(k, deserialize_type(t, ctx)) for k, t in data["final_names"]], [], ) + module.capsules = set(data["capsules"]) + return module def deserialize_modules(data: dict[str, JsonDict], ctx: DeserMaps) -> dict[str, ModuleIR]: diff --git a/mypyc/ir/ops.py b/mypyc/ir/ops.py index 79a08dc2a9f5b..2153d47e68747 100644 --- a/mypyc/ir/ops.py +++ b/mypyc/ir/ops.py @@ -708,6 +708,7 @@ def __init__( priority: int, is_pure: bool, experimental: bool, + capsule: str | None, ) -> None: # Each primitive much have a distinct name, but otherwise they are arbitrary. self.name: Final = name @@ -733,6 +734,9 @@ def __init__( # Experimental primitives are not used unless mypyc experimental features are # explicitly enabled self.experimental = experimental + # Capsule that needs to imported and configured to call the primitive + # (name of the target module, e.g. "librt.base64"). + self.capsule = capsule def __repr__(self) -> str: return f"" @@ -1233,6 +1237,7 @@ def __init__( *, is_pure: bool = False, returns_null: bool = False, + capsule: str | None = None, ) -> None: self.error_kind = error_kind super().__init__(line) @@ -1250,6 +1255,9 @@ def __init__( # The function might return a null value that does not indicate # an error. self.returns_null = returns_null + # A capsule from this module must be imported and initialized before calling this + # function (used for C functions exported from librt). Example value: "librt.base64" + self.capsule = capsule if is_pure or returns_null: assert error_kind == ERR_NEVER diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index e19bf54e07441..fd66288dbcc57 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -2075,6 +2075,7 @@ def call_c( var_arg_idx, is_pure=desc.is_pure, returns_null=desc.returns_null, + capsule=desc.capsule, ) ) if desc.is_borrowed: @@ -2159,6 +2160,7 @@ def primitive_op( desc.priority, is_pure=desc.is_pure, returns_null=False, + capsule=desc.capsule, ) return self.call_c(c_desc, args, line, result_type=result_type) diff --git a/mypyc/options.py b/mypyc/options.py index f0290cec4f06d..9f16c07dc231a 100644 --- a/mypyc/options.py +++ b/mypyc/options.py @@ -18,7 +18,6 @@ def __init__( group_name: str | None = None, log_trace: bool = False, depends_on_librt_internal: bool = False, - depends_on_librt_base64: bool = False, experimental_features: bool = False, ) -> None: self.strip_asserts = strip_asserts @@ -57,7 +56,6 @@ def __init__( # only for mypy itself, third-party code compiled with mypyc should not use # librt.internal. self.depends_on_librt_internal = depends_on_librt_internal - self.depends_on_librt_base64 = depends_on_librt_base64 # Some experimental features are only available when building librt in # experimental mode (e.g. use _experimental suffix in librt run test). # These can't be used with a librt wheel installed from PyPI. diff --git a/mypyc/primitives/misc_ops.py b/mypyc/primitives/misc_ops.py index b12ed3a2c523e..bb225a76acd88 100644 --- a/mypyc/primitives/misc_ops.py +++ b/mypyc/primitives/misc_ops.py @@ -473,4 +473,5 @@ c_function_name="LibRTBase64_b64encode_internal", error_kind=ERR_MAGIC, experimental=True, + capsule="librt.base64", ) diff --git a/mypyc/primitives/registry.py b/mypyc/primitives/registry.py index 36b9e8d0835c7..2f66b19155013 100644 --- a/mypyc/primitives/registry.py +++ b/mypyc/primitives/registry.py @@ -62,6 +62,7 @@ class CFunctionDescription(NamedTuple): priority: int is_pure: bool returns_null: bool + capsule: str | None # A description for C load operations including LoadGlobal and LoadAddress @@ -100,6 +101,7 @@ def method_op( is_borrowed: bool = False, priority: int = 1, is_pure: bool = False, + capsule: str | None = None, ) -> PrimitiveDescription: """Define a c function call op that replaces a method call. @@ -145,6 +147,7 @@ def method_op( priority, is_pure=is_pure, experimental=False, + capsule=capsule, ) ops.append(desc) return desc @@ -164,6 +167,7 @@ def function_op( is_borrowed: bool = False, priority: int = 1, experimental: bool = False, + capsule: str | None = None, ) -> PrimitiveDescription: """Define a C function call op that replaces a function call. @@ -193,6 +197,7 @@ def function_op( priority=priority, is_pure=False, experimental=experimental, + capsule=capsule, ) ops.append(desc) return desc @@ -212,6 +217,7 @@ def binary_op( steals: StealsDescription = False, is_borrowed: bool = False, priority: int = 1, + capsule: str | None = None, ) -> PrimitiveDescription: """Define a c function call op for a binary operation. @@ -240,6 +246,7 @@ def binary_op( priority=priority, is_pure=False, experimental=False, + capsule=capsule, ) ops.append(desc) return desc @@ -281,6 +288,7 @@ def custom_op( 0, is_pure=is_pure, returns_null=returns_null, + capsule=None, ) @@ -297,6 +305,7 @@ def custom_primitive_op( steals: StealsDescription = False, is_borrowed: bool = False, is_pure: bool = False, + capsule: str | None = None, ) -> PrimitiveDescription: """Define a primitive op that can't be automatically generated based on the AST. @@ -319,6 +328,7 @@ def custom_primitive_op( priority=0, is_pure=is_pure, experimental=False, + capsule=capsule, ) @@ -335,6 +345,7 @@ def unary_op( is_borrowed: bool = False, priority: int = 1, is_pure: bool = False, + capsule: str | None = None, ) -> PrimitiveDescription: """Define a primitive op for an unary operation. @@ -361,6 +372,7 @@ def unary_op( priority=priority, is_pure=is_pure, experimental=False, + capsule=capsule, ) ops.append(desc) return desc diff --git a/mypyc/test-data/irbuild-base64.test b/mypyc/test-data/irbuild-base64.test new file mode 100644 index 0000000000000..bc73b7e354316 --- /dev/null +++ b/mypyc/test-data/irbuild-base64.test @@ -0,0 +1,37 @@ +[case testBase64_experimental] +from librt.base64 import b64encode + +def enc(b: bytes) -> bytes: + return b64encode(b) +[out] +def enc(b): + b, r0 :: bytes +L0: + r0 = LibRTBase64_b64encode_internal(b) + return r0 + +[case testBase64ExperimentalDisabled] +from librt.base64 import b64encode + +def enc(b: bytes) -> bytes: + return b64encode(b) +[out] +def enc(b): + b :: bytes + r0 :: dict + r1 :: str + r2 :: object + r3 :: object[1] + r4 :: object_ptr + r5 :: object + r6 :: bytes +L0: + r0 = __main__.globals :: static + r1 = 'b64encode' + r2 = CPyDict_GetItem(r0, r1) + r3 = [b] + r4 = load_address r3 + r5 = PyObject_Vectorcall(r2, r4, 1, 0) + keep_alive b + r6 = cast(bytes, r5) + return r6 diff --git a/mypyc/test-data/run-base64.test b/mypyc/test-data/run-base64.test index 31997d6305962..0f9151c2b00bb 100644 --- a/mypyc/test-data/run-base64.test +++ b/mypyc/test-data/run-base64.test @@ -1,4 +1,4 @@ -[case testAllBase64Features_librt_base64_experimental] +[case testAllBase64Features_librt_experimental] from typing import Any import base64 @@ -50,3 +50,12 @@ import librt.base64 def test_b64encode_not_available() -> None: assert not hasattr(librt.base64, "b64encode") + +[case testBase64UsedAtTopLevelOnly_librt_experimental] +from librt.base64 import b64encode + +# The only reference to b64encode is at module top level +encoded = b64encode(b"x") + +def test_top_level_only_encode() -> None: + assert encoded == b"eA==" diff --git a/mypyc/test/test_irbuild.py b/mypyc/test/test_irbuild.py index e79cbec392f45..7c248640246d1 100644 --- a/mypyc/test/test_irbuild.py +++ b/mypyc/test/test_irbuild.py @@ -54,6 +54,7 @@ "irbuild-glue-methods.test", "irbuild-math.test", "irbuild-weakref.test", + "irbuild-base64.test", ] if sys.version_info >= (3, 10): diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py index 26d8e44b784bd..6b63a4d546d07 100644 --- a/mypyc/test/test_run.py +++ b/mypyc/test/test_run.py @@ -244,7 +244,6 @@ def run_case_step(self, testcase: DataDrivenTestCase, incremental_step: int) -> # Use _librt_internal to test mypy-specific parts of librt (they have # some special-casing in mypyc), for everything else use _librt suffix. librt_internal = testcase.name.endswith("_librt_internal") - librt_base64 = "_librt_base64" in testcase.name librt = testcase.name.endswith("_librt") or "_librt_" in testcase.name # Enable experimental features (local librt build also includes experimental features) experimental_features = testcase.name.endswith("_experimental") @@ -254,7 +253,6 @@ def run_case_step(self, testcase: DataDrivenTestCase, incremental_step: int) -> separate=self.separate, strict_dunder_typing=self.strict_dunder_typing, depends_on_librt_internal=librt_internal, - depends_on_librt_base64=librt_base64, experimental_features=experimental_features, ) result = emitmodule.parse_and_typecheck( diff --git a/mypyc/test/testutil.py b/mypyc/test/testutil.py index 80a06204bb9da..3e9abc231d9a0 100644 --- a/mypyc/test/testutil.py +++ b/mypyc/test/testutil.py @@ -281,4 +281,6 @@ def infer_ir_build_options_from_test_name(name: str) -> CompilerOptions | None: options.python_version = options.capi_version elif "_py" in name or "_Python" in name: assert False, f"Invalid _py* suffix (should be _pythonX_Y): {name}" + if re.search("_experimental(_|$)", name): + options.experimental_features = True return options From 8e2ce962dba5a83128c9ee8f55fc001a8fd4d28d Mon Sep 17 00:00:00 2001 From: KarelKenens <143591762+KarelKenens@users.noreply.github.com> Date: Thu, 13 Nov 2025 19:51:19 +0100 Subject: [PATCH 396/424] Fix annotated with function as type keyword list parameter (#20094) Fixes #20090 --- mypy/exprtotype.py | 61 +++++++++++++++++++++----- test-data/unit/check-python312.test | 19 ++++++++ test-data/unit/check-type-aliases.test | 19 ++++++++ 3 files changed, 87 insertions(+), 12 deletions(-) diff --git a/mypy/exprtotype.py b/mypy/exprtotype.py index 506194a4b285b..6fd43c08dbac2 100644 --- a/mypy/exprtotype.py +++ b/mypy/exprtotype.py @@ -100,7 +100,9 @@ def expr_to_unanalyzed_type( else: raise TypeTranslationError() elif isinstance(expr, IndexExpr): - base = expr_to_unanalyzed_type(expr.base, options, allow_new_syntax, expr) + base = expr_to_unanalyzed_type( + expr.base, options, allow_new_syntax, expr, lookup_qualified=lookup_qualified + ) if isinstance(base, UnboundType): if base.args: raise TypeTranslationError() @@ -124,9 +126,18 @@ def expr_to_unanalyzed_type( # TODO: this is not the optimal solution as we are basically getting rid # of the Annotation definition and only returning the type information, # losing all the annotations. - return expr_to_unanalyzed_type(args[0], options, allow_new_syntax, expr) + return expr_to_unanalyzed_type( + args[0], options, allow_new_syntax, expr, lookup_qualified=lookup_qualified + ) base.args = tuple( - expr_to_unanalyzed_type(arg, options, allow_new_syntax, expr, allow_unpack=True) + expr_to_unanalyzed_type( + arg, + options, + allow_new_syntax, + expr, + allow_unpack=True, + lookup_qualified=lookup_qualified, + ) for arg in args ) if not base.args: @@ -141,8 +152,12 @@ def expr_to_unanalyzed_type( ): return UnionType( [ - expr_to_unanalyzed_type(expr.left, options, allow_new_syntax), - expr_to_unanalyzed_type(expr.right, options, allow_new_syntax), + expr_to_unanalyzed_type( + expr.left, options, allow_new_syntax, lookup_qualified=lookup_qualified + ), + expr_to_unanalyzed_type( + expr.right, options, allow_new_syntax, lookup_qualified=lookup_qualified + ), ], uses_pep604_syntax=True, ) @@ -178,12 +193,16 @@ def expr_to_unanalyzed_type( if typ is not default_type: # Two types raise TypeTranslationError() - typ = expr_to_unanalyzed_type(arg, options, allow_new_syntax, expr) + typ = expr_to_unanalyzed_type( + arg, options, allow_new_syntax, expr, lookup_qualified=lookup_qualified + ) continue else: raise TypeTranslationError() elif i == 0: - typ = expr_to_unanalyzed_type(arg, options, allow_new_syntax, expr) + typ = expr_to_unanalyzed_type( + arg, options, allow_new_syntax, expr, lookup_qualified=lookup_qualified + ) elif i == 1: name = _extract_argument_name(arg) else: @@ -192,7 +211,14 @@ def expr_to_unanalyzed_type( elif isinstance(expr, ListExpr): return TypeList( [ - expr_to_unanalyzed_type(t, options, allow_new_syntax, expr, allow_unpack=True) + expr_to_unanalyzed_type( + t, + options, + allow_new_syntax, + expr, + allow_unpack=True, + lookup_qualified=lookup_qualified, + ) for t in expr.items ], line=expr.line, @@ -203,7 +229,9 @@ def expr_to_unanalyzed_type( elif isinstance(expr, BytesExpr): return parse_type_string(expr.value, "builtins.bytes", expr.line, expr.column) elif isinstance(expr, UnaryExpr): - typ = expr_to_unanalyzed_type(expr.expr, options, allow_new_syntax) + typ = expr_to_unanalyzed_type( + expr.expr, options, allow_new_syntax, lookup_qualified=lookup_qualified + ) if isinstance(typ, RawExpressionType): if isinstance(typ.literal_value, int): if expr.op == "-": @@ -225,7 +253,10 @@ def expr_to_unanalyzed_type( return EllipsisType(expr.line) elif allow_unpack and isinstance(expr, StarExpr): return UnpackType( - expr_to_unanalyzed_type(expr.expr, options, allow_new_syntax), from_star_syntax=True + expr_to_unanalyzed_type( + expr.expr, options, allow_new_syntax, lookup_qualified=lookup_qualified + ), + from_star_syntax=True, ) elif isinstance(expr, DictExpr): if not expr.items: @@ -236,12 +267,18 @@ def expr_to_unanalyzed_type( if not isinstance(item_name, StrExpr): if item_name is None: extra_items_from.append( - expr_to_unanalyzed_type(value, options, allow_new_syntax, expr) + expr_to_unanalyzed_type( + value, + options, + allow_new_syntax, + expr, + lookup_qualified=lookup_qualified, + ) ) continue raise TypeTranslationError() items[item_name.value] = expr_to_unanalyzed_type( - value, options, allow_new_syntax, expr + value, options, allow_new_syntax, expr, lookup_qualified=lookup_qualified ) result = TypedDictType( items, set(), set(), Instance(MISSING_FALLBACK, ()), expr.line, expr.column diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index c8b306b6bd551..c501962967d51 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -2170,6 +2170,25 @@ reveal_type(x[0]) # N: Revealed type is "Any" [builtins fixtures/tuple.pyi] [typing fixtures/typing-full.pyi] +[case testAnnotatedWithCallableAsParameterTypeKeyword] +from typing_extensions import Annotated + +def something() -> None: ... + +type A = list[Annotated[str, something()]] +a: A +reveal_type(a) # N: Revealed type is "builtins.list[builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testAnnotatedWithCallableAsParameterTypeKeywordDeeper] +from typing_extensions import Annotated + +def something() -> None: ... + +type A = list[Annotated[Annotated[str, something()], something()]] +a: A +reveal_type(a) # N: Revealed type is "builtins.list[builtins.str]" +[builtins fixtures/tuple.pyi] [case testPEP695TypeAliasRecursiveInParameterBound] from typing import Any diff --git a/test-data/unit/check-type-aliases.test b/test-data/unit/check-type-aliases.test index 06e2237161895..6923b0d8f0064 100644 --- a/test-data/unit/check-type-aliases.test +++ b/test-data/unit/check-type-aliases.test @@ -1319,6 +1319,25 @@ from typing_extensions import TypeAlias Foo: TypeAlias = ClassVar[int] # E: ClassVar[...] can't be used inside a type alias [builtins fixtures/tuple.pyi] +[case testAnnotatedWithCallableAsParameterTypeAlias] +from typing_extensions import Annotated, TypeAlias + +def something() -> None: ... + +A: TypeAlias = list[Annotated[str, something()]] +a: A +reveal_type(a) # N: Revealed type is "builtins.list[builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testAnnotatedWithCallableAsParameterTypeAliasDeeper] +from typing_extensions import Annotated, TypeAlias + +def something() -> None: ... + +A: TypeAlias = list[Annotated[Annotated[str, something()], something()]] +a: A +reveal_type(a) # N: Revealed type is "builtins.list[builtins.str]" +[builtins fixtures/tuple.pyi] [case testTypeAliasDict] D = dict[str, int] From 92864259a75ff91fc8ebf2fe743ff6d7a331519b Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Sun, 16 Nov 2025 01:20:10 +0100 Subject: [PATCH 397/424] Sync typeshed (#20241) Source commit: https://github.com/python/typeshed/commit/ebce8d766b41fbf4d83cf47c1297563a9508ff60 --- mypy/typeshed/stdlib/_compression.pyi | 15 ++- mypy/typeshed/stdlib/asyncio/protocols.pyi | 4 +- mypy/typeshed/stdlib/builtins.pyi | 23 ++-- .../stdlib/compression/_common/_streams.pyi | 15 ++- mypy/typeshed/stdlib/html/parser.pyi | 3 +- mypy/typeshed/stdlib/http/client.pyi | 1 + mypy/typeshed/stdlib/imaplib.pyi | 1 + mypy/typeshed/stdlib/importlib/util.pyi | 16 ++- mypy/typeshed/stdlib/locale.pyi | 4 + mypy/typeshed/stdlib/os/__init__.pyi | 27 ++--- mypy/typeshed/stdlib/parser.pyi | 8 +- mypy/typeshed/stdlib/select.pyi | 12 +- mypy/typeshed/stdlib/ssl.pyi | 3 + mypy/typeshed/stdlib/stat.pyi | 109 +++++++++++++++++- mypy/typeshed/stdlib/sys/__init__.pyi | 15 ++- mypy/typeshed/stdlib/sys/_monitoring.pyi | 8 +- mypy/typeshed/stdlib/threading.pyi | 6 +- mypy/typeshed/stdlib/types.pyi | 4 +- mypy/typeshed/stdlib/typing.pyi | 4 - mypy/typeshed/stdlib/unittest/util.pyi | 21 +++- mypy/typeshed/stdlib/uuid.pyi | 11 +- mypy/typeshed/stdlib/winreg.pyi | 6 +- test-data/unit/pythoneval.test | 2 +- 23 files changed, 251 insertions(+), 67 deletions(-) diff --git a/mypy/typeshed/stdlib/_compression.pyi b/mypy/typeshed/stdlib/_compression.pyi index aa67df2ab4787..6015bcb13f1cc 100644 --- a/mypy/typeshed/stdlib/_compression.pyi +++ b/mypy/typeshed/stdlib/_compression.pyi @@ -1,6 +1,6 @@ # _compression is replaced by compression._common._streams on Python 3.14+ (PEP-784) -from _typeshed import Incomplete, WriteableBuffer +from _typeshed import ReadableBuffer, WriteableBuffer from collections.abc import Callable from io import DEFAULT_BUFFER_SIZE, BufferedIOBase, RawIOBase from typing import Any, Protocol, type_check_only @@ -13,13 +13,24 @@ class _Reader(Protocol): def seekable(self) -> bool: ... def seek(self, n: int, /) -> Any: ... +@type_check_only +class _Decompressor(Protocol): + def decompress(self, data: ReadableBuffer, /, max_length: int = ...) -> bytes: ... + @property + def unused_data(self) -> bytes: ... + @property + def eof(self) -> bool: ... + # `zlib._Decompress` does not have next property, but `DecompressReader` calls it: + # @property + # def needs_input(self) -> bool: ... + class BaseStream(BufferedIOBase): ... class DecompressReader(RawIOBase): def __init__( self, fp: _Reader, - decomp_factory: Callable[..., Incomplete], + decomp_factory: Callable[..., _Decompressor], trailing_error: type[Exception] | tuple[type[Exception], ...] = (), **decomp_args: Any, # These are passed to decomp_factory. ) -> None: ... diff --git a/mypy/typeshed/stdlib/asyncio/protocols.pyi b/mypy/typeshed/stdlib/asyncio/protocols.pyi index 2c52ad4be4102..3a8965f03e29c 100644 --- a/mypy/typeshed/stdlib/asyncio/protocols.pyi +++ b/mypy/typeshed/stdlib/asyncio/protocols.pyi @@ -14,7 +14,7 @@ class BaseProtocol: class Protocol(BaseProtocol): # Need annotation or mypy will complain about 'Cannot determine type of "__slots__" in base class' - __slots__: tuple[()] = () + __slots__: tuple[str, ...] = () def data_received(self, data: bytes) -> None: ... def eof_received(self) -> bool | None: ... @@ -35,7 +35,7 @@ class DatagramProtocol(BaseProtocol): def error_received(self, exc: Exception) -> None: ... class SubprocessProtocol(BaseProtocol): - __slots__: tuple[()] = () + __slots__: tuple[str, ...] = () def pipe_data_received(self, fd: int, data: bytes) -> None: ... def pipe_connection_lost(self, fd: int, exc: Exception | None) -> None: ... def process_exited(self) -> None: ... diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi index e03a92ce3d91d..30dfb9dbb25c9 100644 --- a/mypy/typeshed/stdlib/builtins.pyi +++ b/mypy/typeshed/stdlib/builtins.pyi @@ -42,6 +42,7 @@ from typing import ( # noqa: Y022,UP035 Any, BinaryIO, ClassVar, + Final, Generic, Mapping, MutableMapping, @@ -188,8 +189,9 @@ class type: __bases__: tuple[type, ...] @property def __basicsize__(self) -> int: ... - @property - def __dict__(self) -> types.MappingProxyType[str, Any]: ... # type: ignore[override] + # type.__dict__ is read-only at runtime, but that can't be expressed currently. + # See https://github.com/python/typeshed/issues/11033 for a discussion. + __dict__: Final[types.MappingProxyType[str, Any]] # type: ignore[assignment] @property def __dictoffset__(self) -> int: ... @property @@ -1267,13 +1269,6 @@ class property: def __set__(self, instance: Any, value: Any, /) -> None: ... def __delete__(self, instance: Any, /) -> None: ... -@final -@type_check_only -class _NotImplementedType(Any): - __call__: None - -NotImplemented: _NotImplementedType - def abs(x: SupportsAbs[_T], /) -> _T: ... def all(iterable: Iterable[object], /) -> bool: ... def any(iterable: Iterable[object], /) -> bool: ... @@ -1932,14 +1927,14 @@ def __import__( def __build_class__(func: Callable[[], CellType | Any], name: str, /, *bases: Any, metaclass: Any = ..., **kwds: Any) -> Any: ... if sys.version_info >= (3, 10): - from types import EllipsisType + from types import EllipsisType, NotImplementedType # Backwards compatibility hack for folks who relied on the ellipsis type # existing in typeshed in Python 3.9 and earlier. ellipsis = EllipsisType Ellipsis: EllipsisType - + NotImplemented: NotImplementedType else: # Actually the type of Ellipsis is , but since it's # not exposed anywhere under that name, we make it private here. @@ -1949,6 +1944,12 @@ else: Ellipsis: ellipsis + @final + @type_check_only + class _NotImplementedType(Any): ... + + NotImplemented: _NotImplementedType + @disjoint_base class BaseException: args: tuple[Any, ...] diff --git a/mypy/typeshed/stdlib/compression/_common/_streams.pyi b/mypy/typeshed/stdlib/compression/_common/_streams.pyi index b8463973ec671..96aec24d1c2d0 100644 --- a/mypy/typeshed/stdlib/compression/_common/_streams.pyi +++ b/mypy/typeshed/stdlib/compression/_common/_streams.pyi @@ -1,4 +1,4 @@ -from _typeshed import Incomplete, WriteableBuffer +from _typeshed import ReadableBuffer, WriteableBuffer from collections.abc import Callable from io import DEFAULT_BUFFER_SIZE, BufferedIOBase, RawIOBase from typing import Any, Protocol, type_check_only @@ -11,13 +11,24 @@ class _Reader(Protocol): def seekable(self) -> bool: ... def seek(self, n: int, /) -> Any: ... +@type_check_only +class _Decompressor(Protocol): + def decompress(self, data: ReadableBuffer, /, max_length: int = ...) -> bytes: ... + @property + def unused_data(self) -> bytes: ... + @property + def eof(self) -> bool: ... + # `zlib._Decompress` does not have next property, but `DecompressReader` calls it: + # @property + # def needs_input(self) -> bool: ... + class BaseStream(BufferedIOBase): ... class DecompressReader(RawIOBase): def __init__( self, fp: _Reader, - decomp_factory: Callable[..., Incomplete], # Consider backporting changes to _compression + decomp_factory: Callable[..., _Decompressor], # Consider backporting changes to _compression trailing_error: type[Exception] | tuple[type[Exception], ...] = (), **decomp_args: Any, # These are passed to decomp_factory. ) -> None: ... diff --git a/mypy/typeshed/stdlib/html/parser.pyi b/mypy/typeshed/stdlib/html/parser.pyi index 7edd39e8c7037..08dc7b9369228 100644 --- a/mypy/typeshed/stdlib/html/parser.pyi +++ b/mypy/typeshed/stdlib/html/parser.pyi @@ -9,7 +9,8 @@ class HTMLParser(ParserBase): # Added in Python 3.9.23, 3.10.18, 3.11.13, 3.12.11, 3.13.6 RCDATA_CONTENT_ELEMENTS: Final[tuple[str, ...]] - def __init__(self, *, convert_charrefs: bool = True) -> None: ... + # `scripting` parameter added in Python 3.9.25, 3.10.20, 3.11.15, 3.12.13, 3.13.10, 3.14.1 + def __init__(self, *, convert_charrefs: bool = True, scripting: bool = False) -> None: ... def feed(self, data: str) -> None: ... def close(self) -> None: ... def get_starttag_text(self) -> str | None: ... diff --git a/mypy/typeshed/stdlib/http/client.pyi b/mypy/typeshed/stdlib/http/client.pyi index d259e84e6f2aa..1568567d58541 100644 --- a/mypy/typeshed/stdlib/http/client.pyi +++ b/mypy/typeshed/stdlib/http/client.pyi @@ -166,6 +166,7 @@ class HTTPResponse(io.BufferedIOBase, BinaryIO): # type: ignore[misc] # incomp def begin(self) -> None: ... class HTTPConnection: + blocksize: int auto_open: int # undocumented debuglevel: int default_port: int # undocumented diff --git a/mypy/typeshed/stdlib/imaplib.pyi b/mypy/typeshed/stdlib/imaplib.pyi index 536985a592b7f..39fd466529bb2 100644 --- a/mypy/typeshed/stdlib/imaplib.pyi +++ b/mypy/typeshed/stdlib/imaplib.pyi @@ -26,6 +26,7 @@ class IMAP4: class error(Exception): ... class abort(error): ... class readonly(abort): ... + utf8_enabled: bool mustquote: Pattern[str] debug: int state: str diff --git a/mypy/typeshed/stdlib/importlib/util.pyi b/mypy/typeshed/stdlib/importlib/util.pyi index 05c4d0d1edb30..577d3a667eca8 100644 --- a/mypy/typeshed/stdlib/importlib/util.pyi +++ b/mypy/typeshed/stdlib/importlib/util.pyi @@ -12,7 +12,9 @@ from importlib._bootstrap_external import ( spec_from_file_location as spec_from_file_location, ) from importlib.abc import Loader -from typing_extensions import ParamSpec, deprecated +from types import TracebackType +from typing import Literal +from typing_extensions import ParamSpec, Self, deprecated _P = ParamSpec("_P") @@ -44,6 +46,18 @@ class LazyLoader(Loader): def source_hash(source_bytes: ReadableBuffer) -> bytes: ... +if sys.version_info >= (3, 12): + class _incompatible_extension_module_restrictions: + def __init__(self, *, disable_check: bool) -> None: ... + disable_check: bool + old: Literal[-1, 0, 1] # exists only while entered + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None + ) -> None: ... + @property + def override(self) -> Literal[-1, 1]: ... # undocumented + if sys.version_info >= (3, 14): __all__ = [ "LazyLoader", diff --git a/mypy/typeshed/stdlib/locale.pyi b/mypy/typeshed/stdlib/locale.pyi index fae9f849b6373..80c39a532dc86 100644 --- a/mypy/typeshed/stdlib/locale.pyi +++ b/mypy/typeshed/stdlib/locale.pyi @@ -153,6 +153,10 @@ if sys.version_info < (3, 12): def format_string(f: _str, val: Any, grouping: bool = False, monetary: bool = False) -> _str: ... def currency(val: float | Decimal, symbol: bool = True, grouping: bool = False, international: bool = False) -> _str: ... def delocalize(string: _str) -> _str: ... + +if sys.version_info >= (3, 10): + def localize(string: _str, grouping: bool = False, monetary: bool = False) -> _str: ... + def atof(string: _str, func: Callable[[_str], float] = ...) -> float: ... def atoi(string: _str) -> int: ... def str(val: float) -> _str: ... diff --git a/mypy/typeshed/stdlib/os/__init__.pyi b/mypy/typeshed/stdlib/os/__init__.pyi index 580452739f7f0..bb0a57153948a 100644 --- a/mypy/typeshed/stdlib/os/__init__.pyi +++ b/mypy/typeshed/stdlib/os/__init__.pyi @@ -41,10 +41,22 @@ from typing import ( runtime_checkable, type_check_only, ) -from typing_extensions import Self, TypeAlias, Unpack, deprecated +from typing_extensions import LiteralString, Self, TypeAlias, Unpack, deprecated from . import path as _path +# Re-export common definitions from os.path to reduce duplication +from .path import ( + altsep as altsep, + curdir as curdir, + defpath as defpath, + devnull as devnull, + extsep as extsep, + pardir as pardir, + pathsep as pathsep, + sep as sep, +) + __all__ = [ "F_OK", "O_APPEND", @@ -674,19 +686,8 @@ if sys.platform != "win32": ST_NOSUID: Final[int] ST_RDONLY: Final[int] -curdir: str -pardir: str -sep: str -if sys.platform == "win32": - altsep: str -else: - altsep: str | None -extsep: str -pathsep: str -defpath: str linesep: Literal["\n", "\r\n"] -devnull: str -name: str +name: LiteralString F_OK: Final = 0 R_OK: Final = 4 diff --git a/mypy/typeshed/stdlib/parser.pyi b/mypy/typeshed/stdlib/parser.pyi index 26140c76248ae..9b287fcc6529d 100644 --- a/mypy/typeshed/stdlib/parser.pyi +++ b/mypy/typeshed/stdlib/parser.pyi @@ -7,8 +7,8 @@ def expr(source: str) -> STType: ... def suite(source: str) -> STType: ... def sequence2st(sequence: Sequence[Any]) -> STType: ... def tuple2st(sequence: Sequence[Any]) -> STType: ... -def st2list(st: STType, line_info: bool = ..., col_info: bool = ...) -> list[Any]: ... -def st2tuple(st: STType, line_info: bool = ..., col_info: bool = ...) -> tuple[Any, ...]: ... +def st2list(st: STType, line_info: bool = False, col_info: bool = False) -> list[Any]: ... +def st2tuple(st: STType, line_info: bool = False, col_info: bool = False) -> tuple[Any, ...]: ... def compilest(st: STType, filename: StrOrBytesPath = ...) -> CodeType: ... def isexpr(st: STType) -> bool: ... def issuite(st: STType) -> bool: ... @@ -21,5 +21,5 @@ class STType: def compile(self, filename: StrOrBytesPath = ...) -> CodeType: ... def isexpr(self) -> bool: ... def issuite(self) -> bool: ... - def tolist(self, line_info: bool = ..., col_info: bool = ...) -> list[Any]: ... - def totuple(self, line_info: bool = ..., col_info: bool = ...) -> tuple[Any, ...]: ... + def tolist(self, line_info: bool = False, col_info: bool = False) -> list[Any]: ... + def totuple(self, line_info: bool = False, col_info: bool = False) -> tuple[Any, ...]: ... diff --git a/mypy/typeshed/stdlib/select.pyi b/mypy/typeshed/stdlib/select.pyi index 587bc75376ef1..43a9e4274b236 100644 --- a/mypy/typeshed/stdlib/select.pyi +++ b/mypy/typeshed/stdlib/select.pyi @@ -2,8 +2,8 @@ import sys from _typeshed import FileDescriptorLike from collections.abc import Iterable from types import TracebackType -from typing import Any, ClassVar, Final, final -from typing_extensions import Self +from typing import Any, ClassVar, Final, TypeVar, final +from typing_extensions import Never, Self if sys.platform != "win32": PIPE_BUF: Final[int] @@ -31,9 +31,13 @@ if sys.platform != "win32": def unregister(self, fd: FileDescriptorLike, /) -> None: ... def poll(self, timeout: float | None = None, /) -> list[tuple[int, int]]: ... +_R = TypeVar("_R", default=Never) +_W = TypeVar("_W", default=Never) +_X = TypeVar("_X", default=Never) + def select( - rlist: Iterable[Any], wlist: Iterable[Any], xlist: Iterable[Any], timeout: float | None = None, / -) -> tuple[list[Any], list[Any], list[Any]]: ... + rlist: Iterable[_R], wlist: Iterable[_W], xlist: Iterable[_X], timeout: float | None = None, / +) -> tuple[list[_R], list[_W], list[_X]]: ... error = OSError diff --git a/mypy/typeshed/stdlib/ssl.pyi b/mypy/typeshed/stdlib/ssl.pyi index faa98cb399200..aa94fc84255e5 100644 --- a/mypy/typeshed/stdlib/ssl.pyi +++ b/mypy/typeshed/stdlib/ssl.pyi @@ -33,6 +33,9 @@ from typing_extensions import Never, Self, TypeAlias, deprecated if sys.version_info >= (3, 13): from _ssl import HAS_PSK as HAS_PSK +if sys.version_info >= (3, 14): + from _ssl import HAS_PHA as HAS_PHA + if sys.version_info < (3, 12): from _ssl import RAND_pseudo_bytes as RAND_pseudo_bytes diff --git a/mypy/typeshed/stdlib/stat.pyi b/mypy/typeshed/stdlib/stat.pyi index face28ab0cbb6..6c26080e06653 100644 --- a/mypy/typeshed/stdlib/stat.pyi +++ b/mypy/typeshed/stdlib/stat.pyi @@ -1,7 +1,114 @@ import sys -from _stat import * +from _stat import ( + S_ENFMT as S_ENFMT, + S_IEXEC as S_IEXEC, + S_IFBLK as S_IFBLK, + S_IFCHR as S_IFCHR, + S_IFDIR as S_IFDIR, + S_IFDOOR as S_IFDOOR, + S_IFIFO as S_IFIFO, + S_IFLNK as S_IFLNK, + S_IFMT as S_IFMT, + S_IFPORT as S_IFPORT, + S_IFREG as S_IFREG, + S_IFSOCK as S_IFSOCK, + S_IFWHT as S_IFWHT, + S_IMODE as S_IMODE, + S_IREAD as S_IREAD, + S_IRGRP as S_IRGRP, + S_IROTH as S_IROTH, + S_IRUSR as S_IRUSR, + S_IRWXG as S_IRWXG, + S_IRWXO as S_IRWXO, + S_IRWXU as S_IRWXU, + S_ISBLK as S_ISBLK, + S_ISCHR as S_ISCHR, + S_ISDIR as S_ISDIR, + S_ISDOOR as S_ISDOOR, + S_ISFIFO as S_ISFIFO, + S_ISGID as S_ISGID, + S_ISLNK as S_ISLNK, + S_ISPORT as S_ISPORT, + S_ISREG as S_ISREG, + S_ISSOCK as S_ISSOCK, + S_ISUID as S_ISUID, + S_ISVTX as S_ISVTX, + S_ISWHT as S_ISWHT, + S_IWGRP as S_IWGRP, + S_IWOTH as S_IWOTH, + S_IWRITE as S_IWRITE, + S_IWUSR as S_IWUSR, + S_IXGRP as S_IXGRP, + S_IXOTH as S_IXOTH, + S_IXUSR as S_IXUSR, + SF_APPEND as SF_APPEND, + SF_ARCHIVED as SF_ARCHIVED, + SF_IMMUTABLE as SF_IMMUTABLE, + SF_NOUNLINK as SF_NOUNLINK, + SF_SNAPSHOT as SF_SNAPSHOT, + ST_ATIME as ST_ATIME, + ST_CTIME as ST_CTIME, + ST_DEV as ST_DEV, + ST_GID as ST_GID, + ST_INO as ST_INO, + ST_MODE as ST_MODE, + ST_MTIME as ST_MTIME, + ST_NLINK as ST_NLINK, + ST_SIZE as ST_SIZE, + ST_UID as ST_UID, + UF_APPEND as UF_APPEND, + UF_COMPRESSED as UF_COMPRESSED, + UF_HIDDEN as UF_HIDDEN, + UF_IMMUTABLE as UF_IMMUTABLE, + UF_NODUMP as UF_NODUMP, + UF_NOUNLINK as UF_NOUNLINK, + UF_OPAQUE as UF_OPAQUE, + filemode as filemode, +) from typing import Final +if sys.platform == "win32": + from _stat import ( + IO_REPARSE_TAG_APPEXECLINK as IO_REPARSE_TAG_APPEXECLINK, + IO_REPARSE_TAG_MOUNT_POINT as IO_REPARSE_TAG_MOUNT_POINT, + IO_REPARSE_TAG_SYMLINK as IO_REPARSE_TAG_SYMLINK, + ) + +if sys.version_info >= (3, 13): + from _stat import ( + SF_DATALESS as SF_DATALESS, + SF_FIRMLINK as SF_FIRMLINK, + SF_SETTABLE as SF_SETTABLE, + UF_DATAVAULT as UF_DATAVAULT, + UF_SETTABLE as UF_SETTABLE, + UF_TRACKED as UF_TRACKED, + ) + + if sys.platform == "darwin": + from _stat import SF_SUPPORTED as SF_SUPPORTED, SF_SYNTHETIC as SF_SYNTHETIC + +# _stat.c defines FILE_ATTRIBUTE_* constants conditionally, +# making them available only at runtime on Windows. +# stat.py unconditionally redefines the same FILE_ATTRIBUTE_* constants +# on all platforms. +FILE_ATTRIBUTE_ARCHIVE: Final = 32 +FILE_ATTRIBUTE_COMPRESSED: Final = 2048 +FILE_ATTRIBUTE_DEVICE: Final = 64 +FILE_ATTRIBUTE_DIRECTORY: Final = 16 +FILE_ATTRIBUTE_ENCRYPTED: Final = 16384 +FILE_ATTRIBUTE_HIDDEN: Final = 2 +FILE_ATTRIBUTE_INTEGRITY_STREAM: Final = 32768 +FILE_ATTRIBUTE_NORMAL: Final = 128 +FILE_ATTRIBUTE_NOT_CONTENT_INDEXED: Final = 8192 +FILE_ATTRIBUTE_NO_SCRUB_DATA: Final = 131072 +FILE_ATTRIBUTE_OFFLINE: Final = 4096 +FILE_ATTRIBUTE_READONLY: Final = 1 +FILE_ATTRIBUTE_REPARSE_POINT: Final = 1024 +FILE_ATTRIBUTE_SPARSE_FILE: Final = 512 +FILE_ATTRIBUTE_SYSTEM: Final = 4 +FILE_ATTRIBUTE_TEMPORARY: Final = 256 +FILE_ATTRIBUTE_VIRTUAL: Final = 65536 + if sys.version_info >= (3, 13): # https://github.com/python/cpython/issues/114081#issuecomment-2119017790 SF_RESTRICTED: Final = 0x00080000 diff --git a/mypy/typeshed/stdlib/sys/__init__.pyi b/mypy/typeshed/stdlib/sys/__init__.pyi index 97e65d3094aae..6abef85dfb7f1 100644 --- a/mypy/typeshed/stdlib/sys/__init__.pyi +++ b/mypy/typeshed/stdlib/sys/__init__.pyi @@ -5,7 +5,7 @@ from builtins import object as _object from collections.abc import AsyncGenerator, Callable, Sequence from io import TextIOWrapper from types import FrameType, ModuleType, TracebackType -from typing import Any, Final, Literal, NoReturn, Protocol, TextIO, TypeVar, final, type_check_only +from typing import Any, Final, Literal, NoReturn, Protocol, TextIO, TypeVar, final, overload, type_check_only from typing_extensions import LiteralString, TypeAlias, deprecated _T = TypeVar("_T") @@ -378,13 +378,13 @@ if sys.platform == "android": # noqa: Y008 def getandroidapilevel() -> int: ... def getallocatedblocks() -> int: ... -def getdefaultencoding() -> str: ... +def getdefaultencoding() -> Literal["utf-8"]: ... if sys.platform != "win32": def getdlopenflags() -> int: ... -def getfilesystemencoding() -> str: ... -def getfilesystemencodeerrors() -> str: ... +def getfilesystemencoding() -> LiteralString: ... +def getfilesystemencodeerrors() -> LiteralString: ... def getrefcount(object: Any, /) -> int: ... def getrecursionlimit() -> int: ... def getsizeof(obj: object, default: int = ...) -> int: ... @@ -422,7 +422,12 @@ if sys.platform == "win32": def getwindowsversion() -> _WinVersion: ... -def intern(string: str, /) -> str: ... +@overload +def intern(string: LiteralString, /) -> LiteralString: ... +@overload +def intern(string: str, /) -> str: ... # type: ignore[misc] + +__interactivehook__: Callable[[], object] if sys.version_info >= (3, 13): def _is_gil_enabled() -> bool: ... diff --git a/mypy/typeshed/stdlib/sys/_monitoring.pyi b/mypy/typeshed/stdlib/sys/_monitoring.pyi index 5d231c7a93b39..db799e6f32f83 100644 --- a/mypy/typeshed/stdlib/sys/_monitoring.pyi +++ b/mypy/typeshed/stdlib/sys/_monitoring.pyi @@ -17,6 +17,10 @@ PROFILER_ID: Final = 2 OPTIMIZER_ID: Final = 5 def use_tool_id(tool_id: int, name: str, /) -> None: ... + +if sys.version_info >= (3, 14): + def clear_tool_id(tool_id: int, /) -> None: ... + def free_tool_id(tool_id: int, /) -> None: ... def get_tool(tool_id: int, /) -> str | None: ... @@ -43,10 +47,10 @@ class _events: STOP_ITERATION: Final[int] if sys.version_info >= (3, 14): BRANCH_LEFT: Final[int] - BRANCH_TAKEN: Final[int] + BRANCH_RIGHT: Final[int] @property - @deprecated("Deprecated since Python 3.14. Use `BRANCH_LEFT` or `BRANCH_TAKEN` instead.") + @deprecated("Deprecated since Python 3.14. Use `BRANCH_LEFT` or `BRANCH_RIGHT` instead.") def BRANCH(self) -> int: ... else: diff --git a/mypy/typeshed/stdlib/threading.pyi b/mypy/typeshed/stdlib/threading.pyi index 28fa5267a9975..7b0f15bdfa2e8 100644 --- a/mypy/typeshed/stdlib/threading.pyi +++ b/mypy/typeshed/stdlib/threading.pyi @@ -1,6 +1,6 @@ import _thread import sys -from _thread import _excepthook, _ExceptHookArgs, get_native_id as get_native_id +from _thread import _ExceptHookArgs, get_native_id as get_native_id from _typeshed import ProfileFunction, TraceFunction from collections.abc import Callable, Iterable, Mapping from contextvars import ContextVar @@ -169,7 +169,9 @@ class Event: def clear(self) -> None: ... def wait(self, timeout: float | None = None) -> bool: ... -excepthook = _excepthook +excepthook: Callable[[_ExceptHookArgs], object] +if sys.version_info >= (3, 10): + __excepthook__: Callable[[_ExceptHookArgs], object] ExceptHookArgs = _ExceptHookArgs class Timer(Thread): diff --git a/mypy/typeshed/stdlib/types.pyi b/mypy/typeshed/stdlib/types.pyi index 649e463ff71f8..0293e5cb0b4bb 100644 --- a/mypy/typeshed/stdlib/types.pyi +++ b/mypy/typeshed/stdlib/types.pyi @@ -717,9 +717,9 @@ if sys.version_info >= (3, 10): @final class EllipsisType: ... - from builtins import _NotImplementedType + @final + class NotImplementedType(Any): ... - NotImplementedType = _NotImplementedType @final class UnionType: @property diff --git a/mypy/typeshed/stdlib/typing.pyi b/mypy/typeshed/stdlib/typing.pyi index 2ca65dad4562f..e3e5d1ff28422 100644 --- a/mypy/typeshed/stdlib/typing.pyi +++ b/mypy/typeshed/stdlib/typing.pyi @@ -222,10 +222,6 @@ class TypeVar: @property def evaluate_default(self) -> EvaluateFunc | None: ... -# Used for an undocumented mypy feature. Does not exist at runtime. -# Obsolete, use _typeshed._type_checker_internals.promote instead. -_promote = object() - # N.B. Keep this definition in sync with typing_extensions._SpecialForm @final class _SpecialForm(_Final): diff --git a/mypy/typeshed/stdlib/unittest/util.pyi b/mypy/typeshed/stdlib/unittest/util.pyi index 31c830e8268a7..763c1478f5e6d 100644 --- a/mypy/typeshed/stdlib/unittest/util.pyi +++ b/mypy/typeshed/stdlib/unittest/util.pyi @@ -1,9 +1,26 @@ from collections.abc import MutableSequence, Sequence -from typing import Any, Final, TypeVar +from typing import Any, Final, Literal, Protocol, TypeVar, type_check_only from typing_extensions import TypeAlias +@type_check_only +class _SupportsDunderLT(Protocol): + def __lt__(self, other: Any, /) -> bool: ... + +@type_check_only +class _SupportsDunderGT(Protocol): + def __gt__(self, other: Any, /) -> bool: ... + +@type_check_only +class _SupportsDunderLE(Protocol): + def __le__(self, other: Any, /) -> bool: ... + +@type_check_only +class _SupportsDunderGE(Protocol): + def __ge__(self, other: Any, /) -> bool: ... + _T = TypeVar("_T") _Mismatch: TypeAlias = tuple[_T, _T, int] +_SupportsComparison: TypeAlias = _SupportsDunderLE | _SupportsDunderGE | _SupportsDunderGT | _SupportsDunderLT _MAX_LENGTH: Final = 80 _PLACEHOLDER_LEN: Final = 12 @@ -18,6 +35,6 @@ def safe_repr(obj: object, short: bool = False) -> str: ... def strclass(cls: type) -> str: ... def sorted_list_difference(expected: Sequence[_T], actual: Sequence[_T]) -> tuple[list[_T], list[_T]]: ... def unorderable_list_difference(expected: MutableSequence[_T], actual: MutableSequence[_T]) -> tuple[list[_T], list[_T]]: ... -def three_way_cmp(x: Any, y: Any) -> int: ... +def three_way_cmp(x: _SupportsComparison, y: _SupportsComparison) -> Literal[-1, 0, 1]: ... def _count_diff_all_purpose(actual: Sequence[_T], expected: Sequence[_T]) -> list[_Mismatch[_T]]: ... def _count_diff_hashable(actual: Sequence[_T], expected: Sequence[_T]) -> list[_Mismatch[_T]]: ... diff --git a/mypy/typeshed/stdlib/uuid.pyi b/mypy/typeshed/stdlib/uuid.pyi index 303fb10eaf537..055f4def311cd 100644 --- a/mypy/typeshed/stdlib/uuid.pyi +++ b/mypy/typeshed/stdlib/uuid.pyi @@ -1,7 +1,8 @@ import builtins import sys +from _typeshed import Unused from enum import Enum -from typing import Final +from typing import Final, NoReturn from typing_extensions import LiteralString, TypeAlias _FieldsType: TypeAlias = tuple[int, int, int, int, int, int] @@ -13,6 +14,9 @@ class SafeUUID(Enum): class UUID: __slots__ = ("int", "is_safe", "__weakref__") + is_safe: Final[SafeUUID] + int: Final[builtins.int] + def __init__( self, hex: str | None = None, @@ -25,8 +29,6 @@ class UUID: is_safe: SafeUUID = SafeUUID.unknown, ) -> None: ... @property - def is_safe(self) -> SafeUUID: ... - @property def bytes(self) -> builtins.bytes: ... @property def bytes_le(self) -> builtins.bytes: ... @@ -41,8 +43,6 @@ class UUID: @property def hex(self) -> str: ... @property - def int(self) -> builtins.int: ... - @property def node(self) -> builtins.int: ... @property def time(self) -> builtins.int: ... @@ -65,6 +65,7 @@ class UUID: def __gt__(self, other: UUID) -> bool: ... def __ge__(self, other: UUID) -> bool: ... def __hash__(self) -> builtins.int: ... + def __setattr__(self, name: Unused, value: Unused) -> NoReturn: ... def getnode() -> int: ... def uuid1(node: int | None = None, clock_seq: int | None = None) -> UUID: ... diff --git a/mypy/typeshed/stdlib/winreg.pyi b/mypy/typeshed/stdlib/winreg.pyi index 53457112ee968..a654bbcdfb615 100644 --- a/mypy/typeshed/stdlib/winreg.pyi +++ b/mypy/typeshed/stdlib/winreg.pyi @@ -18,13 +18,13 @@ if sys.platform == "win32": def ExpandEnvironmentStrings(string: str, /) -> str: ... def FlushKey(key: _KeyType, /) -> None: ... def LoadKey(key: _KeyType, sub_key: str, file_name: str, /) -> None: ... - def OpenKey(key: _KeyType, sub_key: str, reserved: int = 0, access: int = 131097) -> HKEYType: ... - def OpenKeyEx(key: _KeyType, sub_key: str, reserved: int = 0, access: int = 131097) -> HKEYType: ... + def OpenKey(key: _KeyType, sub_key: str | None, reserved: int = 0, access: int = 131097) -> HKEYType: ... + def OpenKeyEx(key: _KeyType, sub_key: str | None, reserved: int = 0, access: int = 131097) -> HKEYType: ... def QueryInfoKey(key: _KeyType, /) -> tuple[int, int, int]: ... def QueryValue(key: _KeyType, sub_key: str | None, /) -> str: ... def QueryValueEx(key: _KeyType, name: str, /) -> tuple[Any, int]: ... def SaveKey(key: _KeyType, file_name: str, /) -> None: ... - def SetValue(key: _KeyType, sub_key: str, type: int, value: str, /) -> None: ... + def SetValue(key: _KeyType, sub_key: str | None, type: int, value: str, /) -> None: ... @overload # type=REG_DWORD|REG_QWORD def SetValueEx( key: _KeyType, value_name: str | None, reserved: Unused, type: Literal[4, 5], value: int | None, / diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index 7dfcf7447b619..8d6a78bfe470f 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -1948,7 +1948,7 @@ Foo().__dict__ = {} [out] _testInferenceOfDunderDictOnClassObjects.py:2: note: Revealed type is "types.MappingProxyType[builtins.str, Any]" _testInferenceOfDunderDictOnClassObjects.py:3: note: Revealed type is "builtins.dict[builtins.str, Any]" -_testInferenceOfDunderDictOnClassObjects.py:4: error: Property "__dict__" defined in "type" is read-only +_testInferenceOfDunderDictOnClassObjects.py:4: error: Cannot assign to final attribute "__dict__" _testInferenceOfDunderDictOnClassObjects.py:4: error: Incompatible types in assignment (expression has type "dict[Never, Never]", variable has type "MappingProxyType[str, Any]") [case testTypeVarTuple] From 8f922b3d879f31e4c52c1c643de70e27b5720654 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 17 Nov 2025 11:20:52 +0000 Subject: [PATCH 398/424] [mypyc] Use faster base64 encode implementation in librt.base64 (#20237) Vendor optimized base64 implementation from https://github.com/aklomp/base64. This is based on commit 9e8ed65048ff0f703fad3deb03bf66ac7f78a4d7 (May 2025). Enable SIMD on macOS (64-bit ARM only). Other platforms probably use a generic version. I'll look into enabling SIMD more generally in a follow-up PR. A `b64encode` micro-benchmark was up to 11 times faster compared to the stdlib `base64` module (on a MacBook Pro). --- LICENSE | 35 + mypyc/build.py | 63 +- mypyc/lib-rt/base64/arch/avx/codec.c | 68 ++ mypyc/lib-rt/base64/arch/avx/enc_loop_asm.c | 264 +++++ mypyc/lib-rt/base64/arch/avx2/codec.c | 58 + mypyc/lib-rt/base64/arch/avx2/dec_loop.c | 110 ++ mypyc/lib-rt/base64/arch/avx2/dec_reshuffle.c | 34 + mypyc/lib-rt/base64/arch/avx2/enc_loop.c | 89 ++ mypyc/lib-rt/base64/arch/avx2/enc_loop_asm.c | 291 +++++ mypyc/lib-rt/base64/arch/avx2/enc_reshuffle.c | 83 ++ mypyc/lib-rt/base64/arch/avx2/enc_translate.c | 30 + mypyc/lib-rt/base64/arch/avx512/codec.c | 44 + mypyc/lib-rt/base64/arch/avx512/enc_loop.c | 61 + .../arch/avx512/enc_reshuffle_translate.c | 50 + .../lib-rt/base64/arch/generic/32/dec_loop.c | 86 ++ .../lib-rt/base64/arch/generic/32/enc_loop.c | 73 ++ .../lib-rt/base64/arch/generic/64/enc_loop.c | 77 ++ mypyc/lib-rt/base64/arch/generic/codec.c | 41 + mypyc/lib-rt/base64/arch/generic/dec_head.c | 37 + mypyc/lib-rt/base64/arch/generic/dec_tail.c | 91 ++ mypyc/lib-rt/base64/arch/generic/enc_head.c | 24 + mypyc/lib-rt/base64/arch/generic/enc_tail.c | 34 + mypyc/lib-rt/base64/arch/neon32/codec.c | 79 ++ mypyc/lib-rt/base64/arch/neon32/dec_loop.c | 106 ++ mypyc/lib-rt/base64/arch/neon32/enc_loop.c | 170 +++ .../lib-rt/base64/arch/neon32/enc_reshuffle.c | 31 + .../lib-rt/base64/arch/neon32/enc_translate.c | 57 + mypyc/lib-rt/base64/arch/neon64/codec.c | 93 ++ mypyc/lib-rt/base64/arch/neon64/dec_loop.c | 129 +++ mypyc/lib-rt/base64/arch/neon64/enc_loop.c | 66 ++ .../lib-rt/base64/arch/neon64/enc_loop_asm.c | 168 +++ .../lib-rt/base64/arch/neon64/enc_reshuffle.c | 31 + mypyc/lib-rt/base64/arch/sse41/codec.c | 58 + mypyc/lib-rt/base64/arch/sse42/codec.c | 58 + mypyc/lib-rt/base64/arch/ssse3/codec.c | 60 + mypyc/lib-rt/base64/arch/ssse3/dec_loop.c | 173 +++ .../lib-rt/base64/arch/ssse3/dec_reshuffle.c | 33 + mypyc/lib-rt/base64/arch/ssse3/enc_loop.c | 67 ++ mypyc/lib-rt/base64/arch/ssse3/enc_loop_asm.c | 268 +++++ .../lib-rt/base64/arch/ssse3/enc_reshuffle.c | 48 + .../lib-rt/base64/arch/ssse3/enc_translate.c | 33 + mypyc/lib-rt/base64/codec_choose.c | 314 +++++ mypyc/lib-rt/base64/codecs.h | 57 + mypyc/lib-rt/base64/config.h | 33 + mypyc/lib-rt/base64/env.h | 84 ++ mypyc/lib-rt/base64/lib.c | 164 +++ mypyc/lib-rt/base64/libbase64.h | 146 +++ mypyc/lib-rt/base64/tables/table_dec_32bit.h | 393 +++++++ mypyc/lib-rt/base64/tables/table_enc_12bit.h | 1031 +++++++++++++++++ mypyc/lib-rt/base64/tables/tables.c | 40 + mypyc/lib-rt/base64/tables/tables.h | 23 + mypyc/lib-rt/librt_base64.c | 66 +- mypyc/lib-rt/setup.py | 19 +- 53 files changed, 5787 insertions(+), 54 deletions(-) create mode 100644 mypyc/lib-rt/base64/arch/avx/codec.c create mode 100644 mypyc/lib-rt/base64/arch/avx/enc_loop_asm.c create mode 100644 mypyc/lib-rt/base64/arch/avx2/codec.c create mode 100644 mypyc/lib-rt/base64/arch/avx2/dec_loop.c create mode 100644 mypyc/lib-rt/base64/arch/avx2/dec_reshuffle.c create mode 100644 mypyc/lib-rt/base64/arch/avx2/enc_loop.c create mode 100644 mypyc/lib-rt/base64/arch/avx2/enc_loop_asm.c create mode 100644 mypyc/lib-rt/base64/arch/avx2/enc_reshuffle.c create mode 100644 mypyc/lib-rt/base64/arch/avx2/enc_translate.c create mode 100644 mypyc/lib-rt/base64/arch/avx512/codec.c create mode 100644 mypyc/lib-rt/base64/arch/avx512/enc_loop.c create mode 100644 mypyc/lib-rt/base64/arch/avx512/enc_reshuffle_translate.c create mode 100644 mypyc/lib-rt/base64/arch/generic/32/dec_loop.c create mode 100644 mypyc/lib-rt/base64/arch/generic/32/enc_loop.c create mode 100644 mypyc/lib-rt/base64/arch/generic/64/enc_loop.c create mode 100644 mypyc/lib-rt/base64/arch/generic/codec.c create mode 100644 mypyc/lib-rt/base64/arch/generic/dec_head.c create mode 100644 mypyc/lib-rt/base64/arch/generic/dec_tail.c create mode 100644 mypyc/lib-rt/base64/arch/generic/enc_head.c create mode 100644 mypyc/lib-rt/base64/arch/generic/enc_tail.c create mode 100644 mypyc/lib-rt/base64/arch/neon32/codec.c create mode 100644 mypyc/lib-rt/base64/arch/neon32/dec_loop.c create mode 100644 mypyc/lib-rt/base64/arch/neon32/enc_loop.c create mode 100644 mypyc/lib-rt/base64/arch/neon32/enc_reshuffle.c create mode 100644 mypyc/lib-rt/base64/arch/neon32/enc_translate.c create mode 100644 mypyc/lib-rt/base64/arch/neon64/codec.c create mode 100644 mypyc/lib-rt/base64/arch/neon64/dec_loop.c create mode 100644 mypyc/lib-rt/base64/arch/neon64/enc_loop.c create mode 100644 mypyc/lib-rt/base64/arch/neon64/enc_loop_asm.c create mode 100644 mypyc/lib-rt/base64/arch/neon64/enc_reshuffle.c create mode 100644 mypyc/lib-rt/base64/arch/sse41/codec.c create mode 100644 mypyc/lib-rt/base64/arch/sse42/codec.c create mode 100644 mypyc/lib-rt/base64/arch/ssse3/codec.c create mode 100644 mypyc/lib-rt/base64/arch/ssse3/dec_loop.c create mode 100644 mypyc/lib-rt/base64/arch/ssse3/dec_reshuffle.c create mode 100644 mypyc/lib-rt/base64/arch/ssse3/enc_loop.c create mode 100644 mypyc/lib-rt/base64/arch/ssse3/enc_loop_asm.c create mode 100644 mypyc/lib-rt/base64/arch/ssse3/enc_reshuffle.c create mode 100644 mypyc/lib-rt/base64/arch/ssse3/enc_translate.c create mode 100644 mypyc/lib-rt/base64/codec_choose.c create mode 100644 mypyc/lib-rt/base64/codecs.h create mode 100644 mypyc/lib-rt/base64/config.h create mode 100644 mypyc/lib-rt/base64/env.h create mode 100644 mypyc/lib-rt/base64/lib.c create mode 100644 mypyc/lib-rt/base64/libbase64.h create mode 100644 mypyc/lib-rt/base64/tables/table_dec_32bit.h create mode 100644 mypyc/lib-rt/base64/tables/table_enc_12bit.h create mode 100644 mypyc/lib-rt/base64/tables/tables.c create mode 100644 mypyc/lib-rt/base64/tables/tables.h diff --git a/LICENSE b/LICENSE index 55d01ee19ad84..080c5a402dbb0 100644 --- a/LICENSE +++ b/LICENSE @@ -227,3 +227,38 @@ FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + += = = = = + +Files under lib-rt/base64 are licensed under the following license. + += = = = = + +Copyright (c) 2005-2007, Nick Galbreath +Copyright (c) 2015-2018, Wojciech Muła +Copyright (c) 2016-2017, Matthieu Darbois +Copyright (c) 2013-2022, Alfred Klomp +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +- Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +- Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/mypyc/build.py b/mypyc/build.py index 351aa87b9264d..8505a2d957017 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -26,7 +26,7 @@ import sys import time from collections.abc import Iterable -from typing import TYPE_CHECKING, Any, NoReturn, Union, cast +from typing import TYPE_CHECKING, Any, NamedTuple, NoReturn, Union, cast from mypy.build import BuildSource from mypy.errors import CompileError @@ -42,7 +42,52 @@ from mypyc.namegen import exported_name from mypyc.options import CompilerOptions -LIBRT_MODULES = [("librt.internal", "librt_internal.c"), ("librt.base64", "librt_base64.c")] + +class ModDesc(NamedTuple): + module: str + c_files: list[str] + other_files: list[str] + include_dirs: list[str] + + +LIBRT_MODULES = [ + ModDesc("librt.internal", ["librt_internal.c"], [], []), + ModDesc( + "librt.base64", + [ + "librt_base64.c", + "base64/lib.c", + "base64/codec_choose.c", + "base64/tables/tables.c", + "base64/arch/generic/codec.c", + "base64/arch/ssse3/codec.c", + "base64/arch/sse41/codec.c", + "base64/arch/sse42/codec.c", + "base64/arch/avx/codec.c", + "base64/arch/avx2/codec.c", + "base64/arch/avx512/codec.c", + "base64/arch/neon32/codec.c", + "base64/arch/neon64/codec.c", + ], + [ + "base64/arch/generic/32/enc_loop.c", + "base64/arch/generic/64/enc_loop.c", + "base64/arch/generic/32/dec_loop.c", + "base64/arch/generic/enc_head.c", + "base64/arch/generic/enc_tail.c", + "base64/arch/generic/dec_head.c", + "base64/arch/generic/dec_tail.c", + "base64/arch/neon64/dec_loop.c", + "base64/arch/neon64/enc_loop_asm.c", + "base64/codecs.h", + "base64/env.h", + "base64/tables/tables.h", + "base64/tables/table_dec_32bit.h", + "base64/tables/table_enc_12bit.h", + ], + ["base64"], + ), +] try: # Import setuptools so that it monkey-patch overrides distutils @@ -677,17 +722,19 @@ def mypycify( rt_file = os.path.join(build_dir, name) with open(os.path.join(include_dir(), name), encoding="utf-8") as f: write_file(rt_file, f.read()) - for mod, file_name in LIBRT_MODULES: - rt_file = os.path.join(build_dir, file_name) - with open(os.path.join(include_dir(), file_name), encoding="utf-8") as f: - write_file(rt_file, f.read()) + for mod, file_names, addit_files, includes in LIBRT_MODULES: + for file_name in file_names + addit_files: + rt_file = os.path.join(build_dir, file_name) + with open(os.path.join(include_dir(), file_name), encoding="utf-8") as f: + write_file(rt_file, f.read()) extensions.append( get_extension()( mod, sources=[ - os.path.join(build_dir, file) for file in [file_name] + RUNTIME_C_FILES + os.path.join(build_dir, file) for file in file_names + RUNTIME_C_FILES ], - include_dirs=[include_dir()], + include_dirs=[include_dir()] + + [os.path.join(include_dir(), d) for d in includes], extra_compile_args=cflags, ) ) diff --git a/mypyc/lib-rt/base64/arch/avx/codec.c b/mypyc/lib-rt/base64/arch/avx/codec.c new file mode 100644 index 0000000000000..8e2ef5c2e7243 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/avx/codec.c @@ -0,0 +1,68 @@ +#include +#include +#include + +#include "libbase64.h" +#include "../../tables/tables.h" +#include "../../codecs.h" +#include "config.h" +#include "../../env.h" + +#if HAVE_AVX +#include + +// Only enable inline assembly on supported compilers and on 64-bit CPUs. +#ifndef BASE64_AVX_USE_ASM +# if (defined(__GNUC__) || defined(__clang__)) && BASE64_WORDSIZE == 64 +# define BASE64_AVX_USE_ASM 1 +# else +# define BASE64_AVX_USE_ASM 0 +# endif +#endif + +#include "../ssse3/dec_reshuffle.c" +#include "../ssse3/dec_loop.c" + +#if BASE64_AVX_USE_ASM +# include "enc_loop_asm.c" +#else +# include "../ssse3/enc_translate.c" +# include "../ssse3/enc_reshuffle.c" +# include "../ssse3/enc_loop.c" +#endif + +#endif // HAVE_AVX + +void +base64_stream_encode_avx BASE64_ENC_PARAMS +{ +#if HAVE_AVX + #include "../generic/enc_head.c" + + // For supported compilers, use a hand-optimized inline assembly + // encoder. Otherwise fall back on the SSSE3 encoder, but compiled with + // AVX flags to generate better optimized AVX code. + +#if BASE64_AVX_USE_ASM + enc_loop_avx(&s, &slen, &o, &olen); +#else + enc_loop_ssse3(&s, &slen, &o, &olen); +#endif + + #include "../generic/enc_tail.c" +#else + base64_enc_stub(state, src, srclen, out, outlen); +#endif +} + +int +base64_stream_decode_avx BASE64_DEC_PARAMS +{ +#if HAVE_AVX + #include "../generic/dec_head.c" + dec_loop_ssse3(&s, &slen, &o, &olen); + #include "../generic/dec_tail.c" +#else + return base64_dec_stub(state, src, srclen, out, outlen); +#endif +} diff --git a/mypyc/lib-rt/base64/arch/avx/enc_loop_asm.c b/mypyc/lib-rt/base64/arch/avx/enc_loop_asm.c new file mode 100644 index 0000000000000..979269af57740 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/avx/enc_loop_asm.c @@ -0,0 +1,264 @@ +// Apologies in advance for combining the preprocessor with inline assembly, +// two notoriously gnarly parts of C, but it was necessary to avoid a lot of +// code repetition. The preprocessor is used to template large sections of +// inline assembly that differ only in the registers used. If the code was +// written out by hand, it would become very large and hard to audit. + +// Generate a block of inline assembly that loads register R0 from memory. The +// offset at which the register is loaded is set by the given round. +#define LOAD(R0, ROUND) \ + "vlddqu ("#ROUND" * 12)(%[src]), %["R0"] \n\t" + +// Generate a block of inline assembly that deinterleaves and shuffles register +// R0 using preloaded constants. Outputs in R0 and R1. +#define SHUF(R0, R1, R2) \ + "vpshufb %[lut0], %["R0"], %["R1"] \n\t" \ + "vpand %["R1"], %[msk0], %["R2"] \n\t" \ + "vpand %["R1"], %[msk2], %["R1"] \n\t" \ + "vpmulhuw %["R2"], %[msk1], %["R2"] \n\t" \ + "vpmullw %["R1"], %[msk3], %["R1"] \n\t" \ + "vpor %["R1"], %["R2"], %["R1"] \n\t" + +// Generate a block of inline assembly that takes R0 and R1 and translates +// their contents to the base64 alphabet, using preloaded constants. +#define TRAN(R0, R1, R2) \ + "vpsubusb %[n51], %["R1"], %["R0"] \n\t" \ + "vpcmpgtb %[n25], %["R1"], %["R2"] \n\t" \ + "vpsubb %["R2"], %["R0"], %["R0"] \n\t" \ + "vpshufb %["R0"], %[lut1], %["R2"] \n\t" \ + "vpaddb %["R1"], %["R2"], %["R0"] \n\t" + +// Generate a block of inline assembly that stores the given register R0 at an +// offset set by the given round. +#define STOR(R0, ROUND) \ + "vmovdqu %["R0"], ("#ROUND" * 16)(%[dst]) \n\t" + +// Generate a block of inline assembly that generates a single self-contained +// encoder round: fetch the data, process it, and store the result. Then update +// the source and destination pointers. +#define ROUND() \ + LOAD("a", 0) \ + SHUF("a", "b", "c") \ + TRAN("a", "b", "c") \ + STOR("a", 0) \ + "add $12, %[src] \n\t" \ + "add $16, %[dst] \n\t" + +// Define a macro that initiates a three-way interleaved encoding round by +// preloading registers a, b and c from memory. +// The register graph shows which registers are in use during each step, and +// is a visual aid for choosing registers for that step. Symbol index: +// +// + indicates that a register is loaded by that step. +// | indicates that a register is in use and must not be touched. +// - indicates that a register is decommissioned by that step. +// x indicates that a register is used as a temporary by that step. +// V indicates that a register is an input or output to the macro. +// +#define ROUND_3_INIT() /* a b c d e f */ \ + LOAD("a", 0) /* + */ \ + SHUF("a", "d", "e") /* | + x */ \ + LOAD("b", 1) /* | + | */ \ + TRAN("a", "d", "e") /* | | - x */ \ + LOAD("c", 2) /* V V V */ + +// Define a macro that translates, shuffles and stores the input registers A, B +// and C, and preloads registers D, E and F for the next round. +// This macro can be arbitrarily daisy-chained by feeding output registers D, E +// and F back into the next round as input registers A, B and C. The macro +// carefully interleaves memory operations with data operations for optimal +// pipelined performance. + +#define ROUND_3(ROUND, A,B,C,D,E,F) /* A B C D E F */ \ + LOAD(D, (ROUND + 3)) /* V V V + */ \ + SHUF(B, E, F) /* | | | | + x */ \ + STOR(A, (ROUND + 0)) /* - | | | | */ \ + TRAN(B, E, F) /* | | | - x */ \ + LOAD(E, (ROUND + 4)) /* | | | + */ \ + SHUF(C, A, F) /* + | | | | x */ \ + STOR(B, (ROUND + 1)) /* | - | | | */ \ + TRAN(C, A, F) /* - | | | x */ \ + LOAD(F, (ROUND + 5)) /* | | | + */ \ + SHUF(D, A, B) /* + x | | | | */ \ + STOR(C, (ROUND + 2)) /* | - | | | */ \ + TRAN(D, A, B) /* - x V V V */ + +// Define a macro that terminates a ROUND_3 macro by taking pre-loaded +// registers D, E and F, and translating, shuffling and storing them. +#define ROUND_3_END(ROUND, A,B,C,D,E,F) /* A B C D E F */ \ + SHUF(E, A, B) /* + x V V V */ \ + STOR(D, (ROUND + 3)) /* | - | | */ \ + TRAN(E, A, B) /* - x | | */ \ + SHUF(F, C, D) /* + x | | */ \ + STOR(E, (ROUND + 4)) /* | - | */ \ + TRAN(F, C, D) /* - x | */ \ + STOR(F, (ROUND + 5)) /* - */ + +// Define a type A round. Inputs are a, b, and c, outputs are d, e, and f. +#define ROUND_3_A(ROUND) \ + ROUND_3(ROUND, "a", "b", "c", "d", "e", "f") + +// Define a type B round. Inputs and outputs are swapped with regard to type A. +#define ROUND_3_B(ROUND) \ + ROUND_3(ROUND, "d", "e", "f", "a", "b", "c") + +// Terminating macro for a type A round. +#define ROUND_3_A_LAST(ROUND) \ + ROUND_3_A(ROUND) \ + ROUND_3_END(ROUND, "a", "b", "c", "d", "e", "f") + +// Terminating macro for a type B round. +#define ROUND_3_B_LAST(ROUND) \ + ROUND_3_B(ROUND) \ + ROUND_3_END(ROUND, "d", "e", "f", "a", "b", "c") + +// Suppress clang's warning that the literal string in the asm statement is +// overlong (longer than the ISO-mandated minimum size of 4095 bytes for C99 +// compilers). It may be true, but the goal here is not C99 portability. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Woverlength-strings" + +static inline void +enc_loop_avx (const uint8_t **s, size_t *slen, uint8_t **o, size_t *olen) +{ + // For a clearer explanation of the algorithm used by this function, + // please refer to the plain (not inline assembly) implementation. This + // function follows the same basic logic. + + if (*slen < 16) { + return; + } + + // Process blocks of 12 bytes at a time. Input is read in blocks of 16 + // bytes, so "reserve" four bytes from the input buffer to ensure that + // we never read beyond the end of the input buffer. + size_t rounds = (*slen - 4) / 12; + + *slen -= rounds * 12; // 12 bytes consumed per round + *olen += rounds * 16; // 16 bytes produced per round + + // Number of times to go through the 36x loop. + size_t loops = rounds / 36; + + // Number of rounds remaining after the 36x loop. + rounds %= 36; + + // Lookup tables. + const __m128i lut0 = _mm_set_epi8( + 10, 11, 9, 10, 7, 8, 6, 7, 4, 5, 3, 4, 1, 2, 0, 1); + + const __m128i lut1 = _mm_setr_epi8( + 65, 71, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -19, -16, 0, 0); + + // Temporary registers. + __m128i a, b, c, d, e, f; + + __asm__ volatile ( + + // If there are 36 rounds or more, enter a 36x unrolled loop of + // interleaved encoding rounds. The rounds interleave memory + // operations (load/store) with data operations (table lookups, + // etc) to maximize pipeline throughput. + " test %[loops], %[loops] \n\t" + " jz 18f \n\t" + " jmp 36f \n\t" + " \n\t" + ".balign 64 \n\t" + "36: " ROUND_3_INIT() + " " ROUND_3_A( 0) + " " ROUND_3_B( 3) + " " ROUND_3_A( 6) + " " ROUND_3_B( 9) + " " ROUND_3_A(12) + " " ROUND_3_B(15) + " " ROUND_3_A(18) + " " ROUND_3_B(21) + " " ROUND_3_A(24) + " " ROUND_3_B(27) + " " ROUND_3_A_LAST(30) + " add $(12 * 36), %[src] \n\t" + " add $(16 * 36), %[dst] \n\t" + " dec %[loops] \n\t" + " jnz 36b \n\t" + + // Enter an 18x unrolled loop for rounds of 18 or more. + "18: cmp $18, %[rounds] \n\t" + " jl 9f \n\t" + " " ROUND_3_INIT() + " " ROUND_3_A(0) + " " ROUND_3_B(3) + " " ROUND_3_A(6) + " " ROUND_3_B(9) + " " ROUND_3_A_LAST(12) + " sub $18, %[rounds] \n\t" + " add $(12 * 18), %[src] \n\t" + " add $(16 * 18), %[dst] \n\t" + + // Enter a 9x unrolled loop for rounds of 9 or more. + "9: cmp $9, %[rounds] \n\t" + " jl 6f \n\t" + " " ROUND_3_INIT() + " " ROUND_3_A(0) + " " ROUND_3_B_LAST(3) + " sub $9, %[rounds] \n\t" + " add $(12 * 9), %[src] \n\t" + " add $(16 * 9), %[dst] \n\t" + + // Enter a 6x unrolled loop for rounds of 6 or more. + "6: cmp $6, %[rounds] \n\t" + " jl 55f \n\t" + " " ROUND_3_INIT() + " " ROUND_3_A_LAST(0) + " sub $6, %[rounds] \n\t" + " add $(12 * 6), %[src] \n\t" + " add $(16 * 6), %[dst] \n\t" + + // Dispatch the remaining rounds 0..5. + "55: cmp $3, %[rounds] \n\t" + " jg 45f \n\t" + " je 3f \n\t" + " cmp $1, %[rounds] \n\t" + " jg 2f \n\t" + " je 1f \n\t" + " jmp 0f \n\t" + + "45: cmp $4, %[rounds] \n\t" + " je 4f \n\t" + + // Block of non-interlaced encoding rounds, which can each + // individually be jumped to. Rounds fall through to the next. + "5: " ROUND() + "4: " ROUND() + "3: " ROUND() + "2: " ROUND() + "1: " ROUND() + "0: \n\t" + + // Outputs (modified). + : [rounds] "+r" (rounds), + [loops] "+r" (loops), + [src] "+r" (*s), + [dst] "+r" (*o), + [a] "=&x" (a), + [b] "=&x" (b), + [c] "=&x" (c), + [d] "=&x" (d), + [e] "=&x" (e), + [f] "=&x" (f) + + // Inputs (not modified). + : [lut0] "x" (lut0), + [lut1] "x" (lut1), + [msk0] "x" (_mm_set1_epi32(0x0FC0FC00)), + [msk1] "x" (_mm_set1_epi32(0x04000040)), + [msk2] "x" (_mm_set1_epi32(0x003F03F0)), + [msk3] "x" (_mm_set1_epi32(0x01000010)), + [n51] "x" (_mm_set1_epi8(51)), + [n25] "x" (_mm_set1_epi8(25)) + + // Clobbers. + : "cc", "memory" + ); +} + +#pragma GCC diagnostic pop diff --git a/mypyc/lib-rt/base64/arch/avx2/codec.c b/mypyc/lib-rt/base64/arch/avx2/codec.c new file mode 100644 index 0000000000000..fe9200296914f --- /dev/null +++ b/mypyc/lib-rt/base64/arch/avx2/codec.c @@ -0,0 +1,58 @@ +#include +#include +#include + +#include "libbase64.h" +#include "../../tables/tables.h" +#include "../../codecs.h" +#include "config.h" +#include "../../env.h" + +#if HAVE_AVX2 +#include + +// Only enable inline assembly on supported compilers and on 64-bit CPUs. +#ifndef BASE64_AVX2_USE_ASM +# if (defined(__GNUC__) || defined(__clang__)) && BASE64_WORDSIZE == 64 +# define BASE64_AVX2_USE_ASM 1 +# else +# define BASE64_AVX2_USE_ASM 0 +# endif +#endif + +#include "dec_reshuffle.c" +#include "dec_loop.c" + +#if BASE64_AVX2_USE_ASM +# include "enc_loop_asm.c" +#else +# include "enc_translate.c" +# include "enc_reshuffle.c" +# include "enc_loop.c" +#endif + +#endif // HAVE_AVX2 + +void +base64_stream_encode_avx2 BASE64_ENC_PARAMS +{ +#if HAVE_AVX2 + #include "../generic/enc_head.c" + enc_loop_avx2(&s, &slen, &o, &olen); + #include "../generic/enc_tail.c" +#else + base64_enc_stub(state, src, srclen, out, outlen); +#endif +} + +int +base64_stream_decode_avx2 BASE64_DEC_PARAMS +{ +#if HAVE_AVX2 + #include "../generic/dec_head.c" + dec_loop_avx2(&s, &slen, &o, &olen); + #include "../generic/dec_tail.c" +#else + return base64_dec_stub(state, src, srclen, out, outlen); +#endif +} diff --git a/mypyc/lib-rt/base64/arch/avx2/dec_loop.c b/mypyc/lib-rt/base64/arch/avx2/dec_loop.c new file mode 100644 index 0000000000000..b8a4ccafd82af --- /dev/null +++ b/mypyc/lib-rt/base64/arch/avx2/dec_loop.c @@ -0,0 +1,110 @@ +static BASE64_FORCE_INLINE int +dec_loop_avx2_inner (const uint8_t **s, uint8_t **o, size_t *rounds) +{ + const __m256i lut_lo = _mm256_setr_epi8( + 0x15, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, + 0x11, 0x11, 0x13, 0x1A, 0x1B, 0x1B, 0x1B, 0x1A, + 0x15, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, + 0x11, 0x11, 0x13, 0x1A, 0x1B, 0x1B, 0x1B, 0x1A); + + const __m256i lut_hi = _mm256_setr_epi8( + 0x10, 0x10, 0x01, 0x02, 0x04, 0x08, 0x04, 0x08, + 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, + 0x10, 0x10, 0x01, 0x02, 0x04, 0x08, 0x04, 0x08, + 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10); + + const __m256i lut_roll = _mm256_setr_epi8( + 0, 16, 19, 4, -65, -65, -71, -71, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 16, 19, 4, -65, -65, -71, -71, + 0, 0, 0, 0, 0, 0, 0, 0); + + const __m256i mask_2F = _mm256_set1_epi8(0x2F); + + // Load input: + __m256i str = _mm256_loadu_si256((__m256i *) *s); + + // See the SSSE3 decoder for an explanation of the algorithm. + const __m256i hi_nibbles = _mm256_and_si256(_mm256_srli_epi32(str, 4), mask_2F); + const __m256i lo_nibbles = _mm256_and_si256(str, mask_2F); + const __m256i hi = _mm256_shuffle_epi8(lut_hi, hi_nibbles); + const __m256i lo = _mm256_shuffle_epi8(lut_lo, lo_nibbles); + + if (!_mm256_testz_si256(lo, hi)) { + return 0; + } + + const __m256i eq_2F = _mm256_cmpeq_epi8(str, mask_2F); + const __m256i roll = _mm256_shuffle_epi8(lut_roll, _mm256_add_epi8(eq_2F, hi_nibbles)); + + // Now simply add the delta values to the input: + str = _mm256_add_epi8(str, roll); + + // Reshuffle the input to packed 12-byte output format: + str = dec_reshuffle(str); + + // Store the output: + _mm256_storeu_si256((__m256i *) *o, str); + + *s += 32; + *o += 24; + *rounds -= 1; + + return 1; +} + +static inline void +dec_loop_avx2 (const uint8_t **s, size_t *slen, uint8_t **o, size_t *olen) +{ + if (*slen < 45) { + return; + } + + // Process blocks of 32 bytes per round. Because 8 extra zero bytes are + // written after the output, ensure that there will be at least 13 + // bytes of input data left to cover the gap. (11 data bytes and up to + // two end-of-string markers.) + size_t rounds = (*slen - 13) / 32; + + *slen -= rounds * 32; // 32 bytes consumed per round + *olen += rounds * 24; // 24 bytes produced per round + + do { + if (rounds >= 8) { + if (dec_loop_avx2_inner(s, o, &rounds) && + dec_loop_avx2_inner(s, o, &rounds) && + dec_loop_avx2_inner(s, o, &rounds) && + dec_loop_avx2_inner(s, o, &rounds) && + dec_loop_avx2_inner(s, o, &rounds) && + dec_loop_avx2_inner(s, o, &rounds) && + dec_loop_avx2_inner(s, o, &rounds) && + dec_loop_avx2_inner(s, o, &rounds)) { + continue; + } + break; + } + if (rounds >= 4) { + if (dec_loop_avx2_inner(s, o, &rounds) && + dec_loop_avx2_inner(s, o, &rounds) && + dec_loop_avx2_inner(s, o, &rounds) && + dec_loop_avx2_inner(s, o, &rounds)) { + continue; + } + break; + } + if (rounds >= 2) { + if (dec_loop_avx2_inner(s, o, &rounds) && + dec_loop_avx2_inner(s, o, &rounds)) { + continue; + } + break; + } + dec_loop_avx2_inner(s, o, &rounds); + break; + + } while (rounds > 0); + + // Adjust for any rounds that were skipped: + *slen += rounds * 32; + *olen -= rounds * 24; +} diff --git a/mypyc/lib-rt/base64/arch/avx2/dec_reshuffle.c b/mypyc/lib-rt/base64/arch/avx2/dec_reshuffle.c new file mode 100644 index 0000000000000..bc875ce9dd256 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/avx2/dec_reshuffle.c @@ -0,0 +1,34 @@ +static BASE64_FORCE_INLINE __m256i +dec_reshuffle (const __m256i in) +{ + // in, lower lane, bits, upper case are most significant bits, lower + // case are least significant bits: + // 00llllll 00kkkkLL 00jjKKKK 00JJJJJJ + // 00iiiiii 00hhhhII 00ggHHHH 00GGGGGG + // 00ffffff 00eeeeFF 00ddEEEE 00DDDDDD + // 00cccccc 00bbbbCC 00aaBBBB 00AAAAAA + + const __m256i merge_ab_and_bc = _mm256_maddubs_epi16(in, _mm256_set1_epi32(0x01400140)); + // 0000kkkk LLllllll 0000JJJJ JJjjKKKK + // 0000hhhh IIiiiiii 0000GGGG GGggHHHH + // 0000eeee FFffffff 0000DDDD DDddEEEE + // 0000bbbb CCcccccc 0000AAAA AAaaBBBB + + __m256i out = _mm256_madd_epi16(merge_ab_and_bc, _mm256_set1_epi32(0x00011000)); + // 00000000 JJJJJJjj KKKKkkkk LLllllll + // 00000000 GGGGGGgg HHHHhhhh IIiiiiii + // 00000000 DDDDDDdd EEEEeeee FFffffff + // 00000000 AAAAAAaa BBBBbbbb CCcccccc + + // Pack bytes together in each lane: + out = _mm256_shuffle_epi8(out, _mm256_setr_epi8( + 2, 1, 0, 6, 5, 4, 10, 9, 8, 14, 13, 12, -1, -1, -1, -1, + 2, 1, 0, 6, 5, 4, 10, 9, 8, 14, 13, 12, -1, -1, -1, -1)); + // 00000000 00000000 00000000 00000000 + // LLllllll KKKKkkkk JJJJJJjj IIiiiiii + // HHHHhhhh GGGGGGgg FFffffff EEEEeeee + // DDDDDDdd CCcccccc BBBBbbbb AAAAAAaa + + // Pack lanes: + return _mm256_permutevar8x32_epi32(out, _mm256_setr_epi32(0, 1, 2, 4, 5, 6, -1, -1)); +} diff --git a/mypyc/lib-rt/base64/arch/avx2/enc_loop.c b/mypyc/lib-rt/base64/arch/avx2/enc_loop.c new file mode 100644 index 0000000000000..6f4aa0ab580cd --- /dev/null +++ b/mypyc/lib-rt/base64/arch/avx2/enc_loop.c @@ -0,0 +1,89 @@ +static BASE64_FORCE_INLINE void +enc_loop_avx2_inner_first (const uint8_t **s, uint8_t **o) +{ + // First load is done at s - 0 to not get a segfault: + __m256i src = _mm256_loadu_si256((__m256i *) *s); + + // Shift by 4 bytes, as required by enc_reshuffle: + src = _mm256_permutevar8x32_epi32(src, _mm256_setr_epi32(0, 0, 1, 2, 3, 4, 5, 6)); + + // Reshuffle, translate, store: + src = enc_reshuffle(src); + src = enc_translate(src); + _mm256_storeu_si256((__m256i *) *o, src); + + // Subsequent loads will be done at s - 4, set pointer for next round: + *s += 20; + *o += 32; +} + +static BASE64_FORCE_INLINE void +enc_loop_avx2_inner (const uint8_t **s, uint8_t **o) +{ + // Load input: + __m256i src = _mm256_loadu_si256((__m256i *) *s); + + // Reshuffle, translate, store: + src = enc_reshuffle(src); + src = enc_translate(src); + _mm256_storeu_si256((__m256i *) *o, src); + + *s += 24; + *o += 32; +} + +static inline void +enc_loop_avx2 (const uint8_t **s, size_t *slen, uint8_t **o, size_t *olen) +{ + if (*slen < 32) { + return; + } + + // Process blocks of 24 bytes at a time. Because blocks are loaded 32 + // bytes at a time an offset of -4, ensure that there will be at least + // 4 remaining bytes after the last round, so that the final read will + // not pass beyond the bounds of the input buffer: + size_t rounds = (*slen - 4) / 24; + + *slen -= rounds * 24; // 24 bytes consumed per round + *olen += rounds * 32; // 32 bytes produced per round + + // The first loop iteration requires special handling to ensure that + // the read, which is done at an offset, does not underflow the buffer: + enc_loop_avx2_inner_first(s, o); + rounds--; + + while (rounds > 0) { + if (rounds >= 8) { + enc_loop_avx2_inner(s, o); + enc_loop_avx2_inner(s, o); + enc_loop_avx2_inner(s, o); + enc_loop_avx2_inner(s, o); + enc_loop_avx2_inner(s, o); + enc_loop_avx2_inner(s, o); + enc_loop_avx2_inner(s, o); + enc_loop_avx2_inner(s, o); + rounds -= 8; + continue; + } + if (rounds >= 4) { + enc_loop_avx2_inner(s, o); + enc_loop_avx2_inner(s, o); + enc_loop_avx2_inner(s, o); + enc_loop_avx2_inner(s, o); + rounds -= 4; + continue; + } + if (rounds >= 2) { + enc_loop_avx2_inner(s, o); + enc_loop_avx2_inner(s, o); + rounds -= 2; + continue; + } + enc_loop_avx2_inner(s, o); + break; + } + + // Add the offset back: + *s += 4; +} diff --git a/mypyc/lib-rt/base64/arch/avx2/enc_loop_asm.c b/mypyc/lib-rt/base64/arch/avx2/enc_loop_asm.c new file mode 100644 index 0000000000000..eb775a1d1f03d --- /dev/null +++ b/mypyc/lib-rt/base64/arch/avx2/enc_loop_asm.c @@ -0,0 +1,291 @@ +// Apologies in advance for combining the preprocessor with inline assembly, +// two notoriously gnarly parts of C, but it was necessary to avoid a lot of +// code repetition. The preprocessor is used to template large sections of +// inline assembly that differ only in the registers used. If the code was +// written out by hand, it would become very large and hard to audit. + +// Generate a block of inline assembly that loads register R0 from memory. The +// offset at which the register is loaded is set by the given round and a +// constant offset. +#define LOAD(R0, ROUND, OFFSET) \ + "vlddqu ("#ROUND" * 24 + "#OFFSET")(%[src]), %["R0"] \n\t" + +// Generate a block of inline assembly that deinterleaves and shuffles register +// R0 using preloaded constants. Outputs in R0 and R1. +#define SHUF(R0, R1, R2) \ + "vpshufb %[lut0], %["R0"], %["R1"] \n\t" \ + "vpand %["R1"], %[msk0], %["R2"] \n\t" \ + "vpand %["R1"], %[msk2], %["R1"] \n\t" \ + "vpmulhuw %["R2"], %[msk1], %["R2"] \n\t" \ + "vpmullw %["R1"], %[msk3], %["R1"] \n\t" \ + "vpor %["R1"], %["R2"], %["R1"] \n\t" + +// Generate a block of inline assembly that takes R0 and R1 and translates +// their contents to the base64 alphabet, using preloaded constants. +#define TRAN(R0, R1, R2) \ + "vpsubusb %[n51], %["R1"], %["R0"] \n\t" \ + "vpcmpgtb %[n25], %["R1"], %["R2"] \n\t" \ + "vpsubb %["R2"], %["R0"], %["R0"] \n\t" \ + "vpshufb %["R0"], %[lut1], %["R2"] \n\t" \ + "vpaddb %["R1"], %["R2"], %["R0"] \n\t" + +// Generate a block of inline assembly that stores the given register R0 at an +// offset set by the given round. +#define STOR(R0, ROUND) \ + "vmovdqu %["R0"], ("#ROUND" * 32)(%[dst]) \n\t" + +// Generate a block of inline assembly that generates a single self-contained +// encoder round: fetch the data, process it, and store the result. Then update +// the source and destination pointers. +#define ROUND() \ + LOAD("a", 0, -4) \ + SHUF("a", "b", "c") \ + TRAN("a", "b", "c") \ + STOR("a", 0) \ + "add $24, %[src] \n\t" \ + "add $32, %[dst] \n\t" + +// Define a macro that initiates a three-way interleaved encoding round by +// preloading registers a, b and c from memory. +// The register graph shows which registers are in use during each step, and +// is a visual aid for choosing registers for that step. Symbol index: +// +// + indicates that a register is loaded by that step. +// | indicates that a register is in use and must not be touched. +// - indicates that a register is decommissioned by that step. +// x indicates that a register is used as a temporary by that step. +// V indicates that a register is an input or output to the macro. +// +#define ROUND_3_INIT() /* a b c d e f */ \ + LOAD("a", 0, -4) /* + */ \ + SHUF("a", "d", "e") /* | + x */ \ + LOAD("b", 1, -4) /* | + | */ \ + TRAN("a", "d", "e") /* | | - x */ \ + LOAD("c", 2, -4) /* V V V */ + +// Define a macro that translates, shuffles and stores the input registers A, B +// and C, and preloads registers D, E and F for the next round. +// This macro can be arbitrarily daisy-chained by feeding output registers D, E +// and F back into the next round as input registers A, B and C. The macro +// carefully interleaves memory operations with data operations for optimal +// pipelined performance. + +#define ROUND_3(ROUND, A,B,C,D,E,F) /* A B C D E F */ \ + LOAD(D, (ROUND + 3), -4) /* V V V + */ \ + SHUF(B, E, F) /* | | | | + x */ \ + STOR(A, (ROUND + 0)) /* - | | | | */ \ + TRAN(B, E, F) /* | | | - x */ \ + LOAD(E, (ROUND + 4), -4) /* | | | + */ \ + SHUF(C, A, F) /* + | | | | x */ \ + STOR(B, (ROUND + 1)) /* | - | | | */ \ + TRAN(C, A, F) /* - | | | x */ \ + LOAD(F, (ROUND + 5), -4) /* | | | + */ \ + SHUF(D, A, B) /* + x | | | | */ \ + STOR(C, (ROUND + 2)) /* | - | | | */ \ + TRAN(D, A, B) /* - x V V V */ + +// Define a macro that terminates a ROUND_3 macro by taking pre-loaded +// registers D, E and F, and translating, shuffling and storing them. +#define ROUND_3_END(ROUND, A,B,C,D,E,F) /* A B C D E F */ \ + SHUF(E, A, B) /* + x V V V */ \ + STOR(D, (ROUND + 3)) /* | - | | */ \ + TRAN(E, A, B) /* - x | | */ \ + SHUF(F, C, D) /* + x | | */ \ + STOR(E, (ROUND + 4)) /* | - | */ \ + TRAN(F, C, D) /* - x | */ \ + STOR(F, (ROUND + 5)) /* - */ + +// Define a type A round. Inputs are a, b, and c, outputs are d, e, and f. +#define ROUND_3_A(ROUND) \ + ROUND_3(ROUND, "a", "b", "c", "d", "e", "f") + +// Define a type B round. Inputs and outputs are swapped with regard to type A. +#define ROUND_3_B(ROUND) \ + ROUND_3(ROUND, "d", "e", "f", "a", "b", "c") + +// Terminating macro for a type A round. +#define ROUND_3_A_LAST(ROUND) \ + ROUND_3_A(ROUND) \ + ROUND_3_END(ROUND, "a", "b", "c", "d", "e", "f") + +// Terminating macro for a type B round. +#define ROUND_3_B_LAST(ROUND) \ + ROUND_3_B(ROUND) \ + ROUND_3_END(ROUND, "d", "e", "f", "a", "b", "c") + +// Suppress clang's warning that the literal string in the asm statement is +// overlong (longer than the ISO-mandated minimum size of 4095 bytes for C99 +// compilers). It may be true, but the goal here is not C99 portability. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Woverlength-strings" + +static inline void +enc_loop_avx2 (const uint8_t **s, size_t *slen, uint8_t **o, size_t *olen) +{ + // For a clearer explanation of the algorithm used by this function, + // please refer to the plain (not inline assembly) implementation. This + // function follows the same basic logic. + + if (*slen < 32) { + return; + } + + // Process blocks of 24 bytes at a time. Because blocks are loaded 32 + // bytes at a time an offset of -4, ensure that there will be at least + // 4 remaining bytes after the last round, so that the final read will + // not pass beyond the bounds of the input buffer. + size_t rounds = (*slen - 4) / 24; + + *slen -= rounds * 24; // 24 bytes consumed per round + *olen += rounds * 32; // 32 bytes produced per round + + // Pre-decrement the number of rounds to get the number of rounds + // *after* the first round, which is handled as a special case. + rounds--; + + // Number of times to go through the 36x loop. + size_t loops = rounds / 36; + + // Number of rounds remaining after the 36x loop. + rounds %= 36; + + // Lookup tables. + const __m256i lut0 = _mm256_set_epi8( + 10, 11, 9, 10, 7, 8, 6, 7, 4, 5, 3, 4, 1, 2, 0, 1, + 14, 15, 13, 14, 11, 12, 10, 11, 8, 9, 7, 8, 5, 6, 4, 5); + + const __m256i lut1 = _mm256_setr_epi8( + 65, 71, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -19, -16, 0, 0, + 65, 71, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -19, -16, 0, 0); + + // Temporary registers. + __m256i a, b, c, d, e; + + // Temporary register f doubles as the shift mask for the first round. + __m256i f = _mm256_setr_epi32(0, 0, 1, 2, 3, 4, 5, 6); + + __asm__ volatile ( + + // The first loop iteration requires special handling to ensure + // that the read, which is normally done at an offset of -4, + // does not underflow the buffer. Load the buffer at an offset + // of 0 and permute the input to achieve the same effect. + LOAD("a", 0, 0) + "vpermd %[a], %[f], %[a] \n\t" + + // Perform the standard shuffling and translation steps. + SHUF("a", "b", "c") + TRAN("a", "b", "c") + + // Store the result and increment the source and dest pointers. + "vmovdqu %[a], (%[dst]) \n\t" + "add $24, %[src] \n\t" + "add $32, %[dst] \n\t" + + // If there are 36 rounds or more, enter a 36x unrolled loop of + // interleaved encoding rounds. The rounds interleave memory + // operations (load/store) with data operations (table lookups, + // etc) to maximize pipeline throughput. + " test %[loops], %[loops] \n\t" + " jz 18f \n\t" + " jmp 36f \n\t" + " \n\t" + ".balign 64 \n\t" + "36: " ROUND_3_INIT() + " " ROUND_3_A( 0) + " " ROUND_3_B( 3) + " " ROUND_3_A( 6) + " " ROUND_3_B( 9) + " " ROUND_3_A(12) + " " ROUND_3_B(15) + " " ROUND_3_A(18) + " " ROUND_3_B(21) + " " ROUND_3_A(24) + " " ROUND_3_B(27) + " " ROUND_3_A_LAST(30) + " add $(24 * 36), %[src] \n\t" + " add $(32 * 36), %[dst] \n\t" + " dec %[loops] \n\t" + " jnz 36b \n\t" + + // Enter an 18x unrolled loop for rounds of 18 or more. + "18: cmp $18, %[rounds] \n\t" + " jl 9f \n\t" + " " ROUND_3_INIT() + " " ROUND_3_A(0) + " " ROUND_3_B(3) + " " ROUND_3_A(6) + " " ROUND_3_B(9) + " " ROUND_3_A_LAST(12) + " sub $18, %[rounds] \n\t" + " add $(24 * 18), %[src] \n\t" + " add $(32 * 18), %[dst] \n\t" + + // Enter a 9x unrolled loop for rounds of 9 or more. + "9: cmp $9, %[rounds] \n\t" + " jl 6f \n\t" + " " ROUND_3_INIT() + " " ROUND_3_A(0) + " " ROUND_3_B_LAST(3) + " sub $9, %[rounds] \n\t" + " add $(24 * 9), %[src] \n\t" + " add $(32 * 9), %[dst] \n\t" + + // Enter a 6x unrolled loop for rounds of 6 or more. + "6: cmp $6, %[rounds] \n\t" + " jl 55f \n\t" + " " ROUND_3_INIT() + " " ROUND_3_A_LAST(0) + " sub $6, %[rounds] \n\t" + " add $(24 * 6), %[src] \n\t" + " add $(32 * 6), %[dst] \n\t" + + // Dispatch the remaining rounds 0..5. + "55: cmp $3, %[rounds] \n\t" + " jg 45f \n\t" + " je 3f \n\t" + " cmp $1, %[rounds] \n\t" + " jg 2f \n\t" + " je 1f \n\t" + " jmp 0f \n\t" + + "45: cmp $4, %[rounds] \n\t" + " je 4f \n\t" + + // Block of non-interlaced encoding rounds, which can each + // individually be jumped to. Rounds fall through to the next. + "5: " ROUND() + "4: " ROUND() + "3: " ROUND() + "2: " ROUND() + "1: " ROUND() + "0: \n\t" + + // Outputs (modified). + : [rounds] "+r" (rounds), + [loops] "+r" (loops), + [src] "+r" (*s), + [dst] "+r" (*o), + [a] "=&x" (a), + [b] "=&x" (b), + [c] "=&x" (c), + [d] "=&x" (d), + [e] "=&x" (e), + [f] "+x" (f) + + // Inputs (not modified). + : [lut0] "x" (lut0), + [lut1] "x" (lut1), + [msk0] "x" (_mm256_set1_epi32(0x0FC0FC00)), + [msk1] "x" (_mm256_set1_epi32(0x04000040)), + [msk2] "x" (_mm256_set1_epi32(0x003F03F0)), + [msk3] "x" (_mm256_set1_epi32(0x01000010)), + [n51] "x" (_mm256_set1_epi8(51)), + [n25] "x" (_mm256_set1_epi8(25)) + + // Clobbers. + : "cc", "memory" + ); +} + +#pragma GCC diagnostic pop diff --git a/mypyc/lib-rt/base64/arch/avx2/enc_reshuffle.c b/mypyc/lib-rt/base64/arch/avx2/enc_reshuffle.c new file mode 100644 index 0000000000000..82c659b39ce74 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/avx2/enc_reshuffle.c @@ -0,0 +1,83 @@ +static BASE64_FORCE_INLINE __m256i +enc_reshuffle (const __m256i input) +{ + // Translation of the SSSE3 reshuffling algorithm to AVX2. This one + // works with shifted (4 bytes) input in order to be able to work + // efficiently in the two 128-bit lanes. + + // Input, bytes MSB to LSB: + // 0 0 0 0 x w v u t s r q p o n m + // l k j i h g f e d c b a 0 0 0 0 + + const __m256i in = _mm256_shuffle_epi8(input, _mm256_set_epi8( + 10, 11, 9, 10, + 7, 8, 6, 7, + 4, 5, 3, 4, + 1, 2, 0, 1, + + 14, 15, 13, 14, + 11, 12, 10, 11, + 8, 9, 7, 8, + 5, 6, 4, 5)); + // in, bytes MSB to LSB: + // w x v w + // t u s t + // q r p q + // n o m n + // k l j k + // h i g h + // e f d e + // b c a b + + const __m256i t0 = _mm256_and_si256(in, _mm256_set1_epi32(0x0FC0FC00)); + // bits, upper case are most significant bits, lower case are least + // significant bits. + // 0000wwww XX000000 VVVVVV00 00000000 + // 0000tttt UU000000 SSSSSS00 00000000 + // 0000qqqq RR000000 PPPPPP00 00000000 + // 0000nnnn OO000000 MMMMMM00 00000000 + // 0000kkkk LL000000 JJJJJJ00 00000000 + // 0000hhhh II000000 GGGGGG00 00000000 + // 0000eeee FF000000 DDDDDD00 00000000 + // 0000bbbb CC000000 AAAAAA00 00000000 + + const __m256i t1 = _mm256_mulhi_epu16(t0, _mm256_set1_epi32(0x04000040)); + // 00000000 00wwwwXX 00000000 00VVVVVV + // 00000000 00ttttUU 00000000 00SSSSSS + // 00000000 00qqqqRR 00000000 00PPPPPP + // 00000000 00nnnnOO 00000000 00MMMMMM + // 00000000 00kkkkLL 00000000 00JJJJJJ + // 00000000 00hhhhII 00000000 00GGGGGG + // 00000000 00eeeeFF 00000000 00DDDDDD + // 00000000 00bbbbCC 00000000 00AAAAAA + + const __m256i t2 = _mm256_and_si256(in, _mm256_set1_epi32(0x003F03F0)); + // 00000000 00xxxxxx 000000vv WWWW0000 + // 00000000 00uuuuuu 000000ss TTTT0000 + // 00000000 00rrrrrr 000000pp QQQQ0000 + // 00000000 00oooooo 000000mm NNNN0000 + // 00000000 00llllll 000000jj KKKK0000 + // 00000000 00iiiiii 000000gg HHHH0000 + // 00000000 00ffffff 000000dd EEEE0000 + // 00000000 00cccccc 000000aa BBBB0000 + + const __m256i t3 = _mm256_mullo_epi16(t2, _mm256_set1_epi32(0x01000010)); + // 00xxxxxx 00000000 00vvWWWW 00000000 + // 00uuuuuu 00000000 00ssTTTT 00000000 + // 00rrrrrr 00000000 00ppQQQQ 00000000 + // 00oooooo 00000000 00mmNNNN 00000000 + // 00llllll 00000000 00jjKKKK 00000000 + // 00iiiiii 00000000 00ggHHHH 00000000 + // 00ffffff 00000000 00ddEEEE 00000000 + // 00cccccc 00000000 00aaBBBB 00000000 + + return _mm256_or_si256(t1, t3); + // 00xxxxxx 00wwwwXX 00vvWWWW 00VVVVVV + // 00uuuuuu 00ttttUU 00ssTTTT 00SSSSSS + // 00rrrrrr 00qqqqRR 00ppQQQQ 00PPPPPP + // 00oooooo 00nnnnOO 00mmNNNN 00MMMMMM + // 00llllll 00kkkkLL 00jjKKKK 00JJJJJJ + // 00iiiiii 00hhhhII 00ggHHHH 00GGGGGG + // 00ffffff 00eeeeFF 00ddEEEE 00DDDDDD + // 00cccccc 00bbbbCC 00aaBBBB 00AAAAAA +} diff --git a/mypyc/lib-rt/base64/arch/avx2/enc_translate.c b/mypyc/lib-rt/base64/arch/avx2/enc_translate.c new file mode 100644 index 0000000000000..370da98f596ca --- /dev/null +++ b/mypyc/lib-rt/base64/arch/avx2/enc_translate.c @@ -0,0 +1,30 @@ +static BASE64_FORCE_INLINE __m256i +enc_translate (const __m256i in) +{ + // A lookup table containing the absolute offsets for all ranges: + const __m256i lut = _mm256_setr_epi8( + 65, 71, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -19, -16, 0, 0, + 65, 71, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -19, -16, 0, 0); + + // Translate values 0..63 to the Base64 alphabet. There are five sets: + // # From To Abs Index Characters + // 0 [0..25] [65..90] +65 0 ABCDEFGHIJKLMNOPQRSTUVWXYZ + // 1 [26..51] [97..122] +71 1 abcdefghijklmnopqrstuvwxyz + // 2 [52..61] [48..57] -4 [2..11] 0123456789 + // 3 [62] [43] -19 12 + + // 4 [63] [47] -16 13 / + + // Create LUT indices from the input. The index for range #0 is right, + // others are 1 less than expected: + __m256i indices = _mm256_subs_epu8(in, _mm256_set1_epi8(51)); + + // mask is 0xFF (-1) for range #[1..4] and 0x00 for range #0: + const __m256i mask = _mm256_cmpgt_epi8(in, _mm256_set1_epi8(25)); + + // Subtract -1, so add 1 to indices for range #[1..4]. All indices are + // now correct: + indices = _mm256_sub_epi8(indices, mask); + + // Add offsets to input values: + return _mm256_add_epi8(in, _mm256_shuffle_epi8(lut, indices)); +} diff --git a/mypyc/lib-rt/base64/arch/avx512/codec.c b/mypyc/lib-rt/base64/arch/avx512/codec.c new file mode 100644 index 0000000000000..98210826a5fe9 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/avx512/codec.c @@ -0,0 +1,44 @@ +#include +#include +#include + +#include "libbase64.h" +#include "../../tables/tables.h" +#include "../../codecs.h" +#include "config.h" +#include "../../env.h" + +#if HAVE_AVX512 +#include + +#include "../avx2/dec_reshuffle.c" +#include "../avx2/dec_loop.c" +#include "enc_reshuffle_translate.c" +#include "enc_loop.c" + +#endif // HAVE_AVX512 + +void +base64_stream_encode_avx512 BASE64_ENC_PARAMS +{ +#if HAVE_AVX512 + #include "../generic/enc_head.c" + enc_loop_avx512(&s, &slen, &o, &olen); + #include "../generic/enc_tail.c" +#else + base64_enc_stub(state, src, srclen, out, outlen); +#endif +} + +// Reuse AVX2 decoding. Not supporting AVX512 at present +int +base64_stream_decode_avx512 BASE64_DEC_PARAMS +{ +#if HAVE_AVX512 + #include "../generic/dec_head.c" + dec_loop_avx2(&s, &slen, &o, &olen); + #include "../generic/dec_tail.c" +#else + return base64_dec_stub(state, src, srclen, out, outlen); +#endif +} diff --git a/mypyc/lib-rt/base64/arch/avx512/enc_loop.c b/mypyc/lib-rt/base64/arch/avx512/enc_loop.c new file mode 100644 index 0000000000000..cb44696ba31a5 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/avx512/enc_loop.c @@ -0,0 +1,61 @@ +static BASE64_FORCE_INLINE void +enc_loop_avx512_inner (const uint8_t **s, uint8_t **o) +{ + // Load input. + __m512i src = _mm512_loadu_si512((__m512i *) *s); + + // Reshuffle, translate, store. + src = enc_reshuffle_translate(src); + _mm512_storeu_si512((__m512i *) *o, src); + + *s += 48; + *o += 64; +} + +static inline void +enc_loop_avx512 (const uint8_t **s, size_t *slen, uint8_t **o, size_t *olen) +{ + if (*slen < 64) { + return; + } + + // Process blocks of 48 bytes at a time. Because blocks are loaded 64 + // bytes at a time, ensure that there will be at least 24 remaining + // bytes after the last round, so that the final read will not pass + // beyond the bounds of the input buffer. + size_t rounds = (*slen - 24) / 48; + + *slen -= rounds * 48; // 48 bytes consumed per round + *olen += rounds * 64; // 64 bytes produced per round + + while (rounds > 0) { + if (rounds >= 8) { + enc_loop_avx512_inner(s, o); + enc_loop_avx512_inner(s, o); + enc_loop_avx512_inner(s, o); + enc_loop_avx512_inner(s, o); + enc_loop_avx512_inner(s, o); + enc_loop_avx512_inner(s, o); + enc_loop_avx512_inner(s, o); + enc_loop_avx512_inner(s, o); + rounds -= 8; + continue; + } + if (rounds >= 4) { + enc_loop_avx512_inner(s, o); + enc_loop_avx512_inner(s, o); + enc_loop_avx512_inner(s, o); + enc_loop_avx512_inner(s, o); + rounds -= 4; + continue; + } + if (rounds >= 2) { + enc_loop_avx512_inner(s, o); + enc_loop_avx512_inner(s, o); + rounds -= 2; + continue; + } + enc_loop_avx512_inner(s, o); + break; + } +} diff --git a/mypyc/lib-rt/base64/arch/avx512/enc_reshuffle_translate.c b/mypyc/lib-rt/base64/arch/avx512/enc_reshuffle_translate.c new file mode 100644 index 0000000000000..ae12b3af25490 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/avx512/enc_reshuffle_translate.c @@ -0,0 +1,50 @@ +// AVX512 algorithm is based on permutevar and multishift. The code is based on +// https://github.com/WojciechMula/base64simd which is under BSD-2 license. + +static BASE64_FORCE_INLINE __m512i +enc_reshuffle_translate (const __m512i input) +{ + // 32-bit input + // [ 0 0 0 0 0 0 0 0|c1 c0 d5 d4 d3 d2 d1 d0| + // b3 b2 b1 b0 c5 c4 c3 c2|a5 a4 a3 a2 a1 a0 b5 b4] + // output order [1, 2, 0, 1] + // [b3 b2 b1 b0 c5 c4 c3 c2|c1 c0 d5 d4 d3 d2 d1 d0| + // a5 a4 a3 a2 a1 a0 b5 b4|b3 b2 b1 b0 c3 c2 c1 c0] + + const __m512i shuffle_input = _mm512_setr_epi32(0x01020001, + 0x04050304, + 0x07080607, + 0x0a0b090a, + 0x0d0e0c0d, + 0x10110f10, + 0x13141213, + 0x16171516, + 0x191a1819, + 0x1c1d1b1c, + 0x1f201e1f, + 0x22232122, + 0x25262425, + 0x28292728, + 0x2b2c2a2b, + 0x2e2f2d2e); + + // Reorder bytes + // [b3 b2 b1 b0 c5 c4 c3 c2|c1 c0 d5 d4 d3 d2 d1 d0| + // a5 a4 a3 a2 a1 a0 b5 b4|b3 b2 b1 b0 c3 c2 c1 c0] + const __m512i in = _mm512_permutexvar_epi8(shuffle_input, input); + + // After multishift a single 32-bit lane has following layout + // [c1 c0 d5 d4 d3 d2 d1 d0|b1 b0 c5 c4 c3 c2 c1 c0| + // a1 a0 b5 b4 b3 b2 b1 b0|d1 d0 a5 a4 a3 a2 a1 a0] + // (a = [10:17], b = [4:11], c = [22:27], d = [16:21]) + + // 48, 54, 36, 42, 16, 22, 4, 10 + const __m512i shifts = _mm512_set1_epi64(0x3036242a1016040alu); + __m512i shuffled_in = _mm512_multishift_epi64_epi8(shifts, in); + + // Translate immediately after reshuffled. + const __m512i lookup = _mm512_loadu_si512(base64_table_enc_6bit); + + // Translation 6-bit values to ASCII. + return _mm512_permutexvar_epi8(shuffled_in, lookup); +} diff --git a/mypyc/lib-rt/base64/arch/generic/32/dec_loop.c b/mypyc/lib-rt/base64/arch/generic/32/dec_loop.c new file mode 100644 index 0000000000000..aa290d7e03a86 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/generic/32/dec_loop.c @@ -0,0 +1,86 @@ +static BASE64_FORCE_INLINE int +dec_loop_generic_32_inner (const uint8_t **s, uint8_t **o, size_t *rounds) +{ + const uint32_t str + = base64_table_dec_32bit_d0[(*s)[0]] + | base64_table_dec_32bit_d1[(*s)[1]] + | base64_table_dec_32bit_d2[(*s)[2]] + | base64_table_dec_32bit_d3[(*s)[3]]; + +#if BASE64_LITTLE_ENDIAN + + // LUTs for little-endian set MSB in case of invalid character: + if (str & UINT32_C(0x80000000)) { + return 0; + } +#else + // LUTs for big-endian set LSB in case of invalid character: + if (str & UINT32_C(1)) { + return 0; + } +#endif + // Store the output: + memcpy(*o, &str, sizeof (str)); + + *s += 4; + *o += 3; + *rounds -= 1; + + return 1; +} + +static inline void +dec_loop_generic_32 (const uint8_t **s, size_t *slen, uint8_t **o, size_t *olen) +{ + if (*slen < 8) { + return; + } + + // Process blocks of 4 bytes per round. Because one extra zero byte is + // written after the output, ensure that there will be at least 4 bytes + // of input data left to cover the gap. (Two data bytes and up to two + // end-of-string markers.) + size_t rounds = (*slen - 4) / 4; + + *slen -= rounds * 4; // 4 bytes consumed per round + *olen += rounds * 3; // 3 bytes produced per round + + do { + if (rounds >= 8) { + if (dec_loop_generic_32_inner(s, o, &rounds) && + dec_loop_generic_32_inner(s, o, &rounds) && + dec_loop_generic_32_inner(s, o, &rounds) && + dec_loop_generic_32_inner(s, o, &rounds) && + dec_loop_generic_32_inner(s, o, &rounds) && + dec_loop_generic_32_inner(s, o, &rounds) && + dec_loop_generic_32_inner(s, o, &rounds) && + dec_loop_generic_32_inner(s, o, &rounds)) { + continue; + } + break; + } + if (rounds >= 4) { + if (dec_loop_generic_32_inner(s, o, &rounds) && + dec_loop_generic_32_inner(s, o, &rounds) && + dec_loop_generic_32_inner(s, o, &rounds) && + dec_loop_generic_32_inner(s, o, &rounds)) { + continue; + } + break; + } + if (rounds >= 2) { + if (dec_loop_generic_32_inner(s, o, &rounds) && + dec_loop_generic_32_inner(s, o, &rounds)) { + continue; + } + break; + } + dec_loop_generic_32_inner(s, o, &rounds); + break; + + } while (rounds > 0); + + // Adjust for any rounds that were skipped: + *slen += rounds * 4; + *olen -= rounds * 3; +} diff --git a/mypyc/lib-rt/base64/arch/generic/32/enc_loop.c b/mypyc/lib-rt/base64/arch/generic/32/enc_loop.c new file mode 100644 index 0000000000000..b5e6eefd9d430 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/generic/32/enc_loop.c @@ -0,0 +1,73 @@ +static BASE64_FORCE_INLINE void +enc_loop_generic_32_inner (const uint8_t **s, uint8_t **o) +{ + uint32_t src; + + // Load input: + memcpy(&src, *s, sizeof (src)); + + // Reorder to 32-bit big-endian, if not already in that format. The + // workset must be in big-endian, otherwise the shifted bits do not + // carry over properly among adjacent bytes: + src = BASE64_HTOBE32(src); + + // Two indices for the 12-bit lookup table: + const size_t index0 = (src >> 20) & 0xFFFU; + const size_t index1 = (src >> 8) & 0xFFFU; + + // Table lookup and store: + memcpy(*o + 0, base64_table_enc_12bit + index0, 2); + memcpy(*o + 2, base64_table_enc_12bit + index1, 2); + + *s += 3; + *o += 4; +} + +static inline void +enc_loop_generic_32 (const uint8_t **s, size_t *slen, uint8_t **o, size_t *olen) +{ + if (*slen < 4) { + return; + } + + // Process blocks of 3 bytes at a time. Because blocks are loaded 4 + // bytes at a time, ensure that there will be at least one remaining + // byte after the last round, so that the final read will not pass + // beyond the bounds of the input buffer: + size_t rounds = (*slen - 1) / 3; + + *slen -= rounds * 3; // 3 bytes consumed per round + *olen += rounds * 4; // 4 bytes produced per round + + do { + if (rounds >= 8) { + enc_loop_generic_32_inner(s, o); + enc_loop_generic_32_inner(s, o); + enc_loop_generic_32_inner(s, o); + enc_loop_generic_32_inner(s, o); + enc_loop_generic_32_inner(s, o); + enc_loop_generic_32_inner(s, o); + enc_loop_generic_32_inner(s, o); + enc_loop_generic_32_inner(s, o); + rounds -= 8; + continue; + } + if (rounds >= 4) { + enc_loop_generic_32_inner(s, o); + enc_loop_generic_32_inner(s, o); + enc_loop_generic_32_inner(s, o); + enc_loop_generic_32_inner(s, o); + rounds -= 4; + continue; + } + if (rounds >= 2) { + enc_loop_generic_32_inner(s, o); + enc_loop_generic_32_inner(s, o); + rounds -= 2; + continue; + } + enc_loop_generic_32_inner(s, o); + break; + + } while (rounds > 0); +} diff --git a/mypyc/lib-rt/base64/arch/generic/64/enc_loop.c b/mypyc/lib-rt/base64/arch/generic/64/enc_loop.c new file mode 100644 index 0000000000000..e6a29cd5ec418 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/generic/64/enc_loop.c @@ -0,0 +1,77 @@ +static BASE64_FORCE_INLINE void +enc_loop_generic_64_inner (const uint8_t **s, uint8_t **o) +{ + uint64_t src; + + // Load input: + memcpy(&src, *s, sizeof (src)); + + // Reorder to 64-bit big-endian, if not already in that format. The + // workset must be in big-endian, otherwise the shifted bits do not + // carry over properly among adjacent bytes: + src = BASE64_HTOBE64(src); + + // Four indices for the 12-bit lookup table: + const size_t index0 = (src >> 52) & 0xFFFU; + const size_t index1 = (src >> 40) & 0xFFFU; + const size_t index2 = (src >> 28) & 0xFFFU; + const size_t index3 = (src >> 16) & 0xFFFU; + + // Table lookup and store: + memcpy(*o + 0, base64_table_enc_12bit + index0, 2); + memcpy(*o + 2, base64_table_enc_12bit + index1, 2); + memcpy(*o + 4, base64_table_enc_12bit + index2, 2); + memcpy(*o + 6, base64_table_enc_12bit + index3, 2); + + *s += 6; + *o += 8; +} + +static inline void +enc_loop_generic_64 (const uint8_t **s, size_t *slen, uint8_t **o, size_t *olen) +{ + if (*slen < 8) { + return; + } + + // Process blocks of 6 bytes at a time. Because blocks are loaded 8 + // bytes at a time, ensure that there will be at least 2 remaining + // bytes after the last round, so that the final read will not pass + // beyond the bounds of the input buffer: + size_t rounds = (*slen - 2) / 6; + + *slen -= rounds * 6; // 6 bytes consumed per round + *olen += rounds * 8; // 8 bytes produced per round + + do { + if (rounds >= 8) { + enc_loop_generic_64_inner(s, o); + enc_loop_generic_64_inner(s, o); + enc_loop_generic_64_inner(s, o); + enc_loop_generic_64_inner(s, o); + enc_loop_generic_64_inner(s, o); + enc_loop_generic_64_inner(s, o); + enc_loop_generic_64_inner(s, o); + enc_loop_generic_64_inner(s, o); + rounds -= 8; + continue; + } + if (rounds >= 4) { + enc_loop_generic_64_inner(s, o); + enc_loop_generic_64_inner(s, o); + enc_loop_generic_64_inner(s, o); + enc_loop_generic_64_inner(s, o); + rounds -= 4; + continue; + } + if (rounds >= 2) { + enc_loop_generic_64_inner(s, o); + enc_loop_generic_64_inner(s, o); + rounds -= 2; + continue; + } + enc_loop_generic_64_inner(s, o); + break; + + } while (rounds > 0); +} diff --git a/mypyc/lib-rt/base64/arch/generic/codec.c b/mypyc/lib-rt/base64/arch/generic/codec.c new file mode 100644 index 0000000000000..1a29be7c836d1 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/generic/codec.c @@ -0,0 +1,41 @@ +#include +#include +#include + +#include "libbase64.h" +#include "../../tables/tables.h" +#include "../../codecs.h" +#include "config.h" +#include "../../env.h" + +#if BASE64_WORDSIZE == 32 +# include "32/enc_loop.c" +#elif BASE64_WORDSIZE == 64 +# include "64/enc_loop.c" +#endif + +#if BASE64_WORDSIZE >= 32 +# include "32/dec_loop.c" +#endif + +void +base64_stream_encode_plain BASE64_ENC_PARAMS +{ + #include "enc_head.c" +#if BASE64_WORDSIZE == 32 + enc_loop_generic_32(&s, &slen, &o, &olen); +#elif BASE64_WORDSIZE == 64 + enc_loop_generic_64(&s, &slen, &o, &olen); +#endif + #include "enc_tail.c" +} + +int +base64_stream_decode_plain BASE64_DEC_PARAMS +{ + #include "dec_head.c" +#if BASE64_WORDSIZE >= 32 + dec_loop_generic_32(&s, &slen, &o, &olen); +#endif + #include "dec_tail.c" +} diff --git a/mypyc/lib-rt/base64/arch/generic/dec_head.c b/mypyc/lib-rt/base64/arch/generic/dec_head.c new file mode 100644 index 0000000000000..179a31b63ff4e --- /dev/null +++ b/mypyc/lib-rt/base64/arch/generic/dec_head.c @@ -0,0 +1,37 @@ +int ret = 0; +const uint8_t *s = (const uint8_t *) src; +uint8_t *o = (uint8_t *) out; +uint8_t q; + +// Use local temporaries to avoid cache thrashing: +size_t olen = 0; +size_t slen = srclen; +struct base64_state st; +st.eof = state->eof; +st.bytes = state->bytes; +st.carry = state->carry; + +// If we previously saw an EOF or an invalid character, bail out: +if (st.eof) { + *outlen = 0; + ret = 0; + // If there was a trailing '=' to check, check it: + if (slen && (st.eof == BASE64_AEOF)) { + state->bytes = 0; + state->eof = BASE64_EOF; + ret = ((base64_table_dec_8bit[*s++] == 254) && (slen == 1)) ? 1 : 0; + } + return ret; +} + +// Turn four 6-bit numbers into three bytes: +// out[0] = 11111122 +// out[1] = 22223333 +// out[2] = 33444444 + +// Duff's device again: +switch (st.bytes) +{ + for (;;) + { + case 0: diff --git a/mypyc/lib-rt/base64/arch/generic/dec_tail.c b/mypyc/lib-rt/base64/arch/generic/dec_tail.c new file mode 100644 index 0000000000000..e64f7247f3f12 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/generic/dec_tail.c @@ -0,0 +1,91 @@ + if (slen-- == 0) { + ret = 1; + break; + } + if ((q = base64_table_dec_8bit[*s++]) >= 254) { + st.eof = BASE64_EOF; + // Treat character '=' as invalid for byte 0: + break; + } + st.carry = q << 2; + st.bytes++; + + // Deliberate fallthrough: + BASE64_FALLTHROUGH + + case 1: if (slen-- == 0) { + ret = 1; + break; + } + if ((q = base64_table_dec_8bit[*s++]) >= 254) { + st.eof = BASE64_EOF; + // Treat character '=' as invalid for byte 1: + break; + } + *o++ = st.carry | (q >> 4); + st.carry = q << 4; + st.bytes++; + olen++; + + // Deliberate fallthrough: + BASE64_FALLTHROUGH + + case 2: if (slen-- == 0) { + ret = 1; + break; + } + if ((q = base64_table_dec_8bit[*s++]) >= 254) { + st.bytes++; + // When q == 254, the input char is '='. + // Check if next byte is also '=': + if (q == 254) { + if (slen-- != 0) { + st.bytes = 0; + // EOF: + st.eof = BASE64_EOF; + q = base64_table_dec_8bit[*s++]; + ret = ((q == 254) && (slen == 0)) ? 1 : 0; + break; + } + else { + // Almost EOF + st.eof = BASE64_AEOF; + ret = 1; + break; + } + } + // If we get here, there was an error: + break; + } + *o++ = st.carry | (q >> 2); + st.carry = q << 6; + st.bytes++; + olen++; + + // Deliberate fallthrough: + BASE64_FALLTHROUGH + + case 3: if (slen-- == 0) { + ret = 1; + break; + } + if ((q = base64_table_dec_8bit[*s++]) >= 254) { + st.bytes = 0; + st.eof = BASE64_EOF; + // When q == 254, the input char is '='. Return 1 and EOF. + // When q == 255, the input char is invalid. Return 0 and EOF. + ret = ((q == 254) && (slen == 0)) ? 1 : 0; + break; + } + *o++ = st.carry | q; + st.carry = 0; + st.bytes = 0; + olen++; + } +} + +state->eof = st.eof; +state->bytes = st.bytes; +state->carry = st.carry; +*outlen = olen; +return ret; diff --git a/mypyc/lib-rt/base64/arch/generic/enc_head.c b/mypyc/lib-rt/base64/arch/generic/enc_head.c new file mode 100644 index 0000000000000..38d60b2c62b53 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/generic/enc_head.c @@ -0,0 +1,24 @@ +// Assume that *out is large enough to contain the output. +// Theoretically it should be 4/3 the length of src. +const uint8_t *s = (const uint8_t *) src; +uint8_t *o = (uint8_t *) out; + +// Use local temporaries to avoid cache thrashing: +size_t olen = 0; +size_t slen = srclen; +struct base64_state st; +st.bytes = state->bytes; +st.carry = state->carry; + +// Turn three bytes into four 6-bit numbers: +// in[0] = 00111111 +// in[1] = 00112222 +// in[2] = 00222233 +// in[3] = 00333333 + +// Duff's device, a for() loop inside a switch() statement. Legal! +switch (st.bytes) +{ + for (;;) + { + case 0: diff --git a/mypyc/lib-rt/base64/arch/generic/enc_tail.c b/mypyc/lib-rt/base64/arch/generic/enc_tail.c new file mode 100644 index 0000000000000..cbd573376812d --- /dev/null +++ b/mypyc/lib-rt/base64/arch/generic/enc_tail.c @@ -0,0 +1,34 @@ + if (slen-- == 0) { + break; + } + *o++ = base64_table_enc_6bit[*s >> 2]; + st.carry = (*s++ << 4) & 0x30; + st.bytes++; + olen += 1; + + // Deliberate fallthrough: + BASE64_FALLTHROUGH + + case 1: if (slen-- == 0) { + break; + } + *o++ = base64_table_enc_6bit[st.carry | (*s >> 4)]; + st.carry = (*s++ << 2) & 0x3C; + st.bytes++; + olen += 1; + + // Deliberate fallthrough: + BASE64_FALLTHROUGH + + case 2: if (slen-- == 0) { + break; + } + *o++ = base64_table_enc_6bit[st.carry | (*s >> 6)]; + *o++ = base64_table_enc_6bit[*s++ & 0x3F]; + st.bytes = 0; + olen += 2; + } +} +state->bytes = st.bytes; +state->carry = st.carry; +*outlen = olen; diff --git a/mypyc/lib-rt/base64/arch/neon32/codec.c b/mypyc/lib-rt/base64/arch/neon32/codec.c new file mode 100644 index 0000000000000..4a32592b9c469 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/neon32/codec.c @@ -0,0 +1,79 @@ +#include +#include +#include + +#include "libbase64.h" +#include "../../tables/tables.h" +#include "../../codecs.h" +#include "config.h" +#include "../../env.h" + +#ifdef __arm__ +# if (defined(__ARM_NEON__) || defined(__ARM_NEON)) && HAVE_NEON32 +# define BASE64_USE_NEON32 +# endif +#endif + +#ifdef BASE64_USE_NEON32 +#include + +// Only enable inline assembly on supported compilers. +#if defined(__GNUC__) || defined(__clang__) +#define BASE64_NEON32_USE_ASM +#endif + +static BASE64_FORCE_INLINE uint8x16_t +vqtbl1q_u8 (const uint8x16_t lut, const uint8x16_t indices) +{ + // NEON32 only supports 64-bit wide lookups in 128-bit tables. Emulate + // the NEON64 `vqtbl1q_u8` intrinsic to do 128-bit wide lookups. + uint8x8x2_t lut2; + uint8x8x2_t result; + + lut2.val[0] = vget_low_u8(lut); + lut2.val[1] = vget_high_u8(lut); + + result.val[0] = vtbl2_u8(lut2, vget_low_u8(indices)); + result.val[1] = vtbl2_u8(lut2, vget_high_u8(indices)); + + return vcombine_u8(result.val[0], result.val[1]); +} + +#include "../generic/32/dec_loop.c" +#include "../generic/32/enc_loop.c" +#include "dec_loop.c" +#include "enc_reshuffle.c" +#include "enc_translate.c" +#include "enc_loop.c" + +#endif // BASE64_USE_NEON32 + +// Stride size is so large on these NEON 32-bit functions +// (48 bytes encode, 32 bytes decode) that we inline the +// uint32 codec to stay performant on smaller inputs. + +void +base64_stream_encode_neon32 BASE64_ENC_PARAMS +{ +#ifdef BASE64_USE_NEON32 + #include "../generic/enc_head.c" + enc_loop_neon32(&s, &slen, &o, &olen); + enc_loop_generic_32(&s, &slen, &o, &olen); + #include "../generic/enc_tail.c" +#else + base64_enc_stub(state, src, srclen, out, outlen); +#endif +} + +int +base64_stream_decode_neon32 BASE64_DEC_PARAMS +{ +#ifdef BASE64_USE_NEON32 + #include "../generic/dec_head.c" + dec_loop_neon32(&s, &slen, &o, &olen); + dec_loop_generic_32(&s, &slen, &o, &olen); + #include "../generic/dec_tail.c" +#else + return base64_dec_stub(state, src, srclen, out, outlen); +#endif +} diff --git a/mypyc/lib-rt/base64/arch/neon32/dec_loop.c b/mypyc/lib-rt/base64/arch/neon32/dec_loop.c new file mode 100644 index 0000000000000..e4caed7a7d1cc --- /dev/null +++ b/mypyc/lib-rt/base64/arch/neon32/dec_loop.c @@ -0,0 +1,106 @@ +static BASE64_FORCE_INLINE int +is_nonzero (const uint8x16_t v) +{ + uint64_t u64; + const uint64x2_t v64 = vreinterpretq_u64_u8(v); + const uint32x2_t v32 = vqmovn_u64(v64); + + vst1_u64(&u64, vreinterpret_u64_u32(v32)); + return u64 != 0; +} + +static BASE64_FORCE_INLINE uint8x16_t +delta_lookup (const uint8x16_t v) +{ + const uint8x8_t lut = { + 0, 16, 19, 4, (uint8_t) -65, (uint8_t) -65, (uint8_t) -71, (uint8_t) -71, + }; + + return vcombine_u8( + vtbl1_u8(lut, vget_low_u8(v)), + vtbl1_u8(lut, vget_high_u8(v))); +} + +static BASE64_FORCE_INLINE uint8x16_t +dec_loop_neon32_lane (uint8x16_t *lane) +{ + // See the SSSE3 decoder for an explanation of the algorithm. + const uint8x16_t lut_lo = { + 0x15, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, + 0x11, 0x11, 0x13, 0x1A, 0x1B, 0x1B, 0x1B, 0x1A + }; + + const uint8x16_t lut_hi = { + 0x10, 0x10, 0x01, 0x02, 0x04, 0x08, 0x04, 0x08, + 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10 + }; + + const uint8x16_t mask_0F = vdupq_n_u8(0x0F); + const uint8x16_t mask_2F = vdupq_n_u8(0x2F); + + const uint8x16_t hi_nibbles = vshrq_n_u8(*lane, 4); + const uint8x16_t lo_nibbles = vandq_u8(*lane, mask_0F); + const uint8x16_t eq_2F = vceqq_u8(*lane, mask_2F); + + const uint8x16_t hi = vqtbl1q_u8(lut_hi, hi_nibbles); + const uint8x16_t lo = vqtbl1q_u8(lut_lo, lo_nibbles); + + // Now simply add the delta values to the input: + *lane = vaddq_u8(*lane, delta_lookup(vaddq_u8(eq_2F, hi_nibbles))); + + // Return the validity mask: + return vandq_u8(lo, hi); +} + +static inline void +dec_loop_neon32 (const uint8_t **s, size_t *slen, uint8_t **o, size_t *olen) +{ + if (*slen < 64) { + return; + } + + // Process blocks of 64 bytes per round. Unlike the SSE codecs, no + // extra trailing zero bytes are written, so it is not necessary to + // reserve extra input bytes: + size_t rounds = *slen / 64; + + *slen -= rounds * 64; // 64 bytes consumed per round + *olen += rounds * 48; // 48 bytes produced per round + + do { + uint8x16x3_t dec; + + // Load 64 bytes and deinterleave: + uint8x16x4_t str = vld4q_u8(*s); + + // Decode each lane, collect a mask of invalid inputs: + const uint8x16_t classified + = dec_loop_neon32_lane(&str.val[0]) + | dec_loop_neon32_lane(&str.val[1]) + | dec_loop_neon32_lane(&str.val[2]) + | dec_loop_neon32_lane(&str.val[3]); + + // Check for invalid input: if any of the delta values are + // zero, fall back on bytewise code to do error checking and + // reporting: + if (is_nonzero(classified)) { + break; + } + + // Compress four bytes into three: + dec.val[0] = vorrq_u8(vshlq_n_u8(str.val[0], 2), vshrq_n_u8(str.val[1], 4)); + dec.val[1] = vorrq_u8(vshlq_n_u8(str.val[1], 4), vshrq_n_u8(str.val[2], 2)); + dec.val[2] = vorrq_u8(vshlq_n_u8(str.val[2], 6), str.val[3]); + + // Interleave and store decoded result: + vst3q_u8(*o, dec); + + *s += 64; + *o += 48; + + } while (--rounds > 0); + + // Adjust for any rounds that were skipped: + *slen += rounds * 64; + *olen -= rounds * 48; +} diff --git a/mypyc/lib-rt/base64/arch/neon32/enc_loop.c b/mypyc/lib-rt/base64/arch/neon32/enc_loop.c new file mode 100644 index 0000000000000..2adff48f2bcc0 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/neon32/enc_loop.c @@ -0,0 +1,170 @@ +#ifdef BASE64_NEON32_USE_ASM +static BASE64_FORCE_INLINE void +enc_loop_neon32_inner_asm (const uint8_t **s, uint8_t **o) +{ + // This function duplicates the functionality of enc_loop_neon32_inner, + // but entirely with inline assembly. This gives a significant speedup + // over using NEON intrinsics, which do not always generate very good + // code. The logic of the assembly is directly lifted from the + // intrinsics version, so it can be used as a guide to this code. + + // Temporary registers, used as scratch space. + uint8x16_t tmp0, tmp1, tmp2, tmp3; + uint8x16_t mask0, mask1, mask2, mask3; + + // A lookup table containing the absolute offsets for all ranges. + const uint8x16_t lut = { + 65U, 71U, 252U, 252U, + 252U, 252U, 252U, 252U, + 252U, 252U, 252U, 252U, + 237U, 240U, 0U, 0U + }; + + // Numeric constants. + const uint8x16_t n51 = vdupq_n_u8(51); + const uint8x16_t n25 = vdupq_n_u8(25); + const uint8x16_t n63 = vdupq_n_u8(63); + + __asm__ ( + + // Load 48 bytes and deinterleave. The bytes are loaded to + // hard-coded registers q12, q13 and q14, to ensure that they + // are contiguous. Increment the source pointer. + "vld3.8 {d24, d26, d28}, [%[src]]! \n\t" + "vld3.8 {d25, d27, d29}, [%[src]]! \n\t" + + // Reshuffle the bytes using temporaries. + "vshr.u8 %q[t0], q12, #2 \n\t" + "vshr.u8 %q[t1], q13, #4 \n\t" + "vshr.u8 %q[t2], q14, #6 \n\t" + "vsli.8 %q[t1], q12, #4 \n\t" + "vsli.8 %q[t2], q13, #2 \n\t" + "vand.u8 %q[t1], %q[t1], %q[n63] \n\t" + "vand.u8 %q[t2], %q[t2], %q[n63] \n\t" + "vand.u8 %q[t3], q14, %q[n63] \n\t" + + // t0..t3 are the reshuffled inputs. Create LUT indices. + "vqsub.u8 q12, %q[t0], %q[n51] \n\t" + "vqsub.u8 q13, %q[t1], %q[n51] \n\t" + "vqsub.u8 q14, %q[t2], %q[n51] \n\t" + "vqsub.u8 q15, %q[t3], %q[n51] \n\t" + + // Create the mask for range #0. + "vcgt.u8 %q[m0], %q[t0], %q[n25] \n\t" + "vcgt.u8 %q[m1], %q[t1], %q[n25] \n\t" + "vcgt.u8 %q[m2], %q[t2], %q[n25] \n\t" + "vcgt.u8 %q[m3], %q[t3], %q[n25] \n\t" + + // Subtract -1 to correct the LUT indices. + "vsub.u8 q12, %q[m0] \n\t" + "vsub.u8 q13, %q[m1] \n\t" + "vsub.u8 q14, %q[m2] \n\t" + "vsub.u8 q15, %q[m3] \n\t" + + // Lookup the delta values. + "vtbl.u8 d24, {%q[lut]}, d24 \n\t" + "vtbl.u8 d25, {%q[lut]}, d25 \n\t" + "vtbl.u8 d26, {%q[lut]}, d26 \n\t" + "vtbl.u8 d27, {%q[lut]}, d27 \n\t" + "vtbl.u8 d28, {%q[lut]}, d28 \n\t" + "vtbl.u8 d29, {%q[lut]}, d29 \n\t" + "vtbl.u8 d30, {%q[lut]}, d30 \n\t" + "vtbl.u8 d31, {%q[lut]}, d31 \n\t" + + // Add the delta values. + "vadd.u8 q12, %q[t0] \n\t" + "vadd.u8 q13, %q[t1] \n\t" + "vadd.u8 q14, %q[t2] \n\t" + "vadd.u8 q15, %q[t3] \n\t" + + // Store 64 bytes and interleave. Increment the dest pointer. + "vst4.8 {d24, d26, d28, d30}, [%[dst]]! \n\t" + "vst4.8 {d25, d27, d29, d31}, [%[dst]]! \n\t" + + // Outputs (modified). + : [src] "+r" (*s), + [dst] "+r" (*o), + [t0] "=&w" (tmp0), + [t1] "=&w" (tmp1), + [t2] "=&w" (tmp2), + [t3] "=&w" (tmp3), + [m0] "=&w" (mask0), + [m1] "=&w" (mask1), + [m2] "=&w" (mask2), + [m3] "=&w" (mask3) + + // Inputs (not modified). + : [lut] "w" (lut), + [n25] "w" (n25), + [n51] "w" (n51), + [n63] "w" (n63) + + // Clobbers. + : "d24", "d25", "d26", "d27", "d28", "d29", "d30", "d31", + "cc", "memory" + ); +} +#endif + +static BASE64_FORCE_INLINE void +enc_loop_neon32_inner (const uint8_t **s, uint8_t **o) +{ +#ifdef BASE64_NEON32_USE_ASM + enc_loop_neon32_inner_asm(s, o); +#else + // Load 48 bytes and deinterleave: + uint8x16x3_t src = vld3q_u8(*s); + + // Reshuffle: + uint8x16x4_t out = enc_reshuffle(src); + + // Translate reshuffled bytes to the Base64 alphabet: + out = enc_translate(out); + + // Interleave and store output: + vst4q_u8(*o, out); + + *s += 48; + *o += 64; +#endif +} + +static inline void +enc_loop_neon32 (const uint8_t **s, size_t *slen, uint8_t **o, size_t *olen) +{ + size_t rounds = *slen / 48; + + *slen -= rounds * 48; // 48 bytes consumed per round + *olen += rounds * 64; // 64 bytes produced per round + + while (rounds > 0) { + if (rounds >= 8) { + enc_loop_neon32_inner(s, o); + enc_loop_neon32_inner(s, o); + enc_loop_neon32_inner(s, o); + enc_loop_neon32_inner(s, o); + enc_loop_neon32_inner(s, o); + enc_loop_neon32_inner(s, o); + enc_loop_neon32_inner(s, o); + enc_loop_neon32_inner(s, o); + rounds -= 8; + continue; + } + if (rounds >= 4) { + enc_loop_neon32_inner(s, o); + enc_loop_neon32_inner(s, o); + enc_loop_neon32_inner(s, o); + enc_loop_neon32_inner(s, o); + rounds -= 4; + continue; + } + if (rounds >= 2) { + enc_loop_neon32_inner(s, o); + enc_loop_neon32_inner(s, o); + rounds -= 2; + continue; + } + enc_loop_neon32_inner(s, o); + break; + } +} diff --git a/mypyc/lib-rt/base64/arch/neon32/enc_reshuffle.c b/mypyc/lib-rt/base64/arch/neon32/enc_reshuffle.c new file mode 100644 index 0000000000000..fa94d2799be64 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/neon32/enc_reshuffle.c @@ -0,0 +1,31 @@ +static BASE64_FORCE_INLINE uint8x16x4_t +enc_reshuffle (uint8x16x3_t in) +{ + uint8x16x4_t out; + + // Input: + // in[0] = a7 a6 a5 a4 a3 a2 a1 a0 + // in[1] = b7 b6 b5 b4 b3 b2 b1 b0 + // in[2] = c7 c6 c5 c4 c3 c2 c1 c0 + + // Output: + // out[0] = 00 00 a7 a6 a5 a4 a3 a2 + // out[1] = 00 00 a1 a0 b7 b6 b5 b4 + // out[2] = 00 00 b3 b2 b1 b0 c7 c6 + // out[3] = 00 00 c5 c4 c3 c2 c1 c0 + + // Move the input bits to where they need to be in the outputs. Except + // for the first output, the high two bits are not cleared. + out.val[0] = vshrq_n_u8(in.val[0], 2); + out.val[1] = vshrq_n_u8(in.val[1], 4); + out.val[2] = vshrq_n_u8(in.val[2], 6); + out.val[1] = vsliq_n_u8(out.val[1], in.val[0], 4); + out.val[2] = vsliq_n_u8(out.val[2], in.val[1], 2); + + // Clear the high two bits in the second, third and fourth output. + out.val[1] = vandq_u8(out.val[1], vdupq_n_u8(0x3F)); + out.val[2] = vandq_u8(out.val[2], vdupq_n_u8(0x3F)); + out.val[3] = vandq_u8(in.val[2], vdupq_n_u8(0x3F)); + + return out; +} diff --git a/mypyc/lib-rt/base64/arch/neon32/enc_translate.c b/mypyc/lib-rt/base64/arch/neon32/enc_translate.c new file mode 100644 index 0000000000000..ff3d88dd152ba --- /dev/null +++ b/mypyc/lib-rt/base64/arch/neon32/enc_translate.c @@ -0,0 +1,57 @@ +static BASE64_FORCE_INLINE uint8x16x4_t +enc_translate (const uint8x16x4_t in) +{ + // A lookup table containing the absolute offsets for all ranges: + const uint8x16_t lut = { + 65U, 71U, 252U, 252U, + 252U, 252U, 252U, 252U, + 252U, 252U, 252U, 252U, + 237U, 240U, 0U, 0U + }; + + const uint8x16_t offset = vdupq_n_u8(51); + + uint8x16x4_t indices, mask, delta, out; + + // Translate values 0..63 to the Base64 alphabet. There are five sets: + // # From To Abs Index Characters + // 0 [0..25] [65..90] +65 0 ABCDEFGHIJKLMNOPQRSTUVWXYZ + // 1 [26..51] [97..122] +71 1 abcdefghijklmnopqrstuvwxyz + // 2 [52..61] [48..57] -4 [2..11] 0123456789 + // 3 [62] [43] -19 12 + + // 4 [63] [47] -16 13 / + + // Create LUT indices from input: + // the index for range #0 is right, others are 1 less than expected: + indices.val[0] = vqsubq_u8(in.val[0], offset); + indices.val[1] = vqsubq_u8(in.val[1], offset); + indices.val[2] = vqsubq_u8(in.val[2], offset); + indices.val[3] = vqsubq_u8(in.val[3], offset); + + // mask is 0xFF (-1) for range #[1..4] and 0x00 for range #0: + mask.val[0] = vcgtq_u8(in.val[0], vdupq_n_u8(25)); + mask.val[1] = vcgtq_u8(in.val[1], vdupq_n_u8(25)); + mask.val[2] = vcgtq_u8(in.val[2], vdupq_n_u8(25)); + mask.val[3] = vcgtq_u8(in.val[3], vdupq_n_u8(25)); + + // Subtract -1, so add 1 to indices for range #[1..4], All indices are + // now correct: + indices.val[0] = vsubq_u8(indices.val[0], mask.val[0]); + indices.val[1] = vsubq_u8(indices.val[1], mask.val[1]); + indices.val[2] = vsubq_u8(indices.val[2], mask.val[2]); + indices.val[3] = vsubq_u8(indices.val[3], mask.val[3]); + + // Lookup delta values: + delta.val[0] = vqtbl1q_u8(lut, indices.val[0]); + delta.val[1] = vqtbl1q_u8(lut, indices.val[1]); + delta.val[2] = vqtbl1q_u8(lut, indices.val[2]); + delta.val[3] = vqtbl1q_u8(lut, indices.val[3]); + + // Add delta values: + out.val[0] = vaddq_u8(in.val[0], delta.val[0]); + out.val[1] = vaddq_u8(in.val[1], delta.val[1]); + out.val[2] = vaddq_u8(in.val[2], delta.val[2]); + out.val[3] = vaddq_u8(in.val[3], delta.val[3]); + + return out; +} diff --git a/mypyc/lib-rt/base64/arch/neon64/codec.c b/mypyc/lib-rt/base64/arch/neon64/codec.c new file mode 100644 index 0000000000000..70dc463de94f8 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/neon64/codec.c @@ -0,0 +1,93 @@ +#include +#include +#include + +#include "libbase64.h" +#include "../../tables/tables.h" +#include "../../codecs.h" +#include "config.h" +#include "../../env.h" + +#if HAVE_NEON64 +#include + +// Only enable inline assembly on supported compilers. +#if defined(__GNUC__) || defined(__clang__) +#define BASE64_NEON64_USE_ASM +#endif + +static BASE64_FORCE_INLINE uint8x16x4_t +load_64byte_table (const uint8_t *p) +{ +#ifdef BASE64_NEON64_USE_ASM + + // Force the table to be loaded into contiguous registers. GCC will not + // normally allocate contiguous registers for a `uint8x16x4_t'. These + // registers are chosen to not conflict with the ones in the enc loop. + register uint8x16_t t0 __asm__ ("v8"); + register uint8x16_t t1 __asm__ ("v9"); + register uint8x16_t t2 __asm__ ("v10"); + register uint8x16_t t3 __asm__ ("v11"); + + __asm__ ( + "ld1 {%[t0].16b, %[t1].16b, %[t2].16b, %[t3].16b}, [%[src]], #64 \n\t" + : [src] "+r" (p), + [t0] "=w" (t0), + [t1] "=w" (t1), + [t2] "=w" (t2), + [t3] "=w" (t3) + ); + + return (uint8x16x4_t) { + .val[0] = t0, + .val[1] = t1, + .val[2] = t2, + .val[3] = t3, + }; +#else + return vld1q_u8_x4(p); +#endif +} + +#include "../generic/32/dec_loop.c" +#include "../generic/64/enc_loop.c" +#include "dec_loop.c" + +#ifdef BASE64_NEON64_USE_ASM +# include "enc_loop_asm.c" +#else +# include "enc_reshuffle.c" +# include "enc_loop.c" +#endif + +#endif // HAVE_NEON64 + +// Stride size is so large on these NEON 64-bit functions +// (48 bytes encode, 64 bytes decode) that we inline the +// uint64 codec to stay performant on smaller inputs. + +void +base64_stream_encode_neon64 BASE64_ENC_PARAMS +{ +#if HAVE_NEON64 + #include "../generic/enc_head.c" + enc_loop_neon64(&s, &slen, &o, &olen); + enc_loop_generic_64(&s, &slen, &o, &olen); + #include "../generic/enc_tail.c" +#else + base64_enc_stub(state, src, srclen, out, outlen); +#endif +} + +int +base64_stream_decode_neon64 BASE64_DEC_PARAMS +{ +#if HAVE_NEON64 + #include "../generic/dec_head.c" + dec_loop_neon64(&s, &slen, &o, &olen); + dec_loop_generic_32(&s, &slen, &o, &olen); + #include "../generic/dec_tail.c" +#else + return base64_dec_stub(state, src, srclen, out, outlen); +#endif +} diff --git a/mypyc/lib-rt/base64/arch/neon64/dec_loop.c b/mypyc/lib-rt/base64/arch/neon64/dec_loop.c new file mode 100644 index 0000000000000..428e0651f8e04 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/neon64/dec_loop.c @@ -0,0 +1,129 @@ +// The input consists of five valid character sets in the Base64 alphabet, +// which we need to map back to the 6-bit values they represent. +// There are three ranges, two singles, and then there's the rest. +// +// # From To LUT Characters +// 1 [0..42] [255] #1 invalid input +// 2 [43] [62] #1 + +// 3 [44..46] [255] #1 invalid input +// 4 [47] [63] #1 / +// 5 [48..57] [52..61] #1 0..9 +// 6 [58..63] [255] #1 invalid input +// 7 [64] [255] #2 invalid input +// 8 [65..90] [0..25] #2 A..Z +// 9 [91..96] [255] #2 invalid input +// 10 [97..122] [26..51] #2 a..z +// 11 [123..126] [255] #2 invalid input +// (12) Everything else => invalid input + +// The first LUT will use the VTBL instruction (out of range indices are set to +// 0 in destination). +static const uint8_t dec_lut1[] = { + 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, + 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, + 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 62U, 255U, 255U, 255U, 63U, + 52U, 53U, 54U, 55U, 56U, 57U, 58U, 59U, 60U, 61U, 255U, 255U, 255U, 255U, 255U, 255U, +}; + +// The second LUT will use the VTBX instruction (out of range indices will be +// unchanged in destination). Input [64..126] will be mapped to index [1..63] +// in this LUT. Index 0 means that value comes from LUT #1. +static const uint8_t dec_lut2[] = { + 0U, 255U, 0U, 1U, 2U, 3U, 4U, 5U, 6U, 7U, 8U, 9U, 10U, 11U, 12U, 13U, + 14U, 15U, 16U, 17U, 18U, 19U, 20U, 21U, 22U, 23U, 24U, 25U, 255U, 255U, 255U, 255U, + 255U, 255U, 26U, 27U, 28U, 29U, 30U, 31U, 32U, 33U, 34U, 35U, 36U, 37U, 38U, 39U, + 40U, 41U, 42U, 43U, 44U, 45U, 46U, 47U, 48U, 49U, 50U, 51U, 255U, 255U, 255U, 255U, +}; + +// All input values in range for the first look-up will be 0U in the second +// look-up result. All input values out of range for the first look-up will be +// 0U in the first look-up result. Thus, the two results can be ORed without +// conflicts. +// +// Invalid characters that are in the valid range for either look-up will be +// set to 255U in the combined result. Other invalid characters will just be +// passed through with the second look-up result (using the VTBX instruction). +// Since the second LUT is 64 bytes, those passed-through values are guaranteed +// to have a value greater than 63U. Therefore, valid characters will be mapped +// to the valid [0..63] range and all invalid characters will be mapped to +// values greater than 63. + +static inline void +dec_loop_neon64 (const uint8_t **s, size_t *slen, uint8_t **o, size_t *olen) +{ + if (*slen < 64) { + return; + } + + // Process blocks of 64 bytes per round. Unlike the SSE codecs, no + // extra trailing zero bytes are written, so it is not necessary to + // reserve extra input bytes: + size_t rounds = *slen / 64; + + *slen -= rounds * 64; // 64 bytes consumed per round + *olen += rounds * 48; // 48 bytes produced per round + + const uint8x16x4_t tbl_dec1 = load_64byte_table(dec_lut1); + const uint8x16x4_t tbl_dec2 = load_64byte_table(dec_lut2); + + do { + const uint8x16_t offset = vdupq_n_u8(63U); + uint8x16x4_t dec1, dec2; + uint8x16x3_t dec; + + // Load 64 bytes and deinterleave: + uint8x16x4_t str = vld4q_u8((uint8_t *) *s); + + // Get indices for second LUT: + dec2.val[0] = vqsubq_u8(str.val[0], offset); + dec2.val[1] = vqsubq_u8(str.val[1], offset); + dec2.val[2] = vqsubq_u8(str.val[2], offset); + dec2.val[3] = vqsubq_u8(str.val[3], offset); + + // Get values from first LUT: + dec1.val[0] = vqtbl4q_u8(tbl_dec1, str.val[0]); + dec1.val[1] = vqtbl4q_u8(tbl_dec1, str.val[1]); + dec1.val[2] = vqtbl4q_u8(tbl_dec1, str.val[2]); + dec1.val[3] = vqtbl4q_u8(tbl_dec1, str.val[3]); + + // Get values from second LUT: + dec2.val[0] = vqtbx4q_u8(dec2.val[0], tbl_dec2, dec2.val[0]); + dec2.val[1] = vqtbx4q_u8(dec2.val[1], tbl_dec2, dec2.val[1]); + dec2.val[2] = vqtbx4q_u8(dec2.val[2], tbl_dec2, dec2.val[2]); + dec2.val[3] = vqtbx4q_u8(dec2.val[3], tbl_dec2, dec2.val[3]); + + // Get final values: + str.val[0] = vorrq_u8(dec1.val[0], dec2.val[0]); + str.val[1] = vorrq_u8(dec1.val[1], dec2.val[1]); + str.val[2] = vorrq_u8(dec1.val[2], dec2.val[2]); + str.val[3] = vorrq_u8(dec1.val[3], dec2.val[3]); + + // Check for invalid input, any value larger than 63: + const uint8x16_t classified + = vorrq_u8( + vorrq_u8(vcgtq_u8(str.val[0], vdupq_n_u8(63)), vcgtq_u8(str.val[1], vdupq_n_u8(63))), + vorrq_u8(vcgtq_u8(str.val[2], vdupq_n_u8(63)), vcgtq_u8(str.val[3], vdupq_n_u8(63))) + ); + + // Check that all bits are zero: + if (vmaxvq_u8(classified) != 0U) { + break; + } + + // Compress four bytes into three: + dec.val[0] = vorrq_u8(vshlq_n_u8(str.val[0], 2), vshrq_n_u8(str.val[1], 4)); + dec.val[1] = vorrq_u8(vshlq_n_u8(str.val[1], 4), vshrq_n_u8(str.val[2], 2)); + dec.val[2] = vorrq_u8(vshlq_n_u8(str.val[2], 6), str.val[3]); + + // Interleave and store decoded result: + vst3q_u8((uint8_t *) *o, dec); + + *s += 64; + *o += 48; + + } while (--rounds > 0); + + // Adjust for any rounds that were skipped: + *slen += rounds * 64; + *olen -= rounds * 48; +} diff --git a/mypyc/lib-rt/base64/arch/neon64/enc_loop.c b/mypyc/lib-rt/base64/arch/neon64/enc_loop.c new file mode 100644 index 0000000000000..8bdd088306532 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/neon64/enc_loop.c @@ -0,0 +1,66 @@ +static BASE64_FORCE_INLINE void +enc_loop_neon64_inner (const uint8_t **s, uint8_t **o, const uint8x16x4_t tbl_enc) +{ + // Load 48 bytes and deinterleave: + uint8x16x3_t src = vld3q_u8(*s); + + // Divide bits of three input bytes over four output bytes: + uint8x16x4_t out = enc_reshuffle(src); + + // The bits have now been shifted to the right locations; + // translate their values 0..63 to the Base64 alphabet. + // Use a 64-byte table lookup: + out.val[0] = vqtbl4q_u8(tbl_enc, out.val[0]); + out.val[1] = vqtbl4q_u8(tbl_enc, out.val[1]); + out.val[2] = vqtbl4q_u8(tbl_enc, out.val[2]); + out.val[3] = vqtbl4q_u8(tbl_enc, out.val[3]); + + // Interleave and store output: + vst4q_u8(*o, out); + + *s += 48; + *o += 64; +} + +static inline void +enc_loop_neon64 (const uint8_t **s, size_t *slen, uint8_t **o, size_t *olen) +{ + size_t rounds = *slen / 48; + + *slen -= rounds * 48; // 48 bytes consumed per round + *olen += rounds * 64; // 64 bytes produced per round + + // Load the encoding table: + const uint8x16x4_t tbl_enc = load_64byte_table(base64_table_enc_6bit); + + while (rounds > 0) { + if (rounds >= 8) { + enc_loop_neon64_inner(s, o, tbl_enc); + enc_loop_neon64_inner(s, o, tbl_enc); + enc_loop_neon64_inner(s, o, tbl_enc); + enc_loop_neon64_inner(s, o, tbl_enc); + enc_loop_neon64_inner(s, o, tbl_enc); + enc_loop_neon64_inner(s, o, tbl_enc); + enc_loop_neon64_inner(s, o, tbl_enc); + enc_loop_neon64_inner(s, o, tbl_enc); + rounds -= 8; + continue; + } + if (rounds >= 4) { + enc_loop_neon64_inner(s, o, tbl_enc); + enc_loop_neon64_inner(s, o, tbl_enc); + enc_loop_neon64_inner(s, o, tbl_enc); + enc_loop_neon64_inner(s, o, tbl_enc); + rounds -= 4; + continue; + } + if (rounds >= 2) { + enc_loop_neon64_inner(s, o, tbl_enc); + enc_loop_neon64_inner(s, o, tbl_enc); + rounds -= 2; + continue; + } + enc_loop_neon64_inner(s, o, tbl_enc); + break; + } +} diff --git a/mypyc/lib-rt/base64/arch/neon64/enc_loop_asm.c b/mypyc/lib-rt/base64/arch/neon64/enc_loop_asm.c new file mode 100644 index 0000000000000..182e9cdf4a17d --- /dev/null +++ b/mypyc/lib-rt/base64/arch/neon64/enc_loop_asm.c @@ -0,0 +1,168 @@ +// Apologies in advance for combining the preprocessor with inline assembly, +// two notoriously gnarly parts of C, but it was necessary to avoid a lot of +// code repetition. The preprocessor is used to template large sections of +// inline assembly that differ only in the registers used. If the code was +// written out by hand, it would become very large and hard to audit. + +// Generate a block of inline assembly that loads three user-defined registers +// A, B, C from memory and deinterleaves them, post-incrementing the src +// pointer. The register set should be sequential. +#define LOAD(A, B, C) \ + "ld3 {"A".16b, "B".16b, "C".16b}, [%[src]], #48 \n\t" + +// Generate a block of inline assembly that takes three deinterleaved registers +// and shuffles the bytes. The output is in temporary registers t0..t3. +#define SHUF(A, B, C) \ + "ushr %[t0].16b, "A".16b, #2 \n\t" \ + "ushr %[t1].16b, "B".16b, #4 \n\t" \ + "ushr %[t2].16b, "C".16b, #6 \n\t" \ + "sli %[t1].16b, "A".16b, #4 \n\t" \ + "sli %[t2].16b, "B".16b, #2 \n\t" \ + "and %[t1].16b, %[t1].16b, %[n63].16b \n\t" \ + "and %[t2].16b, %[t2].16b, %[n63].16b \n\t" \ + "and %[t3].16b, "C".16b, %[n63].16b \n\t" + +// Generate a block of inline assembly that takes temporary registers t0..t3 +// and translates them to the base64 alphabet, using a table loaded into +// v8..v11. The output is in user-defined registers A..D. +#define TRAN(A, B, C, D) \ + "tbl "A".16b, {v8.16b-v11.16b}, %[t0].16b \n\t" \ + "tbl "B".16b, {v8.16b-v11.16b}, %[t1].16b \n\t" \ + "tbl "C".16b, {v8.16b-v11.16b}, %[t2].16b \n\t" \ + "tbl "D".16b, {v8.16b-v11.16b}, %[t3].16b \n\t" + +// Generate a block of inline assembly that interleaves four registers and +// stores them, post-incrementing the destination pointer. +#define STOR(A, B, C, D) \ + "st4 {"A".16b, "B".16b, "C".16b, "D".16b}, [%[dst]], #64 \n\t" + +// Generate a block of inline assembly that generates a single self-contained +// encoder round: fetch the data, process it, and store the result. +#define ROUND() \ + LOAD("v12", "v13", "v14") \ + SHUF("v12", "v13", "v14") \ + TRAN("v12", "v13", "v14", "v15") \ + STOR("v12", "v13", "v14", "v15") + +// Generate a block of assembly that generates a type A interleaved encoder +// round. It uses registers that were loaded by the previous type B round, and +// in turn loads registers for the next type B round. +#define ROUND_A() \ + SHUF("v2", "v3", "v4") \ + LOAD("v12", "v13", "v14") \ + TRAN("v2", "v3", "v4", "v5") \ + STOR("v2", "v3", "v4", "v5") + +// Type B interleaved encoder round. Same as type A, but register sets swapped. +#define ROUND_B() \ + SHUF("v12", "v13", "v14") \ + LOAD("v2", "v3", "v4") \ + TRAN("v12", "v13", "v14", "v15") \ + STOR("v12", "v13", "v14", "v15") + +// The first type A round needs to load its own registers. +#define ROUND_A_FIRST() \ + LOAD("v2", "v3", "v4") \ + ROUND_A() + +// The last type B round omits the load for the next step. +#define ROUND_B_LAST() \ + SHUF("v12", "v13", "v14") \ + TRAN("v12", "v13", "v14", "v15") \ + STOR("v12", "v13", "v14", "v15") + +// Suppress clang's warning that the literal string in the asm statement is +// overlong (longer than the ISO-mandated minimum size of 4095 bytes for C99 +// compilers). It may be true, but the goal here is not C99 portability. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Woverlength-strings" + +static inline void +enc_loop_neon64 (const uint8_t **s, size_t *slen, uint8_t **o, size_t *olen) +{ + size_t rounds = *slen / 48; + + if (rounds == 0) { + return; + } + + *slen -= rounds * 48; // 48 bytes consumed per round. + *olen += rounds * 64; // 64 bytes produced per round. + + // Number of times to go through the 8x loop. + size_t loops = rounds / 8; + + // Number of rounds remaining after the 8x loop. + rounds %= 8; + + // Temporary registers, used as scratch space. + uint8x16_t tmp0, tmp1, tmp2, tmp3; + + __asm__ volatile ( + + // Load the encoding table into v8..v11. + " ld1 {v8.16b-v11.16b}, [%[tbl]] \n\t" + + // If there are eight rounds or more, enter an 8x unrolled loop + // of interleaved encoding rounds. The rounds interleave memory + // operations (load/store) with data operations to maximize + // pipeline throughput. + " cbz %[loops], 4f \n\t" + + // The SIMD instructions do not touch the flags. + "88: subs %[loops], %[loops], #1 \n\t" + " " ROUND_A_FIRST() + " " ROUND_B() + " " ROUND_A() + " " ROUND_B() + " " ROUND_A() + " " ROUND_B() + " " ROUND_A() + " " ROUND_B_LAST() + " b.ne 88b \n\t" + + // Enter a 4x unrolled loop for rounds of 4 or more. + "4: cmp %[rounds], #4 \n\t" + " b.lt 30f \n\t" + " " ROUND_A_FIRST() + " " ROUND_B() + " " ROUND_A() + " " ROUND_B_LAST() + " sub %[rounds], %[rounds], #4 \n\t" + + // Dispatch the remaining rounds 0..3. + "30: cbz %[rounds], 0f \n\t" + " cmp %[rounds], #2 \n\t" + " b.eq 2f \n\t" + " b.lt 1f \n\t" + + // Block of non-interlaced encoding rounds, which can each + // individually be jumped to. Rounds fall through to the next. + "3: " ROUND() + "2: " ROUND() + "1: " ROUND() + "0: \n\t" + + // Outputs (modified). + : [loops] "+r" (loops), + [src] "+r" (*s), + [dst] "+r" (*o), + [t0] "=&w" (tmp0), + [t1] "=&w" (tmp1), + [t2] "=&w" (tmp2), + [t3] "=&w" (tmp3) + + // Inputs (not modified). + : [rounds] "r" (rounds), + [tbl] "r" (base64_table_enc_6bit), + [n63] "w" (vdupq_n_u8(63)) + + // Clobbers. + : "v2", "v3", "v4", "v5", + "v8", "v9", "v10", "v11", + "v12", "v13", "v14", "v15", + "cc", "memory" + ); +} + +#pragma GCC diagnostic pop diff --git a/mypyc/lib-rt/base64/arch/neon64/enc_reshuffle.c b/mypyc/lib-rt/base64/arch/neon64/enc_reshuffle.c new file mode 100644 index 0000000000000..2655df10f3eb3 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/neon64/enc_reshuffle.c @@ -0,0 +1,31 @@ +static BASE64_FORCE_INLINE uint8x16x4_t +enc_reshuffle (const uint8x16x3_t in) +{ + uint8x16x4_t out; + + // Input: + // in[0] = a7 a6 a5 a4 a3 a2 a1 a0 + // in[1] = b7 b6 b5 b4 b3 b2 b1 b0 + // in[2] = c7 c6 c5 c4 c3 c2 c1 c0 + + // Output: + // out[0] = 00 00 a7 a6 a5 a4 a3 a2 + // out[1] = 00 00 a1 a0 b7 b6 b5 b4 + // out[2] = 00 00 b3 b2 b1 b0 c7 c6 + // out[3] = 00 00 c5 c4 c3 c2 c1 c0 + + // Move the input bits to where they need to be in the outputs. Except + // for the first output, the high two bits are not cleared. + out.val[0] = vshrq_n_u8(in.val[0], 2); + out.val[1] = vshrq_n_u8(in.val[1], 4); + out.val[2] = vshrq_n_u8(in.val[2], 6); + out.val[1] = vsliq_n_u8(out.val[1], in.val[0], 4); + out.val[2] = vsliq_n_u8(out.val[2], in.val[1], 2); + + // Clear the high two bits in the second, third and fourth output. + out.val[1] = vandq_u8(out.val[1], vdupq_n_u8(0x3F)); + out.val[2] = vandq_u8(out.val[2], vdupq_n_u8(0x3F)); + out.val[3] = vandq_u8(in.val[2], vdupq_n_u8(0x3F)); + + return out; +} diff --git a/mypyc/lib-rt/base64/arch/sse41/codec.c b/mypyc/lib-rt/base64/arch/sse41/codec.c new file mode 100644 index 0000000000000..c627db5f726d4 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/sse41/codec.c @@ -0,0 +1,58 @@ +#include +#include +#include + +#include "libbase64.h" +#include "../../tables/tables.h" +#include "../../codecs.h" +#include "config.h" +#include "../../env.h" + +#if HAVE_SSE41 +#include + +// Only enable inline assembly on supported compilers and on 64-bit CPUs. +#ifndef BASE64_SSE41_USE_ASM +# if (defined(__GNUC__) || defined(__clang__)) && BASE64_WORDSIZE == 64 +# define BASE64_SSE41_USE_ASM 1 +# else +# define BASE64_SSE41_USE_ASM 0 +# endif +#endif + +#include "../ssse3/dec_reshuffle.c" +#include "../ssse3/dec_loop.c" + +#if BASE64_SSE41_USE_ASM +# include "../ssse3/enc_loop_asm.c" +#else +# include "../ssse3/enc_translate.c" +# include "../ssse3/enc_reshuffle.c" +# include "../ssse3/enc_loop.c" +#endif + +#endif // HAVE_SSE41 + +void +base64_stream_encode_sse41 BASE64_ENC_PARAMS +{ +#if HAVE_SSE41 + #include "../generic/enc_head.c" + enc_loop_ssse3(&s, &slen, &o, &olen); + #include "../generic/enc_tail.c" +#else + base64_enc_stub(state, src, srclen, out, outlen); +#endif +} + +int +base64_stream_decode_sse41 BASE64_DEC_PARAMS +{ +#if HAVE_SSE41 + #include "../generic/dec_head.c" + dec_loop_ssse3(&s, &slen, &o, &olen); + #include "../generic/dec_tail.c" +#else + return base64_dec_stub(state, src, srclen, out, outlen); +#endif +} diff --git a/mypyc/lib-rt/base64/arch/sse42/codec.c b/mypyc/lib-rt/base64/arch/sse42/codec.c new file mode 100644 index 0000000000000..2fe4e2997aa14 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/sse42/codec.c @@ -0,0 +1,58 @@ +#include +#include +#include + +#include "libbase64.h" +#include "../../tables/tables.h" +#include "../../codecs.h" +#include "config.h" +#include "../../env.h" + +#if HAVE_SSE42 +#include + +// Only enable inline assembly on supported compilers and on 64-bit CPUs. +#ifndef BASE64_SSE42_USE_ASM +# if (defined(__GNUC__) || defined(__clang__)) && BASE64_WORDSIZE == 64 +# define BASE64_SSE42_USE_ASM 1 +# else +# define BASE64_SSE42_USE_ASM 0 +# endif +#endif + +#include "../ssse3/dec_reshuffle.c" +#include "../ssse3/dec_loop.c" + +#if BASE64_SSE42_USE_ASM +# include "../ssse3/enc_loop_asm.c" +#else +# include "../ssse3/enc_translate.c" +# include "../ssse3/enc_reshuffle.c" +# include "../ssse3/enc_loop.c" +#endif + +#endif // HAVE_SSE42 + +void +base64_stream_encode_sse42 BASE64_ENC_PARAMS +{ +#if HAVE_SSE42 + #include "../generic/enc_head.c" + enc_loop_ssse3(&s, &slen, &o, &olen); + #include "../generic/enc_tail.c" +#else + base64_enc_stub(state, src, srclen, out, outlen); +#endif +} + +int +base64_stream_decode_sse42 BASE64_DEC_PARAMS +{ +#if HAVE_SSE42 + #include "../generic/dec_head.c" + dec_loop_ssse3(&s, &slen, &o, &olen); + #include "../generic/dec_tail.c" +#else + return base64_dec_stub(state, src, srclen, out, outlen); +#endif +} diff --git a/mypyc/lib-rt/base64/arch/ssse3/codec.c b/mypyc/lib-rt/base64/arch/ssse3/codec.c new file mode 100644 index 0000000000000..e51b3dfdb1677 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/ssse3/codec.c @@ -0,0 +1,60 @@ +#include +#include +#include + +#include "libbase64.h" +#include "../../tables/tables.h" +#include "../../codecs.h" +#include "config.h" +#include "../../env.h" + +#if HAVE_SSSE3 +#include + +// Only enable inline assembly on supported compilers and on 64-bit CPUs. +// 32-bit CPUs with SSSE3 support, such as low-end Atoms, only have eight XMM +// registers, which is not enough to run the inline assembly. +#ifndef BASE64_SSSE3_USE_ASM +# if (defined(__GNUC__) || defined(__clang__)) && BASE64_WORDSIZE == 64 +# define BASE64_SSSE3_USE_ASM 1 +# else +# define BASE64_SSSE3_USE_ASM 0 +# endif +#endif + +#include "dec_reshuffle.c" +#include "dec_loop.c" + +#if BASE64_SSSE3_USE_ASM +# include "enc_loop_asm.c" +#else +# include "enc_reshuffle.c" +# include "enc_translate.c" +# include "enc_loop.c" +#endif + +#endif // HAVE_SSSE3 + +void +base64_stream_encode_ssse3 BASE64_ENC_PARAMS +{ +#if HAVE_SSSE3 + #include "../generic/enc_head.c" + enc_loop_ssse3(&s, &slen, &o, &olen); + #include "../generic/enc_tail.c" +#else + base64_enc_stub(state, src, srclen, out, outlen); +#endif +} + +int +base64_stream_decode_ssse3 BASE64_DEC_PARAMS +{ +#if HAVE_SSSE3 + #include "../generic/dec_head.c" + dec_loop_ssse3(&s, &slen, &o, &olen); + #include "../generic/dec_tail.c" +#else + return base64_dec_stub(state, src, srclen, out, outlen); +#endif +} diff --git a/mypyc/lib-rt/base64/arch/ssse3/dec_loop.c b/mypyc/lib-rt/base64/arch/ssse3/dec_loop.c new file mode 100644 index 0000000000000..7ddb73bf88149 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/ssse3/dec_loop.c @@ -0,0 +1,173 @@ +// The input consists of six character sets in the Base64 alphabet, which we +// need to map back to the 6-bit values they represent. There are three ranges, +// two singles, and then there's the rest. +// +// # From To Add Characters +// 1 [43] [62] +19 + +// 2 [47] [63] +16 / +// 3 [48..57] [52..61] +4 0..9 +// 4 [65..90] [0..25] -65 A..Z +// 5 [97..122] [26..51] -71 a..z +// (6) Everything else => invalid input +// +// We will use lookup tables for character validation and offset computation. +// Remember that 0x2X and 0x0X are the same index for _mm_shuffle_epi8, this +// allows to mask with 0x2F instead of 0x0F and thus save one constant +// declaration (register and/or memory access). +// +// For offsets: +// Perfect hash for lut = ((src >> 4) & 0x2F) + ((src == 0x2F) ? 0xFF : 0x00) +// 0000 = garbage +// 0001 = / +// 0010 = + +// 0011 = 0-9 +// 0100 = A-Z +// 0101 = A-Z +// 0110 = a-z +// 0111 = a-z +// 1000 >= garbage +// +// For validation, here's the table. +// A character is valid if and only if the AND of the 2 lookups equals 0: +// +// hi \ lo 0000 0001 0010 0011 0100 0101 0110 0111 1000 1001 1010 1011 1100 1101 1110 1111 +// LUT 0x15 0x11 0x11 0x11 0x11 0x11 0x11 0x11 0x11 0x11 0x13 0x1A 0x1B 0x1B 0x1B 0x1A +// +// 0000 0x10 char NUL SOH STX ETX EOT ENQ ACK BEL BS HT LF VT FF CR SO SI +// andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 +// +// 0001 0x10 char DLE DC1 DC2 DC3 DC4 NAK SYN ETB CAN EM SUB ESC FS GS RS US +// andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 +// +// 0010 0x01 char ! " # $ % & ' ( ) * + , - . / +// andlut 0x01 0x01 0x01 0x01 0x01 0x01 0x01 0x01 0x01 0x01 0x01 0x00 0x01 0x01 0x01 0x00 +// +// 0011 0x02 char 0 1 2 3 4 5 6 7 8 9 : ; < = > ? +// andlut 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x02 0x02 0x02 0x02 0x02 0x02 +// +// 0100 0x04 char @ A B C D E F G H I J K L M N O +// andlut 0x04 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 +// +// 0101 0x08 char P Q R S T U V W X Y Z [ \ ] ^ _ +// andlut 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x08 0x08 0x08 0x08 0x08 +// +// 0110 0x04 char ` a b c d e f g h i j k l m n o +// andlut 0x04 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 +// 0111 0x08 char p q r s t u v w x y z { | } ~ +// andlut 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x08 0x08 0x08 0x08 0x08 +// +// 1000 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 +// 1001 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 +// 1010 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 +// 1011 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 +// 1100 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 +// 1101 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 +// 1110 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 +// 1111 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 + +static BASE64_FORCE_INLINE int +dec_loop_ssse3_inner (const uint8_t **s, uint8_t **o, size_t *rounds) +{ + const __m128i lut_lo = _mm_setr_epi8( + 0x15, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, + 0x11, 0x11, 0x13, 0x1A, 0x1B, 0x1B, 0x1B, 0x1A); + + const __m128i lut_hi = _mm_setr_epi8( + 0x10, 0x10, 0x01, 0x02, 0x04, 0x08, 0x04, 0x08, + 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10); + + const __m128i lut_roll = _mm_setr_epi8( + 0, 16, 19, 4, -65, -65, -71, -71, + 0, 0, 0, 0, 0, 0, 0, 0); + + const __m128i mask_2F = _mm_set1_epi8(0x2F); + + // Load input: + __m128i str = _mm_loadu_si128((__m128i *) *s); + + // Table lookups: + const __m128i hi_nibbles = _mm_and_si128(_mm_srli_epi32(str, 4), mask_2F); + const __m128i lo_nibbles = _mm_and_si128(str, mask_2F); + const __m128i hi = _mm_shuffle_epi8(lut_hi, hi_nibbles); + const __m128i lo = _mm_shuffle_epi8(lut_lo, lo_nibbles); + + // Check for invalid input: if any "and" values from lo and hi are not + // zero, fall back on bytewise code to do error checking and reporting: + if (_mm_movemask_epi8(_mm_cmpgt_epi8(_mm_and_si128(lo, hi), _mm_setzero_si128())) != 0) { + return 0; + } + + const __m128i eq_2F = _mm_cmpeq_epi8(str, mask_2F); + const __m128i roll = _mm_shuffle_epi8(lut_roll, _mm_add_epi8(eq_2F, hi_nibbles)); + + // Now simply add the delta values to the input: + str = _mm_add_epi8(str, roll); + + // Reshuffle the input to packed 12-byte output format: + str = dec_reshuffle(str); + + // Store the output: + _mm_storeu_si128((__m128i *) *o, str); + + *s += 16; + *o += 12; + *rounds -= 1; + + return 1; +} + +static inline void +dec_loop_ssse3 (const uint8_t **s, size_t *slen, uint8_t **o, size_t *olen) +{ + if (*slen < 24) { + return; + } + + // Process blocks of 16 bytes per round. Because 4 extra zero bytes are + // written after the output, ensure that there will be at least 8 bytes + // of input data left to cover the gap. (6 data bytes and up to two + // end-of-string markers.) + size_t rounds = (*slen - 8) / 16; + + *slen -= rounds * 16; // 16 bytes consumed per round + *olen += rounds * 12; // 12 bytes produced per round + + do { + if (rounds >= 8) { + if (dec_loop_ssse3_inner(s, o, &rounds) && + dec_loop_ssse3_inner(s, o, &rounds) && + dec_loop_ssse3_inner(s, o, &rounds) && + dec_loop_ssse3_inner(s, o, &rounds) && + dec_loop_ssse3_inner(s, o, &rounds) && + dec_loop_ssse3_inner(s, o, &rounds) && + dec_loop_ssse3_inner(s, o, &rounds) && + dec_loop_ssse3_inner(s, o, &rounds)) { + continue; + } + break; + } + if (rounds >= 4) { + if (dec_loop_ssse3_inner(s, o, &rounds) && + dec_loop_ssse3_inner(s, o, &rounds) && + dec_loop_ssse3_inner(s, o, &rounds) && + dec_loop_ssse3_inner(s, o, &rounds)) { + continue; + } + break; + } + if (rounds >= 2) { + if (dec_loop_ssse3_inner(s, o, &rounds) && + dec_loop_ssse3_inner(s, o, &rounds)) { + continue; + } + break; + } + dec_loop_ssse3_inner(s, o, &rounds); + break; + + } while (rounds > 0); + + // Adjust for any rounds that were skipped: + *slen += rounds * 16; + *olen -= rounds * 12; +} diff --git a/mypyc/lib-rt/base64/arch/ssse3/dec_reshuffle.c b/mypyc/lib-rt/base64/arch/ssse3/dec_reshuffle.c new file mode 100644 index 0000000000000..d3dd395427ae0 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/ssse3/dec_reshuffle.c @@ -0,0 +1,33 @@ +static BASE64_FORCE_INLINE __m128i +dec_reshuffle (const __m128i in) +{ + // in, bits, upper case are most significant bits, lower case are least significant bits + // 00llllll 00kkkkLL 00jjKKKK 00JJJJJJ + // 00iiiiii 00hhhhII 00ggHHHH 00GGGGGG + // 00ffffff 00eeeeFF 00ddEEEE 00DDDDDD + // 00cccccc 00bbbbCC 00aaBBBB 00AAAAAA + + const __m128i merge_ab_and_bc = _mm_maddubs_epi16(in, _mm_set1_epi32(0x01400140)); + // 0000kkkk LLllllll 0000JJJJ JJjjKKKK + // 0000hhhh IIiiiiii 0000GGGG GGggHHHH + // 0000eeee FFffffff 0000DDDD DDddEEEE + // 0000bbbb CCcccccc 0000AAAA AAaaBBBB + + const __m128i out = _mm_madd_epi16(merge_ab_and_bc, _mm_set1_epi32(0x00011000)); + // 00000000 JJJJJJjj KKKKkkkk LLllllll + // 00000000 GGGGGGgg HHHHhhhh IIiiiiii + // 00000000 DDDDDDdd EEEEeeee FFffffff + // 00000000 AAAAAAaa BBBBbbbb CCcccccc + + // Pack bytes together: + return _mm_shuffle_epi8(out, _mm_setr_epi8( + 2, 1, 0, + 6, 5, 4, + 10, 9, 8, + 14, 13, 12, + -1, -1, -1, -1)); + // 00000000 00000000 00000000 00000000 + // LLllllll KKKKkkkk JJJJJJjj IIiiiiii + // HHHHhhhh GGGGGGgg FFffffff EEEEeeee + // DDDDDDdd CCcccccc BBBBbbbb AAAAAAaa +} diff --git a/mypyc/lib-rt/base64/arch/ssse3/enc_loop.c b/mypyc/lib-rt/base64/arch/ssse3/enc_loop.c new file mode 100644 index 0000000000000..9b67b70db1fd5 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/ssse3/enc_loop.c @@ -0,0 +1,67 @@ +static BASE64_FORCE_INLINE void +enc_loop_ssse3_inner (const uint8_t **s, uint8_t **o) +{ + // Load input: + __m128i str = _mm_loadu_si128((__m128i *) *s); + + // Reshuffle: + str = enc_reshuffle(str); + + // Translate reshuffled bytes to the Base64 alphabet: + str = enc_translate(str); + + // Store: + _mm_storeu_si128((__m128i *) *o, str); + + *s += 12; + *o += 16; +} + +static inline void +enc_loop_ssse3 (const uint8_t **s, size_t *slen, uint8_t **o, size_t *olen) +{ + if (*slen < 16) { + return; + } + + // Process blocks of 12 bytes at a time. Because blocks are loaded 16 + // bytes at a time, ensure that there will be at least 4 remaining + // bytes after the last round, so that the final read will not pass + // beyond the bounds of the input buffer: + size_t rounds = (*slen - 4) / 12; + + *slen -= rounds * 12; // 12 bytes consumed per round + *olen += rounds * 16; // 16 bytes produced per round + + do { + if (rounds >= 8) { + enc_loop_ssse3_inner(s, o); + enc_loop_ssse3_inner(s, o); + enc_loop_ssse3_inner(s, o); + enc_loop_ssse3_inner(s, o); + enc_loop_ssse3_inner(s, o); + enc_loop_ssse3_inner(s, o); + enc_loop_ssse3_inner(s, o); + enc_loop_ssse3_inner(s, o); + rounds -= 8; + continue; + } + if (rounds >= 4) { + enc_loop_ssse3_inner(s, o); + enc_loop_ssse3_inner(s, o); + enc_loop_ssse3_inner(s, o); + enc_loop_ssse3_inner(s, o); + rounds -= 4; + continue; + } + if (rounds >= 2) { + enc_loop_ssse3_inner(s, o); + enc_loop_ssse3_inner(s, o); + rounds -= 2; + continue; + } + enc_loop_ssse3_inner(s, o); + break; + + } while (rounds > 0); +} diff --git a/mypyc/lib-rt/base64/arch/ssse3/enc_loop_asm.c b/mypyc/lib-rt/base64/arch/ssse3/enc_loop_asm.c new file mode 100644 index 0000000000000..0cdb340a63b7f --- /dev/null +++ b/mypyc/lib-rt/base64/arch/ssse3/enc_loop_asm.c @@ -0,0 +1,268 @@ +// Apologies in advance for combining the preprocessor with inline assembly, +// two notoriously gnarly parts of C, but it was necessary to avoid a lot of +// code repetition. The preprocessor is used to template large sections of +// inline assembly that differ only in the registers used. If the code was +// written out by hand, it would become very large and hard to audit. + +// Generate a block of inline assembly that loads register R0 from memory. The +// offset at which the register is loaded is set by the given round. +#define LOAD(R0, ROUND) \ + "lddqu ("#ROUND" * 12)(%[src]), %["R0"] \n\t" + +// Generate a block of inline assembly that deinterleaves and shuffles register +// R0 using preloaded constants. Outputs in R0 and R1. +#define SHUF(R0, R1) \ + "pshufb %[lut0], %["R0"] \n\t" \ + "movdqa %["R0"], %["R1"] \n\t" \ + "pand %[msk0], %["R0"] \n\t" \ + "pand %[msk2], %["R1"] \n\t" \ + "pmulhuw %[msk1], %["R0"] \n\t" \ + "pmullw %[msk3], %["R1"] \n\t" \ + "por %["R1"], %["R0"] \n\t" + +// Generate a block of inline assembly that takes R0 and R1 and translates +// their contents to the base64 alphabet, using preloaded constants. +#define TRAN(R0, R1, R2) \ + "movdqa %["R0"], %["R1"] \n\t" \ + "movdqa %["R0"], %["R2"] \n\t" \ + "psubusb %[n51], %["R1"] \n\t" \ + "pcmpgtb %[n25], %["R2"] \n\t" \ + "psubb %["R2"], %["R1"] \n\t" \ + "movdqa %[lut1], %["R2"] \n\t" \ + "pshufb %["R1"], %["R2"] \n\t" \ + "paddb %["R2"], %["R0"] \n\t" + +// Generate a block of inline assembly that stores the given register R0 at an +// offset set by the given round. +#define STOR(R0, ROUND) \ + "movdqu %["R0"], ("#ROUND" * 16)(%[dst]) \n\t" + +// Generate a block of inline assembly that generates a single self-contained +// encoder round: fetch the data, process it, and store the result. Then update +// the source and destination pointers. +#define ROUND() \ + LOAD("a", 0) \ + SHUF("a", "b") \ + TRAN("a", "b", "c") \ + STOR("a", 0) \ + "add $12, %[src] \n\t" \ + "add $16, %[dst] \n\t" + +// Define a macro that initiates a three-way interleaved encoding round by +// preloading registers a, b and c from memory. +// The register graph shows which registers are in use during each step, and +// is a visual aid for choosing registers for that step. Symbol index: +// +// + indicates that a register is loaded by that step. +// | indicates that a register is in use and must not be touched. +// - indicates that a register is decommissioned by that step. +// x indicates that a register is used as a temporary by that step. +// V indicates that a register is an input or output to the macro. +// +#define ROUND_3_INIT() /* a b c d e f */ \ + LOAD("a", 0) /* + */ \ + SHUF("a", "d") /* | + */ \ + LOAD("b", 1) /* | + | */ \ + TRAN("a", "d", "e") /* | | - x */ \ + LOAD("c", 2) /* V V V */ + +// Define a macro that translates, shuffles and stores the input registers A, B +// and C, and preloads registers D, E and F for the next round. +// This macro can be arbitrarily daisy-chained by feeding output registers D, E +// and F back into the next round as input registers A, B and C. The macro +// carefully interleaves memory operations with data operations for optimal +// pipelined performance. + +#define ROUND_3(ROUND, A,B,C,D,E,F) /* A B C D E F */ \ + LOAD(D, (ROUND + 3)) /* V V V + */ \ + SHUF(B, E) /* | | | | + */ \ + STOR(A, (ROUND + 0)) /* - | | | | */ \ + TRAN(B, E, F) /* | | | - x */ \ + LOAD(E, (ROUND + 4)) /* | | | + */ \ + SHUF(C, A) /* + | | | | */ \ + STOR(B, (ROUND + 1)) /* | - | | | */ \ + TRAN(C, A, F) /* - | | | x */ \ + LOAD(F, (ROUND + 5)) /* | | | + */ \ + SHUF(D, A) /* + | | | | */ \ + STOR(C, (ROUND + 2)) /* | - | | | */ \ + TRAN(D, A, B) /* - x V V V */ + +// Define a macro that terminates a ROUND_3 macro by taking pre-loaded +// registers D, E and F, and translating, shuffling and storing them. +#define ROUND_3_END(ROUND, A,B,C,D,E,F) /* A B C D E F */ \ + SHUF(E, A) /* + V V V */ \ + STOR(D, (ROUND + 3)) /* | - | | */ \ + TRAN(E, A, B) /* - x | | */ \ + SHUF(F, C) /* + | | */ \ + STOR(E, (ROUND + 4)) /* | - | */ \ + TRAN(F, C, D) /* - x | */ \ + STOR(F, (ROUND + 5)) /* - */ + +// Define a type A round. Inputs are a, b, and c, outputs are d, e, and f. +#define ROUND_3_A(ROUND) \ + ROUND_3(ROUND, "a", "b", "c", "d", "e", "f") + +// Define a type B round. Inputs and outputs are swapped with regard to type A. +#define ROUND_3_B(ROUND) \ + ROUND_3(ROUND, "d", "e", "f", "a", "b", "c") + +// Terminating macro for a type A round. +#define ROUND_3_A_LAST(ROUND) \ + ROUND_3_A(ROUND) \ + ROUND_3_END(ROUND, "a", "b", "c", "d", "e", "f") + +// Terminating macro for a type B round. +#define ROUND_3_B_LAST(ROUND) \ + ROUND_3_B(ROUND) \ + ROUND_3_END(ROUND, "d", "e", "f", "a", "b", "c") + +// Suppress clang's warning that the literal string in the asm statement is +// overlong (longer than the ISO-mandated minimum size of 4095 bytes for C99 +// compilers). It may be true, but the goal here is not C99 portability. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Woverlength-strings" + +static inline void +enc_loop_ssse3 (const uint8_t **s, size_t *slen, uint8_t **o, size_t *olen) +{ + // For a clearer explanation of the algorithm used by this function, + // please refer to the plain (not inline assembly) implementation. This + // function follows the same basic logic. + + if (*slen < 16) { + return; + } + + // Process blocks of 12 bytes at a time. Input is read in blocks of 16 + // bytes, so "reserve" four bytes from the input buffer to ensure that + // we never read beyond the end of the input buffer. + size_t rounds = (*slen - 4) / 12; + + *slen -= rounds * 12; // 12 bytes consumed per round + *olen += rounds * 16; // 16 bytes produced per round + + // Number of times to go through the 36x loop. + size_t loops = rounds / 36; + + // Number of rounds remaining after the 36x loop. + rounds %= 36; + + // Lookup tables. + const __m128i lut0 = _mm_set_epi8( + 10, 11, 9, 10, 7, 8, 6, 7, 4, 5, 3, 4, 1, 2, 0, 1); + + const __m128i lut1 = _mm_setr_epi8( + 65, 71, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -19, -16, 0, 0); + + // Temporary registers. + __m128i a, b, c, d, e, f; + + __asm__ volatile ( + + // If there are 36 rounds or more, enter a 36x unrolled loop of + // interleaved encoding rounds. The rounds interleave memory + // operations (load/store) with data operations (table lookups, + // etc) to maximize pipeline throughput. + " test %[loops], %[loops] \n\t" + " jz 18f \n\t" + " jmp 36f \n\t" + " \n\t" + ".balign 64 \n\t" + "36: " ROUND_3_INIT() + " " ROUND_3_A( 0) + " " ROUND_3_B( 3) + " " ROUND_3_A( 6) + " " ROUND_3_B( 9) + " " ROUND_3_A(12) + " " ROUND_3_B(15) + " " ROUND_3_A(18) + " " ROUND_3_B(21) + " " ROUND_3_A(24) + " " ROUND_3_B(27) + " " ROUND_3_A_LAST(30) + " add $(12 * 36), %[src] \n\t" + " add $(16 * 36), %[dst] \n\t" + " dec %[loops] \n\t" + " jnz 36b \n\t" + + // Enter an 18x unrolled loop for rounds of 18 or more. + "18: cmp $18, %[rounds] \n\t" + " jl 9f \n\t" + " " ROUND_3_INIT() + " " ROUND_3_A(0) + " " ROUND_3_B(3) + " " ROUND_3_A(6) + " " ROUND_3_B(9) + " " ROUND_3_A_LAST(12) + " sub $18, %[rounds] \n\t" + " add $(12 * 18), %[src] \n\t" + " add $(16 * 18), %[dst] \n\t" + + // Enter a 9x unrolled loop for rounds of 9 or more. + "9: cmp $9, %[rounds] \n\t" + " jl 6f \n\t" + " " ROUND_3_INIT() + " " ROUND_3_A(0) + " " ROUND_3_B_LAST(3) + " sub $9, %[rounds] \n\t" + " add $(12 * 9), %[src] \n\t" + " add $(16 * 9), %[dst] \n\t" + + // Enter a 6x unrolled loop for rounds of 6 or more. + "6: cmp $6, %[rounds] \n\t" + " jl 55f \n\t" + " " ROUND_3_INIT() + " " ROUND_3_A_LAST(0) + " sub $6, %[rounds] \n\t" + " add $(12 * 6), %[src] \n\t" + " add $(16 * 6), %[dst] \n\t" + + // Dispatch the remaining rounds 0..5. + "55: cmp $3, %[rounds] \n\t" + " jg 45f \n\t" + " je 3f \n\t" + " cmp $1, %[rounds] \n\t" + " jg 2f \n\t" + " je 1f \n\t" + " jmp 0f \n\t" + + "45: cmp $4, %[rounds] \n\t" + " je 4f \n\t" + + // Block of non-interlaced encoding rounds, which can each + // individually be jumped to. Rounds fall through to the next. + "5: " ROUND() + "4: " ROUND() + "3: " ROUND() + "2: " ROUND() + "1: " ROUND() + "0: \n\t" + + // Outputs (modified). + : [rounds] "+r" (rounds), + [loops] "+r" (loops), + [src] "+r" (*s), + [dst] "+r" (*o), + [a] "=&x" (a), + [b] "=&x" (b), + [c] "=&x" (c), + [d] "=&x" (d), + [e] "=&x" (e), + [f] "=&x" (f) + + // Inputs (not modified). + : [lut0] "x" (lut0), + [lut1] "x" (lut1), + [msk0] "x" (_mm_set1_epi32(0x0FC0FC00)), + [msk1] "x" (_mm_set1_epi32(0x04000040)), + [msk2] "x" (_mm_set1_epi32(0x003F03F0)), + [msk3] "x" (_mm_set1_epi32(0x01000010)), + [n51] "x" (_mm_set1_epi8(51)), + [n25] "x" (_mm_set1_epi8(25)) + + // Clobbers. + : "cc", "memory" + ); +} + +#pragma GCC diagnostic pop diff --git a/mypyc/lib-rt/base64/arch/ssse3/enc_reshuffle.c b/mypyc/lib-rt/base64/arch/ssse3/enc_reshuffle.c new file mode 100644 index 0000000000000..f9dc949f255dc --- /dev/null +++ b/mypyc/lib-rt/base64/arch/ssse3/enc_reshuffle.c @@ -0,0 +1,48 @@ +static BASE64_FORCE_INLINE __m128i +enc_reshuffle (__m128i in) +{ + // Input, bytes MSB to LSB: + // 0 0 0 0 l k j i h g f e d c b a + + in = _mm_shuffle_epi8(in, _mm_set_epi8( + 10, 11, 9, 10, + 7, 8, 6, 7, + 4, 5, 3, 4, + 1, 2, 0, 1)); + // in, bytes MSB to LSB: + // k l j k + // h i g h + // e f d e + // b c a b + + const __m128i t0 = _mm_and_si128(in, _mm_set1_epi32(0x0FC0FC00)); + // bits, upper case are most significant bits, lower case are least significant bits + // 0000kkkk LL000000 JJJJJJ00 00000000 + // 0000hhhh II000000 GGGGGG00 00000000 + // 0000eeee FF000000 DDDDDD00 00000000 + // 0000bbbb CC000000 AAAAAA00 00000000 + + const __m128i t1 = _mm_mulhi_epu16(t0, _mm_set1_epi32(0x04000040)); + // 00000000 00kkkkLL 00000000 00JJJJJJ + // 00000000 00hhhhII 00000000 00GGGGGG + // 00000000 00eeeeFF 00000000 00DDDDDD + // 00000000 00bbbbCC 00000000 00AAAAAA + + const __m128i t2 = _mm_and_si128(in, _mm_set1_epi32(0x003F03F0)); + // 00000000 00llllll 000000jj KKKK0000 + // 00000000 00iiiiii 000000gg HHHH0000 + // 00000000 00ffffff 000000dd EEEE0000 + // 00000000 00cccccc 000000aa BBBB0000 + + const __m128i t3 = _mm_mullo_epi16(t2, _mm_set1_epi32(0x01000010)); + // 00llllll 00000000 00jjKKKK 00000000 + // 00iiiiii 00000000 00ggHHHH 00000000 + // 00ffffff 00000000 00ddEEEE 00000000 + // 00cccccc 00000000 00aaBBBB 00000000 + + return _mm_or_si128(t1, t3); + // 00llllll 00kkkkLL 00jjKKKK 00JJJJJJ + // 00iiiiii 00hhhhII 00ggHHHH 00GGGGGG + // 00ffffff 00eeeeFF 00ddEEEE 00DDDDDD + // 00cccccc 00bbbbCC 00aaBBBB 00AAAAAA +} diff --git a/mypyc/lib-rt/base64/arch/ssse3/enc_translate.c b/mypyc/lib-rt/base64/arch/ssse3/enc_translate.c new file mode 100644 index 0000000000000..60d9a42b8a303 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/ssse3/enc_translate.c @@ -0,0 +1,33 @@ +static BASE64_FORCE_INLINE __m128i +enc_translate (const __m128i in) +{ + // A lookup table containing the absolute offsets for all ranges: + const __m128i lut = _mm_setr_epi8( + 65, 71, -4, -4, + -4, -4, -4, -4, + -4, -4, -4, -4, + -19, -16, 0, 0 + ); + + // Translate values 0..63 to the Base64 alphabet. There are five sets: + // # From To Abs Index Characters + // 0 [0..25] [65..90] +65 0 ABCDEFGHIJKLMNOPQRSTUVWXYZ + // 1 [26..51] [97..122] +71 1 abcdefghijklmnopqrstuvwxyz + // 2 [52..61] [48..57] -4 [2..11] 0123456789 + // 3 [62] [43] -19 12 + + // 4 [63] [47] -16 13 / + + // Create LUT indices from the input. The index for range #0 is right, + // others are 1 less than expected: + __m128i indices = _mm_subs_epu8(in, _mm_set1_epi8(51)); + + // mask is 0xFF (-1) for range #[1..4] and 0x00 for range #0: + __m128i mask = _mm_cmpgt_epi8(in, _mm_set1_epi8(25)); + + // Subtract -1, so add 1 to indices for range #[1..4]. All indices are + // now correct: + indices = _mm_sub_epi8(indices, mask); + + // Add offsets to input values: + return _mm_add_epi8(in, _mm_shuffle_epi8(lut, indices)); +} diff --git a/mypyc/lib-rt/base64/codec_choose.c b/mypyc/lib-rt/base64/codec_choose.c new file mode 100644 index 0000000000000..74b0aac7b2d29 --- /dev/null +++ b/mypyc/lib-rt/base64/codec_choose.c @@ -0,0 +1,314 @@ +#include +#include +#include +#include +#include + +#include "libbase64.h" +#include "codecs.h" +#include "config.h" +#include "env.h" + +#if (__x86_64__ || __i386__ || _M_X86 || _M_X64) + #define BASE64_X86 + #if (HAVE_SSSE3 || HAVE_SSE41 || HAVE_SSE42 || HAVE_AVX || HAVE_AVX2 || HAVE_AVX512) + #define BASE64_X86_SIMD + #endif +#endif + +#ifdef BASE64_X86 +#ifdef _MSC_VER + #include + #define __cpuid_count(__level, __count, __eax, __ebx, __ecx, __edx) \ + { \ + int info[4]; \ + __cpuidex(info, __level, __count); \ + __eax = info[0]; \ + __ebx = info[1]; \ + __ecx = info[2]; \ + __edx = info[3]; \ + } + #define __cpuid(__level, __eax, __ebx, __ecx, __edx) \ + __cpuid_count(__level, 0, __eax, __ebx, __ecx, __edx) +#else + #include + #if HAVE_AVX512 || HAVE_AVX2 || HAVE_AVX + #if ((__GNUC__ > 4 || __GNUC__ == 4 && __GNUC_MINOR__ >= 2) || (__clang_major__ >= 3)) + static inline uint64_t _xgetbv (uint32_t index) + { + uint32_t eax, edx; + __asm__ __volatile__("xgetbv" : "=a"(eax), "=d"(edx) : "c"(index)); + return ((uint64_t)edx << 32) | eax; + } + #else + #error "Platform not supported" + #endif + #endif +#endif + +#ifndef bit_AVX512vl +#define bit_AVX512vl (1 << 31) +#endif +#ifndef bit_AVX512vbmi +#define bit_AVX512vbmi (1 << 1) +#endif +#ifndef bit_AVX2 +#define bit_AVX2 (1 << 5) +#endif +#ifndef bit_SSSE3 +#define bit_SSSE3 (1 << 9) +#endif +#ifndef bit_SSE41 +#define bit_SSE41 (1 << 19) +#endif +#ifndef bit_SSE42 +#define bit_SSE42 (1 << 20) +#endif +#ifndef bit_AVX +#define bit_AVX (1 << 28) +#endif + +#define bit_XSAVE_XRSTORE (1 << 27) + +#ifndef _XCR_XFEATURE_ENABLED_MASK +#define _XCR_XFEATURE_ENABLED_MASK 0 +#endif + +#define bit_XMM (1 << 1) +#define bit_YMM (1 << 2) +#define bit_OPMASK (1 << 5) +#define bit_ZMM (1 << 6) +#define bit_HIGH_ZMM (1 << 7) + +#define _XCR_XMM_AND_YMM_STATE_ENABLED_BY_OS (bit_XMM | bit_YMM) + +#define _AVX_512_ENABLED_BY_OS (bit_XMM | bit_YMM | bit_OPMASK | bit_ZMM | bit_HIGH_ZMM) + +#endif + +// Function declarations: +#define BASE64_CODEC_FUNCS(arch) \ + extern void base64_stream_encode_ ## arch BASE64_ENC_PARAMS; \ + extern int base64_stream_decode_ ## arch BASE64_DEC_PARAMS; + +BASE64_CODEC_FUNCS(avx512) +BASE64_CODEC_FUNCS(avx2) +BASE64_CODEC_FUNCS(neon32) +BASE64_CODEC_FUNCS(neon64) +BASE64_CODEC_FUNCS(plain) +BASE64_CODEC_FUNCS(ssse3) +BASE64_CODEC_FUNCS(sse41) +BASE64_CODEC_FUNCS(sse42) +BASE64_CODEC_FUNCS(avx) + +static bool +codec_choose_forced (struct codec *codec, int flags) +{ + // If the user wants to use a certain codec, + // always allow it, even if the codec is a no-op. + // For testing purposes. + + if (!(flags & 0xFFFF)) { + return false; + } + + if (flags & BASE64_FORCE_AVX2) { + codec->enc = base64_stream_encode_avx2; + codec->dec = base64_stream_decode_avx2; + return true; + } + if (flags & BASE64_FORCE_NEON32) { + codec->enc = base64_stream_encode_neon32; + codec->dec = base64_stream_decode_neon32; + return true; + } + if (flags & BASE64_FORCE_NEON64) { + codec->enc = base64_stream_encode_neon64; + codec->dec = base64_stream_decode_neon64; + return true; + } + if (flags & BASE64_FORCE_PLAIN) { + codec->enc = base64_stream_encode_plain; + codec->dec = base64_stream_decode_plain; + return true; + } + if (flags & BASE64_FORCE_SSSE3) { + codec->enc = base64_stream_encode_ssse3; + codec->dec = base64_stream_decode_ssse3; + return true; + } + if (flags & BASE64_FORCE_SSE41) { + codec->enc = base64_stream_encode_sse41; + codec->dec = base64_stream_decode_sse41; + return true; + } + if (flags & BASE64_FORCE_SSE42) { + codec->enc = base64_stream_encode_sse42; + codec->dec = base64_stream_decode_sse42; + return true; + } + if (flags & BASE64_FORCE_AVX) { + codec->enc = base64_stream_encode_avx; + codec->dec = base64_stream_decode_avx; + return true; + } + if (flags & BASE64_FORCE_AVX512) { + codec->enc = base64_stream_encode_avx512; + codec->dec = base64_stream_decode_avx512; + return true; + } + return false; +} + +static bool +codec_choose_arm (struct codec *codec) +{ +#if HAVE_NEON64 || ((defined(__ARM_NEON__) || defined(__ARM_NEON)) && HAVE_NEON32) + + // Unfortunately there is no portable way to check for NEON + // support at runtime from userland in the same way that x86 + // has cpuid, so just stick to the compile-time configuration: + + #if HAVE_NEON64 + codec->enc = base64_stream_encode_neon64; + codec->dec = base64_stream_decode_neon64; + #else + codec->enc = base64_stream_encode_neon32; + codec->dec = base64_stream_decode_neon32; + #endif + + return true; + +#else + (void)codec; + return false; +#endif +} + +static bool +codec_choose_x86 (struct codec *codec) +{ +#ifdef BASE64_X86_SIMD + + unsigned int eax, ebx = 0, ecx = 0, edx; + unsigned int max_level; + + #ifdef _MSC_VER + int info[4]; + __cpuidex(info, 0, 0); + max_level = info[0]; + #else + max_level = __get_cpuid_max(0, NULL); + #endif + + #if HAVE_AVX512 || HAVE_AVX2 || HAVE_AVX + // Check for AVX/AVX2/AVX512 support: + // Checking for AVX requires 3 things: + // 1) CPUID indicates that the OS uses XSAVE and XRSTORE instructions + // (allowing saving YMM registers on context switch) + // 2) CPUID indicates support for AVX + // 3) XGETBV indicates the AVX registers will be saved and restored on + // context switch + // + // Note that XGETBV is only available on 686 or later CPUs, so the + // instruction needs to be conditionally run. + if (max_level >= 1) { + __cpuid_count(1, 0, eax, ebx, ecx, edx); + if (ecx & bit_XSAVE_XRSTORE) { + uint64_t xcr_mask; + xcr_mask = _xgetbv(_XCR_XFEATURE_ENABLED_MASK); + if ((xcr_mask & _XCR_XMM_AND_YMM_STATE_ENABLED_BY_OS) == _XCR_XMM_AND_YMM_STATE_ENABLED_BY_OS) { // check multiple bits at once + #if HAVE_AVX512 + if (max_level >= 7 && ((xcr_mask & _AVX_512_ENABLED_BY_OS) == _AVX_512_ENABLED_BY_OS)) { + __cpuid_count(7, 0, eax, ebx, ecx, edx); + if ((ebx & bit_AVX512vl) && (ecx & bit_AVX512vbmi)) { + codec->enc = base64_stream_encode_avx512; + codec->dec = base64_stream_decode_avx512; + return true; + } + } + #endif + #if HAVE_AVX2 + if (max_level >= 7) { + __cpuid_count(7, 0, eax, ebx, ecx, edx); + if (ebx & bit_AVX2) { + codec->enc = base64_stream_encode_avx2; + codec->dec = base64_stream_decode_avx2; + return true; + } + } + #endif + #if HAVE_AVX + __cpuid_count(1, 0, eax, ebx, ecx, edx); + if (ecx & bit_AVX) { + codec->enc = base64_stream_encode_avx; + codec->dec = base64_stream_decode_avx; + return true; + } + #endif + } + } + } + #endif + + #if HAVE_SSE42 + // Check for SSE42 support: + if (max_level >= 1) { + __cpuid(1, eax, ebx, ecx, edx); + if (ecx & bit_SSE42) { + codec->enc = base64_stream_encode_sse42; + codec->dec = base64_stream_decode_sse42; + return true; + } + } + #endif + + #if HAVE_SSE41 + // Check for SSE41 support: + if (max_level >= 1) { + __cpuid(1, eax, ebx, ecx, edx); + if (ecx & bit_SSE41) { + codec->enc = base64_stream_encode_sse41; + codec->dec = base64_stream_decode_sse41; + return true; + } + } + #endif + + #if HAVE_SSSE3 + // Check for SSSE3 support: + if (max_level >= 1) { + __cpuid(1, eax, ebx, ecx, edx); + if (ecx & bit_SSSE3) { + codec->enc = base64_stream_encode_ssse3; + codec->dec = base64_stream_decode_ssse3; + return true; + } + } + #endif + +#else + (void)codec; +#endif + + return false; +} + +void +codec_choose (struct codec *codec, int flags) +{ + // User forced a codec: + if (codec_choose_forced(codec, flags)) { + return; + } + + // Runtime feature detection: + if (codec_choose_arm(codec)) { + return; + } + if (codec_choose_x86(codec)) { + return; + } + codec->enc = base64_stream_encode_plain; + codec->dec = base64_stream_decode_plain; +} diff --git a/mypyc/lib-rt/base64/codecs.h b/mypyc/lib-rt/base64/codecs.h new file mode 100644 index 0000000000000..34d54dc8fde94 --- /dev/null +++ b/mypyc/lib-rt/base64/codecs.h @@ -0,0 +1,57 @@ +#include "libbase64.h" + +// Function parameters for encoding functions: +#define BASE64_ENC_PARAMS \ + ( struct base64_state *state \ + , const char *src \ + , size_t srclen \ + , char *out \ + , size_t *outlen \ + ) + +// Function parameters for decoding functions: +#define BASE64_DEC_PARAMS \ + ( struct base64_state *state \ + , const char *src \ + , size_t srclen \ + , char *out \ + , size_t *outlen \ + ) + +// This function is used as a stub when a certain encoder is not compiled in. +// It discards the inputs and returns zero output bytes. +static inline void +base64_enc_stub BASE64_ENC_PARAMS +{ + (void) state; + (void) src; + (void) srclen; + (void) out; + + *outlen = 0; +} + +// This function is used as a stub when a certain decoder is not compiled in. +// It discards the inputs and returns an invalid decoding result. +static inline int +base64_dec_stub BASE64_DEC_PARAMS +{ + (void) state; + (void) src; + (void) srclen; + (void) out; + (void) outlen; + + return -1; +} + +typedef void (* base64_enc_fn) BASE64_ENC_PARAMS; +typedef int (* base64_dec_fn) BASE64_DEC_PARAMS; + +struct codec +{ + base64_enc_fn enc; + base64_dec_fn dec; +}; + +extern void codec_choose (struct codec *, int flags); diff --git a/mypyc/lib-rt/base64/config.h b/mypyc/lib-rt/base64/config.h new file mode 100644 index 0000000000000..fd516c4be2d60 --- /dev/null +++ b/mypyc/lib-rt/base64/config.h @@ -0,0 +1,33 @@ +#ifndef BASE64_CONFIG_H +#define BASE64_CONFIG_H + +#define BASE64_WITH_SSSE3 0 +#define HAVE_SSSE3 BASE64_WITH_SSSE3 + +#define BASE64_WITH_SSE41 0 +#define HAVE_SSE41 BASE64_WITH_SSE41 + +#define BASE64_WITH_SSE42 0 +#define HAVE_SSE42 BASE64_WITH_SSE42 + +#define BASE64_WITH_AVX 0 +#define HAVE_AVX BASE64_WITH_AVX + +#define BASE64_WITH_AVX2 0 +#define HAVE_AVX2 BASE64_WITH_AVX2 + +#define BASE64_WITH_AVX512 0 +#define HAVE_AVX512 BASE64_WITH_AVX512 + +#define BASE64_WITH_NEON32 0 +#define HAVE_NEON32 BASE64_WITH_NEON32 + +#if defined(__APPLE__) && defined(__aarch64__) +#define BASE64_WITH_NEON64 1 +#else +#define BASE64_WITH_NEON64 0 +#endif + +#define HAVE_NEON64 BASE64_WITH_NEON64 + +#endif // BASE64_CONFIG_H diff --git a/mypyc/lib-rt/base64/env.h b/mypyc/lib-rt/base64/env.h new file mode 100644 index 0000000000000..083706507900b --- /dev/null +++ b/mypyc/lib-rt/base64/env.h @@ -0,0 +1,84 @@ +#ifndef BASE64_ENV_H +#define BASE64_ENV_H + +#include + +// This header file contains macro definitions that describe certain aspects of +// the compile-time environment. Compatibility and portability macros go here. + +// Define machine endianness. This is for GCC: +#if (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) +# define BASE64_LITTLE_ENDIAN 1 +#else +# define BASE64_LITTLE_ENDIAN 0 +#endif + +// This is for Clang: +#ifdef __LITTLE_ENDIAN__ +# define BASE64_LITTLE_ENDIAN 1 +#endif + +#ifdef __BIG_ENDIAN__ +# define BASE64_LITTLE_ENDIAN 0 +#endif + +// MSVC++ needs intrin.h for _byteswap_uint64 (issue #68): +#if BASE64_LITTLE_ENDIAN && defined(_MSC_VER) +# include +#endif + +// Endian conversion functions: +#if BASE64_LITTLE_ENDIAN +# ifdef _MSC_VER +// Microsoft Visual C++: +# define BASE64_HTOBE32(x) _byteswap_ulong(x) +# define BASE64_HTOBE64(x) _byteswap_uint64(x) +# else +// GCC and Clang: +# define BASE64_HTOBE32(x) __builtin_bswap32(x) +# define BASE64_HTOBE64(x) __builtin_bswap64(x) +# endif +#else +// No conversion needed: +# define BASE64_HTOBE32(x) (x) +# define BASE64_HTOBE64(x) (x) +#endif + +// Detect word size: +#if defined (__x86_64__) +// This also works for the x32 ABI, which has a 64-bit word size. +# define BASE64_WORDSIZE 64 +#elif SIZE_MAX == UINT32_MAX +# define BASE64_WORDSIZE 32 +#elif SIZE_MAX == UINT64_MAX +# define BASE64_WORDSIZE 64 +#else +# error BASE64_WORDSIZE_NOT_DEFINED +#endif + +// End-of-file definitions. +// Almost end-of-file when waiting for the last '=' character: +#define BASE64_AEOF 1 +// End-of-file when stream end has been reached or invalid input provided: +#define BASE64_EOF 2 + +// GCC 7 defaults to issuing a warning for fallthrough in switch statements, +// unless the fallthrough cases are marked with an attribute. As we use +// fallthrough deliberately, define an alias for the attribute: +#if __GNUC__ >= 7 +# define BASE64_FALLTHROUGH __attribute__((fallthrough)); +#else +# define BASE64_FALLTHROUGH +#endif + +// Declare macros to ensure that functions that are intended to be inlined, are +// actually inlined, even when no optimization is applied. A lot of inner loop +// code is factored into separate functions for reasons of readability, but +// that code should always be inlined (and optimized) in the main loop. +#ifdef _MSC_VER +# define BASE64_FORCE_INLINE __forceinline +#else +# define BASE64_FORCE_INLINE inline __attribute__((always_inline)) +#endif + +#endif // BASE64_ENV_H diff --git a/mypyc/lib-rt/base64/lib.c b/mypyc/lib-rt/base64/lib.c new file mode 100644 index 0000000000000..0f24d52e99912 --- /dev/null +++ b/mypyc/lib-rt/base64/lib.c @@ -0,0 +1,164 @@ +#include +#include +#ifdef _OPENMP +#include +#endif + +#include "libbase64.h" +#include "tables/tables.h" +#include "codecs.h" +#include "env.h" + +// These static function pointers are initialized once when the library is +// first used, and remain in use for the remaining lifetime of the program. +// The idea being that CPU features don't change at runtime. +static struct codec codec = { NULL, NULL }; + +void +base64_stream_encode_init (struct base64_state *state, int flags) +{ + // If any of the codec flags are set, redo choice: + if (codec.enc == NULL || flags & 0xFF) { + codec_choose(&codec, flags); + } + state->eof = 0; + state->bytes = 0; + state->carry = 0; + state->flags = flags; +} + +void +base64_stream_encode + ( struct base64_state *state + , const char *src + , size_t srclen + , char *out + , size_t *outlen + ) +{ + codec.enc(state, src, srclen, out, outlen); +} + +void +base64_stream_encode_final + ( struct base64_state *state + , char *out + , size_t *outlen + ) +{ + uint8_t *o = (uint8_t *)out; + + if (state->bytes == 1) { + *o++ = base64_table_enc_6bit[state->carry]; + *o++ = '='; + *o++ = '='; + *outlen = 3; + return; + } + if (state->bytes == 2) { + *o++ = base64_table_enc_6bit[state->carry]; + *o++ = '='; + *outlen = 2; + return; + } + *outlen = 0; +} + +void +base64_stream_decode_init (struct base64_state *state, int flags) +{ + // If any of the codec flags are set, redo choice: + if (codec.dec == NULL || flags & 0xFFFF) { + codec_choose(&codec, flags); + } + state->eof = 0; + state->bytes = 0; + state->carry = 0; + state->flags = flags; +} + +int +base64_stream_decode + ( struct base64_state *state + , const char *src + , size_t srclen + , char *out + , size_t *outlen + ) +{ + return codec.dec(state, src, srclen, out, outlen); +} + +#ifdef _OPENMP + + // Due to the overhead of initializing OpenMP and creating a team of + // threads, we require the data length to be larger than a threshold: + #define OMP_THRESHOLD 20000 + + // Conditionally include OpenMP-accelerated codec implementations: + #include "lib_openmp.c" +#endif + +void +base64_encode + ( const char *src + , size_t srclen + , char *out + , size_t *outlen + , int flags + ) +{ + size_t s; + size_t t; + struct base64_state state; + + #ifdef _OPENMP + if (srclen >= OMP_THRESHOLD) { + base64_encode_openmp(src, srclen, out, outlen, flags); + return; + } + #endif + + // Init the stream reader: + base64_stream_encode_init(&state, flags); + + // Feed the whole string to the stream reader: + base64_stream_encode(&state, src, srclen, out, &s); + + // Finalize the stream by writing trailer if any: + base64_stream_encode_final(&state, out + s, &t); + + // Final output length is stream length plus tail: + *outlen = s + t; +} + +int +base64_decode + ( const char *src + , size_t srclen + , char *out + , size_t *outlen + , int flags + ) +{ + int ret; + struct base64_state state; + + #ifdef _OPENMP + if (srclen >= OMP_THRESHOLD) { + return base64_decode_openmp(src, srclen, out, outlen, flags); + } + #endif + + // Init the stream reader: + base64_stream_decode_init(&state, flags); + + // Feed the whole string to the stream reader: + ret = base64_stream_decode(&state, src, srclen, out, outlen); + + // If when decoding a whole block, we're still waiting for input then fail: + if (ret && (state.bytes == 0)) { + return ret; + } + return 0; +} diff --git a/mypyc/lib-rt/base64/libbase64.h b/mypyc/lib-rt/base64/libbase64.h new file mode 100644 index 0000000000000..c5908973c5e73 --- /dev/null +++ b/mypyc/lib-rt/base64/libbase64.h @@ -0,0 +1,146 @@ +#ifndef LIBBASE64_H +#define LIBBASE64_H + +#include /* size_t */ + + +#if defined(_WIN32) || defined(__CYGWIN__) +#define BASE64_SYMBOL_IMPORT __declspec(dllimport) +#define BASE64_SYMBOL_EXPORT __declspec(dllexport) +#define BASE64_SYMBOL_PRIVATE + +#elif __GNUC__ >= 4 +#define BASE64_SYMBOL_IMPORT __attribute__ ((visibility ("default"))) +#define BASE64_SYMBOL_EXPORT __attribute__ ((visibility ("default"))) +#define BASE64_SYMBOL_PRIVATE __attribute__ ((visibility ("hidden"))) + +#else +#define BASE64_SYMBOL_IMPORT +#define BASE64_SYMBOL_EXPORT +#define BASE64_SYMBOL_PRIVATE +#endif + +#if defined(BASE64_STATIC_DEFINE) +#define BASE64_EXPORT +#define BASE64_NO_EXPORT + +#else +#if defined(BASE64_EXPORTS) // defined if we are building the shared library +#define BASE64_EXPORT BASE64_SYMBOL_EXPORT + +#else +#define BASE64_EXPORT BASE64_SYMBOL_IMPORT +#endif + +#define BASE64_NO_EXPORT BASE64_SYMBOL_PRIVATE +#endif + + +#ifdef __cplusplus +extern "C" { +#endif + +/* These are the flags that can be passed in the `flags` argument. The values + * below force the use of a given codec, even if that codec is a no-op in the + * current build. Used in testing. Set to 0 for the default behavior, which is + * runtime feature detection on x86, a compile-time fixed codec on ARM, and + * the plain codec on other platforms: */ +#define BASE64_FORCE_AVX2 (1 << 0) +#define BASE64_FORCE_NEON32 (1 << 1) +#define BASE64_FORCE_NEON64 (1 << 2) +#define BASE64_FORCE_PLAIN (1 << 3) +#define BASE64_FORCE_SSSE3 (1 << 4) +#define BASE64_FORCE_SSE41 (1 << 5) +#define BASE64_FORCE_SSE42 (1 << 6) +#define BASE64_FORCE_AVX (1 << 7) +#define BASE64_FORCE_AVX512 (1 << 8) + +struct base64_state { + int eof; + int bytes; + int flags; + unsigned char carry; +}; + +/* Wrapper function to encode a plain string of given length. Output is written + * to *out without trailing zero. Output length in bytes is written to *outlen. + * The buffer in `out` has been allocated by the caller and is at least 4/3 the + * size of the input. See above for `flags`; set to 0 for default operation: */ +void BASE64_EXPORT base64_encode + ( const char *src + , size_t srclen + , char *out + , size_t *outlen + , int flags + ) ; + +/* Call this before calling base64_stream_encode() to init the state. See above + * for `flags`; set to 0 for default operation: */ +void BASE64_EXPORT base64_stream_encode_init + ( struct base64_state *state + , int flags + ) ; + +/* Encodes the block of data of given length at `src`, into the buffer at + * `out`. Caller is responsible for allocating a large enough out-buffer; it + * must be at least 4/3 the size of the in-buffer, but take some margin. Places + * the number of new bytes written into `outlen` (which is set to zero when the + * function starts). Does not zero-terminate or finalize the output. */ +void BASE64_EXPORT base64_stream_encode + ( struct base64_state *state + , const char *src + , size_t srclen + , char *out + , size_t *outlen + ) ; + +/* Finalizes the output begun by previous calls to `base64_stream_encode()`. + * Adds the required end-of-stream markers if appropriate. `outlen` is modified + * and will contain the number of new bytes written at `out` (which will quite + * often be zero). */ +void BASE64_EXPORT base64_stream_encode_final + ( struct base64_state *state + , char *out + , size_t *outlen + ) ; + +/* Wrapper function to decode a plain string of given length. Output is written + * to *out without trailing zero. Output length in bytes is written to *outlen. + * The buffer in `out` has been allocated by the caller and is at least 3/4 the + * size of the input. See above for `flags`, set to 0 for default operation: */ +int BASE64_EXPORT base64_decode + ( const char *src + , size_t srclen + , char *out + , size_t *outlen + , int flags + ) ; + +/* Call this before calling base64_stream_decode() to init the state. See above + * for `flags`; set to 0 for default operation: */ +void BASE64_EXPORT base64_stream_decode_init + ( struct base64_state *state + , int flags + ) ; + +/* Decodes the block of data of given length at `src`, into the buffer at + * `out`. Caller is responsible for allocating a large enough out-buffer; it + * must be at least 3/4 the size of the in-buffer, but take some margin. Places + * the number of new bytes written into `outlen` (which is set to zero when the + * function starts). Does not zero-terminate the output. Returns 1 if all is + * well, and 0 if a decoding error was found, such as an invalid character. + * Returns -1 if the chosen codec is not included in the current build. Used by + * the test harness to check whether a codec is available for testing. */ +int BASE64_EXPORT base64_stream_decode + ( struct base64_state *state + , const char *src + , size_t srclen + , char *out + , size_t *outlen + ) ; + +#ifdef __cplusplus +} +#endif + +#endif /* LIBBASE64_H */ diff --git a/mypyc/lib-rt/base64/tables/table_dec_32bit.h b/mypyc/lib-rt/base64/tables/table_dec_32bit.h new file mode 100644 index 0000000000000..f5d951fa79c71 --- /dev/null +++ b/mypyc/lib-rt/base64/tables/table_dec_32bit.h @@ -0,0 +1,393 @@ +#include +#define CHAR62 '+' +#define CHAR63 '/' +#define CHARPAD '=' + + +#if BASE64_LITTLE_ENDIAN + + +/* SPECIAL DECODE TABLES FOR LITTLE ENDIAN (INTEL) CPUS */ + +const uint32_t base64_table_dec_32bit_d0[256] = { +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0x000000f8, 0xffffffff, 0xffffffff, 0xffffffff, 0x000000fc, +0x000000d0, 0x000000d4, 0x000000d8, 0x000000dc, 0x000000e0, 0x000000e4, +0x000000e8, 0x000000ec, 0x000000f0, 0x000000f4, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000, +0x00000004, 0x00000008, 0x0000000c, 0x00000010, 0x00000014, 0x00000018, +0x0000001c, 0x00000020, 0x00000024, 0x00000028, 0x0000002c, 0x00000030, +0x00000034, 0x00000038, 0x0000003c, 0x00000040, 0x00000044, 0x00000048, +0x0000004c, 0x00000050, 0x00000054, 0x00000058, 0x0000005c, 0x00000060, +0x00000064, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0x00000068, 0x0000006c, 0x00000070, 0x00000074, 0x00000078, +0x0000007c, 0x00000080, 0x00000084, 0x00000088, 0x0000008c, 0x00000090, +0x00000094, 0x00000098, 0x0000009c, 0x000000a0, 0x000000a4, 0x000000a8, +0x000000ac, 0x000000b0, 0x000000b4, 0x000000b8, 0x000000bc, 0x000000c0, +0x000000c4, 0x000000c8, 0x000000cc, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff +}; + + +const uint32_t base64_table_dec_32bit_d1[256] = { +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0x0000e003, 0xffffffff, 0xffffffff, 0xffffffff, 0x0000f003, +0x00004003, 0x00005003, 0x00006003, 0x00007003, 0x00008003, 0x00009003, +0x0000a003, 0x0000b003, 0x0000c003, 0x0000d003, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000, +0x00001000, 0x00002000, 0x00003000, 0x00004000, 0x00005000, 0x00006000, +0x00007000, 0x00008000, 0x00009000, 0x0000a000, 0x0000b000, 0x0000c000, +0x0000d000, 0x0000e000, 0x0000f000, 0x00000001, 0x00001001, 0x00002001, +0x00003001, 0x00004001, 0x00005001, 0x00006001, 0x00007001, 0x00008001, +0x00009001, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0x0000a001, 0x0000b001, 0x0000c001, 0x0000d001, 0x0000e001, +0x0000f001, 0x00000002, 0x00001002, 0x00002002, 0x00003002, 0x00004002, +0x00005002, 0x00006002, 0x00007002, 0x00008002, 0x00009002, 0x0000a002, +0x0000b002, 0x0000c002, 0x0000d002, 0x0000e002, 0x0000f002, 0x00000003, +0x00001003, 0x00002003, 0x00003003, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff +}; + + +const uint32_t base64_table_dec_32bit_d2[256] = { +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0x00800f00, 0xffffffff, 0xffffffff, 0xffffffff, 0x00c00f00, +0x00000d00, 0x00400d00, 0x00800d00, 0x00c00d00, 0x00000e00, 0x00400e00, +0x00800e00, 0x00c00e00, 0x00000f00, 0x00400f00, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000, +0x00400000, 0x00800000, 0x00c00000, 0x00000100, 0x00400100, 0x00800100, +0x00c00100, 0x00000200, 0x00400200, 0x00800200, 0x00c00200, 0x00000300, +0x00400300, 0x00800300, 0x00c00300, 0x00000400, 0x00400400, 0x00800400, +0x00c00400, 0x00000500, 0x00400500, 0x00800500, 0x00c00500, 0x00000600, +0x00400600, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0x00800600, 0x00c00600, 0x00000700, 0x00400700, 0x00800700, +0x00c00700, 0x00000800, 0x00400800, 0x00800800, 0x00c00800, 0x00000900, +0x00400900, 0x00800900, 0x00c00900, 0x00000a00, 0x00400a00, 0x00800a00, +0x00c00a00, 0x00000b00, 0x00400b00, 0x00800b00, 0x00c00b00, 0x00000c00, +0x00400c00, 0x00800c00, 0x00c00c00, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff +}; + + +const uint32_t base64_table_dec_32bit_d3[256] = { +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0x003e0000, 0xffffffff, 0xffffffff, 0xffffffff, 0x003f0000, +0x00340000, 0x00350000, 0x00360000, 0x00370000, 0x00380000, 0x00390000, +0x003a0000, 0x003b0000, 0x003c0000, 0x003d0000, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000, +0x00010000, 0x00020000, 0x00030000, 0x00040000, 0x00050000, 0x00060000, +0x00070000, 0x00080000, 0x00090000, 0x000a0000, 0x000b0000, 0x000c0000, +0x000d0000, 0x000e0000, 0x000f0000, 0x00100000, 0x00110000, 0x00120000, +0x00130000, 0x00140000, 0x00150000, 0x00160000, 0x00170000, 0x00180000, +0x00190000, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0x001a0000, 0x001b0000, 0x001c0000, 0x001d0000, 0x001e0000, +0x001f0000, 0x00200000, 0x00210000, 0x00220000, 0x00230000, 0x00240000, +0x00250000, 0x00260000, 0x00270000, 0x00280000, 0x00290000, 0x002a0000, +0x002b0000, 0x002c0000, 0x002d0000, 0x002e0000, 0x002f0000, 0x00300000, +0x00310000, 0x00320000, 0x00330000, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff +}; + + +#else + + +/* SPECIAL DECODE TABLES FOR BIG ENDIAN (IBM/MOTOROLA/SUN) CPUS */ + +const uint32_t base64_table_dec_32bit_d0[256] = { +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xf8000000, 0xffffffff, 0xffffffff, 0xffffffff, 0xfc000000, +0xd0000000, 0xd4000000, 0xd8000000, 0xdc000000, 0xe0000000, 0xe4000000, +0xe8000000, 0xec000000, 0xf0000000, 0xf4000000, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000, +0x04000000, 0x08000000, 0x0c000000, 0x10000000, 0x14000000, 0x18000000, +0x1c000000, 0x20000000, 0x24000000, 0x28000000, 0x2c000000, 0x30000000, +0x34000000, 0x38000000, 0x3c000000, 0x40000000, 0x44000000, 0x48000000, +0x4c000000, 0x50000000, 0x54000000, 0x58000000, 0x5c000000, 0x60000000, +0x64000000, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0x68000000, 0x6c000000, 0x70000000, 0x74000000, 0x78000000, +0x7c000000, 0x80000000, 0x84000000, 0x88000000, 0x8c000000, 0x90000000, +0x94000000, 0x98000000, 0x9c000000, 0xa0000000, 0xa4000000, 0xa8000000, +0xac000000, 0xb0000000, 0xb4000000, 0xb8000000, 0xbc000000, 0xc0000000, +0xc4000000, 0xc8000000, 0xcc000000, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff +}; + + +const uint32_t base64_table_dec_32bit_d1[256] = { +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0x03e00000, 0xffffffff, 0xffffffff, 0xffffffff, 0x03f00000, +0x03400000, 0x03500000, 0x03600000, 0x03700000, 0x03800000, 0x03900000, +0x03a00000, 0x03b00000, 0x03c00000, 0x03d00000, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000, +0x00100000, 0x00200000, 0x00300000, 0x00400000, 0x00500000, 0x00600000, +0x00700000, 0x00800000, 0x00900000, 0x00a00000, 0x00b00000, 0x00c00000, +0x00d00000, 0x00e00000, 0x00f00000, 0x01000000, 0x01100000, 0x01200000, +0x01300000, 0x01400000, 0x01500000, 0x01600000, 0x01700000, 0x01800000, +0x01900000, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0x01a00000, 0x01b00000, 0x01c00000, 0x01d00000, 0x01e00000, +0x01f00000, 0x02000000, 0x02100000, 0x02200000, 0x02300000, 0x02400000, +0x02500000, 0x02600000, 0x02700000, 0x02800000, 0x02900000, 0x02a00000, +0x02b00000, 0x02c00000, 0x02d00000, 0x02e00000, 0x02f00000, 0x03000000, +0x03100000, 0x03200000, 0x03300000, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff +}; + + +const uint32_t base64_table_dec_32bit_d2[256] = { +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0x000f8000, 0xffffffff, 0xffffffff, 0xffffffff, 0x000fc000, +0x000d0000, 0x000d4000, 0x000d8000, 0x000dc000, 0x000e0000, 0x000e4000, +0x000e8000, 0x000ec000, 0x000f0000, 0x000f4000, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000, +0x00004000, 0x00008000, 0x0000c000, 0x00010000, 0x00014000, 0x00018000, +0x0001c000, 0x00020000, 0x00024000, 0x00028000, 0x0002c000, 0x00030000, +0x00034000, 0x00038000, 0x0003c000, 0x00040000, 0x00044000, 0x00048000, +0x0004c000, 0x00050000, 0x00054000, 0x00058000, 0x0005c000, 0x00060000, +0x00064000, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0x00068000, 0x0006c000, 0x00070000, 0x00074000, 0x00078000, +0x0007c000, 0x00080000, 0x00084000, 0x00088000, 0x0008c000, 0x00090000, +0x00094000, 0x00098000, 0x0009c000, 0x000a0000, 0x000a4000, 0x000a8000, +0x000ac000, 0x000b0000, 0x000b4000, 0x000b8000, 0x000bc000, 0x000c0000, +0x000c4000, 0x000c8000, 0x000cc000, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff +}; + + +const uint32_t base64_table_dec_32bit_d3[256] = { +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0x00003e00, 0xffffffff, 0xffffffff, 0xffffffff, 0x00003f00, +0x00003400, 0x00003500, 0x00003600, 0x00003700, 0x00003800, 0x00003900, +0x00003a00, 0x00003b00, 0x00003c00, 0x00003d00, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000, +0x00000100, 0x00000200, 0x00000300, 0x00000400, 0x00000500, 0x00000600, +0x00000700, 0x00000800, 0x00000900, 0x00000a00, 0x00000b00, 0x00000c00, +0x00000d00, 0x00000e00, 0x00000f00, 0x00001000, 0x00001100, 0x00001200, +0x00001300, 0x00001400, 0x00001500, 0x00001600, 0x00001700, 0x00001800, +0x00001900, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0x00001a00, 0x00001b00, 0x00001c00, 0x00001d00, 0x00001e00, +0x00001f00, 0x00002000, 0x00002100, 0x00002200, 0x00002300, 0x00002400, +0x00002500, 0x00002600, 0x00002700, 0x00002800, 0x00002900, 0x00002a00, +0x00002b00, 0x00002c00, 0x00002d00, 0x00002e00, 0x00002f00, 0x00003000, +0x00003100, 0x00003200, 0x00003300, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff +}; + + +#endif diff --git a/mypyc/lib-rt/base64/tables/table_enc_12bit.h b/mypyc/lib-rt/base64/tables/table_enc_12bit.h new file mode 100644 index 0000000000000..2bc0d23068753 --- /dev/null +++ b/mypyc/lib-rt/base64/tables/table_enc_12bit.h @@ -0,0 +1,1031 @@ +#include + +const uint16_t base64_table_enc_12bit[] = { +#if BASE64_LITTLE_ENDIAN + 0x4141U, 0x4241U, 0x4341U, 0x4441U, 0x4541U, 0x4641U, 0x4741U, 0x4841U, + 0x4941U, 0x4A41U, 0x4B41U, 0x4C41U, 0x4D41U, 0x4E41U, 0x4F41U, 0x5041U, + 0x5141U, 0x5241U, 0x5341U, 0x5441U, 0x5541U, 0x5641U, 0x5741U, 0x5841U, + 0x5941U, 0x5A41U, 0x6141U, 0x6241U, 0x6341U, 0x6441U, 0x6541U, 0x6641U, + 0x6741U, 0x6841U, 0x6941U, 0x6A41U, 0x6B41U, 0x6C41U, 0x6D41U, 0x6E41U, + 0x6F41U, 0x7041U, 0x7141U, 0x7241U, 0x7341U, 0x7441U, 0x7541U, 0x7641U, + 0x7741U, 0x7841U, 0x7941U, 0x7A41U, 0x3041U, 0x3141U, 0x3241U, 0x3341U, + 0x3441U, 0x3541U, 0x3641U, 0x3741U, 0x3841U, 0x3941U, 0x2B41U, 0x2F41U, + 0x4142U, 0x4242U, 0x4342U, 0x4442U, 0x4542U, 0x4642U, 0x4742U, 0x4842U, + 0x4942U, 0x4A42U, 0x4B42U, 0x4C42U, 0x4D42U, 0x4E42U, 0x4F42U, 0x5042U, + 0x5142U, 0x5242U, 0x5342U, 0x5442U, 0x5542U, 0x5642U, 0x5742U, 0x5842U, + 0x5942U, 0x5A42U, 0x6142U, 0x6242U, 0x6342U, 0x6442U, 0x6542U, 0x6642U, + 0x6742U, 0x6842U, 0x6942U, 0x6A42U, 0x6B42U, 0x6C42U, 0x6D42U, 0x6E42U, + 0x6F42U, 0x7042U, 0x7142U, 0x7242U, 0x7342U, 0x7442U, 0x7542U, 0x7642U, + 0x7742U, 0x7842U, 0x7942U, 0x7A42U, 0x3042U, 0x3142U, 0x3242U, 0x3342U, + 0x3442U, 0x3542U, 0x3642U, 0x3742U, 0x3842U, 0x3942U, 0x2B42U, 0x2F42U, + 0x4143U, 0x4243U, 0x4343U, 0x4443U, 0x4543U, 0x4643U, 0x4743U, 0x4843U, + 0x4943U, 0x4A43U, 0x4B43U, 0x4C43U, 0x4D43U, 0x4E43U, 0x4F43U, 0x5043U, + 0x5143U, 0x5243U, 0x5343U, 0x5443U, 0x5543U, 0x5643U, 0x5743U, 0x5843U, + 0x5943U, 0x5A43U, 0x6143U, 0x6243U, 0x6343U, 0x6443U, 0x6543U, 0x6643U, + 0x6743U, 0x6843U, 0x6943U, 0x6A43U, 0x6B43U, 0x6C43U, 0x6D43U, 0x6E43U, + 0x6F43U, 0x7043U, 0x7143U, 0x7243U, 0x7343U, 0x7443U, 0x7543U, 0x7643U, + 0x7743U, 0x7843U, 0x7943U, 0x7A43U, 0x3043U, 0x3143U, 0x3243U, 0x3343U, + 0x3443U, 0x3543U, 0x3643U, 0x3743U, 0x3843U, 0x3943U, 0x2B43U, 0x2F43U, + 0x4144U, 0x4244U, 0x4344U, 0x4444U, 0x4544U, 0x4644U, 0x4744U, 0x4844U, + 0x4944U, 0x4A44U, 0x4B44U, 0x4C44U, 0x4D44U, 0x4E44U, 0x4F44U, 0x5044U, + 0x5144U, 0x5244U, 0x5344U, 0x5444U, 0x5544U, 0x5644U, 0x5744U, 0x5844U, + 0x5944U, 0x5A44U, 0x6144U, 0x6244U, 0x6344U, 0x6444U, 0x6544U, 0x6644U, + 0x6744U, 0x6844U, 0x6944U, 0x6A44U, 0x6B44U, 0x6C44U, 0x6D44U, 0x6E44U, + 0x6F44U, 0x7044U, 0x7144U, 0x7244U, 0x7344U, 0x7444U, 0x7544U, 0x7644U, + 0x7744U, 0x7844U, 0x7944U, 0x7A44U, 0x3044U, 0x3144U, 0x3244U, 0x3344U, + 0x3444U, 0x3544U, 0x3644U, 0x3744U, 0x3844U, 0x3944U, 0x2B44U, 0x2F44U, + 0x4145U, 0x4245U, 0x4345U, 0x4445U, 0x4545U, 0x4645U, 0x4745U, 0x4845U, + 0x4945U, 0x4A45U, 0x4B45U, 0x4C45U, 0x4D45U, 0x4E45U, 0x4F45U, 0x5045U, + 0x5145U, 0x5245U, 0x5345U, 0x5445U, 0x5545U, 0x5645U, 0x5745U, 0x5845U, + 0x5945U, 0x5A45U, 0x6145U, 0x6245U, 0x6345U, 0x6445U, 0x6545U, 0x6645U, + 0x6745U, 0x6845U, 0x6945U, 0x6A45U, 0x6B45U, 0x6C45U, 0x6D45U, 0x6E45U, + 0x6F45U, 0x7045U, 0x7145U, 0x7245U, 0x7345U, 0x7445U, 0x7545U, 0x7645U, + 0x7745U, 0x7845U, 0x7945U, 0x7A45U, 0x3045U, 0x3145U, 0x3245U, 0x3345U, + 0x3445U, 0x3545U, 0x3645U, 0x3745U, 0x3845U, 0x3945U, 0x2B45U, 0x2F45U, + 0x4146U, 0x4246U, 0x4346U, 0x4446U, 0x4546U, 0x4646U, 0x4746U, 0x4846U, + 0x4946U, 0x4A46U, 0x4B46U, 0x4C46U, 0x4D46U, 0x4E46U, 0x4F46U, 0x5046U, + 0x5146U, 0x5246U, 0x5346U, 0x5446U, 0x5546U, 0x5646U, 0x5746U, 0x5846U, + 0x5946U, 0x5A46U, 0x6146U, 0x6246U, 0x6346U, 0x6446U, 0x6546U, 0x6646U, + 0x6746U, 0x6846U, 0x6946U, 0x6A46U, 0x6B46U, 0x6C46U, 0x6D46U, 0x6E46U, + 0x6F46U, 0x7046U, 0x7146U, 0x7246U, 0x7346U, 0x7446U, 0x7546U, 0x7646U, + 0x7746U, 0x7846U, 0x7946U, 0x7A46U, 0x3046U, 0x3146U, 0x3246U, 0x3346U, + 0x3446U, 0x3546U, 0x3646U, 0x3746U, 0x3846U, 0x3946U, 0x2B46U, 0x2F46U, + 0x4147U, 0x4247U, 0x4347U, 0x4447U, 0x4547U, 0x4647U, 0x4747U, 0x4847U, + 0x4947U, 0x4A47U, 0x4B47U, 0x4C47U, 0x4D47U, 0x4E47U, 0x4F47U, 0x5047U, + 0x5147U, 0x5247U, 0x5347U, 0x5447U, 0x5547U, 0x5647U, 0x5747U, 0x5847U, + 0x5947U, 0x5A47U, 0x6147U, 0x6247U, 0x6347U, 0x6447U, 0x6547U, 0x6647U, + 0x6747U, 0x6847U, 0x6947U, 0x6A47U, 0x6B47U, 0x6C47U, 0x6D47U, 0x6E47U, + 0x6F47U, 0x7047U, 0x7147U, 0x7247U, 0x7347U, 0x7447U, 0x7547U, 0x7647U, + 0x7747U, 0x7847U, 0x7947U, 0x7A47U, 0x3047U, 0x3147U, 0x3247U, 0x3347U, + 0x3447U, 0x3547U, 0x3647U, 0x3747U, 0x3847U, 0x3947U, 0x2B47U, 0x2F47U, + 0x4148U, 0x4248U, 0x4348U, 0x4448U, 0x4548U, 0x4648U, 0x4748U, 0x4848U, + 0x4948U, 0x4A48U, 0x4B48U, 0x4C48U, 0x4D48U, 0x4E48U, 0x4F48U, 0x5048U, + 0x5148U, 0x5248U, 0x5348U, 0x5448U, 0x5548U, 0x5648U, 0x5748U, 0x5848U, + 0x5948U, 0x5A48U, 0x6148U, 0x6248U, 0x6348U, 0x6448U, 0x6548U, 0x6648U, + 0x6748U, 0x6848U, 0x6948U, 0x6A48U, 0x6B48U, 0x6C48U, 0x6D48U, 0x6E48U, + 0x6F48U, 0x7048U, 0x7148U, 0x7248U, 0x7348U, 0x7448U, 0x7548U, 0x7648U, + 0x7748U, 0x7848U, 0x7948U, 0x7A48U, 0x3048U, 0x3148U, 0x3248U, 0x3348U, + 0x3448U, 0x3548U, 0x3648U, 0x3748U, 0x3848U, 0x3948U, 0x2B48U, 0x2F48U, + 0x4149U, 0x4249U, 0x4349U, 0x4449U, 0x4549U, 0x4649U, 0x4749U, 0x4849U, + 0x4949U, 0x4A49U, 0x4B49U, 0x4C49U, 0x4D49U, 0x4E49U, 0x4F49U, 0x5049U, + 0x5149U, 0x5249U, 0x5349U, 0x5449U, 0x5549U, 0x5649U, 0x5749U, 0x5849U, + 0x5949U, 0x5A49U, 0x6149U, 0x6249U, 0x6349U, 0x6449U, 0x6549U, 0x6649U, + 0x6749U, 0x6849U, 0x6949U, 0x6A49U, 0x6B49U, 0x6C49U, 0x6D49U, 0x6E49U, + 0x6F49U, 0x7049U, 0x7149U, 0x7249U, 0x7349U, 0x7449U, 0x7549U, 0x7649U, + 0x7749U, 0x7849U, 0x7949U, 0x7A49U, 0x3049U, 0x3149U, 0x3249U, 0x3349U, + 0x3449U, 0x3549U, 0x3649U, 0x3749U, 0x3849U, 0x3949U, 0x2B49U, 0x2F49U, + 0x414AU, 0x424AU, 0x434AU, 0x444AU, 0x454AU, 0x464AU, 0x474AU, 0x484AU, + 0x494AU, 0x4A4AU, 0x4B4AU, 0x4C4AU, 0x4D4AU, 0x4E4AU, 0x4F4AU, 0x504AU, + 0x514AU, 0x524AU, 0x534AU, 0x544AU, 0x554AU, 0x564AU, 0x574AU, 0x584AU, + 0x594AU, 0x5A4AU, 0x614AU, 0x624AU, 0x634AU, 0x644AU, 0x654AU, 0x664AU, + 0x674AU, 0x684AU, 0x694AU, 0x6A4AU, 0x6B4AU, 0x6C4AU, 0x6D4AU, 0x6E4AU, + 0x6F4AU, 0x704AU, 0x714AU, 0x724AU, 0x734AU, 0x744AU, 0x754AU, 0x764AU, + 0x774AU, 0x784AU, 0x794AU, 0x7A4AU, 0x304AU, 0x314AU, 0x324AU, 0x334AU, + 0x344AU, 0x354AU, 0x364AU, 0x374AU, 0x384AU, 0x394AU, 0x2B4AU, 0x2F4AU, + 0x414BU, 0x424BU, 0x434BU, 0x444BU, 0x454BU, 0x464BU, 0x474BU, 0x484BU, + 0x494BU, 0x4A4BU, 0x4B4BU, 0x4C4BU, 0x4D4BU, 0x4E4BU, 0x4F4BU, 0x504BU, + 0x514BU, 0x524BU, 0x534BU, 0x544BU, 0x554BU, 0x564BU, 0x574BU, 0x584BU, + 0x594BU, 0x5A4BU, 0x614BU, 0x624BU, 0x634BU, 0x644BU, 0x654BU, 0x664BU, + 0x674BU, 0x684BU, 0x694BU, 0x6A4BU, 0x6B4BU, 0x6C4BU, 0x6D4BU, 0x6E4BU, + 0x6F4BU, 0x704BU, 0x714BU, 0x724BU, 0x734BU, 0x744BU, 0x754BU, 0x764BU, + 0x774BU, 0x784BU, 0x794BU, 0x7A4BU, 0x304BU, 0x314BU, 0x324BU, 0x334BU, + 0x344BU, 0x354BU, 0x364BU, 0x374BU, 0x384BU, 0x394BU, 0x2B4BU, 0x2F4BU, + 0x414CU, 0x424CU, 0x434CU, 0x444CU, 0x454CU, 0x464CU, 0x474CU, 0x484CU, + 0x494CU, 0x4A4CU, 0x4B4CU, 0x4C4CU, 0x4D4CU, 0x4E4CU, 0x4F4CU, 0x504CU, + 0x514CU, 0x524CU, 0x534CU, 0x544CU, 0x554CU, 0x564CU, 0x574CU, 0x584CU, + 0x594CU, 0x5A4CU, 0x614CU, 0x624CU, 0x634CU, 0x644CU, 0x654CU, 0x664CU, + 0x674CU, 0x684CU, 0x694CU, 0x6A4CU, 0x6B4CU, 0x6C4CU, 0x6D4CU, 0x6E4CU, + 0x6F4CU, 0x704CU, 0x714CU, 0x724CU, 0x734CU, 0x744CU, 0x754CU, 0x764CU, + 0x774CU, 0x784CU, 0x794CU, 0x7A4CU, 0x304CU, 0x314CU, 0x324CU, 0x334CU, + 0x344CU, 0x354CU, 0x364CU, 0x374CU, 0x384CU, 0x394CU, 0x2B4CU, 0x2F4CU, + 0x414DU, 0x424DU, 0x434DU, 0x444DU, 0x454DU, 0x464DU, 0x474DU, 0x484DU, + 0x494DU, 0x4A4DU, 0x4B4DU, 0x4C4DU, 0x4D4DU, 0x4E4DU, 0x4F4DU, 0x504DU, + 0x514DU, 0x524DU, 0x534DU, 0x544DU, 0x554DU, 0x564DU, 0x574DU, 0x584DU, + 0x594DU, 0x5A4DU, 0x614DU, 0x624DU, 0x634DU, 0x644DU, 0x654DU, 0x664DU, + 0x674DU, 0x684DU, 0x694DU, 0x6A4DU, 0x6B4DU, 0x6C4DU, 0x6D4DU, 0x6E4DU, + 0x6F4DU, 0x704DU, 0x714DU, 0x724DU, 0x734DU, 0x744DU, 0x754DU, 0x764DU, + 0x774DU, 0x784DU, 0x794DU, 0x7A4DU, 0x304DU, 0x314DU, 0x324DU, 0x334DU, + 0x344DU, 0x354DU, 0x364DU, 0x374DU, 0x384DU, 0x394DU, 0x2B4DU, 0x2F4DU, + 0x414EU, 0x424EU, 0x434EU, 0x444EU, 0x454EU, 0x464EU, 0x474EU, 0x484EU, + 0x494EU, 0x4A4EU, 0x4B4EU, 0x4C4EU, 0x4D4EU, 0x4E4EU, 0x4F4EU, 0x504EU, + 0x514EU, 0x524EU, 0x534EU, 0x544EU, 0x554EU, 0x564EU, 0x574EU, 0x584EU, + 0x594EU, 0x5A4EU, 0x614EU, 0x624EU, 0x634EU, 0x644EU, 0x654EU, 0x664EU, + 0x674EU, 0x684EU, 0x694EU, 0x6A4EU, 0x6B4EU, 0x6C4EU, 0x6D4EU, 0x6E4EU, + 0x6F4EU, 0x704EU, 0x714EU, 0x724EU, 0x734EU, 0x744EU, 0x754EU, 0x764EU, + 0x774EU, 0x784EU, 0x794EU, 0x7A4EU, 0x304EU, 0x314EU, 0x324EU, 0x334EU, + 0x344EU, 0x354EU, 0x364EU, 0x374EU, 0x384EU, 0x394EU, 0x2B4EU, 0x2F4EU, + 0x414FU, 0x424FU, 0x434FU, 0x444FU, 0x454FU, 0x464FU, 0x474FU, 0x484FU, + 0x494FU, 0x4A4FU, 0x4B4FU, 0x4C4FU, 0x4D4FU, 0x4E4FU, 0x4F4FU, 0x504FU, + 0x514FU, 0x524FU, 0x534FU, 0x544FU, 0x554FU, 0x564FU, 0x574FU, 0x584FU, + 0x594FU, 0x5A4FU, 0x614FU, 0x624FU, 0x634FU, 0x644FU, 0x654FU, 0x664FU, + 0x674FU, 0x684FU, 0x694FU, 0x6A4FU, 0x6B4FU, 0x6C4FU, 0x6D4FU, 0x6E4FU, + 0x6F4FU, 0x704FU, 0x714FU, 0x724FU, 0x734FU, 0x744FU, 0x754FU, 0x764FU, + 0x774FU, 0x784FU, 0x794FU, 0x7A4FU, 0x304FU, 0x314FU, 0x324FU, 0x334FU, + 0x344FU, 0x354FU, 0x364FU, 0x374FU, 0x384FU, 0x394FU, 0x2B4FU, 0x2F4FU, + 0x4150U, 0x4250U, 0x4350U, 0x4450U, 0x4550U, 0x4650U, 0x4750U, 0x4850U, + 0x4950U, 0x4A50U, 0x4B50U, 0x4C50U, 0x4D50U, 0x4E50U, 0x4F50U, 0x5050U, + 0x5150U, 0x5250U, 0x5350U, 0x5450U, 0x5550U, 0x5650U, 0x5750U, 0x5850U, + 0x5950U, 0x5A50U, 0x6150U, 0x6250U, 0x6350U, 0x6450U, 0x6550U, 0x6650U, + 0x6750U, 0x6850U, 0x6950U, 0x6A50U, 0x6B50U, 0x6C50U, 0x6D50U, 0x6E50U, + 0x6F50U, 0x7050U, 0x7150U, 0x7250U, 0x7350U, 0x7450U, 0x7550U, 0x7650U, + 0x7750U, 0x7850U, 0x7950U, 0x7A50U, 0x3050U, 0x3150U, 0x3250U, 0x3350U, + 0x3450U, 0x3550U, 0x3650U, 0x3750U, 0x3850U, 0x3950U, 0x2B50U, 0x2F50U, + 0x4151U, 0x4251U, 0x4351U, 0x4451U, 0x4551U, 0x4651U, 0x4751U, 0x4851U, + 0x4951U, 0x4A51U, 0x4B51U, 0x4C51U, 0x4D51U, 0x4E51U, 0x4F51U, 0x5051U, + 0x5151U, 0x5251U, 0x5351U, 0x5451U, 0x5551U, 0x5651U, 0x5751U, 0x5851U, + 0x5951U, 0x5A51U, 0x6151U, 0x6251U, 0x6351U, 0x6451U, 0x6551U, 0x6651U, + 0x6751U, 0x6851U, 0x6951U, 0x6A51U, 0x6B51U, 0x6C51U, 0x6D51U, 0x6E51U, + 0x6F51U, 0x7051U, 0x7151U, 0x7251U, 0x7351U, 0x7451U, 0x7551U, 0x7651U, + 0x7751U, 0x7851U, 0x7951U, 0x7A51U, 0x3051U, 0x3151U, 0x3251U, 0x3351U, + 0x3451U, 0x3551U, 0x3651U, 0x3751U, 0x3851U, 0x3951U, 0x2B51U, 0x2F51U, + 0x4152U, 0x4252U, 0x4352U, 0x4452U, 0x4552U, 0x4652U, 0x4752U, 0x4852U, + 0x4952U, 0x4A52U, 0x4B52U, 0x4C52U, 0x4D52U, 0x4E52U, 0x4F52U, 0x5052U, + 0x5152U, 0x5252U, 0x5352U, 0x5452U, 0x5552U, 0x5652U, 0x5752U, 0x5852U, + 0x5952U, 0x5A52U, 0x6152U, 0x6252U, 0x6352U, 0x6452U, 0x6552U, 0x6652U, + 0x6752U, 0x6852U, 0x6952U, 0x6A52U, 0x6B52U, 0x6C52U, 0x6D52U, 0x6E52U, + 0x6F52U, 0x7052U, 0x7152U, 0x7252U, 0x7352U, 0x7452U, 0x7552U, 0x7652U, + 0x7752U, 0x7852U, 0x7952U, 0x7A52U, 0x3052U, 0x3152U, 0x3252U, 0x3352U, + 0x3452U, 0x3552U, 0x3652U, 0x3752U, 0x3852U, 0x3952U, 0x2B52U, 0x2F52U, + 0x4153U, 0x4253U, 0x4353U, 0x4453U, 0x4553U, 0x4653U, 0x4753U, 0x4853U, + 0x4953U, 0x4A53U, 0x4B53U, 0x4C53U, 0x4D53U, 0x4E53U, 0x4F53U, 0x5053U, + 0x5153U, 0x5253U, 0x5353U, 0x5453U, 0x5553U, 0x5653U, 0x5753U, 0x5853U, + 0x5953U, 0x5A53U, 0x6153U, 0x6253U, 0x6353U, 0x6453U, 0x6553U, 0x6653U, + 0x6753U, 0x6853U, 0x6953U, 0x6A53U, 0x6B53U, 0x6C53U, 0x6D53U, 0x6E53U, + 0x6F53U, 0x7053U, 0x7153U, 0x7253U, 0x7353U, 0x7453U, 0x7553U, 0x7653U, + 0x7753U, 0x7853U, 0x7953U, 0x7A53U, 0x3053U, 0x3153U, 0x3253U, 0x3353U, + 0x3453U, 0x3553U, 0x3653U, 0x3753U, 0x3853U, 0x3953U, 0x2B53U, 0x2F53U, + 0x4154U, 0x4254U, 0x4354U, 0x4454U, 0x4554U, 0x4654U, 0x4754U, 0x4854U, + 0x4954U, 0x4A54U, 0x4B54U, 0x4C54U, 0x4D54U, 0x4E54U, 0x4F54U, 0x5054U, + 0x5154U, 0x5254U, 0x5354U, 0x5454U, 0x5554U, 0x5654U, 0x5754U, 0x5854U, + 0x5954U, 0x5A54U, 0x6154U, 0x6254U, 0x6354U, 0x6454U, 0x6554U, 0x6654U, + 0x6754U, 0x6854U, 0x6954U, 0x6A54U, 0x6B54U, 0x6C54U, 0x6D54U, 0x6E54U, + 0x6F54U, 0x7054U, 0x7154U, 0x7254U, 0x7354U, 0x7454U, 0x7554U, 0x7654U, + 0x7754U, 0x7854U, 0x7954U, 0x7A54U, 0x3054U, 0x3154U, 0x3254U, 0x3354U, + 0x3454U, 0x3554U, 0x3654U, 0x3754U, 0x3854U, 0x3954U, 0x2B54U, 0x2F54U, + 0x4155U, 0x4255U, 0x4355U, 0x4455U, 0x4555U, 0x4655U, 0x4755U, 0x4855U, + 0x4955U, 0x4A55U, 0x4B55U, 0x4C55U, 0x4D55U, 0x4E55U, 0x4F55U, 0x5055U, + 0x5155U, 0x5255U, 0x5355U, 0x5455U, 0x5555U, 0x5655U, 0x5755U, 0x5855U, + 0x5955U, 0x5A55U, 0x6155U, 0x6255U, 0x6355U, 0x6455U, 0x6555U, 0x6655U, + 0x6755U, 0x6855U, 0x6955U, 0x6A55U, 0x6B55U, 0x6C55U, 0x6D55U, 0x6E55U, + 0x6F55U, 0x7055U, 0x7155U, 0x7255U, 0x7355U, 0x7455U, 0x7555U, 0x7655U, + 0x7755U, 0x7855U, 0x7955U, 0x7A55U, 0x3055U, 0x3155U, 0x3255U, 0x3355U, + 0x3455U, 0x3555U, 0x3655U, 0x3755U, 0x3855U, 0x3955U, 0x2B55U, 0x2F55U, + 0x4156U, 0x4256U, 0x4356U, 0x4456U, 0x4556U, 0x4656U, 0x4756U, 0x4856U, + 0x4956U, 0x4A56U, 0x4B56U, 0x4C56U, 0x4D56U, 0x4E56U, 0x4F56U, 0x5056U, + 0x5156U, 0x5256U, 0x5356U, 0x5456U, 0x5556U, 0x5656U, 0x5756U, 0x5856U, + 0x5956U, 0x5A56U, 0x6156U, 0x6256U, 0x6356U, 0x6456U, 0x6556U, 0x6656U, + 0x6756U, 0x6856U, 0x6956U, 0x6A56U, 0x6B56U, 0x6C56U, 0x6D56U, 0x6E56U, + 0x6F56U, 0x7056U, 0x7156U, 0x7256U, 0x7356U, 0x7456U, 0x7556U, 0x7656U, + 0x7756U, 0x7856U, 0x7956U, 0x7A56U, 0x3056U, 0x3156U, 0x3256U, 0x3356U, + 0x3456U, 0x3556U, 0x3656U, 0x3756U, 0x3856U, 0x3956U, 0x2B56U, 0x2F56U, + 0x4157U, 0x4257U, 0x4357U, 0x4457U, 0x4557U, 0x4657U, 0x4757U, 0x4857U, + 0x4957U, 0x4A57U, 0x4B57U, 0x4C57U, 0x4D57U, 0x4E57U, 0x4F57U, 0x5057U, + 0x5157U, 0x5257U, 0x5357U, 0x5457U, 0x5557U, 0x5657U, 0x5757U, 0x5857U, + 0x5957U, 0x5A57U, 0x6157U, 0x6257U, 0x6357U, 0x6457U, 0x6557U, 0x6657U, + 0x6757U, 0x6857U, 0x6957U, 0x6A57U, 0x6B57U, 0x6C57U, 0x6D57U, 0x6E57U, + 0x6F57U, 0x7057U, 0x7157U, 0x7257U, 0x7357U, 0x7457U, 0x7557U, 0x7657U, + 0x7757U, 0x7857U, 0x7957U, 0x7A57U, 0x3057U, 0x3157U, 0x3257U, 0x3357U, + 0x3457U, 0x3557U, 0x3657U, 0x3757U, 0x3857U, 0x3957U, 0x2B57U, 0x2F57U, + 0x4158U, 0x4258U, 0x4358U, 0x4458U, 0x4558U, 0x4658U, 0x4758U, 0x4858U, + 0x4958U, 0x4A58U, 0x4B58U, 0x4C58U, 0x4D58U, 0x4E58U, 0x4F58U, 0x5058U, + 0x5158U, 0x5258U, 0x5358U, 0x5458U, 0x5558U, 0x5658U, 0x5758U, 0x5858U, + 0x5958U, 0x5A58U, 0x6158U, 0x6258U, 0x6358U, 0x6458U, 0x6558U, 0x6658U, + 0x6758U, 0x6858U, 0x6958U, 0x6A58U, 0x6B58U, 0x6C58U, 0x6D58U, 0x6E58U, + 0x6F58U, 0x7058U, 0x7158U, 0x7258U, 0x7358U, 0x7458U, 0x7558U, 0x7658U, + 0x7758U, 0x7858U, 0x7958U, 0x7A58U, 0x3058U, 0x3158U, 0x3258U, 0x3358U, + 0x3458U, 0x3558U, 0x3658U, 0x3758U, 0x3858U, 0x3958U, 0x2B58U, 0x2F58U, + 0x4159U, 0x4259U, 0x4359U, 0x4459U, 0x4559U, 0x4659U, 0x4759U, 0x4859U, + 0x4959U, 0x4A59U, 0x4B59U, 0x4C59U, 0x4D59U, 0x4E59U, 0x4F59U, 0x5059U, + 0x5159U, 0x5259U, 0x5359U, 0x5459U, 0x5559U, 0x5659U, 0x5759U, 0x5859U, + 0x5959U, 0x5A59U, 0x6159U, 0x6259U, 0x6359U, 0x6459U, 0x6559U, 0x6659U, + 0x6759U, 0x6859U, 0x6959U, 0x6A59U, 0x6B59U, 0x6C59U, 0x6D59U, 0x6E59U, + 0x6F59U, 0x7059U, 0x7159U, 0x7259U, 0x7359U, 0x7459U, 0x7559U, 0x7659U, + 0x7759U, 0x7859U, 0x7959U, 0x7A59U, 0x3059U, 0x3159U, 0x3259U, 0x3359U, + 0x3459U, 0x3559U, 0x3659U, 0x3759U, 0x3859U, 0x3959U, 0x2B59U, 0x2F59U, + 0x415AU, 0x425AU, 0x435AU, 0x445AU, 0x455AU, 0x465AU, 0x475AU, 0x485AU, + 0x495AU, 0x4A5AU, 0x4B5AU, 0x4C5AU, 0x4D5AU, 0x4E5AU, 0x4F5AU, 0x505AU, + 0x515AU, 0x525AU, 0x535AU, 0x545AU, 0x555AU, 0x565AU, 0x575AU, 0x585AU, + 0x595AU, 0x5A5AU, 0x615AU, 0x625AU, 0x635AU, 0x645AU, 0x655AU, 0x665AU, + 0x675AU, 0x685AU, 0x695AU, 0x6A5AU, 0x6B5AU, 0x6C5AU, 0x6D5AU, 0x6E5AU, + 0x6F5AU, 0x705AU, 0x715AU, 0x725AU, 0x735AU, 0x745AU, 0x755AU, 0x765AU, + 0x775AU, 0x785AU, 0x795AU, 0x7A5AU, 0x305AU, 0x315AU, 0x325AU, 0x335AU, + 0x345AU, 0x355AU, 0x365AU, 0x375AU, 0x385AU, 0x395AU, 0x2B5AU, 0x2F5AU, + 0x4161U, 0x4261U, 0x4361U, 0x4461U, 0x4561U, 0x4661U, 0x4761U, 0x4861U, + 0x4961U, 0x4A61U, 0x4B61U, 0x4C61U, 0x4D61U, 0x4E61U, 0x4F61U, 0x5061U, + 0x5161U, 0x5261U, 0x5361U, 0x5461U, 0x5561U, 0x5661U, 0x5761U, 0x5861U, + 0x5961U, 0x5A61U, 0x6161U, 0x6261U, 0x6361U, 0x6461U, 0x6561U, 0x6661U, + 0x6761U, 0x6861U, 0x6961U, 0x6A61U, 0x6B61U, 0x6C61U, 0x6D61U, 0x6E61U, + 0x6F61U, 0x7061U, 0x7161U, 0x7261U, 0x7361U, 0x7461U, 0x7561U, 0x7661U, + 0x7761U, 0x7861U, 0x7961U, 0x7A61U, 0x3061U, 0x3161U, 0x3261U, 0x3361U, + 0x3461U, 0x3561U, 0x3661U, 0x3761U, 0x3861U, 0x3961U, 0x2B61U, 0x2F61U, + 0x4162U, 0x4262U, 0x4362U, 0x4462U, 0x4562U, 0x4662U, 0x4762U, 0x4862U, + 0x4962U, 0x4A62U, 0x4B62U, 0x4C62U, 0x4D62U, 0x4E62U, 0x4F62U, 0x5062U, + 0x5162U, 0x5262U, 0x5362U, 0x5462U, 0x5562U, 0x5662U, 0x5762U, 0x5862U, + 0x5962U, 0x5A62U, 0x6162U, 0x6262U, 0x6362U, 0x6462U, 0x6562U, 0x6662U, + 0x6762U, 0x6862U, 0x6962U, 0x6A62U, 0x6B62U, 0x6C62U, 0x6D62U, 0x6E62U, + 0x6F62U, 0x7062U, 0x7162U, 0x7262U, 0x7362U, 0x7462U, 0x7562U, 0x7662U, + 0x7762U, 0x7862U, 0x7962U, 0x7A62U, 0x3062U, 0x3162U, 0x3262U, 0x3362U, + 0x3462U, 0x3562U, 0x3662U, 0x3762U, 0x3862U, 0x3962U, 0x2B62U, 0x2F62U, + 0x4163U, 0x4263U, 0x4363U, 0x4463U, 0x4563U, 0x4663U, 0x4763U, 0x4863U, + 0x4963U, 0x4A63U, 0x4B63U, 0x4C63U, 0x4D63U, 0x4E63U, 0x4F63U, 0x5063U, + 0x5163U, 0x5263U, 0x5363U, 0x5463U, 0x5563U, 0x5663U, 0x5763U, 0x5863U, + 0x5963U, 0x5A63U, 0x6163U, 0x6263U, 0x6363U, 0x6463U, 0x6563U, 0x6663U, + 0x6763U, 0x6863U, 0x6963U, 0x6A63U, 0x6B63U, 0x6C63U, 0x6D63U, 0x6E63U, + 0x6F63U, 0x7063U, 0x7163U, 0x7263U, 0x7363U, 0x7463U, 0x7563U, 0x7663U, + 0x7763U, 0x7863U, 0x7963U, 0x7A63U, 0x3063U, 0x3163U, 0x3263U, 0x3363U, + 0x3463U, 0x3563U, 0x3663U, 0x3763U, 0x3863U, 0x3963U, 0x2B63U, 0x2F63U, + 0x4164U, 0x4264U, 0x4364U, 0x4464U, 0x4564U, 0x4664U, 0x4764U, 0x4864U, + 0x4964U, 0x4A64U, 0x4B64U, 0x4C64U, 0x4D64U, 0x4E64U, 0x4F64U, 0x5064U, + 0x5164U, 0x5264U, 0x5364U, 0x5464U, 0x5564U, 0x5664U, 0x5764U, 0x5864U, + 0x5964U, 0x5A64U, 0x6164U, 0x6264U, 0x6364U, 0x6464U, 0x6564U, 0x6664U, + 0x6764U, 0x6864U, 0x6964U, 0x6A64U, 0x6B64U, 0x6C64U, 0x6D64U, 0x6E64U, + 0x6F64U, 0x7064U, 0x7164U, 0x7264U, 0x7364U, 0x7464U, 0x7564U, 0x7664U, + 0x7764U, 0x7864U, 0x7964U, 0x7A64U, 0x3064U, 0x3164U, 0x3264U, 0x3364U, + 0x3464U, 0x3564U, 0x3664U, 0x3764U, 0x3864U, 0x3964U, 0x2B64U, 0x2F64U, + 0x4165U, 0x4265U, 0x4365U, 0x4465U, 0x4565U, 0x4665U, 0x4765U, 0x4865U, + 0x4965U, 0x4A65U, 0x4B65U, 0x4C65U, 0x4D65U, 0x4E65U, 0x4F65U, 0x5065U, + 0x5165U, 0x5265U, 0x5365U, 0x5465U, 0x5565U, 0x5665U, 0x5765U, 0x5865U, + 0x5965U, 0x5A65U, 0x6165U, 0x6265U, 0x6365U, 0x6465U, 0x6565U, 0x6665U, + 0x6765U, 0x6865U, 0x6965U, 0x6A65U, 0x6B65U, 0x6C65U, 0x6D65U, 0x6E65U, + 0x6F65U, 0x7065U, 0x7165U, 0x7265U, 0x7365U, 0x7465U, 0x7565U, 0x7665U, + 0x7765U, 0x7865U, 0x7965U, 0x7A65U, 0x3065U, 0x3165U, 0x3265U, 0x3365U, + 0x3465U, 0x3565U, 0x3665U, 0x3765U, 0x3865U, 0x3965U, 0x2B65U, 0x2F65U, + 0x4166U, 0x4266U, 0x4366U, 0x4466U, 0x4566U, 0x4666U, 0x4766U, 0x4866U, + 0x4966U, 0x4A66U, 0x4B66U, 0x4C66U, 0x4D66U, 0x4E66U, 0x4F66U, 0x5066U, + 0x5166U, 0x5266U, 0x5366U, 0x5466U, 0x5566U, 0x5666U, 0x5766U, 0x5866U, + 0x5966U, 0x5A66U, 0x6166U, 0x6266U, 0x6366U, 0x6466U, 0x6566U, 0x6666U, + 0x6766U, 0x6866U, 0x6966U, 0x6A66U, 0x6B66U, 0x6C66U, 0x6D66U, 0x6E66U, + 0x6F66U, 0x7066U, 0x7166U, 0x7266U, 0x7366U, 0x7466U, 0x7566U, 0x7666U, + 0x7766U, 0x7866U, 0x7966U, 0x7A66U, 0x3066U, 0x3166U, 0x3266U, 0x3366U, + 0x3466U, 0x3566U, 0x3666U, 0x3766U, 0x3866U, 0x3966U, 0x2B66U, 0x2F66U, + 0x4167U, 0x4267U, 0x4367U, 0x4467U, 0x4567U, 0x4667U, 0x4767U, 0x4867U, + 0x4967U, 0x4A67U, 0x4B67U, 0x4C67U, 0x4D67U, 0x4E67U, 0x4F67U, 0x5067U, + 0x5167U, 0x5267U, 0x5367U, 0x5467U, 0x5567U, 0x5667U, 0x5767U, 0x5867U, + 0x5967U, 0x5A67U, 0x6167U, 0x6267U, 0x6367U, 0x6467U, 0x6567U, 0x6667U, + 0x6767U, 0x6867U, 0x6967U, 0x6A67U, 0x6B67U, 0x6C67U, 0x6D67U, 0x6E67U, + 0x6F67U, 0x7067U, 0x7167U, 0x7267U, 0x7367U, 0x7467U, 0x7567U, 0x7667U, + 0x7767U, 0x7867U, 0x7967U, 0x7A67U, 0x3067U, 0x3167U, 0x3267U, 0x3367U, + 0x3467U, 0x3567U, 0x3667U, 0x3767U, 0x3867U, 0x3967U, 0x2B67U, 0x2F67U, + 0x4168U, 0x4268U, 0x4368U, 0x4468U, 0x4568U, 0x4668U, 0x4768U, 0x4868U, + 0x4968U, 0x4A68U, 0x4B68U, 0x4C68U, 0x4D68U, 0x4E68U, 0x4F68U, 0x5068U, + 0x5168U, 0x5268U, 0x5368U, 0x5468U, 0x5568U, 0x5668U, 0x5768U, 0x5868U, + 0x5968U, 0x5A68U, 0x6168U, 0x6268U, 0x6368U, 0x6468U, 0x6568U, 0x6668U, + 0x6768U, 0x6868U, 0x6968U, 0x6A68U, 0x6B68U, 0x6C68U, 0x6D68U, 0x6E68U, + 0x6F68U, 0x7068U, 0x7168U, 0x7268U, 0x7368U, 0x7468U, 0x7568U, 0x7668U, + 0x7768U, 0x7868U, 0x7968U, 0x7A68U, 0x3068U, 0x3168U, 0x3268U, 0x3368U, + 0x3468U, 0x3568U, 0x3668U, 0x3768U, 0x3868U, 0x3968U, 0x2B68U, 0x2F68U, + 0x4169U, 0x4269U, 0x4369U, 0x4469U, 0x4569U, 0x4669U, 0x4769U, 0x4869U, + 0x4969U, 0x4A69U, 0x4B69U, 0x4C69U, 0x4D69U, 0x4E69U, 0x4F69U, 0x5069U, + 0x5169U, 0x5269U, 0x5369U, 0x5469U, 0x5569U, 0x5669U, 0x5769U, 0x5869U, + 0x5969U, 0x5A69U, 0x6169U, 0x6269U, 0x6369U, 0x6469U, 0x6569U, 0x6669U, + 0x6769U, 0x6869U, 0x6969U, 0x6A69U, 0x6B69U, 0x6C69U, 0x6D69U, 0x6E69U, + 0x6F69U, 0x7069U, 0x7169U, 0x7269U, 0x7369U, 0x7469U, 0x7569U, 0x7669U, + 0x7769U, 0x7869U, 0x7969U, 0x7A69U, 0x3069U, 0x3169U, 0x3269U, 0x3369U, + 0x3469U, 0x3569U, 0x3669U, 0x3769U, 0x3869U, 0x3969U, 0x2B69U, 0x2F69U, + 0x416AU, 0x426AU, 0x436AU, 0x446AU, 0x456AU, 0x466AU, 0x476AU, 0x486AU, + 0x496AU, 0x4A6AU, 0x4B6AU, 0x4C6AU, 0x4D6AU, 0x4E6AU, 0x4F6AU, 0x506AU, + 0x516AU, 0x526AU, 0x536AU, 0x546AU, 0x556AU, 0x566AU, 0x576AU, 0x586AU, + 0x596AU, 0x5A6AU, 0x616AU, 0x626AU, 0x636AU, 0x646AU, 0x656AU, 0x666AU, + 0x676AU, 0x686AU, 0x696AU, 0x6A6AU, 0x6B6AU, 0x6C6AU, 0x6D6AU, 0x6E6AU, + 0x6F6AU, 0x706AU, 0x716AU, 0x726AU, 0x736AU, 0x746AU, 0x756AU, 0x766AU, + 0x776AU, 0x786AU, 0x796AU, 0x7A6AU, 0x306AU, 0x316AU, 0x326AU, 0x336AU, + 0x346AU, 0x356AU, 0x366AU, 0x376AU, 0x386AU, 0x396AU, 0x2B6AU, 0x2F6AU, + 0x416BU, 0x426BU, 0x436BU, 0x446BU, 0x456BU, 0x466BU, 0x476BU, 0x486BU, + 0x496BU, 0x4A6BU, 0x4B6BU, 0x4C6BU, 0x4D6BU, 0x4E6BU, 0x4F6BU, 0x506BU, + 0x516BU, 0x526BU, 0x536BU, 0x546BU, 0x556BU, 0x566BU, 0x576BU, 0x586BU, + 0x596BU, 0x5A6BU, 0x616BU, 0x626BU, 0x636BU, 0x646BU, 0x656BU, 0x666BU, + 0x676BU, 0x686BU, 0x696BU, 0x6A6BU, 0x6B6BU, 0x6C6BU, 0x6D6BU, 0x6E6BU, + 0x6F6BU, 0x706BU, 0x716BU, 0x726BU, 0x736BU, 0x746BU, 0x756BU, 0x766BU, + 0x776BU, 0x786BU, 0x796BU, 0x7A6BU, 0x306BU, 0x316BU, 0x326BU, 0x336BU, + 0x346BU, 0x356BU, 0x366BU, 0x376BU, 0x386BU, 0x396BU, 0x2B6BU, 0x2F6BU, + 0x416CU, 0x426CU, 0x436CU, 0x446CU, 0x456CU, 0x466CU, 0x476CU, 0x486CU, + 0x496CU, 0x4A6CU, 0x4B6CU, 0x4C6CU, 0x4D6CU, 0x4E6CU, 0x4F6CU, 0x506CU, + 0x516CU, 0x526CU, 0x536CU, 0x546CU, 0x556CU, 0x566CU, 0x576CU, 0x586CU, + 0x596CU, 0x5A6CU, 0x616CU, 0x626CU, 0x636CU, 0x646CU, 0x656CU, 0x666CU, + 0x676CU, 0x686CU, 0x696CU, 0x6A6CU, 0x6B6CU, 0x6C6CU, 0x6D6CU, 0x6E6CU, + 0x6F6CU, 0x706CU, 0x716CU, 0x726CU, 0x736CU, 0x746CU, 0x756CU, 0x766CU, + 0x776CU, 0x786CU, 0x796CU, 0x7A6CU, 0x306CU, 0x316CU, 0x326CU, 0x336CU, + 0x346CU, 0x356CU, 0x366CU, 0x376CU, 0x386CU, 0x396CU, 0x2B6CU, 0x2F6CU, + 0x416DU, 0x426DU, 0x436DU, 0x446DU, 0x456DU, 0x466DU, 0x476DU, 0x486DU, + 0x496DU, 0x4A6DU, 0x4B6DU, 0x4C6DU, 0x4D6DU, 0x4E6DU, 0x4F6DU, 0x506DU, + 0x516DU, 0x526DU, 0x536DU, 0x546DU, 0x556DU, 0x566DU, 0x576DU, 0x586DU, + 0x596DU, 0x5A6DU, 0x616DU, 0x626DU, 0x636DU, 0x646DU, 0x656DU, 0x666DU, + 0x676DU, 0x686DU, 0x696DU, 0x6A6DU, 0x6B6DU, 0x6C6DU, 0x6D6DU, 0x6E6DU, + 0x6F6DU, 0x706DU, 0x716DU, 0x726DU, 0x736DU, 0x746DU, 0x756DU, 0x766DU, + 0x776DU, 0x786DU, 0x796DU, 0x7A6DU, 0x306DU, 0x316DU, 0x326DU, 0x336DU, + 0x346DU, 0x356DU, 0x366DU, 0x376DU, 0x386DU, 0x396DU, 0x2B6DU, 0x2F6DU, + 0x416EU, 0x426EU, 0x436EU, 0x446EU, 0x456EU, 0x466EU, 0x476EU, 0x486EU, + 0x496EU, 0x4A6EU, 0x4B6EU, 0x4C6EU, 0x4D6EU, 0x4E6EU, 0x4F6EU, 0x506EU, + 0x516EU, 0x526EU, 0x536EU, 0x546EU, 0x556EU, 0x566EU, 0x576EU, 0x586EU, + 0x596EU, 0x5A6EU, 0x616EU, 0x626EU, 0x636EU, 0x646EU, 0x656EU, 0x666EU, + 0x676EU, 0x686EU, 0x696EU, 0x6A6EU, 0x6B6EU, 0x6C6EU, 0x6D6EU, 0x6E6EU, + 0x6F6EU, 0x706EU, 0x716EU, 0x726EU, 0x736EU, 0x746EU, 0x756EU, 0x766EU, + 0x776EU, 0x786EU, 0x796EU, 0x7A6EU, 0x306EU, 0x316EU, 0x326EU, 0x336EU, + 0x346EU, 0x356EU, 0x366EU, 0x376EU, 0x386EU, 0x396EU, 0x2B6EU, 0x2F6EU, + 0x416FU, 0x426FU, 0x436FU, 0x446FU, 0x456FU, 0x466FU, 0x476FU, 0x486FU, + 0x496FU, 0x4A6FU, 0x4B6FU, 0x4C6FU, 0x4D6FU, 0x4E6FU, 0x4F6FU, 0x506FU, + 0x516FU, 0x526FU, 0x536FU, 0x546FU, 0x556FU, 0x566FU, 0x576FU, 0x586FU, + 0x596FU, 0x5A6FU, 0x616FU, 0x626FU, 0x636FU, 0x646FU, 0x656FU, 0x666FU, + 0x676FU, 0x686FU, 0x696FU, 0x6A6FU, 0x6B6FU, 0x6C6FU, 0x6D6FU, 0x6E6FU, + 0x6F6FU, 0x706FU, 0x716FU, 0x726FU, 0x736FU, 0x746FU, 0x756FU, 0x766FU, + 0x776FU, 0x786FU, 0x796FU, 0x7A6FU, 0x306FU, 0x316FU, 0x326FU, 0x336FU, + 0x346FU, 0x356FU, 0x366FU, 0x376FU, 0x386FU, 0x396FU, 0x2B6FU, 0x2F6FU, + 0x4170U, 0x4270U, 0x4370U, 0x4470U, 0x4570U, 0x4670U, 0x4770U, 0x4870U, + 0x4970U, 0x4A70U, 0x4B70U, 0x4C70U, 0x4D70U, 0x4E70U, 0x4F70U, 0x5070U, + 0x5170U, 0x5270U, 0x5370U, 0x5470U, 0x5570U, 0x5670U, 0x5770U, 0x5870U, + 0x5970U, 0x5A70U, 0x6170U, 0x6270U, 0x6370U, 0x6470U, 0x6570U, 0x6670U, + 0x6770U, 0x6870U, 0x6970U, 0x6A70U, 0x6B70U, 0x6C70U, 0x6D70U, 0x6E70U, + 0x6F70U, 0x7070U, 0x7170U, 0x7270U, 0x7370U, 0x7470U, 0x7570U, 0x7670U, + 0x7770U, 0x7870U, 0x7970U, 0x7A70U, 0x3070U, 0x3170U, 0x3270U, 0x3370U, + 0x3470U, 0x3570U, 0x3670U, 0x3770U, 0x3870U, 0x3970U, 0x2B70U, 0x2F70U, + 0x4171U, 0x4271U, 0x4371U, 0x4471U, 0x4571U, 0x4671U, 0x4771U, 0x4871U, + 0x4971U, 0x4A71U, 0x4B71U, 0x4C71U, 0x4D71U, 0x4E71U, 0x4F71U, 0x5071U, + 0x5171U, 0x5271U, 0x5371U, 0x5471U, 0x5571U, 0x5671U, 0x5771U, 0x5871U, + 0x5971U, 0x5A71U, 0x6171U, 0x6271U, 0x6371U, 0x6471U, 0x6571U, 0x6671U, + 0x6771U, 0x6871U, 0x6971U, 0x6A71U, 0x6B71U, 0x6C71U, 0x6D71U, 0x6E71U, + 0x6F71U, 0x7071U, 0x7171U, 0x7271U, 0x7371U, 0x7471U, 0x7571U, 0x7671U, + 0x7771U, 0x7871U, 0x7971U, 0x7A71U, 0x3071U, 0x3171U, 0x3271U, 0x3371U, + 0x3471U, 0x3571U, 0x3671U, 0x3771U, 0x3871U, 0x3971U, 0x2B71U, 0x2F71U, + 0x4172U, 0x4272U, 0x4372U, 0x4472U, 0x4572U, 0x4672U, 0x4772U, 0x4872U, + 0x4972U, 0x4A72U, 0x4B72U, 0x4C72U, 0x4D72U, 0x4E72U, 0x4F72U, 0x5072U, + 0x5172U, 0x5272U, 0x5372U, 0x5472U, 0x5572U, 0x5672U, 0x5772U, 0x5872U, + 0x5972U, 0x5A72U, 0x6172U, 0x6272U, 0x6372U, 0x6472U, 0x6572U, 0x6672U, + 0x6772U, 0x6872U, 0x6972U, 0x6A72U, 0x6B72U, 0x6C72U, 0x6D72U, 0x6E72U, + 0x6F72U, 0x7072U, 0x7172U, 0x7272U, 0x7372U, 0x7472U, 0x7572U, 0x7672U, + 0x7772U, 0x7872U, 0x7972U, 0x7A72U, 0x3072U, 0x3172U, 0x3272U, 0x3372U, + 0x3472U, 0x3572U, 0x3672U, 0x3772U, 0x3872U, 0x3972U, 0x2B72U, 0x2F72U, + 0x4173U, 0x4273U, 0x4373U, 0x4473U, 0x4573U, 0x4673U, 0x4773U, 0x4873U, + 0x4973U, 0x4A73U, 0x4B73U, 0x4C73U, 0x4D73U, 0x4E73U, 0x4F73U, 0x5073U, + 0x5173U, 0x5273U, 0x5373U, 0x5473U, 0x5573U, 0x5673U, 0x5773U, 0x5873U, + 0x5973U, 0x5A73U, 0x6173U, 0x6273U, 0x6373U, 0x6473U, 0x6573U, 0x6673U, + 0x6773U, 0x6873U, 0x6973U, 0x6A73U, 0x6B73U, 0x6C73U, 0x6D73U, 0x6E73U, + 0x6F73U, 0x7073U, 0x7173U, 0x7273U, 0x7373U, 0x7473U, 0x7573U, 0x7673U, + 0x7773U, 0x7873U, 0x7973U, 0x7A73U, 0x3073U, 0x3173U, 0x3273U, 0x3373U, + 0x3473U, 0x3573U, 0x3673U, 0x3773U, 0x3873U, 0x3973U, 0x2B73U, 0x2F73U, + 0x4174U, 0x4274U, 0x4374U, 0x4474U, 0x4574U, 0x4674U, 0x4774U, 0x4874U, + 0x4974U, 0x4A74U, 0x4B74U, 0x4C74U, 0x4D74U, 0x4E74U, 0x4F74U, 0x5074U, + 0x5174U, 0x5274U, 0x5374U, 0x5474U, 0x5574U, 0x5674U, 0x5774U, 0x5874U, + 0x5974U, 0x5A74U, 0x6174U, 0x6274U, 0x6374U, 0x6474U, 0x6574U, 0x6674U, + 0x6774U, 0x6874U, 0x6974U, 0x6A74U, 0x6B74U, 0x6C74U, 0x6D74U, 0x6E74U, + 0x6F74U, 0x7074U, 0x7174U, 0x7274U, 0x7374U, 0x7474U, 0x7574U, 0x7674U, + 0x7774U, 0x7874U, 0x7974U, 0x7A74U, 0x3074U, 0x3174U, 0x3274U, 0x3374U, + 0x3474U, 0x3574U, 0x3674U, 0x3774U, 0x3874U, 0x3974U, 0x2B74U, 0x2F74U, + 0x4175U, 0x4275U, 0x4375U, 0x4475U, 0x4575U, 0x4675U, 0x4775U, 0x4875U, + 0x4975U, 0x4A75U, 0x4B75U, 0x4C75U, 0x4D75U, 0x4E75U, 0x4F75U, 0x5075U, + 0x5175U, 0x5275U, 0x5375U, 0x5475U, 0x5575U, 0x5675U, 0x5775U, 0x5875U, + 0x5975U, 0x5A75U, 0x6175U, 0x6275U, 0x6375U, 0x6475U, 0x6575U, 0x6675U, + 0x6775U, 0x6875U, 0x6975U, 0x6A75U, 0x6B75U, 0x6C75U, 0x6D75U, 0x6E75U, + 0x6F75U, 0x7075U, 0x7175U, 0x7275U, 0x7375U, 0x7475U, 0x7575U, 0x7675U, + 0x7775U, 0x7875U, 0x7975U, 0x7A75U, 0x3075U, 0x3175U, 0x3275U, 0x3375U, + 0x3475U, 0x3575U, 0x3675U, 0x3775U, 0x3875U, 0x3975U, 0x2B75U, 0x2F75U, + 0x4176U, 0x4276U, 0x4376U, 0x4476U, 0x4576U, 0x4676U, 0x4776U, 0x4876U, + 0x4976U, 0x4A76U, 0x4B76U, 0x4C76U, 0x4D76U, 0x4E76U, 0x4F76U, 0x5076U, + 0x5176U, 0x5276U, 0x5376U, 0x5476U, 0x5576U, 0x5676U, 0x5776U, 0x5876U, + 0x5976U, 0x5A76U, 0x6176U, 0x6276U, 0x6376U, 0x6476U, 0x6576U, 0x6676U, + 0x6776U, 0x6876U, 0x6976U, 0x6A76U, 0x6B76U, 0x6C76U, 0x6D76U, 0x6E76U, + 0x6F76U, 0x7076U, 0x7176U, 0x7276U, 0x7376U, 0x7476U, 0x7576U, 0x7676U, + 0x7776U, 0x7876U, 0x7976U, 0x7A76U, 0x3076U, 0x3176U, 0x3276U, 0x3376U, + 0x3476U, 0x3576U, 0x3676U, 0x3776U, 0x3876U, 0x3976U, 0x2B76U, 0x2F76U, + 0x4177U, 0x4277U, 0x4377U, 0x4477U, 0x4577U, 0x4677U, 0x4777U, 0x4877U, + 0x4977U, 0x4A77U, 0x4B77U, 0x4C77U, 0x4D77U, 0x4E77U, 0x4F77U, 0x5077U, + 0x5177U, 0x5277U, 0x5377U, 0x5477U, 0x5577U, 0x5677U, 0x5777U, 0x5877U, + 0x5977U, 0x5A77U, 0x6177U, 0x6277U, 0x6377U, 0x6477U, 0x6577U, 0x6677U, + 0x6777U, 0x6877U, 0x6977U, 0x6A77U, 0x6B77U, 0x6C77U, 0x6D77U, 0x6E77U, + 0x6F77U, 0x7077U, 0x7177U, 0x7277U, 0x7377U, 0x7477U, 0x7577U, 0x7677U, + 0x7777U, 0x7877U, 0x7977U, 0x7A77U, 0x3077U, 0x3177U, 0x3277U, 0x3377U, + 0x3477U, 0x3577U, 0x3677U, 0x3777U, 0x3877U, 0x3977U, 0x2B77U, 0x2F77U, + 0x4178U, 0x4278U, 0x4378U, 0x4478U, 0x4578U, 0x4678U, 0x4778U, 0x4878U, + 0x4978U, 0x4A78U, 0x4B78U, 0x4C78U, 0x4D78U, 0x4E78U, 0x4F78U, 0x5078U, + 0x5178U, 0x5278U, 0x5378U, 0x5478U, 0x5578U, 0x5678U, 0x5778U, 0x5878U, + 0x5978U, 0x5A78U, 0x6178U, 0x6278U, 0x6378U, 0x6478U, 0x6578U, 0x6678U, + 0x6778U, 0x6878U, 0x6978U, 0x6A78U, 0x6B78U, 0x6C78U, 0x6D78U, 0x6E78U, + 0x6F78U, 0x7078U, 0x7178U, 0x7278U, 0x7378U, 0x7478U, 0x7578U, 0x7678U, + 0x7778U, 0x7878U, 0x7978U, 0x7A78U, 0x3078U, 0x3178U, 0x3278U, 0x3378U, + 0x3478U, 0x3578U, 0x3678U, 0x3778U, 0x3878U, 0x3978U, 0x2B78U, 0x2F78U, + 0x4179U, 0x4279U, 0x4379U, 0x4479U, 0x4579U, 0x4679U, 0x4779U, 0x4879U, + 0x4979U, 0x4A79U, 0x4B79U, 0x4C79U, 0x4D79U, 0x4E79U, 0x4F79U, 0x5079U, + 0x5179U, 0x5279U, 0x5379U, 0x5479U, 0x5579U, 0x5679U, 0x5779U, 0x5879U, + 0x5979U, 0x5A79U, 0x6179U, 0x6279U, 0x6379U, 0x6479U, 0x6579U, 0x6679U, + 0x6779U, 0x6879U, 0x6979U, 0x6A79U, 0x6B79U, 0x6C79U, 0x6D79U, 0x6E79U, + 0x6F79U, 0x7079U, 0x7179U, 0x7279U, 0x7379U, 0x7479U, 0x7579U, 0x7679U, + 0x7779U, 0x7879U, 0x7979U, 0x7A79U, 0x3079U, 0x3179U, 0x3279U, 0x3379U, + 0x3479U, 0x3579U, 0x3679U, 0x3779U, 0x3879U, 0x3979U, 0x2B79U, 0x2F79U, + 0x417AU, 0x427AU, 0x437AU, 0x447AU, 0x457AU, 0x467AU, 0x477AU, 0x487AU, + 0x497AU, 0x4A7AU, 0x4B7AU, 0x4C7AU, 0x4D7AU, 0x4E7AU, 0x4F7AU, 0x507AU, + 0x517AU, 0x527AU, 0x537AU, 0x547AU, 0x557AU, 0x567AU, 0x577AU, 0x587AU, + 0x597AU, 0x5A7AU, 0x617AU, 0x627AU, 0x637AU, 0x647AU, 0x657AU, 0x667AU, + 0x677AU, 0x687AU, 0x697AU, 0x6A7AU, 0x6B7AU, 0x6C7AU, 0x6D7AU, 0x6E7AU, + 0x6F7AU, 0x707AU, 0x717AU, 0x727AU, 0x737AU, 0x747AU, 0x757AU, 0x767AU, + 0x777AU, 0x787AU, 0x797AU, 0x7A7AU, 0x307AU, 0x317AU, 0x327AU, 0x337AU, + 0x347AU, 0x357AU, 0x367AU, 0x377AU, 0x387AU, 0x397AU, 0x2B7AU, 0x2F7AU, + 0x4130U, 0x4230U, 0x4330U, 0x4430U, 0x4530U, 0x4630U, 0x4730U, 0x4830U, + 0x4930U, 0x4A30U, 0x4B30U, 0x4C30U, 0x4D30U, 0x4E30U, 0x4F30U, 0x5030U, + 0x5130U, 0x5230U, 0x5330U, 0x5430U, 0x5530U, 0x5630U, 0x5730U, 0x5830U, + 0x5930U, 0x5A30U, 0x6130U, 0x6230U, 0x6330U, 0x6430U, 0x6530U, 0x6630U, + 0x6730U, 0x6830U, 0x6930U, 0x6A30U, 0x6B30U, 0x6C30U, 0x6D30U, 0x6E30U, + 0x6F30U, 0x7030U, 0x7130U, 0x7230U, 0x7330U, 0x7430U, 0x7530U, 0x7630U, + 0x7730U, 0x7830U, 0x7930U, 0x7A30U, 0x3030U, 0x3130U, 0x3230U, 0x3330U, + 0x3430U, 0x3530U, 0x3630U, 0x3730U, 0x3830U, 0x3930U, 0x2B30U, 0x2F30U, + 0x4131U, 0x4231U, 0x4331U, 0x4431U, 0x4531U, 0x4631U, 0x4731U, 0x4831U, + 0x4931U, 0x4A31U, 0x4B31U, 0x4C31U, 0x4D31U, 0x4E31U, 0x4F31U, 0x5031U, + 0x5131U, 0x5231U, 0x5331U, 0x5431U, 0x5531U, 0x5631U, 0x5731U, 0x5831U, + 0x5931U, 0x5A31U, 0x6131U, 0x6231U, 0x6331U, 0x6431U, 0x6531U, 0x6631U, + 0x6731U, 0x6831U, 0x6931U, 0x6A31U, 0x6B31U, 0x6C31U, 0x6D31U, 0x6E31U, + 0x6F31U, 0x7031U, 0x7131U, 0x7231U, 0x7331U, 0x7431U, 0x7531U, 0x7631U, + 0x7731U, 0x7831U, 0x7931U, 0x7A31U, 0x3031U, 0x3131U, 0x3231U, 0x3331U, + 0x3431U, 0x3531U, 0x3631U, 0x3731U, 0x3831U, 0x3931U, 0x2B31U, 0x2F31U, + 0x4132U, 0x4232U, 0x4332U, 0x4432U, 0x4532U, 0x4632U, 0x4732U, 0x4832U, + 0x4932U, 0x4A32U, 0x4B32U, 0x4C32U, 0x4D32U, 0x4E32U, 0x4F32U, 0x5032U, + 0x5132U, 0x5232U, 0x5332U, 0x5432U, 0x5532U, 0x5632U, 0x5732U, 0x5832U, + 0x5932U, 0x5A32U, 0x6132U, 0x6232U, 0x6332U, 0x6432U, 0x6532U, 0x6632U, + 0x6732U, 0x6832U, 0x6932U, 0x6A32U, 0x6B32U, 0x6C32U, 0x6D32U, 0x6E32U, + 0x6F32U, 0x7032U, 0x7132U, 0x7232U, 0x7332U, 0x7432U, 0x7532U, 0x7632U, + 0x7732U, 0x7832U, 0x7932U, 0x7A32U, 0x3032U, 0x3132U, 0x3232U, 0x3332U, + 0x3432U, 0x3532U, 0x3632U, 0x3732U, 0x3832U, 0x3932U, 0x2B32U, 0x2F32U, + 0x4133U, 0x4233U, 0x4333U, 0x4433U, 0x4533U, 0x4633U, 0x4733U, 0x4833U, + 0x4933U, 0x4A33U, 0x4B33U, 0x4C33U, 0x4D33U, 0x4E33U, 0x4F33U, 0x5033U, + 0x5133U, 0x5233U, 0x5333U, 0x5433U, 0x5533U, 0x5633U, 0x5733U, 0x5833U, + 0x5933U, 0x5A33U, 0x6133U, 0x6233U, 0x6333U, 0x6433U, 0x6533U, 0x6633U, + 0x6733U, 0x6833U, 0x6933U, 0x6A33U, 0x6B33U, 0x6C33U, 0x6D33U, 0x6E33U, + 0x6F33U, 0x7033U, 0x7133U, 0x7233U, 0x7333U, 0x7433U, 0x7533U, 0x7633U, + 0x7733U, 0x7833U, 0x7933U, 0x7A33U, 0x3033U, 0x3133U, 0x3233U, 0x3333U, + 0x3433U, 0x3533U, 0x3633U, 0x3733U, 0x3833U, 0x3933U, 0x2B33U, 0x2F33U, + 0x4134U, 0x4234U, 0x4334U, 0x4434U, 0x4534U, 0x4634U, 0x4734U, 0x4834U, + 0x4934U, 0x4A34U, 0x4B34U, 0x4C34U, 0x4D34U, 0x4E34U, 0x4F34U, 0x5034U, + 0x5134U, 0x5234U, 0x5334U, 0x5434U, 0x5534U, 0x5634U, 0x5734U, 0x5834U, + 0x5934U, 0x5A34U, 0x6134U, 0x6234U, 0x6334U, 0x6434U, 0x6534U, 0x6634U, + 0x6734U, 0x6834U, 0x6934U, 0x6A34U, 0x6B34U, 0x6C34U, 0x6D34U, 0x6E34U, + 0x6F34U, 0x7034U, 0x7134U, 0x7234U, 0x7334U, 0x7434U, 0x7534U, 0x7634U, + 0x7734U, 0x7834U, 0x7934U, 0x7A34U, 0x3034U, 0x3134U, 0x3234U, 0x3334U, + 0x3434U, 0x3534U, 0x3634U, 0x3734U, 0x3834U, 0x3934U, 0x2B34U, 0x2F34U, + 0x4135U, 0x4235U, 0x4335U, 0x4435U, 0x4535U, 0x4635U, 0x4735U, 0x4835U, + 0x4935U, 0x4A35U, 0x4B35U, 0x4C35U, 0x4D35U, 0x4E35U, 0x4F35U, 0x5035U, + 0x5135U, 0x5235U, 0x5335U, 0x5435U, 0x5535U, 0x5635U, 0x5735U, 0x5835U, + 0x5935U, 0x5A35U, 0x6135U, 0x6235U, 0x6335U, 0x6435U, 0x6535U, 0x6635U, + 0x6735U, 0x6835U, 0x6935U, 0x6A35U, 0x6B35U, 0x6C35U, 0x6D35U, 0x6E35U, + 0x6F35U, 0x7035U, 0x7135U, 0x7235U, 0x7335U, 0x7435U, 0x7535U, 0x7635U, + 0x7735U, 0x7835U, 0x7935U, 0x7A35U, 0x3035U, 0x3135U, 0x3235U, 0x3335U, + 0x3435U, 0x3535U, 0x3635U, 0x3735U, 0x3835U, 0x3935U, 0x2B35U, 0x2F35U, + 0x4136U, 0x4236U, 0x4336U, 0x4436U, 0x4536U, 0x4636U, 0x4736U, 0x4836U, + 0x4936U, 0x4A36U, 0x4B36U, 0x4C36U, 0x4D36U, 0x4E36U, 0x4F36U, 0x5036U, + 0x5136U, 0x5236U, 0x5336U, 0x5436U, 0x5536U, 0x5636U, 0x5736U, 0x5836U, + 0x5936U, 0x5A36U, 0x6136U, 0x6236U, 0x6336U, 0x6436U, 0x6536U, 0x6636U, + 0x6736U, 0x6836U, 0x6936U, 0x6A36U, 0x6B36U, 0x6C36U, 0x6D36U, 0x6E36U, + 0x6F36U, 0x7036U, 0x7136U, 0x7236U, 0x7336U, 0x7436U, 0x7536U, 0x7636U, + 0x7736U, 0x7836U, 0x7936U, 0x7A36U, 0x3036U, 0x3136U, 0x3236U, 0x3336U, + 0x3436U, 0x3536U, 0x3636U, 0x3736U, 0x3836U, 0x3936U, 0x2B36U, 0x2F36U, + 0x4137U, 0x4237U, 0x4337U, 0x4437U, 0x4537U, 0x4637U, 0x4737U, 0x4837U, + 0x4937U, 0x4A37U, 0x4B37U, 0x4C37U, 0x4D37U, 0x4E37U, 0x4F37U, 0x5037U, + 0x5137U, 0x5237U, 0x5337U, 0x5437U, 0x5537U, 0x5637U, 0x5737U, 0x5837U, + 0x5937U, 0x5A37U, 0x6137U, 0x6237U, 0x6337U, 0x6437U, 0x6537U, 0x6637U, + 0x6737U, 0x6837U, 0x6937U, 0x6A37U, 0x6B37U, 0x6C37U, 0x6D37U, 0x6E37U, + 0x6F37U, 0x7037U, 0x7137U, 0x7237U, 0x7337U, 0x7437U, 0x7537U, 0x7637U, + 0x7737U, 0x7837U, 0x7937U, 0x7A37U, 0x3037U, 0x3137U, 0x3237U, 0x3337U, + 0x3437U, 0x3537U, 0x3637U, 0x3737U, 0x3837U, 0x3937U, 0x2B37U, 0x2F37U, + 0x4138U, 0x4238U, 0x4338U, 0x4438U, 0x4538U, 0x4638U, 0x4738U, 0x4838U, + 0x4938U, 0x4A38U, 0x4B38U, 0x4C38U, 0x4D38U, 0x4E38U, 0x4F38U, 0x5038U, + 0x5138U, 0x5238U, 0x5338U, 0x5438U, 0x5538U, 0x5638U, 0x5738U, 0x5838U, + 0x5938U, 0x5A38U, 0x6138U, 0x6238U, 0x6338U, 0x6438U, 0x6538U, 0x6638U, + 0x6738U, 0x6838U, 0x6938U, 0x6A38U, 0x6B38U, 0x6C38U, 0x6D38U, 0x6E38U, + 0x6F38U, 0x7038U, 0x7138U, 0x7238U, 0x7338U, 0x7438U, 0x7538U, 0x7638U, + 0x7738U, 0x7838U, 0x7938U, 0x7A38U, 0x3038U, 0x3138U, 0x3238U, 0x3338U, + 0x3438U, 0x3538U, 0x3638U, 0x3738U, 0x3838U, 0x3938U, 0x2B38U, 0x2F38U, + 0x4139U, 0x4239U, 0x4339U, 0x4439U, 0x4539U, 0x4639U, 0x4739U, 0x4839U, + 0x4939U, 0x4A39U, 0x4B39U, 0x4C39U, 0x4D39U, 0x4E39U, 0x4F39U, 0x5039U, + 0x5139U, 0x5239U, 0x5339U, 0x5439U, 0x5539U, 0x5639U, 0x5739U, 0x5839U, + 0x5939U, 0x5A39U, 0x6139U, 0x6239U, 0x6339U, 0x6439U, 0x6539U, 0x6639U, + 0x6739U, 0x6839U, 0x6939U, 0x6A39U, 0x6B39U, 0x6C39U, 0x6D39U, 0x6E39U, + 0x6F39U, 0x7039U, 0x7139U, 0x7239U, 0x7339U, 0x7439U, 0x7539U, 0x7639U, + 0x7739U, 0x7839U, 0x7939U, 0x7A39U, 0x3039U, 0x3139U, 0x3239U, 0x3339U, + 0x3439U, 0x3539U, 0x3639U, 0x3739U, 0x3839U, 0x3939U, 0x2B39U, 0x2F39U, + 0x412BU, 0x422BU, 0x432BU, 0x442BU, 0x452BU, 0x462BU, 0x472BU, 0x482BU, + 0x492BU, 0x4A2BU, 0x4B2BU, 0x4C2BU, 0x4D2BU, 0x4E2BU, 0x4F2BU, 0x502BU, + 0x512BU, 0x522BU, 0x532BU, 0x542BU, 0x552BU, 0x562BU, 0x572BU, 0x582BU, + 0x592BU, 0x5A2BU, 0x612BU, 0x622BU, 0x632BU, 0x642BU, 0x652BU, 0x662BU, + 0x672BU, 0x682BU, 0x692BU, 0x6A2BU, 0x6B2BU, 0x6C2BU, 0x6D2BU, 0x6E2BU, + 0x6F2BU, 0x702BU, 0x712BU, 0x722BU, 0x732BU, 0x742BU, 0x752BU, 0x762BU, + 0x772BU, 0x782BU, 0x792BU, 0x7A2BU, 0x302BU, 0x312BU, 0x322BU, 0x332BU, + 0x342BU, 0x352BU, 0x362BU, 0x372BU, 0x382BU, 0x392BU, 0x2B2BU, 0x2F2BU, + 0x412FU, 0x422FU, 0x432FU, 0x442FU, 0x452FU, 0x462FU, 0x472FU, 0x482FU, + 0x492FU, 0x4A2FU, 0x4B2FU, 0x4C2FU, 0x4D2FU, 0x4E2FU, 0x4F2FU, 0x502FU, + 0x512FU, 0x522FU, 0x532FU, 0x542FU, 0x552FU, 0x562FU, 0x572FU, 0x582FU, + 0x592FU, 0x5A2FU, 0x612FU, 0x622FU, 0x632FU, 0x642FU, 0x652FU, 0x662FU, + 0x672FU, 0x682FU, 0x692FU, 0x6A2FU, 0x6B2FU, 0x6C2FU, 0x6D2FU, 0x6E2FU, + 0x6F2FU, 0x702FU, 0x712FU, 0x722FU, 0x732FU, 0x742FU, 0x752FU, 0x762FU, + 0x772FU, 0x782FU, 0x792FU, 0x7A2FU, 0x302FU, 0x312FU, 0x322FU, 0x332FU, + 0x342FU, 0x352FU, 0x362FU, 0x372FU, 0x382FU, 0x392FU, 0x2B2FU, 0x2F2FU, +#else + 0x4141U, 0x4142U, 0x4143U, 0x4144U, 0x4145U, 0x4146U, 0x4147U, 0x4148U, + 0x4149U, 0x414AU, 0x414BU, 0x414CU, 0x414DU, 0x414EU, 0x414FU, 0x4150U, + 0x4151U, 0x4152U, 0x4153U, 0x4154U, 0x4155U, 0x4156U, 0x4157U, 0x4158U, + 0x4159U, 0x415AU, 0x4161U, 0x4162U, 0x4163U, 0x4164U, 0x4165U, 0x4166U, + 0x4167U, 0x4168U, 0x4169U, 0x416AU, 0x416BU, 0x416CU, 0x416DU, 0x416EU, + 0x416FU, 0x4170U, 0x4171U, 0x4172U, 0x4173U, 0x4174U, 0x4175U, 0x4176U, + 0x4177U, 0x4178U, 0x4179U, 0x417AU, 0x4130U, 0x4131U, 0x4132U, 0x4133U, + 0x4134U, 0x4135U, 0x4136U, 0x4137U, 0x4138U, 0x4139U, 0x412BU, 0x412FU, + 0x4241U, 0x4242U, 0x4243U, 0x4244U, 0x4245U, 0x4246U, 0x4247U, 0x4248U, + 0x4249U, 0x424AU, 0x424BU, 0x424CU, 0x424DU, 0x424EU, 0x424FU, 0x4250U, + 0x4251U, 0x4252U, 0x4253U, 0x4254U, 0x4255U, 0x4256U, 0x4257U, 0x4258U, + 0x4259U, 0x425AU, 0x4261U, 0x4262U, 0x4263U, 0x4264U, 0x4265U, 0x4266U, + 0x4267U, 0x4268U, 0x4269U, 0x426AU, 0x426BU, 0x426CU, 0x426DU, 0x426EU, + 0x426FU, 0x4270U, 0x4271U, 0x4272U, 0x4273U, 0x4274U, 0x4275U, 0x4276U, + 0x4277U, 0x4278U, 0x4279U, 0x427AU, 0x4230U, 0x4231U, 0x4232U, 0x4233U, + 0x4234U, 0x4235U, 0x4236U, 0x4237U, 0x4238U, 0x4239U, 0x422BU, 0x422FU, + 0x4341U, 0x4342U, 0x4343U, 0x4344U, 0x4345U, 0x4346U, 0x4347U, 0x4348U, + 0x4349U, 0x434AU, 0x434BU, 0x434CU, 0x434DU, 0x434EU, 0x434FU, 0x4350U, + 0x4351U, 0x4352U, 0x4353U, 0x4354U, 0x4355U, 0x4356U, 0x4357U, 0x4358U, + 0x4359U, 0x435AU, 0x4361U, 0x4362U, 0x4363U, 0x4364U, 0x4365U, 0x4366U, + 0x4367U, 0x4368U, 0x4369U, 0x436AU, 0x436BU, 0x436CU, 0x436DU, 0x436EU, + 0x436FU, 0x4370U, 0x4371U, 0x4372U, 0x4373U, 0x4374U, 0x4375U, 0x4376U, + 0x4377U, 0x4378U, 0x4379U, 0x437AU, 0x4330U, 0x4331U, 0x4332U, 0x4333U, + 0x4334U, 0x4335U, 0x4336U, 0x4337U, 0x4338U, 0x4339U, 0x432BU, 0x432FU, + 0x4441U, 0x4442U, 0x4443U, 0x4444U, 0x4445U, 0x4446U, 0x4447U, 0x4448U, + 0x4449U, 0x444AU, 0x444BU, 0x444CU, 0x444DU, 0x444EU, 0x444FU, 0x4450U, + 0x4451U, 0x4452U, 0x4453U, 0x4454U, 0x4455U, 0x4456U, 0x4457U, 0x4458U, + 0x4459U, 0x445AU, 0x4461U, 0x4462U, 0x4463U, 0x4464U, 0x4465U, 0x4466U, + 0x4467U, 0x4468U, 0x4469U, 0x446AU, 0x446BU, 0x446CU, 0x446DU, 0x446EU, + 0x446FU, 0x4470U, 0x4471U, 0x4472U, 0x4473U, 0x4474U, 0x4475U, 0x4476U, + 0x4477U, 0x4478U, 0x4479U, 0x447AU, 0x4430U, 0x4431U, 0x4432U, 0x4433U, + 0x4434U, 0x4435U, 0x4436U, 0x4437U, 0x4438U, 0x4439U, 0x442BU, 0x442FU, + 0x4541U, 0x4542U, 0x4543U, 0x4544U, 0x4545U, 0x4546U, 0x4547U, 0x4548U, + 0x4549U, 0x454AU, 0x454BU, 0x454CU, 0x454DU, 0x454EU, 0x454FU, 0x4550U, + 0x4551U, 0x4552U, 0x4553U, 0x4554U, 0x4555U, 0x4556U, 0x4557U, 0x4558U, + 0x4559U, 0x455AU, 0x4561U, 0x4562U, 0x4563U, 0x4564U, 0x4565U, 0x4566U, + 0x4567U, 0x4568U, 0x4569U, 0x456AU, 0x456BU, 0x456CU, 0x456DU, 0x456EU, + 0x456FU, 0x4570U, 0x4571U, 0x4572U, 0x4573U, 0x4574U, 0x4575U, 0x4576U, + 0x4577U, 0x4578U, 0x4579U, 0x457AU, 0x4530U, 0x4531U, 0x4532U, 0x4533U, + 0x4534U, 0x4535U, 0x4536U, 0x4537U, 0x4538U, 0x4539U, 0x452BU, 0x452FU, + 0x4641U, 0x4642U, 0x4643U, 0x4644U, 0x4645U, 0x4646U, 0x4647U, 0x4648U, + 0x4649U, 0x464AU, 0x464BU, 0x464CU, 0x464DU, 0x464EU, 0x464FU, 0x4650U, + 0x4651U, 0x4652U, 0x4653U, 0x4654U, 0x4655U, 0x4656U, 0x4657U, 0x4658U, + 0x4659U, 0x465AU, 0x4661U, 0x4662U, 0x4663U, 0x4664U, 0x4665U, 0x4666U, + 0x4667U, 0x4668U, 0x4669U, 0x466AU, 0x466BU, 0x466CU, 0x466DU, 0x466EU, + 0x466FU, 0x4670U, 0x4671U, 0x4672U, 0x4673U, 0x4674U, 0x4675U, 0x4676U, + 0x4677U, 0x4678U, 0x4679U, 0x467AU, 0x4630U, 0x4631U, 0x4632U, 0x4633U, + 0x4634U, 0x4635U, 0x4636U, 0x4637U, 0x4638U, 0x4639U, 0x462BU, 0x462FU, + 0x4741U, 0x4742U, 0x4743U, 0x4744U, 0x4745U, 0x4746U, 0x4747U, 0x4748U, + 0x4749U, 0x474AU, 0x474BU, 0x474CU, 0x474DU, 0x474EU, 0x474FU, 0x4750U, + 0x4751U, 0x4752U, 0x4753U, 0x4754U, 0x4755U, 0x4756U, 0x4757U, 0x4758U, + 0x4759U, 0x475AU, 0x4761U, 0x4762U, 0x4763U, 0x4764U, 0x4765U, 0x4766U, + 0x4767U, 0x4768U, 0x4769U, 0x476AU, 0x476BU, 0x476CU, 0x476DU, 0x476EU, + 0x476FU, 0x4770U, 0x4771U, 0x4772U, 0x4773U, 0x4774U, 0x4775U, 0x4776U, + 0x4777U, 0x4778U, 0x4779U, 0x477AU, 0x4730U, 0x4731U, 0x4732U, 0x4733U, + 0x4734U, 0x4735U, 0x4736U, 0x4737U, 0x4738U, 0x4739U, 0x472BU, 0x472FU, + 0x4841U, 0x4842U, 0x4843U, 0x4844U, 0x4845U, 0x4846U, 0x4847U, 0x4848U, + 0x4849U, 0x484AU, 0x484BU, 0x484CU, 0x484DU, 0x484EU, 0x484FU, 0x4850U, + 0x4851U, 0x4852U, 0x4853U, 0x4854U, 0x4855U, 0x4856U, 0x4857U, 0x4858U, + 0x4859U, 0x485AU, 0x4861U, 0x4862U, 0x4863U, 0x4864U, 0x4865U, 0x4866U, + 0x4867U, 0x4868U, 0x4869U, 0x486AU, 0x486BU, 0x486CU, 0x486DU, 0x486EU, + 0x486FU, 0x4870U, 0x4871U, 0x4872U, 0x4873U, 0x4874U, 0x4875U, 0x4876U, + 0x4877U, 0x4878U, 0x4879U, 0x487AU, 0x4830U, 0x4831U, 0x4832U, 0x4833U, + 0x4834U, 0x4835U, 0x4836U, 0x4837U, 0x4838U, 0x4839U, 0x482BU, 0x482FU, + 0x4941U, 0x4942U, 0x4943U, 0x4944U, 0x4945U, 0x4946U, 0x4947U, 0x4948U, + 0x4949U, 0x494AU, 0x494BU, 0x494CU, 0x494DU, 0x494EU, 0x494FU, 0x4950U, + 0x4951U, 0x4952U, 0x4953U, 0x4954U, 0x4955U, 0x4956U, 0x4957U, 0x4958U, + 0x4959U, 0x495AU, 0x4961U, 0x4962U, 0x4963U, 0x4964U, 0x4965U, 0x4966U, + 0x4967U, 0x4968U, 0x4969U, 0x496AU, 0x496BU, 0x496CU, 0x496DU, 0x496EU, + 0x496FU, 0x4970U, 0x4971U, 0x4972U, 0x4973U, 0x4974U, 0x4975U, 0x4976U, + 0x4977U, 0x4978U, 0x4979U, 0x497AU, 0x4930U, 0x4931U, 0x4932U, 0x4933U, + 0x4934U, 0x4935U, 0x4936U, 0x4937U, 0x4938U, 0x4939U, 0x492BU, 0x492FU, + 0x4A41U, 0x4A42U, 0x4A43U, 0x4A44U, 0x4A45U, 0x4A46U, 0x4A47U, 0x4A48U, + 0x4A49U, 0x4A4AU, 0x4A4BU, 0x4A4CU, 0x4A4DU, 0x4A4EU, 0x4A4FU, 0x4A50U, + 0x4A51U, 0x4A52U, 0x4A53U, 0x4A54U, 0x4A55U, 0x4A56U, 0x4A57U, 0x4A58U, + 0x4A59U, 0x4A5AU, 0x4A61U, 0x4A62U, 0x4A63U, 0x4A64U, 0x4A65U, 0x4A66U, + 0x4A67U, 0x4A68U, 0x4A69U, 0x4A6AU, 0x4A6BU, 0x4A6CU, 0x4A6DU, 0x4A6EU, + 0x4A6FU, 0x4A70U, 0x4A71U, 0x4A72U, 0x4A73U, 0x4A74U, 0x4A75U, 0x4A76U, + 0x4A77U, 0x4A78U, 0x4A79U, 0x4A7AU, 0x4A30U, 0x4A31U, 0x4A32U, 0x4A33U, + 0x4A34U, 0x4A35U, 0x4A36U, 0x4A37U, 0x4A38U, 0x4A39U, 0x4A2BU, 0x4A2FU, + 0x4B41U, 0x4B42U, 0x4B43U, 0x4B44U, 0x4B45U, 0x4B46U, 0x4B47U, 0x4B48U, + 0x4B49U, 0x4B4AU, 0x4B4BU, 0x4B4CU, 0x4B4DU, 0x4B4EU, 0x4B4FU, 0x4B50U, + 0x4B51U, 0x4B52U, 0x4B53U, 0x4B54U, 0x4B55U, 0x4B56U, 0x4B57U, 0x4B58U, + 0x4B59U, 0x4B5AU, 0x4B61U, 0x4B62U, 0x4B63U, 0x4B64U, 0x4B65U, 0x4B66U, + 0x4B67U, 0x4B68U, 0x4B69U, 0x4B6AU, 0x4B6BU, 0x4B6CU, 0x4B6DU, 0x4B6EU, + 0x4B6FU, 0x4B70U, 0x4B71U, 0x4B72U, 0x4B73U, 0x4B74U, 0x4B75U, 0x4B76U, + 0x4B77U, 0x4B78U, 0x4B79U, 0x4B7AU, 0x4B30U, 0x4B31U, 0x4B32U, 0x4B33U, + 0x4B34U, 0x4B35U, 0x4B36U, 0x4B37U, 0x4B38U, 0x4B39U, 0x4B2BU, 0x4B2FU, + 0x4C41U, 0x4C42U, 0x4C43U, 0x4C44U, 0x4C45U, 0x4C46U, 0x4C47U, 0x4C48U, + 0x4C49U, 0x4C4AU, 0x4C4BU, 0x4C4CU, 0x4C4DU, 0x4C4EU, 0x4C4FU, 0x4C50U, + 0x4C51U, 0x4C52U, 0x4C53U, 0x4C54U, 0x4C55U, 0x4C56U, 0x4C57U, 0x4C58U, + 0x4C59U, 0x4C5AU, 0x4C61U, 0x4C62U, 0x4C63U, 0x4C64U, 0x4C65U, 0x4C66U, + 0x4C67U, 0x4C68U, 0x4C69U, 0x4C6AU, 0x4C6BU, 0x4C6CU, 0x4C6DU, 0x4C6EU, + 0x4C6FU, 0x4C70U, 0x4C71U, 0x4C72U, 0x4C73U, 0x4C74U, 0x4C75U, 0x4C76U, + 0x4C77U, 0x4C78U, 0x4C79U, 0x4C7AU, 0x4C30U, 0x4C31U, 0x4C32U, 0x4C33U, + 0x4C34U, 0x4C35U, 0x4C36U, 0x4C37U, 0x4C38U, 0x4C39U, 0x4C2BU, 0x4C2FU, + 0x4D41U, 0x4D42U, 0x4D43U, 0x4D44U, 0x4D45U, 0x4D46U, 0x4D47U, 0x4D48U, + 0x4D49U, 0x4D4AU, 0x4D4BU, 0x4D4CU, 0x4D4DU, 0x4D4EU, 0x4D4FU, 0x4D50U, + 0x4D51U, 0x4D52U, 0x4D53U, 0x4D54U, 0x4D55U, 0x4D56U, 0x4D57U, 0x4D58U, + 0x4D59U, 0x4D5AU, 0x4D61U, 0x4D62U, 0x4D63U, 0x4D64U, 0x4D65U, 0x4D66U, + 0x4D67U, 0x4D68U, 0x4D69U, 0x4D6AU, 0x4D6BU, 0x4D6CU, 0x4D6DU, 0x4D6EU, + 0x4D6FU, 0x4D70U, 0x4D71U, 0x4D72U, 0x4D73U, 0x4D74U, 0x4D75U, 0x4D76U, + 0x4D77U, 0x4D78U, 0x4D79U, 0x4D7AU, 0x4D30U, 0x4D31U, 0x4D32U, 0x4D33U, + 0x4D34U, 0x4D35U, 0x4D36U, 0x4D37U, 0x4D38U, 0x4D39U, 0x4D2BU, 0x4D2FU, + 0x4E41U, 0x4E42U, 0x4E43U, 0x4E44U, 0x4E45U, 0x4E46U, 0x4E47U, 0x4E48U, + 0x4E49U, 0x4E4AU, 0x4E4BU, 0x4E4CU, 0x4E4DU, 0x4E4EU, 0x4E4FU, 0x4E50U, + 0x4E51U, 0x4E52U, 0x4E53U, 0x4E54U, 0x4E55U, 0x4E56U, 0x4E57U, 0x4E58U, + 0x4E59U, 0x4E5AU, 0x4E61U, 0x4E62U, 0x4E63U, 0x4E64U, 0x4E65U, 0x4E66U, + 0x4E67U, 0x4E68U, 0x4E69U, 0x4E6AU, 0x4E6BU, 0x4E6CU, 0x4E6DU, 0x4E6EU, + 0x4E6FU, 0x4E70U, 0x4E71U, 0x4E72U, 0x4E73U, 0x4E74U, 0x4E75U, 0x4E76U, + 0x4E77U, 0x4E78U, 0x4E79U, 0x4E7AU, 0x4E30U, 0x4E31U, 0x4E32U, 0x4E33U, + 0x4E34U, 0x4E35U, 0x4E36U, 0x4E37U, 0x4E38U, 0x4E39U, 0x4E2BU, 0x4E2FU, + 0x4F41U, 0x4F42U, 0x4F43U, 0x4F44U, 0x4F45U, 0x4F46U, 0x4F47U, 0x4F48U, + 0x4F49U, 0x4F4AU, 0x4F4BU, 0x4F4CU, 0x4F4DU, 0x4F4EU, 0x4F4FU, 0x4F50U, + 0x4F51U, 0x4F52U, 0x4F53U, 0x4F54U, 0x4F55U, 0x4F56U, 0x4F57U, 0x4F58U, + 0x4F59U, 0x4F5AU, 0x4F61U, 0x4F62U, 0x4F63U, 0x4F64U, 0x4F65U, 0x4F66U, + 0x4F67U, 0x4F68U, 0x4F69U, 0x4F6AU, 0x4F6BU, 0x4F6CU, 0x4F6DU, 0x4F6EU, + 0x4F6FU, 0x4F70U, 0x4F71U, 0x4F72U, 0x4F73U, 0x4F74U, 0x4F75U, 0x4F76U, + 0x4F77U, 0x4F78U, 0x4F79U, 0x4F7AU, 0x4F30U, 0x4F31U, 0x4F32U, 0x4F33U, + 0x4F34U, 0x4F35U, 0x4F36U, 0x4F37U, 0x4F38U, 0x4F39U, 0x4F2BU, 0x4F2FU, + 0x5041U, 0x5042U, 0x5043U, 0x5044U, 0x5045U, 0x5046U, 0x5047U, 0x5048U, + 0x5049U, 0x504AU, 0x504BU, 0x504CU, 0x504DU, 0x504EU, 0x504FU, 0x5050U, + 0x5051U, 0x5052U, 0x5053U, 0x5054U, 0x5055U, 0x5056U, 0x5057U, 0x5058U, + 0x5059U, 0x505AU, 0x5061U, 0x5062U, 0x5063U, 0x5064U, 0x5065U, 0x5066U, + 0x5067U, 0x5068U, 0x5069U, 0x506AU, 0x506BU, 0x506CU, 0x506DU, 0x506EU, + 0x506FU, 0x5070U, 0x5071U, 0x5072U, 0x5073U, 0x5074U, 0x5075U, 0x5076U, + 0x5077U, 0x5078U, 0x5079U, 0x507AU, 0x5030U, 0x5031U, 0x5032U, 0x5033U, + 0x5034U, 0x5035U, 0x5036U, 0x5037U, 0x5038U, 0x5039U, 0x502BU, 0x502FU, + 0x5141U, 0x5142U, 0x5143U, 0x5144U, 0x5145U, 0x5146U, 0x5147U, 0x5148U, + 0x5149U, 0x514AU, 0x514BU, 0x514CU, 0x514DU, 0x514EU, 0x514FU, 0x5150U, + 0x5151U, 0x5152U, 0x5153U, 0x5154U, 0x5155U, 0x5156U, 0x5157U, 0x5158U, + 0x5159U, 0x515AU, 0x5161U, 0x5162U, 0x5163U, 0x5164U, 0x5165U, 0x5166U, + 0x5167U, 0x5168U, 0x5169U, 0x516AU, 0x516BU, 0x516CU, 0x516DU, 0x516EU, + 0x516FU, 0x5170U, 0x5171U, 0x5172U, 0x5173U, 0x5174U, 0x5175U, 0x5176U, + 0x5177U, 0x5178U, 0x5179U, 0x517AU, 0x5130U, 0x5131U, 0x5132U, 0x5133U, + 0x5134U, 0x5135U, 0x5136U, 0x5137U, 0x5138U, 0x5139U, 0x512BU, 0x512FU, + 0x5241U, 0x5242U, 0x5243U, 0x5244U, 0x5245U, 0x5246U, 0x5247U, 0x5248U, + 0x5249U, 0x524AU, 0x524BU, 0x524CU, 0x524DU, 0x524EU, 0x524FU, 0x5250U, + 0x5251U, 0x5252U, 0x5253U, 0x5254U, 0x5255U, 0x5256U, 0x5257U, 0x5258U, + 0x5259U, 0x525AU, 0x5261U, 0x5262U, 0x5263U, 0x5264U, 0x5265U, 0x5266U, + 0x5267U, 0x5268U, 0x5269U, 0x526AU, 0x526BU, 0x526CU, 0x526DU, 0x526EU, + 0x526FU, 0x5270U, 0x5271U, 0x5272U, 0x5273U, 0x5274U, 0x5275U, 0x5276U, + 0x5277U, 0x5278U, 0x5279U, 0x527AU, 0x5230U, 0x5231U, 0x5232U, 0x5233U, + 0x5234U, 0x5235U, 0x5236U, 0x5237U, 0x5238U, 0x5239U, 0x522BU, 0x522FU, + 0x5341U, 0x5342U, 0x5343U, 0x5344U, 0x5345U, 0x5346U, 0x5347U, 0x5348U, + 0x5349U, 0x534AU, 0x534BU, 0x534CU, 0x534DU, 0x534EU, 0x534FU, 0x5350U, + 0x5351U, 0x5352U, 0x5353U, 0x5354U, 0x5355U, 0x5356U, 0x5357U, 0x5358U, + 0x5359U, 0x535AU, 0x5361U, 0x5362U, 0x5363U, 0x5364U, 0x5365U, 0x5366U, + 0x5367U, 0x5368U, 0x5369U, 0x536AU, 0x536BU, 0x536CU, 0x536DU, 0x536EU, + 0x536FU, 0x5370U, 0x5371U, 0x5372U, 0x5373U, 0x5374U, 0x5375U, 0x5376U, + 0x5377U, 0x5378U, 0x5379U, 0x537AU, 0x5330U, 0x5331U, 0x5332U, 0x5333U, + 0x5334U, 0x5335U, 0x5336U, 0x5337U, 0x5338U, 0x5339U, 0x532BU, 0x532FU, + 0x5441U, 0x5442U, 0x5443U, 0x5444U, 0x5445U, 0x5446U, 0x5447U, 0x5448U, + 0x5449U, 0x544AU, 0x544BU, 0x544CU, 0x544DU, 0x544EU, 0x544FU, 0x5450U, + 0x5451U, 0x5452U, 0x5453U, 0x5454U, 0x5455U, 0x5456U, 0x5457U, 0x5458U, + 0x5459U, 0x545AU, 0x5461U, 0x5462U, 0x5463U, 0x5464U, 0x5465U, 0x5466U, + 0x5467U, 0x5468U, 0x5469U, 0x546AU, 0x546BU, 0x546CU, 0x546DU, 0x546EU, + 0x546FU, 0x5470U, 0x5471U, 0x5472U, 0x5473U, 0x5474U, 0x5475U, 0x5476U, + 0x5477U, 0x5478U, 0x5479U, 0x547AU, 0x5430U, 0x5431U, 0x5432U, 0x5433U, + 0x5434U, 0x5435U, 0x5436U, 0x5437U, 0x5438U, 0x5439U, 0x542BU, 0x542FU, + 0x5541U, 0x5542U, 0x5543U, 0x5544U, 0x5545U, 0x5546U, 0x5547U, 0x5548U, + 0x5549U, 0x554AU, 0x554BU, 0x554CU, 0x554DU, 0x554EU, 0x554FU, 0x5550U, + 0x5551U, 0x5552U, 0x5553U, 0x5554U, 0x5555U, 0x5556U, 0x5557U, 0x5558U, + 0x5559U, 0x555AU, 0x5561U, 0x5562U, 0x5563U, 0x5564U, 0x5565U, 0x5566U, + 0x5567U, 0x5568U, 0x5569U, 0x556AU, 0x556BU, 0x556CU, 0x556DU, 0x556EU, + 0x556FU, 0x5570U, 0x5571U, 0x5572U, 0x5573U, 0x5574U, 0x5575U, 0x5576U, + 0x5577U, 0x5578U, 0x5579U, 0x557AU, 0x5530U, 0x5531U, 0x5532U, 0x5533U, + 0x5534U, 0x5535U, 0x5536U, 0x5537U, 0x5538U, 0x5539U, 0x552BU, 0x552FU, + 0x5641U, 0x5642U, 0x5643U, 0x5644U, 0x5645U, 0x5646U, 0x5647U, 0x5648U, + 0x5649U, 0x564AU, 0x564BU, 0x564CU, 0x564DU, 0x564EU, 0x564FU, 0x5650U, + 0x5651U, 0x5652U, 0x5653U, 0x5654U, 0x5655U, 0x5656U, 0x5657U, 0x5658U, + 0x5659U, 0x565AU, 0x5661U, 0x5662U, 0x5663U, 0x5664U, 0x5665U, 0x5666U, + 0x5667U, 0x5668U, 0x5669U, 0x566AU, 0x566BU, 0x566CU, 0x566DU, 0x566EU, + 0x566FU, 0x5670U, 0x5671U, 0x5672U, 0x5673U, 0x5674U, 0x5675U, 0x5676U, + 0x5677U, 0x5678U, 0x5679U, 0x567AU, 0x5630U, 0x5631U, 0x5632U, 0x5633U, + 0x5634U, 0x5635U, 0x5636U, 0x5637U, 0x5638U, 0x5639U, 0x562BU, 0x562FU, + 0x5741U, 0x5742U, 0x5743U, 0x5744U, 0x5745U, 0x5746U, 0x5747U, 0x5748U, + 0x5749U, 0x574AU, 0x574BU, 0x574CU, 0x574DU, 0x574EU, 0x574FU, 0x5750U, + 0x5751U, 0x5752U, 0x5753U, 0x5754U, 0x5755U, 0x5756U, 0x5757U, 0x5758U, + 0x5759U, 0x575AU, 0x5761U, 0x5762U, 0x5763U, 0x5764U, 0x5765U, 0x5766U, + 0x5767U, 0x5768U, 0x5769U, 0x576AU, 0x576BU, 0x576CU, 0x576DU, 0x576EU, + 0x576FU, 0x5770U, 0x5771U, 0x5772U, 0x5773U, 0x5774U, 0x5775U, 0x5776U, + 0x5777U, 0x5778U, 0x5779U, 0x577AU, 0x5730U, 0x5731U, 0x5732U, 0x5733U, + 0x5734U, 0x5735U, 0x5736U, 0x5737U, 0x5738U, 0x5739U, 0x572BU, 0x572FU, + 0x5841U, 0x5842U, 0x5843U, 0x5844U, 0x5845U, 0x5846U, 0x5847U, 0x5848U, + 0x5849U, 0x584AU, 0x584BU, 0x584CU, 0x584DU, 0x584EU, 0x584FU, 0x5850U, + 0x5851U, 0x5852U, 0x5853U, 0x5854U, 0x5855U, 0x5856U, 0x5857U, 0x5858U, + 0x5859U, 0x585AU, 0x5861U, 0x5862U, 0x5863U, 0x5864U, 0x5865U, 0x5866U, + 0x5867U, 0x5868U, 0x5869U, 0x586AU, 0x586BU, 0x586CU, 0x586DU, 0x586EU, + 0x586FU, 0x5870U, 0x5871U, 0x5872U, 0x5873U, 0x5874U, 0x5875U, 0x5876U, + 0x5877U, 0x5878U, 0x5879U, 0x587AU, 0x5830U, 0x5831U, 0x5832U, 0x5833U, + 0x5834U, 0x5835U, 0x5836U, 0x5837U, 0x5838U, 0x5839U, 0x582BU, 0x582FU, + 0x5941U, 0x5942U, 0x5943U, 0x5944U, 0x5945U, 0x5946U, 0x5947U, 0x5948U, + 0x5949U, 0x594AU, 0x594BU, 0x594CU, 0x594DU, 0x594EU, 0x594FU, 0x5950U, + 0x5951U, 0x5952U, 0x5953U, 0x5954U, 0x5955U, 0x5956U, 0x5957U, 0x5958U, + 0x5959U, 0x595AU, 0x5961U, 0x5962U, 0x5963U, 0x5964U, 0x5965U, 0x5966U, + 0x5967U, 0x5968U, 0x5969U, 0x596AU, 0x596BU, 0x596CU, 0x596DU, 0x596EU, + 0x596FU, 0x5970U, 0x5971U, 0x5972U, 0x5973U, 0x5974U, 0x5975U, 0x5976U, + 0x5977U, 0x5978U, 0x5979U, 0x597AU, 0x5930U, 0x5931U, 0x5932U, 0x5933U, + 0x5934U, 0x5935U, 0x5936U, 0x5937U, 0x5938U, 0x5939U, 0x592BU, 0x592FU, + 0x5A41U, 0x5A42U, 0x5A43U, 0x5A44U, 0x5A45U, 0x5A46U, 0x5A47U, 0x5A48U, + 0x5A49U, 0x5A4AU, 0x5A4BU, 0x5A4CU, 0x5A4DU, 0x5A4EU, 0x5A4FU, 0x5A50U, + 0x5A51U, 0x5A52U, 0x5A53U, 0x5A54U, 0x5A55U, 0x5A56U, 0x5A57U, 0x5A58U, + 0x5A59U, 0x5A5AU, 0x5A61U, 0x5A62U, 0x5A63U, 0x5A64U, 0x5A65U, 0x5A66U, + 0x5A67U, 0x5A68U, 0x5A69U, 0x5A6AU, 0x5A6BU, 0x5A6CU, 0x5A6DU, 0x5A6EU, + 0x5A6FU, 0x5A70U, 0x5A71U, 0x5A72U, 0x5A73U, 0x5A74U, 0x5A75U, 0x5A76U, + 0x5A77U, 0x5A78U, 0x5A79U, 0x5A7AU, 0x5A30U, 0x5A31U, 0x5A32U, 0x5A33U, + 0x5A34U, 0x5A35U, 0x5A36U, 0x5A37U, 0x5A38U, 0x5A39U, 0x5A2BU, 0x5A2FU, + 0x6141U, 0x6142U, 0x6143U, 0x6144U, 0x6145U, 0x6146U, 0x6147U, 0x6148U, + 0x6149U, 0x614AU, 0x614BU, 0x614CU, 0x614DU, 0x614EU, 0x614FU, 0x6150U, + 0x6151U, 0x6152U, 0x6153U, 0x6154U, 0x6155U, 0x6156U, 0x6157U, 0x6158U, + 0x6159U, 0x615AU, 0x6161U, 0x6162U, 0x6163U, 0x6164U, 0x6165U, 0x6166U, + 0x6167U, 0x6168U, 0x6169U, 0x616AU, 0x616BU, 0x616CU, 0x616DU, 0x616EU, + 0x616FU, 0x6170U, 0x6171U, 0x6172U, 0x6173U, 0x6174U, 0x6175U, 0x6176U, + 0x6177U, 0x6178U, 0x6179U, 0x617AU, 0x6130U, 0x6131U, 0x6132U, 0x6133U, + 0x6134U, 0x6135U, 0x6136U, 0x6137U, 0x6138U, 0x6139U, 0x612BU, 0x612FU, + 0x6241U, 0x6242U, 0x6243U, 0x6244U, 0x6245U, 0x6246U, 0x6247U, 0x6248U, + 0x6249U, 0x624AU, 0x624BU, 0x624CU, 0x624DU, 0x624EU, 0x624FU, 0x6250U, + 0x6251U, 0x6252U, 0x6253U, 0x6254U, 0x6255U, 0x6256U, 0x6257U, 0x6258U, + 0x6259U, 0x625AU, 0x6261U, 0x6262U, 0x6263U, 0x6264U, 0x6265U, 0x6266U, + 0x6267U, 0x6268U, 0x6269U, 0x626AU, 0x626BU, 0x626CU, 0x626DU, 0x626EU, + 0x626FU, 0x6270U, 0x6271U, 0x6272U, 0x6273U, 0x6274U, 0x6275U, 0x6276U, + 0x6277U, 0x6278U, 0x6279U, 0x627AU, 0x6230U, 0x6231U, 0x6232U, 0x6233U, + 0x6234U, 0x6235U, 0x6236U, 0x6237U, 0x6238U, 0x6239U, 0x622BU, 0x622FU, + 0x6341U, 0x6342U, 0x6343U, 0x6344U, 0x6345U, 0x6346U, 0x6347U, 0x6348U, + 0x6349U, 0x634AU, 0x634BU, 0x634CU, 0x634DU, 0x634EU, 0x634FU, 0x6350U, + 0x6351U, 0x6352U, 0x6353U, 0x6354U, 0x6355U, 0x6356U, 0x6357U, 0x6358U, + 0x6359U, 0x635AU, 0x6361U, 0x6362U, 0x6363U, 0x6364U, 0x6365U, 0x6366U, + 0x6367U, 0x6368U, 0x6369U, 0x636AU, 0x636BU, 0x636CU, 0x636DU, 0x636EU, + 0x636FU, 0x6370U, 0x6371U, 0x6372U, 0x6373U, 0x6374U, 0x6375U, 0x6376U, + 0x6377U, 0x6378U, 0x6379U, 0x637AU, 0x6330U, 0x6331U, 0x6332U, 0x6333U, + 0x6334U, 0x6335U, 0x6336U, 0x6337U, 0x6338U, 0x6339U, 0x632BU, 0x632FU, + 0x6441U, 0x6442U, 0x6443U, 0x6444U, 0x6445U, 0x6446U, 0x6447U, 0x6448U, + 0x6449U, 0x644AU, 0x644BU, 0x644CU, 0x644DU, 0x644EU, 0x644FU, 0x6450U, + 0x6451U, 0x6452U, 0x6453U, 0x6454U, 0x6455U, 0x6456U, 0x6457U, 0x6458U, + 0x6459U, 0x645AU, 0x6461U, 0x6462U, 0x6463U, 0x6464U, 0x6465U, 0x6466U, + 0x6467U, 0x6468U, 0x6469U, 0x646AU, 0x646BU, 0x646CU, 0x646DU, 0x646EU, + 0x646FU, 0x6470U, 0x6471U, 0x6472U, 0x6473U, 0x6474U, 0x6475U, 0x6476U, + 0x6477U, 0x6478U, 0x6479U, 0x647AU, 0x6430U, 0x6431U, 0x6432U, 0x6433U, + 0x6434U, 0x6435U, 0x6436U, 0x6437U, 0x6438U, 0x6439U, 0x642BU, 0x642FU, + 0x6541U, 0x6542U, 0x6543U, 0x6544U, 0x6545U, 0x6546U, 0x6547U, 0x6548U, + 0x6549U, 0x654AU, 0x654BU, 0x654CU, 0x654DU, 0x654EU, 0x654FU, 0x6550U, + 0x6551U, 0x6552U, 0x6553U, 0x6554U, 0x6555U, 0x6556U, 0x6557U, 0x6558U, + 0x6559U, 0x655AU, 0x6561U, 0x6562U, 0x6563U, 0x6564U, 0x6565U, 0x6566U, + 0x6567U, 0x6568U, 0x6569U, 0x656AU, 0x656BU, 0x656CU, 0x656DU, 0x656EU, + 0x656FU, 0x6570U, 0x6571U, 0x6572U, 0x6573U, 0x6574U, 0x6575U, 0x6576U, + 0x6577U, 0x6578U, 0x6579U, 0x657AU, 0x6530U, 0x6531U, 0x6532U, 0x6533U, + 0x6534U, 0x6535U, 0x6536U, 0x6537U, 0x6538U, 0x6539U, 0x652BU, 0x652FU, + 0x6641U, 0x6642U, 0x6643U, 0x6644U, 0x6645U, 0x6646U, 0x6647U, 0x6648U, + 0x6649U, 0x664AU, 0x664BU, 0x664CU, 0x664DU, 0x664EU, 0x664FU, 0x6650U, + 0x6651U, 0x6652U, 0x6653U, 0x6654U, 0x6655U, 0x6656U, 0x6657U, 0x6658U, + 0x6659U, 0x665AU, 0x6661U, 0x6662U, 0x6663U, 0x6664U, 0x6665U, 0x6666U, + 0x6667U, 0x6668U, 0x6669U, 0x666AU, 0x666BU, 0x666CU, 0x666DU, 0x666EU, + 0x666FU, 0x6670U, 0x6671U, 0x6672U, 0x6673U, 0x6674U, 0x6675U, 0x6676U, + 0x6677U, 0x6678U, 0x6679U, 0x667AU, 0x6630U, 0x6631U, 0x6632U, 0x6633U, + 0x6634U, 0x6635U, 0x6636U, 0x6637U, 0x6638U, 0x6639U, 0x662BU, 0x662FU, + 0x6741U, 0x6742U, 0x6743U, 0x6744U, 0x6745U, 0x6746U, 0x6747U, 0x6748U, + 0x6749U, 0x674AU, 0x674BU, 0x674CU, 0x674DU, 0x674EU, 0x674FU, 0x6750U, + 0x6751U, 0x6752U, 0x6753U, 0x6754U, 0x6755U, 0x6756U, 0x6757U, 0x6758U, + 0x6759U, 0x675AU, 0x6761U, 0x6762U, 0x6763U, 0x6764U, 0x6765U, 0x6766U, + 0x6767U, 0x6768U, 0x6769U, 0x676AU, 0x676BU, 0x676CU, 0x676DU, 0x676EU, + 0x676FU, 0x6770U, 0x6771U, 0x6772U, 0x6773U, 0x6774U, 0x6775U, 0x6776U, + 0x6777U, 0x6778U, 0x6779U, 0x677AU, 0x6730U, 0x6731U, 0x6732U, 0x6733U, + 0x6734U, 0x6735U, 0x6736U, 0x6737U, 0x6738U, 0x6739U, 0x672BU, 0x672FU, + 0x6841U, 0x6842U, 0x6843U, 0x6844U, 0x6845U, 0x6846U, 0x6847U, 0x6848U, + 0x6849U, 0x684AU, 0x684BU, 0x684CU, 0x684DU, 0x684EU, 0x684FU, 0x6850U, + 0x6851U, 0x6852U, 0x6853U, 0x6854U, 0x6855U, 0x6856U, 0x6857U, 0x6858U, + 0x6859U, 0x685AU, 0x6861U, 0x6862U, 0x6863U, 0x6864U, 0x6865U, 0x6866U, + 0x6867U, 0x6868U, 0x6869U, 0x686AU, 0x686BU, 0x686CU, 0x686DU, 0x686EU, + 0x686FU, 0x6870U, 0x6871U, 0x6872U, 0x6873U, 0x6874U, 0x6875U, 0x6876U, + 0x6877U, 0x6878U, 0x6879U, 0x687AU, 0x6830U, 0x6831U, 0x6832U, 0x6833U, + 0x6834U, 0x6835U, 0x6836U, 0x6837U, 0x6838U, 0x6839U, 0x682BU, 0x682FU, + 0x6941U, 0x6942U, 0x6943U, 0x6944U, 0x6945U, 0x6946U, 0x6947U, 0x6948U, + 0x6949U, 0x694AU, 0x694BU, 0x694CU, 0x694DU, 0x694EU, 0x694FU, 0x6950U, + 0x6951U, 0x6952U, 0x6953U, 0x6954U, 0x6955U, 0x6956U, 0x6957U, 0x6958U, + 0x6959U, 0x695AU, 0x6961U, 0x6962U, 0x6963U, 0x6964U, 0x6965U, 0x6966U, + 0x6967U, 0x6968U, 0x6969U, 0x696AU, 0x696BU, 0x696CU, 0x696DU, 0x696EU, + 0x696FU, 0x6970U, 0x6971U, 0x6972U, 0x6973U, 0x6974U, 0x6975U, 0x6976U, + 0x6977U, 0x6978U, 0x6979U, 0x697AU, 0x6930U, 0x6931U, 0x6932U, 0x6933U, + 0x6934U, 0x6935U, 0x6936U, 0x6937U, 0x6938U, 0x6939U, 0x692BU, 0x692FU, + 0x6A41U, 0x6A42U, 0x6A43U, 0x6A44U, 0x6A45U, 0x6A46U, 0x6A47U, 0x6A48U, + 0x6A49U, 0x6A4AU, 0x6A4BU, 0x6A4CU, 0x6A4DU, 0x6A4EU, 0x6A4FU, 0x6A50U, + 0x6A51U, 0x6A52U, 0x6A53U, 0x6A54U, 0x6A55U, 0x6A56U, 0x6A57U, 0x6A58U, + 0x6A59U, 0x6A5AU, 0x6A61U, 0x6A62U, 0x6A63U, 0x6A64U, 0x6A65U, 0x6A66U, + 0x6A67U, 0x6A68U, 0x6A69U, 0x6A6AU, 0x6A6BU, 0x6A6CU, 0x6A6DU, 0x6A6EU, + 0x6A6FU, 0x6A70U, 0x6A71U, 0x6A72U, 0x6A73U, 0x6A74U, 0x6A75U, 0x6A76U, + 0x6A77U, 0x6A78U, 0x6A79U, 0x6A7AU, 0x6A30U, 0x6A31U, 0x6A32U, 0x6A33U, + 0x6A34U, 0x6A35U, 0x6A36U, 0x6A37U, 0x6A38U, 0x6A39U, 0x6A2BU, 0x6A2FU, + 0x6B41U, 0x6B42U, 0x6B43U, 0x6B44U, 0x6B45U, 0x6B46U, 0x6B47U, 0x6B48U, + 0x6B49U, 0x6B4AU, 0x6B4BU, 0x6B4CU, 0x6B4DU, 0x6B4EU, 0x6B4FU, 0x6B50U, + 0x6B51U, 0x6B52U, 0x6B53U, 0x6B54U, 0x6B55U, 0x6B56U, 0x6B57U, 0x6B58U, + 0x6B59U, 0x6B5AU, 0x6B61U, 0x6B62U, 0x6B63U, 0x6B64U, 0x6B65U, 0x6B66U, + 0x6B67U, 0x6B68U, 0x6B69U, 0x6B6AU, 0x6B6BU, 0x6B6CU, 0x6B6DU, 0x6B6EU, + 0x6B6FU, 0x6B70U, 0x6B71U, 0x6B72U, 0x6B73U, 0x6B74U, 0x6B75U, 0x6B76U, + 0x6B77U, 0x6B78U, 0x6B79U, 0x6B7AU, 0x6B30U, 0x6B31U, 0x6B32U, 0x6B33U, + 0x6B34U, 0x6B35U, 0x6B36U, 0x6B37U, 0x6B38U, 0x6B39U, 0x6B2BU, 0x6B2FU, + 0x6C41U, 0x6C42U, 0x6C43U, 0x6C44U, 0x6C45U, 0x6C46U, 0x6C47U, 0x6C48U, + 0x6C49U, 0x6C4AU, 0x6C4BU, 0x6C4CU, 0x6C4DU, 0x6C4EU, 0x6C4FU, 0x6C50U, + 0x6C51U, 0x6C52U, 0x6C53U, 0x6C54U, 0x6C55U, 0x6C56U, 0x6C57U, 0x6C58U, + 0x6C59U, 0x6C5AU, 0x6C61U, 0x6C62U, 0x6C63U, 0x6C64U, 0x6C65U, 0x6C66U, + 0x6C67U, 0x6C68U, 0x6C69U, 0x6C6AU, 0x6C6BU, 0x6C6CU, 0x6C6DU, 0x6C6EU, + 0x6C6FU, 0x6C70U, 0x6C71U, 0x6C72U, 0x6C73U, 0x6C74U, 0x6C75U, 0x6C76U, + 0x6C77U, 0x6C78U, 0x6C79U, 0x6C7AU, 0x6C30U, 0x6C31U, 0x6C32U, 0x6C33U, + 0x6C34U, 0x6C35U, 0x6C36U, 0x6C37U, 0x6C38U, 0x6C39U, 0x6C2BU, 0x6C2FU, + 0x6D41U, 0x6D42U, 0x6D43U, 0x6D44U, 0x6D45U, 0x6D46U, 0x6D47U, 0x6D48U, + 0x6D49U, 0x6D4AU, 0x6D4BU, 0x6D4CU, 0x6D4DU, 0x6D4EU, 0x6D4FU, 0x6D50U, + 0x6D51U, 0x6D52U, 0x6D53U, 0x6D54U, 0x6D55U, 0x6D56U, 0x6D57U, 0x6D58U, + 0x6D59U, 0x6D5AU, 0x6D61U, 0x6D62U, 0x6D63U, 0x6D64U, 0x6D65U, 0x6D66U, + 0x6D67U, 0x6D68U, 0x6D69U, 0x6D6AU, 0x6D6BU, 0x6D6CU, 0x6D6DU, 0x6D6EU, + 0x6D6FU, 0x6D70U, 0x6D71U, 0x6D72U, 0x6D73U, 0x6D74U, 0x6D75U, 0x6D76U, + 0x6D77U, 0x6D78U, 0x6D79U, 0x6D7AU, 0x6D30U, 0x6D31U, 0x6D32U, 0x6D33U, + 0x6D34U, 0x6D35U, 0x6D36U, 0x6D37U, 0x6D38U, 0x6D39U, 0x6D2BU, 0x6D2FU, + 0x6E41U, 0x6E42U, 0x6E43U, 0x6E44U, 0x6E45U, 0x6E46U, 0x6E47U, 0x6E48U, + 0x6E49U, 0x6E4AU, 0x6E4BU, 0x6E4CU, 0x6E4DU, 0x6E4EU, 0x6E4FU, 0x6E50U, + 0x6E51U, 0x6E52U, 0x6E53U, 0x6E54U, 0x6E55U, 0x6E56U, 0x6E57U, 0x6E58U, + 0x6E59U, 0x6E5AU, 0x6E61U, 0x6E62U, 0x6E63U, 0x6E64U, 0x6E65U, 0x6E66U, + 0x6E67U, 0x6E68U, 0x6E69U, 0x6E6AU, 0x6E6BU, 0x6E6CU, 0x6E6DU, 0x6E6EU, + 0x6E6FU, 0x6E70U, 0x6E71U, 0x6E72U, 0x6E73U, 0x6E74U, 0x6E75U, 0x6E76U, + 0x6E77U, 0x6E78U, 0x6E79U, 0x6E7AU, 0x6E30U, 0x6E31U, 0x6E32U, 0x6E33U, + 0x6E34U, 0x6E35U, 0x6E36U, 0x6E37U, 0x6E38U, 0x6E39U, 0x6E2BU, 0x6E2FU, + 0x6F41U, 0x6F42U, 0x6F43U, 0x6F44U, 0x6F45U, 0x6F46U, 0x6F47U, 0x6F48U, + 0x6F49U, 0x6F4AU, 0x6F4BU, 0x6F4CU, 0x6F4DU, 0x6F4EU, 0x6F4FU, 0x6F50U, + 0x6F51U, 0x6F52U, 0x6F53U, 0x6F54U, 0x6F55U, 0x6F56U, 0x6F57U, 0x6F58U, + 0x6F59U, 0x6F5AU, 0x6F61U, 0x6F62U, 0x6F63U, 0x6F64U, 0x6F65U, 0x6F66U, + 0x6F67U, 0x6F68U, 0x6F69U, 0x6F6AU, 0x6F6BU, 0x6F6CU, 0x6F6DU, 0x6F6EU, + 0x6F6FU, 0x6F70U, 0x6F71U, 0x6F72U, 0x6F73U, 0x6F74U, 0x6F75U, 0x6F76U, + 0x6F77U, 0x6F78U, 0x6F79U, 0x6F7AU, 0x6F30U, 0x6F31U, 0x6F32U, 0x6F33U, + 0x6F34U, 0x6F35U, 0x6F36U, 0x6F37U, 0x6F38U, 0x6F39U, 0x6F2BU, 0x6F2FU, + 0x7041U, 0x7042U, 0x7043U, 0x7044U, 0x7045U, 0x7046U, 0x7047U, 0x7048U, + 0x7049U, 0x704AU, 0x704BU, 0x704CU, 0x704DU, 0x704EU, 0x704FU, 0x7050U, + 0x7051U, 0x7052U, 0x7053U, 0x7054U, 0x7055U, 0x7056U, 0x7057U, 0x7058U, + 0x7059U, 0x705AU, 0x7061U, 0x7062U, 0x7063U, 0x7064U, 0x7065U, 0x7066U, + 0x7067U, 0x7068U, 0x7069U, 0x706AU, 0x706BU, 0x706CU, 0x706DU, 0x706EU, + 0x706FU, 0x7070U, 0x7071U, 0x7072U, 0x7073U, 0x7074U, 0x7075U, 0x7076U, + 0x7077U, 0x7078U, 0x7079U, 0x707AU, 0x7030U, 0x7031U, 0x7032U, 0x7033U, + 0x7034U, 0x7035U, 0x7036U, 0x7037U, 0x7038U, 0x7039U, 0x702BU, 0x702FU, + 0x7141U, 0x7142U, 0x7143U, 0x7144U, 0x7145U, 0x7146U, 0x7147U, 0x7148U, + 0x7149U, 0x714AU, 0x714BU, 0x714CU, 0x714DU, 0x714EU, 0x714FU, 0x7150U, + 0x7151U, 0x7152U, 0x7153U, 0x7154U, 0x7155U, 0x7156U, 0x7157U, 0x7158U, + 0x7159U, 0x715AU, 0x7161U, 0x7162U, 0x7163U, 0x7164U, 0x7165U, 0x7166U, + 0x7167U, 0x7168U, 0x7169U, 0x716AU, 0x716BU, 0x716CU, 0x716DU, 0x716EU, + 0x716FU, 0x7170U, 0x7171U, 0x7172U, 0x7173U, 0x7174U, 0x7175U, 0x7176U, + 0x7177U, 0x7178U, 0x7179U, 0x717AU, 0x7130U, 0x7131U, 0x7132U, 0x7133U, + 0x7134U, 0x7135U, 0x7136U, 0x7137U, 0x7138U, 0x7139U, 0x712BU, 0x712FU, + 0x7241U, 0x7242U, 0x7243U, 0x7244U, 0x7245U, 0x7246U, 0x7247U, 0x7248U, + 0x7249U, 0x724AU, 0x724BU, 0x724CU, 0x724DU, 0x724EU, 0x724FU, 0x7250U, + 0x7251U, 0x7252U, 0x7253U, 0x7254U, 0x7255U, 0x7256U, 0x7257U, 0x7258U, + 0x7259U, 0x725AU, 0x7261U, 0x7262U, 0x7263U, 0x7264U, 0x7265U, 0x7266U, + 0x7267U, 0x7268U, 0x7269U, 0x726AU, 0x726BU, 0x726CU, 0x726DU, 0x726EU, + 0x726FU, 0x7270U, 0x7271U, 0x7272U, 0x7273U, 0x7274U, 0x7275U, 0x7276U, + 0x7277U, 0x7278U, 0x7279U, 0x727AU, 0x7230U, 0x7231U, 0x7232U, 0x7233U, + 0x7234U, 0x7235U, 0x7236U, 0x7237U, 0x7238U, 0x7239U, 0x722BU, 0x722FU, + 0x7341U, 0x7342U, 0x7343U, 0x7344U, 0x7345U, 0x7346U, 0x7347U, 0x7348U, + 0x7349U, 0x734AU, 0x734BU, 0x734CU, 0x734DU, 0x734EU, 0x734FU, 0x7350U, + 0x7351U, 0x7352U, 0x7353U, 0x7354U, 0x7355U, 0x7356U, 0x7357U, 0x7358U, + 0x7359U, 0x735AU, 0x7361U, 0x7362U, 0x7363U, 0x7364U, 0x7365U, 0x7366U, + 0x7367U, 0x7368U, 0x7369U, 0x736AU, 0x736BU, 0x736CU, 0x736DU, 0x736EU, + 0x736FU, 0x7370U, 0x7371U, 0x7372U, 0x7373U, 0x7374U, 0x7375U, 0x7376U, + 0x7377U, 0x7378U, 0x7379U, 0x737AU, 0x7330U, 0x7331U, 0x7332U, 0x7333U, + 0x7334U, 0x7335U, 0x7336U, 0x7337U, 0x7338U, 0x7339U, 0x732BU, 0x732FU, + 0x7441U, 0x7442U, 0x7443U, 0x7444U, 0x7445U, 0x7446U, 0x7447U, 0x7448U, + 0x7449U, 0x744AU, 0x744BU, 0x744CU, 0x744DU, 0x744EU, 0x744FU, 0x7450U, + 0x7451U, 0x7452U, 0x7453U, 0x7454U, 0x7455U, 0x7456U, 0x7457U, 0x7458U, + 0x7459U, 0x745AU, 0x7461U, 0x7462U, 0x7463U, 0x7464U, 0x7465U, 0x7466U, + 0x7467U, 0x7468U, 0x7469U, 0x746AU, 0x746BU, 0x746CU, 0x746DU, 0x746EU, + 0x746FU, 0x7470U, 0x7471U, 0x7472U, 0x7473U, 0x7474U, 0x7475U, 0x7476U, + 0x7477U, 0x7478U, 0x7479U, 0x747AU, 0x7430U, 0x7431U, 0x7432U, 0x7433U, + 0x7434U, 0x7435U, 0x7436U, 0x7437U, 0x7438U, 0x7439U, 0x742BU, 0x742FU, + 0x7541U, 0x7542U, 0x7543U, 0x7544U, 0x7545U, 0x7546U, 0x7547U, 0x7548U, + 0x7549U, 0x754AU, 0x754BU, 0x754CU, 0x754DU, 0x754EU, 0x754FU, 0x7550U, + 0x7551U, 0x7552U, 0x7553U, 0x7554U, 0x7555U, 0x7556U, 0x7557U, 0x7558U, + 0x7559U, 0x755AU, 0x7561U, 0x7562U, 0x7563U, 0x7564U, 0x7565U, 0x7566U, + 0x7567U, 0x7568U, 0x7569U, 0x756AU, 0x756BU, 0x756CU, 0x756DU, 0x756EU, + 0x756FU, 0x7570U, 0x7571U, 0x7572U, 0x7573U, 0x7574U, 0x7575U, 0x7576U, + 0x7577U, 0x7578U, 0x7579U, 0x757AU, 0x7530U, 0x7531U, 0x7532U, 0x7533U, + 0x7534U, 0x7535U, 0x7536U, 0x7537U, 0x7538U, 0x7539U, 0x752BU, 0x752FU, + 0x7641U, 0x7642U, 0x7643U, 0x7644U, 0x7645U, 0x7646U, 0x7647U, 0x7648U, + 0x7649U, 0x764AU, 0x764BU, 0x764CU, 0x764DU, 0x764EU, 0x764FU, 0x7650U, + 0x7651U, 0x7652U, 0x7653U, 0x7654U, 0x7655U, 0x7656U, 0x7657U, 0x7658U, + 0x7659U, 0x765AU, 0x7661U, 0x7662U, 0x7663U, 0x7664U, 0x7665U, 0x7666U, + 0x7667U, 0x7668U, 0x7669U, 0x766AU, 0x766BU, 0x766CU, 0x766DU, 0x766EU, + 0x766FU, 0x7670U, 0x7671U, 0x7672U, 0x7673U, 0x7674U, 0x7675U, 0x7676U, + 0x7677U, 0x7678U, 0x7679U, 0x767AU, 0x7630U, 0x7631U, 0x7632U, 0x7633U, + 0x7634U, 0x7635U, 0x7636U, 0x7637U, 0x7638U, 0x7639U, 0x762BU, 0x762FU, + 0x7741U, 0x7742U, 0x7743U, 0x7744U, 0x7745U, 0x7746U, 0x7747U, 0x7748U, + 0x7749U, 0x774AU, 0x774BU, 0x774CU, 0x774DU, 0x774EU, 0x774FU, 0x7750U, + 0x7751U, 0x7752U, 0x7753U, 0x7754U, 0x7755U, 0x7756U, 0x7757U, 0x7758U, + 0x7759U, 0x775AU, 0x7761U, 0x7762U, 0x7763U, 0x7764U, 0x7765U, 0x7766U, + 0x7767U, 0x7768U, 0x7769U, 0x776AU, 0x776BU, 0x776CU, 0x776DU, 0x776EU, + 0x776FU, 0x7770U, 0x7771U, 0x7772U, 0x7773U, 0x7774U, 0x7775U, 0x7776U, + 0x7777U, 0x7778U, 0x7779U, 0x777AU, 0x7730U, 0x7731U, 0x7732U, 0x7733U, + 0x7734U, 0x7735U, 0x7736U, 0x7737U, 0x7738U, 0x7739U, 0x772BU, 0x772FU, + 0x7841U, 0x7842U, 0x7843U, 0x7844U, 0x7845U, 0x7846U, 0x7847U, 0x7848U, + 0x7849U, 0x784AU, 0x784BU, 0x784CU, 0x784DU, 0x784EU, 0x784FU, 0x7850U, + 0x7851U, 0x7852U, 0x7853U, 0x7854U, 0x7855U, 0x7856U, 0x7857U, 0x7858U, + 0x7859U, 0x785AU, 0x7861U, 0x7862U, 0x7863U, 0x7864U, 0x7865U, 0x7866U, + 0x7867U, 0x7868U, 0x7869U, 0x786AU, 0x786BU, 0x786CU, 0x786DU, 0x786EU, + 0x786FU, 0x7870U, 0x7871U, 0x7872U, 0x7873U, 0x7874U, 0x7875U, 0x7876U, + 0x7877U, 0x7878U, 0x7879U, 0x787AU, 0x7830U, 0x7831U, 0x7832U, 0x7833U, + 0x7834U, 0x7835U, 0x7836U, 0x7837U, 0x7838U, 0x7839U, 0x782BU, 0x782FU, + 0x7941U, 0x7942U, 0x7943U, 0x7944U, 0x7945U, 0x7946U, 0x7947U, 0x7948U, + 0x7949U, 0x794AU, 0x794BU, 0x794CU, 0x794DU, 0x794EU, 0x794FU, 0x7950U, + 0x7951U, 0x7952U, 0x7953U, 0x7954U, 0x7955U, 0x7956U, 0x7957U, 0x7958U, + 0x7959U, 0x795AU, 0x7961U, 0x7962U, 0x7963U, 0x7964U, 0x7965U, 0x7966U, + 0x7967U, 0x7968U, 0x7969U, 0x796AU, 0x796BU, 0x796CU, 0x796DU, 0x796EU, + 0x796FU, 0x7970U, 0x7971U, 0x7972U, 0x7973U, 0x7974U, 0x7975U, 0x7976U, + 0x7977U, 0x7978U, 0x7979U, 0x797AU, 0x7930U, 0x7931U, 0x7932U, 0x7933U, + 0x7934U, 0x7935U, 0x7936U, 0x7937U, 0x7938U, 0x7939U, 0x792BU, 0x792FU, + 0x7A41U, 0x7A42U, 0x7A43U, 0x7A44U, 0x7A45U, 0x7A46U, 0x7A47U, 0x7A48U, + 0x7A49U, 0x7A4AU, 0x7A4BU, 0x7A4CU, 0x7A4DU, 0x7A4EU, 0x7A4FU, 0x7A50U, + 0x7A51U, 0x7A52U, 0x7A53U, 0x7A54U, 0x7A55U, 0x7A56U, 0x7A57U, 0x7A58U, + 0x7A59U, 0x7A5AU, 0x7A61U, 0x7A62U, 0x7A63U, 0x7A64U, 0x7A65U, 0x7A66U, + 0x7A67U, 0x7A68U, 0x7A69U, 0x7A6AU, 0x7A6BU, 0x7A6CU, 0x7A6DU, 0x7A6EU, + 0x7A6FU, 0x7A70U, 0x7A71U, 0x7A72U, 0x7A73U, 0x7A74U, 0x7A75U, 0x7A76U, + 0x7A77U, 0x7A78U, 0x7A79U, 0x7A7AU, 0x7A30U, 0x7A31U, 0x7A32U, 0x7A33U, + 0x7A34U, 0x7A35U, 0x7A36U, 0x7A37U, 0x7A38U, 0x7A39U, 0x7A2BU, 0x7A2FU, + 0x3041U, 0x3042U, 0x3043U, 0x3044U, 0x3045U, 0x3046U, 0x3047U, 0x3048U, + 0x3049U, 0x304AU, 0x304BU, 0x304CU, 0x304DU, 0x304EU, 0x304FU, 0x3050U, + 0x3051U, 0x3052U, 0x3053U, 0x3054U, 0x3055U, 0x3056U, 0x3057U, 0x3058U, + 0x3059U, 0x305AU, 0x3061U, 0x3062U, 0x3063U, 0x3064U, 0x3065U, 0x3066U, + 0x3067U, 0x3068U, 0x3069U, 0x306AU, 0x306BU, 0x306CU, 0x306DU, 0x306EU, + 0x306FU, 0x3070U, 0x3071U, 0x3072U, 0x3073U, 0x3074U, 0x3075U, 0x3076U, + 0x3077U, 0x3078U, 0x3079U, 0x307AU, 0x3030U, 0x3031U, 0x3032U, 0x3033U, + 0x3034U, 0x3035U, 0x3036U, 0x3037U, 0x3038U, 0x3039U, 0x302BU, 0x302FU, + 0x3141U, 0x3142U, 0x3143U, 0x3144U, 0x3145U, 0x3146U, 0x3147U, 0x3148U, + 0x3149U, 0x314AU, 0x314BU, 0x314CU, 0x314DU, 0x314EU, 0x314FU, 0x3150U, + 0x3151U, 0x3152U, 0x3153U, 0x3154U, 0x3155U, 0x3156U, 0x3157U, 0x3158U, + 0x3159U, 0x315AU, 0x3161U, 0x3162U, 0x3163U, 0x3164U, 0x3165U, 0x3166U, + 0x3167U, 0x3168U, 0x3169U, 0x316AU, 0x316BU, 0x316CU, 0x316DU, 0x316EU, + 0x316FU, 0x3170U, 0x3171U, 0x3172U, 0x3173U, 0x3174U, 0x3175U, 0x3176U, + 0x3177U, 0x3178U, 0x3179U, 0x317AU, 0x3130U, 0x3131U, 0x3132U, 0x3133U, + 0x3134U, 0x3135U, 0x3136U, 0x3137U, 0x3138U, 0x3139U, 0x312BU, 0x312FU, + 0x3241U, 0x3242U, 0x3243U, 0x3244U, 0x3245U, 0x3246U, 0x3247U, 0x3248U, + 0x3249U, 0x324AU, 0x324BU, 0x324CU, 0x324DU, 0x324EU, 0x324FU, 0x3250U, + 0x3251U, 0x3252U, 0x3253U, 0x3254U, 0x3255U, 0x3256U, 0x3257U, 0x3258U, + 0x3259U, 0x325AU, 0x3261U, 0x3262U, 0x3263U, 0x3264U, 0x3265U, 0x3266U, + 0x3267U, 0x3268U, 0x3269U, 0x326AU, 0x326BU, 0x326CU, 0x326DU, 0x326EU, + 0x326FU, 0x3270U, 0x3271U, 0x3272U, 0x3273U, 0x3274U, 0x3275U, 0x3276U, + 0x3277U, 0x3278U, 0x3279U, 0x327AU, 0x3230U, 0x3231U, 0x3232U, 0x3233U, + 0x3234U, 0x3235U, 0x3236U, 0x3237U, 0x3238U, 0x3239U, 0x322BU, 0x322FU, + 0x3341U, 0x3342U, 0x3343U, 0x3344U, 0x3345U, 0x3346U, 0x3347U, 0x3348U, + 0x3349U, 0x334AU, 0x334BU, 0x334CU, 0x334DU, 0x334EU, 0x334FU, 0x3350U, + 0x3351U, 0x3352U, 0x3353U, 0x3354U, 0x3355U, 0x3356U, 0x3357U, 0x3358U, + 0x3359U, 0x335AU, 0x3361U, 0x3362U, 0x3363U, 0x3364U, 0x3365U, 0x3366U, + 0x3367U, 0x3368U, 0x3369U, 0x336AU, 0x336BU, 0x336CU, 0x336DU, 0x336EU, + 0x336FU, 0x3370U, 0x3371U, 0x3372U, 0x3373U, 0x3374U, 0x3375U, 0x3376U, + 0x3377U, 0x3378U, 0x3379U, 0x337AU, 0x3330U, 0x3331U, 0x3332U, 0x3333U, + 0x3334U, 0x3335U, 0x3336U, 0x3337U, 0x3338U, 0x3339U, 0x332BU, 0x332FU, + 0x3441U, 0x3442U, 0x3443U, 0x3444U, 0x3445U, 0x3446U, 0x3447U, 0x3448U, + 0x3449U, 0x344AU, 0x344BU, 0x344CU, 0x344DU, 0x344EU, 0x344FU, 0x3450U, + 0x3451U, 0x3452U, 0x3453U, 0x3454U, 0x3455U, 0x3456U, 0x3457U, 0x3458U, + 0x3459U, 0x345AU, 0x3461U, 0x3462U, 0x3463U, 0x3464U, 0x3465U, 0x3466U, + 0x3467U, 0x3468U, 0x3469U, 0x346AU, 0x346BU, 0x346CU, 0x346DU, 0x346EU, + 0x346FU, 0x3470U, 0x3471U, 0x3472U, 0x3473U, 0x3474U, 0x3475U, 0x3476U, + 0x3477U, 0x3478U, 0x3479U, 0x347AU, 0x3430U, 0x3431U, 0x3432U, 0x3433U, + 0x3434U, 0x3435U, 0x3436U, 0x3437U, 0x3438U, 0x3439U, 0x342BU, 0x342FU, + 0x3541U, 0x3542U, 0x3543U, 0x3544U, 0x3545U, 0x3546U, 0x3547U, 0x3548U, + 0x3549U, 0x354AU, 0x354BU, 0x354CU, 0x354DU, 0x354EU, 0x354FU, 0x3550U, + 0x3551U, 0x3552U, 0x3553U, 0x3554U, 0x3555U, 0x3556U, 0x3557U, 0x3558U, + 0x3559U, 0x355AU, 0x3561U, 0x3562U, 0x3563U, 0x3564U, 0x3565U, 0x3566U, + 0x3567U, 0x3568U, 0x3569U, 0x356AU, 0x356BU, 0x356CU, 0x356DU, 0x356EU, + 0x356FU, 0x3570U, 0x3571U, 0x3572U, 0x3573U, 0x3574U, 0x3575U, 0x3576U, + 0x3577U, 0x3578U, 0x3579U, 0x357AU, 0x3530U, 0x3531U, 0x3532U, 0x3533U, + 0x3534U, 0x3535U, 0x3536U, 0x3537U, 0x3538U, 0x3539U, 0x352BU, 0x352FU, + 0x3641U, 0x3642U, 0x3643U, 0x3644U, 0x3645U, 0x3646U, 0x3647U, 0x3648U, + 0x3649U, 0x364AU, 0x364BU, 0x364CU, 0x364DU, 0x364EU, 0x364FU, 0x3650U, + 0x3651U, 0x3652U, 0x3653U, 0x3654U, 0x3655U, 0x3656U, 0x3657U, 0x3658U, + 0x3659U, 0x365AU, 0x3661U, 0x3662U, 0x3663U, 0x3664U, 0x3665U, 0x3666U, + 0x3667U, 0x3668U, 0x3669U, 0x366AU, 0x366BU, 0x366CU, 0x366DU, 0x366EU, + 0x366FU, 0x3670U, 0x3671U, 0x3672U, 0x3673U, 0x3674U, 0x3675U, 0x3676U, + 0x3677U, 0x3678U, 0x3679U, 0x367AU, 0x3630U, 0x3631U, 0x3632U, 0x3633U, + 0x3634U, 0x3635U, 0x3636U, 0x3637U, 0x3638U, 0x3639U, 0x362BU, 0x362FU, + 0x3741U, 0x3742U, 0x3743U, 0x3744U, 0x3745U, 0x3746U, 0x3747U, 0x3748U, + 0x3749U, 0x374AU, 0x374BU, 0x374CU, 0x374DU, 0x374EU, 0x374FU, 0x3750U, + 0x3751U, 0x3752U, 0x3753U, 0x3754U, 0x3755U, 0x3756U, 0x3757U, 0x3758U, + 0x3759U, 0x375AU, 0x3761U, 0x3762U, 0x3763U, 0x3764U, 0x3765U, 0x3766U, + 0x3767U, 0x3768U, 0x3769U, 0x376AU, 0x376BU, 0x376CU, 0x376DU, 0x376EU, + 0x376FU, 0x3770U, 0x3771U, 0x3772U, 0x3773U, 0x3774U, 0x3775U, 0x3776U, + 0x3777U, 0x3778U, 0x3779U, 0x377AU, 0x3730U, 0x3731U, 0x3732U, 0x3733U, + 0x3734U, 0x3735U, 0x3736U, 0x3737U, 0x3738U, 0x3739U, 0x372BU, 0x372FU, + 0x3841U, 0x3842U, 0x3843U, 0x3844U, 0x3845U, 0x3846U, 0x3847U, 0x3848U, + 0x3849U, 0x384AU, 0x384BU, 0x384CU, 0x384DU, 0x384EU, 0x384FU, 0x3850U, + 0x3851U, 0x3852U, 0x3853U, 0x3854U, 0x3855U, 0x3856U, 0x3857U, 0x3858U, + 0x3859U, 0x385AU, 0x3861U, 0x3862U, 0x3863U, 0x3864U, 0x3865U, 0x3866U, + 0x3867U, 0x3868U, 0x3869U, 0x386AU, 0x386BU, 0x386CU, 0x386DU, 0x386EU, + 0x386FU, 0x3870U, 0x3871U, 0x3872U, 0x3873U, 0x3874U, 0x3875U, 0x3876U, + 0x3877U, 0x3878U, 0x3879U, 0x387AU, 0x3830U, 0x3831U, 0x3832U, 0x3833U, + 0x3834U, 0x3835U, 0x3836U, 0x3837U, 0x3838U, 0x3839U, 0x382BU, 0x382FU, + 0x3941U, 0x3942U, 0x3943U, 0x3944U, 0x3945U, 0x3946U, 0x3947U, 0x3948U, + 0x3949U, 0x394AU, 0x394BU, 0x394CU, 0x394DU, 0x394EU, 0x394FU, 0x3950U, + 0x3951U, 0x3952U, 0x3953U, 0x3954U, 0x3955U, 0x3956U, 0x3957U, 0x3958U, + 0x3959U, 0x395AU, 0x3961U, 0x3962U, 0x3963U, 0x3964U, 0x3965U, 0x3966U, + 0x3967U, 0x3968U, 0x3969U, 0x396AU, 0x396BU, 0x396CU, 0x396DU, 0x396EU, + 0x396FU, 0x3970U, 0x3971U, 0x3972U, 0x3973U, 0x3974U, 0x3975U, 0x3976U, + 0x3977U, 0x3978U, 0x3979U, 0x397AU, 0x3930U, 0x3931U, 0x3932U, 0x3933U, + 0x3934U, 0x3935U, 0x3936U, 0x3937U, 0x3938U, 0x3939U, 0x392BU, 0x392FU, + 0x2B41U, 0x2B42U, 0x2B43U, 0x2B44U, 0x2B45U, 0x2B46U, 0x2B47U, 0x2B48U, + 0x2B49U, 0x2B4AU, 0x2B4BU, 0x2B4CU, 0x2B4DU, 0x2B4EU, 0x2B4FU, 0x2B50U, + 0x2B51U, 0x2B52U, 0x2B53U, 0x2B54U, 0x2B55U, 0x2B56U, 0x2B57U, 0x2B58U, + 0x2B59U, 0x2B5AU, 0x2B61U, 0x2B62U, 0x2B63U, 0x2B64U, 0x2B65U, 0x2B66U, + 0x2B67U, 0x2B68U, 0x2B69U, 0x2B6AU, 0x2B6BU, 0x2B6CU, 0x2B6DU, 0x2B6EU, + 0x2B6FU, 0x2B70U, 0x2B71U, 0x2B72U, 0x2B73U, 0x2B74U, 0x2B75U, 0x2B76U, + 0x2B77U, 0x2B78U, 0x2B79U, 0x2B7AU, 0x2B30U, 0x2B31U, 0x2B32U, 0x2B33U, + 0x2B34U, 0x2B35U, 0x2B36U, 0x2B37U, 0x2B38U, 0x2B39U, 0x2B2BU, 0x2B2FU, + 0x2F41U, 0x2F42U, 0x2F43U, 0x2F44U, 0x2F45U, 0x2F46U, 0x2F47U, 0x2F48U, + 0x2F49U, 0x2F4AU, 0x2F4BU, 0x2F4CU, 0x2F4DU, 0x2F4EU, 0x2F4FU, 0x2F50U, + 0x2F51U, 0x2F52U, 0x2F53U, 0x2F54U, 0x2F55U, 0x2F56U, 0x2F57U, 0x2F58U, + 0x2F59U, 0x2F5AU, 0x2F61U, 0x2F62U, 0x2F63U, 0x2F64U, 0x2F65U, 0x2F66U, + 0x2F67U, 0x2F68U, 0x2F69U, 0x2F6AU, 0x2F6BU, 0x2F6CU, 0x2F6DU, 0x2F6EU, + 0x2F6FU, 0x2F70U, 0x2F71U, 0x2F72U, 0x2F73U, 0x2F74U, 0x2F75U, 0x2F76U, + 0x2F77U, 0x2F78U, 0x2F79U, 0x2F7AU, 0x2F30U, 0x2F31U, 0x2F32U, 0x2F33U, + 0x2F34U, 0x2F35U, 0x2F36U, 0x2F37U, 0x2F38U, 0x2F39U, 0x2F2BU, 0x2F2FU, +#endif +}; diff --git a/mypyc/lib-rt/base64/tables/tables.c b/mypyc/lib-rt/base64/tables/tables.c new file mode 100644 index 0000000000000..45778b6befdd6 --- /dev/null +++ b/mypyc/lib-rt/base64/tables/tables.c @@ -0,0 +1,40 @@ +#include "tables.h" + +const uint8_t +base64_table_enc_6bit[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789" + "+/"; + +// In the lookup table below, note that the value for '=' (character 61) is +// 254, not 255. This character is used for in-band signaling of the end of +// the datastream, and we will use that later. The characters A-Z, a-z, 0-9 +// and + / are mapped to their "decoded" values. The other bytes all map to +// the value 255, which flags them as "invalid input". + +const uint8_t +base64_table_dec_8bit[] = +{ + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, // 0..15 + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, // 16..31 + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 255, 255, 255, 63, // 32..47 + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, 255, 254, 255, 255, // 48..63 + 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, // 64..79 + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 255, 255, 255, 255, 255, // 80..95 + 255, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, // 96..111 + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 255, 255, 255, 255, 255, // 112..127 + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, // 128..143 + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, +}; + +#if BASE64_WORDSIZE >= 32 +# include "table_dec_32bit.h" +# include "table_enc_12bit.h" +#endif diff --git a/mypyc/lib-rt/base64/tables/tables.h b/mypyc/lib-rt/base64/tables/tables.h new file mode 100644 index 0000000000000..cb74268a4bf12 --- /dev/null +++ b/mypyc/lib-rt/base64/tables/tables.h @@ -0,0 +1,23 @@ +#ifndef BASE64_TABLES_H +#define BASE64_TABLES_H + +#include + +#include "../env.h" + +// These tables are used by all codecs for fallback plain encoding/decoding: +extern const uint8_t base64_table_enc_6bit[]; +extern const uint8_t base64_table_dec_8bit[]; + +// These tables are used for the 32-bit and 64-bit generic decoders: +#if BASE64_WORDSIZE >= 32 +extern const uint32_t base64_table_dec_32bit_d0[]; +extern const uint32_t base64_table_dec_32bit_d1[]; +extern const uint32_t base64_table_dec_32bit_d2[]; +extern const uint32_t base64_table_dec_32bit_d3[]; + +// This table is used by the 32 and 64-bit generic encoders: +extern const uint16_t base64_table_enc_12bit[]; +#endif + +#endif // BASE64_TABLES_H diff --git a/mypyc/lib-rt/librt_base64.c b/mypyc/lib-rt/librt_base64.c index 1c3a6f8d01a59..020a56e412f4b 100644 --- a/mypyc/lib-rt/librt_base64.c +++ b/mypyc/lib-rt/librt_base64.c @@ -1,24 +1,19 @@ #define PY_SSIZE_T_CLEAN #include #include "librt_base64.h" +#include "libbase64.h" #include "pythoncapi_compat.h" #ifdef MYPYC_EXPERIMENTAL -// b64encode_internal below is adapted from the CPython 3.14.0 binascii module - -static const unsigned char table_b2a_base64[] = -"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - -#define BASE64_PAD '=' - -/* Max binary chunk size; limited only by available memory */ #define BASE64_MAXBIN ((PY_SSIZE_T_MAX - 3) / 2) +#define STACK_BUFFER_SIZE 1024 + static PyObject * b64encode_internal(PyObject *obj) { unsigned char *ascii_data; - const unsigned char *bin_data; + char *bin_data; int leftbits = 0; unsigned char this_ch; unsigned int leftchar = 0; @@ -31,51 +26,32 @@ b64encode_internal(PyObject *obj) { return NULL; } - bin_data = (const unsigned char *)PyBytes_AS_STRING(obj); + bin_data = PyBytes_AS_STRING(obj); bin_len = PyBytes_GET_SIZE(obj); - assert(bin_len >= 0); - if ( bin_len > BASE64_MAXBIN ) { + if (bin_len > BASE64_MAXBIN) { PyErr_SetString(PyExc_ValueError, "Too much data for base64 line"); return NULL; } - /* We're lazy and allocate too much (fixed up later). - "+2" leaves room for up to two pad characters. - Note that 'b' gets encoded as 'Yg==\n' (1 in, 5 out). */ - out_len = bin_len*2 + 2; - if (newline) - out_len++; - writer = PyBytesWriter_Create(out_len); - ascii_data = PyBytesWriter_GetData(writer); - if (writer == NULL) - return NULL; - - for( ; bin_len > 0 ; bin_len--, bin_data++ ) { - /* Shift the data into our buffer */ - leftchar = (leftchar << 8) | *bin_data; - leftbits += 8; - - /* See if there are 6-bit groups ready */ - while ( leftbits >= 6 ) { - this_ch = (leftchar >> (leftbits-6)) & 0x3f; - leftbits -= 6; - *ascii_data++ = table_b2a_base64[this_ch]; + Py_ssize_t buflen = 4 * bin_len / 3 + 4; + char *buf; + char stack_buf[STACK_BUFFER_SIZE]; + if (buflen <= STACK_BUFFER_SIZE) { + buf = stack_buf; + } else { + buf = PyMem_Malloc(buflen); + if (buf == NULL) { + return PyErr_NoMemory(); } } - if ( leftbits == 2 ) { - *ascii_data++ = table_b2a_base64[(leftchar&3) << 4]; - *ascii_data++ = BASE64_PAD; - *ascii_data++ = BASE64_PAD; - } else if ( leftbits == 4 ) { - *ascii_data++ = table_b2a_base64[(leftchar&0xf) << 2]; - *ascii_data++ = BASE64_PAD; - } - if (newline) - *ascii_data++ = '\n'; /* Append a courtesy newline */ - - return PyBytesWriter_FinishWithSize(writer, ascii_data - (unsigned char *)PyBytesWriter_GetData(writer)); + size_t actual_len; + base64_encode(bin_data, bin_len, buf, &actual_len, 0); + PyObject *res = PyBytes_FromStringAndSize(buf, actual_len); + if (buflen > STACK_BUFFER_SIZE) + PyMem_Free(buf); + return res; } static PyObject* diff --git a/mypyc/lib-rt/setup.py b/mypyc/lib-rt/setup.py index 3c08c7dbb5ee0..acd61458e5160 100644 --- a/mypyc/lib-rt/setup.py +++ b/mypyc/lib-rt/setup.py @@ -98,7 +98,24 @@ def run(self) -> None: extra_compile_args=cflags, ), Extension( - "librt.base64", ["librt_base64.c"], include_dirs=["."], extra_compile_args=cflags + "librt.base64", + [ + "librt_base64.c", + "base64/lib.c", + "base64/codec_choose.c", + "base64/tables/tables.c", + "base64/arch/generic/codec.c", + "base64/arch/ssse3/codec.c", + "base64/arch/sse41/codec.c", + "base64/arch/sse42/codec.c", + "base64/arch/avx/codec.c", + "base64/arch/avx2/codec.c", + "base64/arch/avx512/codec.c", + "base64/arch/neon32/codec.c", + "base64/arch/neon64/codec.c", + ], + include_dirs=[".", "base64"], + extra_compile_args=cflags, ), ] ) From 1b6ebb17b7fe64488a7b3c3b4b0187bb14fe331b Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 17 Nov 2025 15:39:31 +0000 Subject: [PATCH 399/424] [mypyc] Enable SIMD for librt.base64 on x86-64 (#20244) Also generally enable SSE4.2 instructions when targeting x86-64. These have been supported by hardware since ~2010, so it seems fine to require them now. This speeds up `b64encode` by up to 100% on Linux running on a recent AMD CPU. Some fairly recent hardware doesn't support AVX2, so it's not enabled. We'd probably need to rely on hardware capability checking for AVX2 support, and we'd need compile different files with different architecture flags probably, and I didn't want to go there (at least not yet). --- mypyc/build.py | 15 ++++++++++++++- mypyc/common.py | 3 +++ mypyc/lib-rt/base64/config.h | 5 +++++ mypyc/lib-rt/setup.py | 7 +++++++ 4 files changed, 29 insertions(+), 1 deletion(-) diff --git a/mypyc/build.py b/mypyc/build.py index 8505a2d957017..02f427c834268 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -36,7 +36,7 @@ from mypy.util import write_junit_xml from mypyc.annotate import generate_annotated_html from mypyc.codegen import emitmodule -from mypyc.common import IS_FREE_THREADED, RUNTIME_C_FILES, shared_lib_name +from mypyc.common import IS_FREE_THREADED, RUNTIME_C_FILES, X86_64, shared_lib_name from mypyc.errors import Errors from mypyc.ir.pprint import format_modules from mypyc.namegen import exported_name @@ -77,6 +77,12 @@ class ModDesc(NamedTuple): "base64/arch/generic/enc_tail.c", "base64/arch/generic/dec_head.c", "base64/arch/generic/dec_tail.c", + "base64/arch/ssse3/dec_reshuffle.c", + "base64/arch/ssse3/dec_loop.c", + "base64/arch/ssse3/enc_loop_asm.c", + "base64/arch/ssse3/enc_translate.c", + "base64/arch/ssse3/enc_reshuffle.c", + "base64/arch/ssse3/enc_loop.c", "base64/arch/neon64/dec_loop.c", "base64/arch/neon64/enc_loop_asm.c", "base64/codecs.h", @@ -655,6 +661,9 @@ def mypycify( # See https://github.com/mypyc/mypyc/issues/956 "-Wno-cpp", ] + if X86_64: + # Enable SIMD extensions. All CPUs released since ~2010 support SSE4.2. + cflags.append("-msse4.2") if log_trace: cflags.append("-DMYPYC_LOG_TRACE") if experimental_features: @@ -683,6 +692,10 @@ def mypycify( # that we actually get the compilation speed and memory # use wins that multi-file mode is intended for. cflags += ["/GL-", "/wd9025"] # warning about overriding /GL + if X86_64: + # Enable SIMD extensions. All CPUs released since ~2010 support SSE4.2. + # Also Windows 11 requires SSE4.2 since 24H2. + cflags.append("/arch:SSE4.2") if log_trace: cflags.append("/DMYPYC_LOG_TRACE") if experimental_features: diff --git a/mypyc/common.py b/mypyc/common.py index 2de63c09bb2ce..98f8a89f6fcb4 100644 --- a/mypyc/common.py +++ b/mypyc/common.py @@ -1,5 +1,6 @@ from __future__ import annotations +import platform import sys import sysconfig from typing import Any, Final @@ -44,6 +45,8 @@ IS_32_BIT_PLATFORM: Final = int(SIZEOF_SIZE_T) == 4 +X86_64: Final = platform.machine() in ("x86_64", "AMD64", "amd64") + PLATFORM_SIZE = 4 if IS_32_BIT_PLATFORM else 8 # Maximum value for a short tagged integer. diff --git a/mypyc/lib-rt/base64/config.h b/mypyc/lib-rt/base64/config.h index fd516c4be2d60..b5e47fb04e756 100644 --- a/mypyc/lib-rt/base64/config.h +++ b/mypyc/lib-rt/base64/config.h @@ -7,7 +7,12 @@ #define BASE64_WITH_SSE41 0 #define HAVE_SSE41 BASE64_WITH_SSE41 +#if defined(__x86_64__) || defined(_M_X64) +#define BASE64_WITH_SSE42 1 +#else #define BASE64_WITH_SSE42 0 +#endif + #define HAVE_SSE42 BASE64_WITH_SSE42 #define BASE64_WITH_AVX 0 diff --git a/mypyc/lib-rt/setup.py b/mypyc/lib-rt/setup.py index acd61458e5160..6a56c65306aeb 100644 --- a/mypyc/lib-rt/setup.py +++ b/mypyc/lib-rt/setup.py @@ -6,6 +6,7 @@ from __future__ import annotations import os +import platform import subprocess import sys from distutils import ccompiler, sysconfig @@ -24,6 +25,8 @@ "pythonsupport.c", ] +X86_64 = platform.machine() in ("x86_64", "AMD64", "amd64") + class BuildExtGtest(build_ext): def get_library_names(self) -> list[str]: @@ -79,8 +82,12 @@ def run(self) -> None: cflags: list[str] = [] if compiler.compiler_type == "unix": cflags += ["-O3"] + if X86_64: + cflags.append("-msse4.2") # Enable SIMD (see also mypyc/build.py) elif compiler.compiler_type == "msvc": cflags += ["/O2"] + if X86_64: + cflags.append("/arch:SSE4.2") # Enable SIMD (see also mypyc/build.py) setup( ext_modules=[ From 0c6593b117ae3be29d9549c7baa81f591953836c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 18 Nov 2025 10:47:34 +0000 Subject: [PATCH 400/424] [mypyc] Fix async or generator methods in traits (#20246) Disable optimization that doesn't work for trait methods. We could probably make the optimization work properly, but it would take significant work, so focus on fixing a regression. Fixes mypyc/mypyc#1141. #20044 was a previous fix attempt that fixes some other use cases, but it didn't address traits. --- mypyc/irbuild/prepare.py | 4 ++++ mypyc/test-data/run-async.test | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index 0f7cc7e3b3c55..9f3c7fc6f2707 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -842,6 +842,10 @@ def adjust_generator_classes_of_methods(mapper: Mapper) -> None: if subcls is None: # Override could be of a different type, so we can't make assumptions. precise_ret_type = False + elif class_ir.is_trait: + # Give up on traits. We could possibly have an abstract base class + # for generator return types to make this use precise types. + precise_ret_type = False else: for s in subcls: if name in s.method_decls: diff --git a/mypyc/test-data/run-async.test b/mypyc/test-data/run-async.test index cf063310fd895..718325b8b7b65 100644 --- a/mypyc/test-data/run-async.test +++ b/mypyc/test-data/run-async.test @@ -1297,6 +1297,8 @@ from typing import final, Coroutine, Any, TypeVar import asyncio +from mypy_extensions import trait + class Base1: async def foo(self) -> int: return 1 @@ -1363,5 +1365,22 @@ def test_override_non_async() -> None: assert asyncio.run(base3_foo(Base3())) == 7 assert asyncio.run(base3_foo(Derived3())) == 8 +class Base4: pass + +@trait +class TraitBase: + async def foo(self, value: int) -> int: + raise NotImplementedError() + +class DerivedFromTrait(Base4, TraitBase): + async def foo(self, value: int) -> int: + return value + 3 + +async def trait_foo(o: TraitBase, x: int) -> int: + return await o.foo(x) + +def test_override_trait() -> None: + assert asyncio.run(trait_foo(DerivedFromTrait(), 7)) == 10 + [file asyncio/__init__.pyi] def run(x: object) -> object: ... From 66797fcdac4ac63b1c7f90ef31cfd104afb0a99b Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 18 Nov 2025 13:55:22 +0000 Subject: [PATCH 401/424] [mypyc] Fix calling base class async method using super() (#20254) Fixes mypyc/mypyc#1154. The next label integer value was incorrectly passed as the `self` argument. --- mypyc/irbuild/expression.py | 4 ++-- mypyc/test-data/run-async.test | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index f6636a0e7b624..86cc2c7fb145d 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -504,12 +504,12 @@ def translate_super_method_call(builder: IRBuilder, expr: CallExpr, callee: Supe if decl.kind == FUNC_CLASSMETHOD: vself = builder.primitive_op(type_op, [vself], expr.line) elif builder.fn_info.is_generator: - # For generator classes, the self target is the 6th value + # For generator classes, the self target is the 7th value # in the symbol table (which is an ordered dict). This is sort # of ugly, but we can't search by name since the 'self' parameter # could be named anything, and it doesn't get added to the # environment indexes. - self_targ = list(builder.symtables[-1].values())[6] + self_targ = list(builder.symtables[-1].values())[7] vself = builder.read(self_targ, builder.fn_info.fitem.line) arg_values.insert(0, vself) arg_kinds.insert(0, ARG_POS) diff --git a/mypyc/test-data/run-async.test b/mypyc/test-data/run-async.test index 718325b8b7b65..39410e00a0247 100644 --- a/mypyc/test-data/run-async.test +++ b/mypyc/test-data/run-async.test @@ -1382,5 +1382,20 @@ async def trait_foo(o: TraitBase, x: int) -> int: def test_override_trait() -> None: assert asyncio.run(trait_foo(DerivedFromTrait(), 7)) == 10 +class Base5: + def __init__(self) -> None: + self._name = "test" + + async def foo(self, x: int) -> int: + assert self._name == "test" + return x + 11 + +class Derived5(Base5): + async def foo(self, x: int) -> int: + return await super().foo(x) + 22 + +def test_call_using_super() -> None: + assert asyncio.run(Derived5().foo(5)) == 38 + [file asyncio/__init__.pyi] def run(x: object) -> object: ... From 7e3fd2479b31109f89b260aae8b124a5422aa2ca Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 18 Nov 2025 17:28:54 +0000 Subject: [PATCH 402/424] Bump librt version (#20256) This pulls the version that contains stubs inside the wheel. I checked that third-party tools can find the stubs, but our local version in typeshed takes precedence as discussed in https://github.com/python/mypy/issues/20245 --- mypy-requirements.txt | 2 +- pyproject.toml | 4 ++-- test-requirements.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index 06e0a9bffb1cb..b0c632dddac56 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -4,4 +4,4 @@ typing_extensions>=4.6.0 mypy_extensions>=1.0.0 pathspec>=0.9.0 tomli>=1.1.0; python_version<'3.11' -librt>=0.6.0 +librt>=0.6.2 diff --git a/pyproject.toml b/pyproject.toml index 336a16c489799..bb41c82b1a3ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ requires = [ "mypy_extensions>=1.0.0", "pathspec>=0.9.0", "tomli>=1.1.0; python_version<'3.11'", - "librt>=0.6.0", + "librt>=0.6.2", # the following is from build-requirements.txt "types-psutil", "types-setuptools", @@ -54,7 +54,7 @@ dependencies = [ "mypy_extensions>=1.0.0", "pathspec>=0.9.0", "tomli>=1.1.0; python_version<'3.11'", - "librt>=0.6.0", + "librt>=0.6.2", ] dynamic = ["version"] diff --git a/test-requirements.txt b/test-requirements.txt index 4d3f644d6bca3..953e7a750c755 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -22,7 +22,7 @@ identify==2.6.15 # via pre-commit iniconfig==2.1.0 # via pytest -librt==0.6.0 +librt==0.6.2 # via -r mypy-requirements.txt lxml==6.0.2 ; python_version < "3.15" # via -r test-requirements.in From 0738db3f9d336622923c7ee143e1c3adf7600a31 Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Tue, 18 Nov 2025 18:33:48 +0100 Subject: [PATCH 403/424] Do not push partial types to the binder (#20202) Fixes #19996. This was unearthed by #19400 where `is_subtype(Partial, Partial)` started returning True - before that `binder.assign_type(expr, Partial, Partial)` just returned early. If I understand correctly, that is how Partial should be handled here: we do not want to push partials to the binder. I do not think we should add a special case for that (both False and True make some sense for a partial type, I am not convinced that either one is marginally better), so I just add an explicit guard to skip adding partial types here. --- mypy/checker.py | 6 +++++- test-data/unit/check-redefine2.test | 10 ++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/mypy/checker.py b/mypy/checker.py index 07f5c520de957..ad7eb3d355683 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -3424,7 +3424,11 @@ def check_assignment( and lvalue_type is not None ): lvalue.node.type = remove_instance_last_known_values(lvalue_type) - elif self.options.allow_redefinition_new and lvalue_type is not None: + elif ( + self.options.allow_redefinition_new + and lvalue_type is not None + and not isinstance(lvalue_type, PartialType) + ): # TODO: Can we use put() here? self.binder.assign_type(lvalue, lvalue_type, lvalue_type) diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 1abe957240b5e..4d99aa17f804d 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -369,6 +369,16 @@ class C4: reveal_type(self.x) # N: Revealed type is "builtins.list[builtins.str]" reveal_type(C4().x) # N: Revealed type is "builtins.list[builtins.str]" + +class C5: + def __init__(self) -> None: + if int(): + self.x = None + return + self.x = [""] + reveal_type(self.x) # N: Revealed type is "builtins.list[builtins.str]" + +reveal_type(C5().x) # N: Revealed type is "Union[builtins.list[builtins.str], None]" [builtins fixtures/list.pyi] [case testNewRedefinePartialGenericTypes] From 094f66dc742cec2d69add9296fb21cdef50624d0 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 18 Nov 2025 18:44:14 +0000 Subject: [PATCH 404/424] [mypyc] Add __repr__ to AssignmentTarget subclasses (#20258) This makes debugging a little easier, since you can see register names etc. --- mypyc/irbuild/targets.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/mypyc/irbuild/targets.py b/mypyc/irbuild/targets.py index 270c2896bc062..8bc9da074f07d 100644 --- a/mypyc/irbuild/targets.py +++ b/mypyc/irbuild/targets.py @@ -20,6 +20,9 @@ def __init__(self, register: Register) -> None: self.register = register self.type = register.type + def __repr__(self) -> str: + return f"AssignmentTargetRegister({self.register.name})" + class AssignmentTargetIndex(AssignmentTarget): """base[index] as assignment target""" @@ -31,6 +34,9 @@ def __init__(self, base: Value, index: Value) -> None: # lvalue type in mypy and use a better type to avoid unneeded boxing. self.type = object_rprimitive + def __repr__(self) -> str: + return f"AssignmentTargetIndex({self.base!r}, {self.index!r})" + class AssignmentTargetAttr(AssignmentTarget): """obj.attr as assignment target""" @@ -48,6 +54,10 @@ def __init__(self, obj: Value, attr: str, can_borrow: bool = False) -> None: self.obj_type = object_rprimitive self.type = object_rprimitive + def __repr__(self) -> str: + can_borrow_str = ", can_borrow=True" if self.can_borrow else "" + return f"AssignmentTargetAttr({self.obj!r}.{self.attr}{can_borrow_str})" + class AssignmentTargetTuple(AssignmentTarget): """x, ..., y as assignment target""" @@ -55,3 +65,6 @@ class AssignmentTargetTuple(AssignmentTarget): def __init__(self, items: list[AssignmentTarget], star_idx: int | None = None) -> None: self.items = items self.star_idx = star_idx + + def __repr__(self) -> str: + return f"AssignmentTargetTuple({self.items}, {self.star_idx})" From 35e843cc38cedc1bdf87d9937c06d51189ad0e45 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 19 Nov 2025 17:18:38 +0000 Subject: [PATCH 405/424] [mypyc] Add efficient librt.base64.b64decode (#20263) The performance can be 10x faster than stdlib if input is valid base64, or if input has extra non-base64 characters only at the end of input. Similar to the base64 encode implementation I added recently, this uses SIMD instructions when available. The implementation first tries to decode the input optimistically assuming valid base64. If this fails, we'll perform a slow path with a preprocessing step that removes extra characters, and we'll perform a strict base64 decode on the cleaned up input. The semantics aren't 100% compatible with stdlib. First, we raise ValueError on invalid padding instead of `binascii.Error`, since I don't want a runtime dependency on the unrelated a`binascii` module. This needs to be documented, but stdlib can already raise ValueError on other conditions, so the deviation is not huge. Also, some invalid inputs are checked more strictly for padding violations. The stdlib implementation has some mysterious behaviors with invalid inputs that didn't seem worth replicating. The function only accepts a single ASCII str or bytes argument for now, since that seems to be by the far the most common use case. The stdlib function also accepts buffer objects and a `validate` argument. The slow path is still somewhat faster than stdlib (on the order of 1.3x to 2x for longer inputs), at least if the input is much smaller than L1 cache size. Got the initial fast path implementation from ChatGPT, but did a bunch of manual edits afterwards and reviewed carefully. --- mypy/typeshed/stubs/librt/librt/base64.pyi | 1 + mypyc/lib-rt/librt_base64.c | 191 ++++++++++++++++++++- mypyc/test-data/run-base64.test | 108 +++++++++++- 3 files changed, 297 insertions(+), 3 deletions(-) diff --git a/mypy/typeshed/stubs/librt/librt/base64.pyi b/mypy/typeshed/stubs/librt/librt/base64.pyi index 36366f5754ce1..1cea838505d67 100644 --- a/mypy/typeshed/stubs/librt/librt/base64.pyi +++ b/mypy/typeshed/stubs/librt/librt/base64.pyi @@ -1 +1,2 @@ def b64encode(s: bytes) -> bytes: ... +def b64decode(s: bytes | str) -> bytes: ... diff --git a/mypyc/lib-rt/librt_base64.c b/mypyc/lib-rt/librt_base64.c index 020a56e412f4b..1720359ef9a69 100644 --- a/mypyc/lib-rt/librt_base64.c +++ b/mypyc/lib-rt/librt_base64.c @@ -1,11 +1,16 @@ #define PY_SSIZE_T_CLEAN #include +#include #include "librt_base64.h" #include "libbase64.h" #include "pythoncapi_compat.h" #ifdef MYPYC_EXPERIMENTAL +static PyObject * +b64decode_handle_invalid_input( + PyObject *out_bytes, char *outbuf, size_t max_out, const char *src, size_t srclen); + #define BASE64_MAXBIN ((PY_SSIZE_T_MAX - 3) / 2) #define STACK_BUFFER_SIZE 1024 @@ -63,11 +68,193 @@ b64encode(PyObject *self, PyObject *const *args, size_t nargs) { return b64encode_internal(args[0]); } +static inline int +is_valid_base64_char(char c, bool allow_padding) { + return ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || + (c >= '0' && c <= '9') || (c == '+') || (c == '/') || (allow_padding && c == '=')); +} + +static PyObject * +b64decode_internal(PyObject *arg) { + const char *src; + Py_ssize_t srclen_ssz; + + // Get input pointer and length + if (PyBytes_Check(arg)) { + src = PyBytes_AS_STRING(arg); + srclen_ssz = PyBytes_GET_SIZE(arg); + } else if (PyUnicode_Check(arg)) { + if (!PyUnicode_IS_ASCII(arg)) { + PyErr_SetString(PyExc_ValueError, + "string argument should contain only ASCII characters"); + return NULL; + } + src = (const char *)PyUnicode_1BYTE_DATA(arg); + srclen_ssz = PyUnicode_GET_LENGTH(arg); + } else { + PyErr_SetString(PyExc_TypeError, + "argument should be a bytes-like object or ASCII string"); + return NULL; + } + + // Fast-path: empty input + if (srclen_ssz == 0) { + return PyBytes_FromStringAndSize(NULL, 0); + } + + // Quickly ignore invalid characters at the end. Other invalid characters + // are also accepted, but they need a slow path. + while (srclen_ssz > 0 && !is_valid_base64_char(src[srclen_ssz - 1], true)) { + srclen_ssz--; + } + + // Compute an output capacity that's at least 3/4 of input, without overflow: + // ceil(3/4 * N) == N - floor(N/4) + size_t srclen = (size_t)srclen_ssz; + size_t max_out = srclen - (srclen / 4); + if (max_out == 0) { + max_out = 1; // defensive (srclen > 0 implies >= 1 anyway) + } + if (max_out > (size_t)PY_SSIZE_T_MAX) { + PyErr_SetString(PyExc_OverflowError, "input too large"); + return NULL; + } + + // Allocate output bytes (uninitialized) of the max capacity + PyObject *out_bytes = PyBytes_FromStringAndSize(NULL, (Py_ssize_t)max_out); + if (out_bytes == NULL) { + return NULL; // Propagate memory error + } + + char *outbuf = PyBytes_AS_STRING(out_bytes); + size_t outlen = max_out; + + int ret = base64_decode(src, srclen, outbuf, &outlen, 0); + + if (ret != 1) { + if (ret == 0) { + // Slow path: handle non-base64 input + return b64decode_handle_invalid_input(out_bytes, outbuf, max_out, src, srclen); + } + Py_DECREF(out_bytes); + if (ret == -1) { + PyErr_SetString(PyExc_NotImplementedError, "base64 codec not available in this build"); + } else { + PyErr_SetString(PyExc_RuntimeError, "base64_decode failed"); + } + return NULL; + } + + // Sanity-check contract (decoder must not overflow our buffer) + if (outlen > max_out) { + Py_DECREF(out_bytes); + PyErr_SetString(PyExc_RuntimeError, "decoder wrote past output buffer"); + return NULL; + } + + // Shrink in place to the actual decoded length + if (_PyBytes_Resize(&out_bytes, (Py_ssize_t)outlen) < 0) { + // _PyBytes_Resize sets an exception and may free the old object + return NULL; + } + return out_bytes; +} + +// Process non-base64 input by ignoring non-base64 characters, for compatibility +// with stdlib b64decode. +static PyObject * +b64decode_handle_invalid_input( + PyObject *out_bytes, char *outbuf, size_t max_out, const char *src, size_t srclen) +{ + // Copy input to a temporary buffer, with non-base64 characters and extra suffix + // characters removed + size_t newbuf_len = 0; + char *newbuf = PyMem_Malloc(srclen); + if (newbuf == NULL) { + Py_DECREF(out_bytes); + return PyErr_NoMemory(); + } + + // Copy base64 characters and some padding to the new buffer + for (size_t i = 0; i < srclen; i++) { + char c = src[i]; + if (is_valid_base64_char(c, false)) { + newbuf[newbuf_len++] = c; + } else if (c == '=') { + // Copy a necessary amount of padding + int remainder = newbuf_len % 4; + if (remainder == 0) { + // No padding needed + break; + } + int numpad = 4 - remainder; + // Check that there is at least the required amount padding (CPython ignores + // extra padding) + while (numpad > 0) { + if (i == srclen || src[i] != '=') { + break; + } + newbuf[newbuf_len++] = '='; + i++; + numpad--; + // Skip non-base64 alphabet characters within padding + while (i < srclen && !is_valid_base64_char(src[i], true)) { + i++; + } + } + break; + } + } + + // Stdlib always performs a non-strict padding check + if (newbuf_len % 4 != 0) { + Py_DECREF(out_bytes); + PyMem_Free(newbuf); + PyErr_SetString(PyExc_ValueError, "Incorrect padding"); + return NULL; + } + + size_t outlen = max_out; + int ret = base64_decode(newbuf, newbuf_len, outbuf, &outlen, 0); + PyMem_Free(newbuf); + + if (ret != 1) { + Py_DECREF(out_bytes); + if (ret == 0) { + PyErr_SetString(PyExc_ValueError, "Only base64 data is allowed"); + } + if (ret == -1) { + PyErr_SetString(PyExc_NotImplementedError, "base64 codec not available in this build"); + } else { + PyErr_SetString(PyExc_RuntimeError, "base64_decode failed"); + } + return NULL; + } + + // Shrink in place to the actual decoded length + if (_PyBytes_Resize(&out_bytes, (Py_ssize_t)outlen) < 0) { + // _PyBytes_Resize sets an exception and may free the old object + return NULL; + } + return out_bytes; +} + + +static PyObject* +b64decode(PyObject *self, PyObject *const *args, size_t nargs) { + if (nargs != 1) { + PyErr_SetString(PyExc_TypeError, "b64decode() takes exactly one argument"); + return 0; + } + return b64decode_internal(args[0]); +} + #endif static PyMethodDef librt_base64_module_methods[] = { #ifdef MYPYC_EXPERIMENTAL - {"b64encode", (PyCFunction)b64encode, METH_FASTCALL, PyDoc_STR("Encode bytes-like object using Base64.")}, + {"b64encode", (PyCFunction)b64encode, METH_FASTCALL, PyDoc_STR("Encode bytes object using Base64.")}, + {"b64decode", (PyCFunction)b64decode, METH_FASTCALL, PyDoc_STR("Decode a Base64 encoded bytes object or ASCII string.")}, #endif {NULL, NULL, 0, NULL} }; @@ -111,7 +298,7 @@ static PyModuleDef_Slot librt_base64_module_slots[] = { static PyModuleDef librt_base64_module = { .m_base = PyModuleDef_HEAD_INIT, .m_name = "base64", - .m_doc = "base64 encoding and decoding optimized for mypyc", + .m_doc = "Fast base64 encoding and decoding optimized for mypyc", .m_size = 0, .m_methods = librt_base64_module_methods, .m_slots = librt_base64_module_slots, diff --git a/mypyc/test-data/run-base64.test b/mypyc/test-data/run-base64.test index 0f9151c2b00bb..8d7eb7c13482d 100644 --- a/mypyc/test-data/run-base64.test +++ b/mypyc/test-data/run-base64.test @@ -1,8 +1,9 @@ [case testAllBase64Features_librt_experimental] from typing import Any import base64 +import binascii -from librt.base64 import b64encode +from librt.base64 import b64encode, b64decode from testutil import assertRaises @@ -44,6 +45,111 @@ def test_encode_wrapper() -> None: with assertRaises(TypeError): enc(b"x", b"y") +def test_decode_basic() -> None: + assert b64decode(b"eA==") == b"x" + + with assertRaises(TypeError): + b64decode(bytearray(b"eA==")) + + for non_ascii in "\x80", "foo\u100bar", "foo\ua1234bar": + with assertRaises(ValueError): + b64decode(non_ascii) + +def check_decode(b: bytes, encoded: bool = False) -> None: + if encoded: + enc = b + else: + enc = b64encode(b) + assert b64decode(enc) == getattr(base64, "b64decode")(enc) + if getattr(enc, "isascii")(): # Test stub has no "isascii" + enc_str = enc.decode("ascii") + assert b64decode(enc_str) == getattr(base64, "b64decode")(enc_str) + +def test_decode_different_strings() -> None: + for i in range(256): + check_decode(bytes([i])) + check_decode(bytes([i]) + b"x") + check_decode(bytes([i]) + b"xy") + check_decode(bytes([i]) + b"xyz") + check_decode(bytes([i]) + b"xyza") + check_decode(b"x" + bytes([i])) + check_decode(b"xy" + bytes([i])) + check_decode(b"xyz" + bytes([i])) + check_decode(b"xyza" + bytes([i])) + + b = b"a\x00\xb7" * 1000 + for i in range(1000): + check_decode(b[:i]) + + for b in b"", b"ab", b"bac", b"1234", b"xyz88", b"abc" * 200: + check_decode(b) + +def is_base64_char(x: int) -> bool: + c = chr(x) + return ('a' <= c <= 'z') or ('A' <= c <= 'Z') or ('0' <= c <= '9') or c in '+/=' + +def test_decode_with_non_base64_chars() -> None: + # For stdlib compatibility, non-base64 characters should be ignored. + + # Invalid characters as a suffix use a fast path. + check_decode(b"eA== ", encoded=True) + check_decode(b"eA==\n", encoded=True) + check_decode(b"eA== \t\n", encoded=True) + check_decode(b"\n", encoded=True) + + check_decode(b" e A = = ", encoded=True) + + # Special case: Two different encodings of the same data + check_decode(b"eAa=", encoded=True) + check_decode(b"eAY=", encoded=True) + + for x in range(256): + if not is_base64_char(x): + b = bytes([x]) + check_decode(b, encoded=True) + check_decode(b"eA==" + b, encoded=True) + check_decode(b"e" + b + b"A==", encoded=True) + check_decode(b"eA=" + b + b"=", encoded=True) + +def check_decode_error(b: bytes, ignore_stdlib: bool = False) -> None: + if not ignore_stdlib: + with assertRaises(binascii.Error): + getattr(base64, "b64decode")(b) + + # The raised error is different, since librt shouldn't depend on binascii + with assertRaises(ValueError): + b64decode(b) + +def test_decode_with_invalid_padding() -> None: + check_decode_error(b"eA") + check_decode_error(b"eA=") + check_decode_error(b"eHk") + check_decode_error(b"eA = ") + + # Here stdlib behavior seems nonsensical, so we don't try to duplicate it + check_decode_error(b"eA=a=", ignore_stdlib=True) + +def test_decode_with_extra_data_after_padding() -> None: + check_decode(b"=", encoded=True) + check_decode(b"==", encoded=True) + check_decode(b"===", encoded=True) + check_decode(b"====", encoded=True) + check_decode(b"eA===", encoded=True) + check_decode(b"eHk==", encoded=True) + check_decode(b"eA==x", encoded=True) + check_decode(b"eHk=x", encoded=True) + check_decode(b"eA==abc=======efg", encoded=True) + +def test_decode_wrapper() -> None: + dec: Any = b64decode + assert dec(b"eA==") == b"x" + + with assertRaises(TypeError): + dec() + + with assertRaises(TypeError): + dec(b"x", b"y") + [case testBase64FeaturesNotAvailableInNonExperimentalBuild_librt_base64] # This also ensures librt.base64 can be built without experimental features import librt.base64 From a087a5894935cfdbc2eba27a6d04ebca38fd6659 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 20 Nov 2025 11:58:01 +0000 Subject: [PATCH 406/424] Update import map when new modules added (#20271) Fixes https://github.com/python/mypy/issues/20209 --- mypy/server/update.py | 2 ++ test-data/unit/fine-grained-modules.test | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/mypy/server/update.py b/mypy/server/update.py index 839090ca45ac9..86ccb57c2a3f0 100644 --- a/mypy/server/update.py +++ b/mypy/server/update.py @@ -631,6 +631,8 @@ def restore(ids: list[str]) -> None: # Find any other modules brought in by imports. changed_modules = [(st.id, st.xpath) for st in new_modules] + for m in new_modules: + manager.import_map[m.id] = set(m.dependencies + m.suppressed) # If there are multiple modules to process, only process one of them and return # the remaining ones to the caller. diff --git a/test-data/unit/fine-grained-modules.test b/test-data/unit/fine-grained-modules.test index f28dbaa1113b9..3ee07a03792f8 100644 --- a/test-data/unit/fine-grained-modules.test +++ b/test-data/unit/fine-grained-modules.test @@ -2244,3 +2244,19 @@ undefined a.py:1: error: Cannot find implementation or library stub for module named "foobar" a.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports == + +[case testDaemonImportMapRefresh] +# cmd: mypy main.py +[file main.py] +[file main.py.2] +import a.b +reveal_type(a.b.foo()) +[file a/__init__.pyi] +[file a/b.pyi] +import a.c +def foo() -> a.c.C: ... +[file a/c.pyi] +class C: ... +[out] +== +main.py:2: note: Revealed type is "a.c.C" From 13369cb25fe450f755f63e59156b86df84c08b3d Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 23 Nov 2025 07:10:10 +0000 Subject: [PATCH 407/424] [mypyc] Fix crash on super in generator (#20291) This is another problem caused by https://github.com/python/mypy/pull/19398 and a missing part of https://github.com/python/mypy/pull/20254 cc @JukkaL @hauntsaninja there is a small chance this may be related to the problems with black. --- mypyc/irbuild/expression.py | 4 ++-- mypyc/test-data/run-classes.test | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index 86cc2c7fb145d..2ed347ca1e797 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -296,8 +296,8 @@ def transform_super_expr(builder: IRBuilder, o: SuperExpr) -> Value: # Grab first argument vself: Value = next(iter_env) if builder.fn_info.is_generator: - # grab sixth argument (see comment in translate_super_method_call) - self_targ = list(builder.symtables[-1].values())[6] + # grab seventh argument (see comment in translate_super_method_call) + self_targ = list(builder.symtables[-1].values())[7] vself = builder.read(self_targ, builder.fn_info.fitem.line) elif not ir.is_ext_class: vself = next(iter_env) # second argument is self if non_extension class diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 2c2eac5057971..da72fe59f456b 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -322,6 +322,17 @@ if sys.version_info[:2] > (3, 5): assert TestEnum.b.name == 'b' assert TestEnum.b.value == 2 +[case testRunSuperYieldFromDict] +from typing import Any, Iterator + +class DictSubclass(dict): + def items(self) -> Iterator[Any]: + yield 1 + yield from super().items() + +def test_sub_dict() -> None: + assert list(DictSubclass().items()) == [1] + [case testGetAttribute] class C: x: int From 1b94fbb9fbc581de7e057d71e9892e3acbf9a7d3 Mon Sep 17 00:00:00 2001 From: Piotr Sawicki Date: Thu, 27 Nov 2025 19:21:10 +0100 Subject: [PATCH 408/424] [mypyc] Fix vtable pointer with inherited dunder new (#20302) Fixes an issue where a subclass would have its vtable pointer set to the base class' vtable when there is a `__new__` method defined in the base class. This resulted in the subclass constructor calling the setup function of the base class because mypyc transforms `object.__new__` into the setup function. The fix is to store the pointers to the setup functions in `tp_methods` of type objects and look them up dynamically when instantiating new objects. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- mypyc/codegen/emitclass.py | 13 ++++++- mypyc/irbuild/specialize.py | 11 +++++- mypyc/lib-rt/CPy.h | 2 + mypyc/lib-rt/generic_ops.c | 20 ++++++++++ mypyc/primitives/generic_ops.py | 7 ++++ mypyc/test-data/irbuild-classes.test | 28 ++++++++++++++ mypyc/test-data/run-classes.test | 55 ++++++++++++++++++++++++++++ 7 files changed, 132 insertions(+), 4 deletions(-) diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index d64940084f12e..e190d45a2e937 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -359,7 +359,7 @@ def emit_line() -> None: if cl.is_trait: generate_new_for_trait(cl, new_name, emitter) - generate_methods_table(cl, methods_name, emitter) + generate_methods_table(cl, methods_name, setup_name if generate_full else None, emitter) emit_line() flags = ["Py_TPFLAGS_DEFAULT", "Py_TPFLAGS_HEAPTYPE", "Py_TPFLAGS_BASETYPE"] @@ -960,8 +960,17 @@ def generate_finalize_for_class( emitter.emit_line("}") -def generate_methods_table(cl: ClassIR, name: str, emitter: Emitter) -> None: +def generate_methods_table( + cl: ClassIR, name: str, setup_name: str | None, emitter: Emitter +) -> None: emitter.emit_line(f"static PyMethodDef {name}[] = {{") + if setup_name: + # Store pointer to the setup function so it can be resolved dynamically + # in case of instance creation in __new__. + # CPy_SetupObject expects this method to be the first one in tp_methods. + emitter.emit_line( + f'{{"__internal_mypyc_setup", (PyCFunction){setup_name}, METH_O, NULL}},' + ) for fn in cl.methods.values(): if fn.decl.is_prop_setter or fn.decl.is_prop_getter or fn.internal: continue diff --git a/mypyc/irbuild/specialize.py b/mypyc/irbuild/specialize.py index e810f11bd079c..b64f51043b138 100644 --- a/mypyc/irbuild/specialize.py +++ b/mypyc/irbuild/specialize.py @@ -99,7 +99,7 @@ isinstance_dict, ) from mypyc.primitives.float_ops import isinstance_float -from mypyc.primitives.generic_ops import generic_setattr +from mypyc.primitives.generic_ops import generic_setattr, setup_object from mypyc.primitives.int_ops import isinstance_int from mypyc.primitives.list_ops import isinstance_list, new_list_set_item_op from mypyc.primitives.misc_ops import isinstance_bool @@ -1103,7 +1103,14 @@ def translate_object_new(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> method_args = fn.fitem.arg_names if isinstance(typ_arg, NameExpr) and len(method_args) > 0 and method_args[0] == typ_arg.name: subtype = builder.accept(expr.args[0]) - return builder.add(Call(ir.setup, [subtype], expr.line)) + subs = ir.subclasses() + if subs is not None and len(subs) == 0: + return builder.add(Call(ir.setup, [subtype], expr.line)) + # Call a function that dynamically resolves the setup function of extension classes from the type object. + # This is necessary because the setup involves default attribute initialization and setting up + # the vtable which are specific to a given type and will not work if a subtype is created using + # the setup function of its base. + return builder.call_c(setup_object, [subtype], expr.line) return None diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index c79923f69e691..6d1e7502a7e7a 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -958,6 +958,8 @@ static inline int CPyObject_GenericSetAttr(PyObject *self, PyObject *name, PyObj return _PyObject_GenericSetAttrWithDict(self, name, value, NULL); } +PyObject *CPy_SetupObject(PyObject *type); + #if CPY_3_11_FEATURES PyObject *CPy_GetName(PyObject *obj); #endif diff --git a/mypyc/lib-rt/generic_ops.c b/mypyc/lib-rt/generic_ops.c index 260cfec5b360d..1e1e184bf290b 100644 --- a/mypyc/lib-rt/generic_ops.c +++ b/mypyc/lib-rt/generic_ops.c @@ -62,3 +62,23 @@ PyObject *CPyObject_GetSlice(PyObject *obj, CPyTagged start, CPyTagged end) { Py_DECREF(slice); return result; } + +typedef PyObject *(*SetupFunction)(PyObject *); + +PyObject *CPy_SetupObject(PyObject *type) { + PyTypeObject *tp = (PyTypeObject *)type; + PyMethodDef *def = NULL; + for(; tp; tp = tp->tp_base) { + def = tp->tp_methods; + if (!def || !def->ml_name) { + continue; + } + + if (!strcmp(def->ml_name, "__internal_mypyc_setup")) { + return ((SetupFunction)(void(*)(void))def->ml_meth)(type); + } + } + + PyErr_SetString(PyExc_RuntimeError, "Internal mypyc error: Unable to find object setup function"); + return NULL; +} diff --git a/mypyc/primitives/generic_ops.py b/mypyc/primitives/generic_ops.py index 16bd074396d2b..1003fda8d9ae8 100644 --- a/mypyc/primitives/generic_ops.py +++ b/mypyc/primitives/generic_ops.py @@ -417,3 +417,10 @@ c_function_name="CPyObject_GenericSetAttr", error_kind=ERR_NEG_INT, ) + +setup_object = custom_op( + arg_types=[object_rprimitive], + return_type=object_rprimitive, + c_function_name="CPy_SetupObject", + error_kind=ERR_MAGIC, +) diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index 0f8ec2b094f05..504e6234de243 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -1685,6 +1685,13 @@ class Test: obj.val = val return obj +class Test2: + def __new__(cls) -> Test2: + return super().__new__(cls) + +class Sub(Test2): + pass + def fn() -> Test: return Test.__new__(Test, 42) @@ -1719,6 +1726,13 @@ L0: obj = r0 obj.val = val; r1 = is_error return obj +def Test2.__new__(cls): + cls, r0 :: object + r1 :: __main__.Test2 +L0: + r0 = CPy_SetupObject(cls) + r1 = cast(__main__.Test2, r0) + return r1 def fn(): r0 :: object r1 :: __main__.Test @@ -1822,6 +1836,13 @@ class Test: obj.val = val return obj +class Test2: + def __new__(cls) -> Test2: + return object.__new__(cls) + +class Sub(Test2): + pass + def fn() -> Test: return Test.__new__(Test, 42) @@ -1874,6 +1895,13 @@ L0: obj = r0 obj.val = val; r1 = is_error return obj +def Test2.__new__(cls): + cls, r0 :: object + r1 :: __main__.Test2 +L0: + r0 = CPy_SetupObject(cls) + r1 = cast(__main__.Test2, r0) + return r1 def fn(): r0 :: object r1 :: __main__.Test diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index da72fe59f456b..02a9934bac713 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -3859,6 +3859,7 @@ Add(1, 0)=1 [case testInheritedDunderNew] from __future__ import annotations from mypy_extensions import mypyc_attr +from testutil import assertRaises from typing_extensions import Self from m import interpreted_subclass @@ -3875,7 +3876,11 @@ class Base: def __init__(self, val: int) -> None: self.init_val = val + def method(self) -> int: + raise NotImplementedError + class Sub(Base): + def __new__(cls, val: int) -> Self: return super().__new__(cls, val + 1) @@ -3883,11 +3888,20 @@ class Sub(Base): super().__init__(val) self.init_val = self.init_val * 2 + def method(self) -> int: + return 0 + class SubWithoutNew(Base): + sub_only_str = "" + sub_only_int: int + def __init__(self, val: int) -> None: super().__init__(val) self.init_val = self.init_val * 2 + def method(self) -> int: + return 1 + class BaseWithoutInterpretedSubclasses: val: int @@ -3899,6 +3913,9 @@ class BaseWithoutInterpretedSubclasses: def __init__(self, val: int) -> None: self.init_val = val + def method(self) -> int: + raise NotImplementedError + class SubNoInterpreted(BaseWithoutInterpretedSubclasses): def __new__(cls, val: int) -> Self: return super().__new__(cls, val + 1) @@ -3907,48 +3924,68 @@ class SubNoInterpreted(BaseWithoutInterpretedSubclasses): super().__init__(val) self.init_val = self.init_val * 2 + def method(self) -> int: + return 0 + class SubNoInterpretedWithoutNew(BaseWithoutInterpretedSubclasses): def __init__(self, val: int) -> None: super().__init__(val) self.init_val = self.init_val * 2 + def method(self) -> int: + return 1 + def test_inherited_dunder_new() -> None: b = Base(42) assert type(b) == Base assert b.val == 43 assert b.init_val == 42 + with assertRaises(NotImplementedError): + b.method() s = Sub(42) assert type(s) == Sub assert s.val == 44 assert s.init_val == 84 + assert s.method() == 0 s2 = SubWithoutNew(42) assert type(s2) == SubWithoutNew assert s2.val == 43 assert s2.init_val == 84 + assert s2.method() == 1 + assert s2.sub_only_str == "" + with assertRaises(AttributeError): + s2.sub_only_int + s2.sub_only_int = 11 + assert s2.sub_only_int == 11 def test_inherited_dunder_new_without_interpreted_subclasses() -> None: b = BaseWithoutInterpretedSubclasses(42) assert type(b) == BaseWithoutInterpretedSubclasses assert b.val == 43 assert b.init_val == 42 + with assertRaises(NotImplementedError): + b.method() s = SubNoInterpreted(42) assert type(s) == SubNoInterpreted assert s.val == 44 assert s.init_val == 84 + assert s.method() == 0 s2 = SubNoInterpretedWithoutNew(42) assert type(s2) == SubNoInterpretedWithoutNew assert s2.val == 43 assert s2.init_val == 84 + assert s2.method() == 1 def test_interpreted_subclass() -> None: interpreted_subclass(Base) [file m.py] from __future__ import annotations +from testutil import assertRaises from typing_extensions import Self def interpreted_subclass(base) -> None: @@ -3956,6 +3993,8 @@ def interpreted_subclass(base) -> None: assert type(b) == base assert b.val == 43 assert b.init_val == 42 + with assertRaises(NotImplementedError): + b.method() class InterpretedSub(base): def __new__(cls, val: int) -> Self: @@ -3965,20 +4004,36 @@ def interpreted_subclass(base) -> None: super().__init__(val) self.init_val : int = self.init_val * 2 + def method(self) -> int: + return 3 + s = InterpretedSub(42) assert type(s) == InterpretedSub assert s.val == 44 assert s.init_val == 84 + assert s.method() == 3 class InterpretedSubWithoutNew(base): + sub_only_str = "" + sub_only_int: int + def __init__(self, val: int) -> None: super().__init__(val) self.init_val : int = self.init_val * 2 + def method(self) -> int: + return 4 + s2 = InterpretedSubWithoutNew(42) assert type(s2) == InterpretedSubWithoutNew assert s2.val == 43 assert s2.init_val == 84 + assert s2.method() == 4 + assert s2.sub_only_str == "" + with assertRaises(AttributeError): + s2.sub_only_int + s2.sub_only_int = 11 + assert s2.sub_only_int == 11 [typing fixtures/typing-full.pyi] From 1999a20e9898f673fa2f4c9a91790c075141ba71 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" <1330696+mr-c@users.noreply.github.com> Date: Fri, 28 Nov 2025 10:58:29 +0100 Subject: [PATCH 409/424] [mypyc] librt base64: use existing SIMD CPU dispatch by customizing build flags (#20253) Fixes the current SSE4.2 requirement added in https://github.com/python/mypy/commit/1b6ebb17b7fe64488a7b3c3b4b0187bb14fe331b / https://github.com/python/mypy/pull/20244 This PR fully enables the existing x86-64 CPU detection and dispatch code for SSSE3, SSE4.1, SSE4.2, AVX, and AVX2 in the base64 module. To use the existing CPU dispatch from the [upstream base64 code](https://github.com/aklomp/base64), one needs to compile the sources in each of the CPU specific codec directories with a specific compiler flag; alas this is difficult to do with setuptools, but I found a solution inspired by https://stackoverflow.com/a/68508804 Note that I did not enable the AVX512 path with this PR, as many intel CPUs that support AVX512 can come with a performance hit if AVX512 is sporadically used; the performance of the AVX512 (encoding) path need to be evaluated in the context of how mypyc uses base64 in various realistic scenarios. (There is no AVX512 accelerated decoding path in the upstream base64 codebase, it falls back to the avx2 decoder). If there are additional performance concerns, then I suggest benchmarking with the openmp feature of base64 turned on, for multi-core processing. --- mypy_self_check.ini | 2 +- mypyc/build.py | 17 ++++---- mypyc/build_setup.py | 62 +++++++++++++++++++++++++++ mypyc/common.py | 3 -- mypyc/lib-rt/base64/arch/avx/codec.c | 2 +- mypyc/lib-rt/base64/arch/avx2/codec.c | 12 +++--- mypyc/lib-rt/base64/config.h | 28 +++--------- mypyc/lib-rt/setup.py | 54 ++++++++++++++++++++--- setup.py | 1 + 9 files changed, 135 insertions(+), 46 deletions(-) create mode 100644 mypyc/build_setup.py diff --git a/mypy_self_check.ini b/mypy_self_check.ini index 0b49b3de862bc..f4f8d2d0e08b2 100644 --- a/mypy_self_check.ini +++ b/mypy_self_check.ini @@ -8,7 +8,7 @@ pretty = True always_false = MYPYC plugins = mypy.plugins.proper_plugin python_version = 3.9 -exclude = mypy/typeshed/|mypyc/test-data/|mypyc/lib-rt/ +exclude = mypy/typeshed/|mypyc/test-data/ enable_error_code = ignore-without-code,redundant-expr enable_incomplete_feature = PreciseTupleTypes show_error_code_links = True diff --git a/mypyc/build.py b/mypyc/build.py index 02f427c834268..69ef6c3bc435d 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -28,6 +28,7 @@ from collections.abc import Iterable from typing import TYPE_CHECKING, Any, NamedTuple, NoReturn, Union, cast +import mypyc.build_setup # noqa: F401 from mypy.build import BuildSource from mypy.errors import CompileError from mypy.fscache import FileSystemCache @@ -36,7 +37,7 @@ from mypy.util import write_junit_xml from mypyc.annotate import generate_annotated_html from mypyc.codegen import emitmodule -from mypyc.common import IS_FREE_THREADED, RUNTIME_C_FILES, X86_64, shared_lib_name +from mypyc.common import IS_FREE_THREADED, RUNTIME_C_FILES, shared_lib_name from mypyc.errors import Errors from mypyc.ir.pprint import format_modules from mypyc.namegen import exported_name @@ -70,6 +71,13 @@ class ModDesc(NamedTuple): "base64/arch/neon64/codec.c", ], [ + "base64/arch/avx/enc_loop_asm.c", + "base64/arch/avx2/enc_loop.c", + "base64/arch/avx2/enc_loop_asm.c", + "base64/arch/avx2/enc_reshuffle.c", + "base64/arch/avx2/enc_translate.c", + "base64/arch/avx2/dec_loop.c", + "base64/arch/avx2/dec_reshuffle.c", "base64/arch/generic/32/enc_loop.c", "base64/arch/generic/64/enc_loop.c", "base64/arch/generic/32/dec_loop.c", @@ -661,9 +669,6 @@ def mypycify( # See https://github.com/mypyc/mypyc/issues/956 "-Wno-cpp", ] - if X86_64: - # Enable SIMD extensions. All CPUs released since ~2010 support SSE4.2. - cflags.append("-msse4.2") if log_trace: cflags.append("-DMYPYC_LOG_TRACE") if experimental_features: @@ -692,10 +697,6 @@ def mypycify( # that we actually get the compilation speed and memory # use wins that multi-file mode is intended for. cflags += ["/GL-", "/wd9025"] # warning about overriding /GL - if X86_64: - # Enable SIMD extensions. All CPUs released since ~2010 support SSE4.2. - # Also Windows 11 requires SSE4.2 since 24H2. - cflags.append("/arch:SSE4.2") if log_trace: cflags.append("/DMYPYC_LOG_TRACE") if experimental_features: diff --git a/mypyc/build_setup.py b/mypyc/build_setup.py new file mode 100644 index 0000000000000..a3e7a669abee9 --- /dev/null +++ b/mypyc/build_setup.py @@ -0,0 +1,62 @@ +import platform +import sys + +try: + # Import setuptools so that it monkey-patch overrides distutils + import setuptools # noqa: F401 +except ImportError: + pass + +if sys.version_info >= (3, 12): + # From setuptools' monkeypatch + from distutils import ccompiler # type: ignore[import-not-found] +else: + from distutils import ccompiler + +EXTRA_FLAGS_PER_COMPILER_TYPE_PER_PATH_COMPONENT = { + "unix": { + "base64/arch/ssse3": ["-mssse3"], + "base64/arch/sse41": ["-msse4.1"], + "base64/arch/sse42": ["-msse4.2"], + "base64/arch/avx2": ["-mavx2"], + "base64/arch/avx": ["-mavx"], + }, + "msvc": { + "base64/arch/sse42": ["/arch:SSE4.2"], + "base64/arch/avx2": ["/arch:AVX2"], + "base64/arch/avx": ["/arch:AVX"], + }, +} + +ccompiler.CCompiler.__spawn = ccompiler.CCompiler.spawn # type: ignore[attr-defined] +X86_64 = platform.machine() in ("x86_64", "AMD64", "amd64") + + +def spawn(self, cmd, **kwargs) -> None: # type: ignore[no-untyped-def] + compiler_type: str = self.compiler_type + extra_options = EXTRA_FLAGS_PER_COMPILER_TYPE_PER_PATH_COMPONENT[compiler_type] + new_cmd = list(cmd) + if X86_64 and extra_options is not None: + # filenames are closer to the end of command line + for argument in reversed(new_cmd): + # Check if the matching argument contains a source filename. + if not str(argument).endswith(".c"): + continue + + for path in extra_options.keys(): + if path in str(argument): + if compiler_type == "bcpp": + compiler = new_cmd.pop() + # Borland accepts a source file name at the end, + # insert the options before it + new_cmd.extend(extra_options[path]) + new_cmd.append(compiler) + else: + new_cmd.extend(extra_options[path]) + + # path component is found, no need to search any further + break + self.__spawn(new_cmd, **kwargs) + + +ccompiler.CCompiler.spawn = spawn # type: ignore[method-assign] diff --git a/mypyc/common.py b/mypyc/common.py index 98f8a89f6fcb4..2de63c09bb2ce 100644 --- a/mypyc/common.py +++ b/mypyc/common.py @@ -1,6 +1,5 @@ from __future__ import annotations -import platform import sys import sysconfig from typing import Any, Final @@ -45,8 +44,6 @@ IS_32_BIT_PLATFORM: Final = int(SIZEOF_SIZE_T) == 4 -X86_64: Final = platform.machine() in ("x86_64", "AMD64", "amd64") - PLATFORM_SIZE = 4 if IS_32_BIT_PLATFORM else 8 # Maximum value for a short tagged integer. diff --git a/mypyc/lib-rt/base64/arch/avx/codec.c b/mypyc/lib-rt/base64/arch/avx/codec.c index 8e2ef5c2e7243..7a64a94be2aff 100644 --- a/mypyc/lib-rt/base64/arch/avx/codec.c +++ b/mypyc/lib-rt/base64/arch/avx/codec.c @@ -24,7 +24,7 @@ #include "../ssse3/dec_loop.c" #if BASE64_AVX_USE_ASM -# include "enc_loop_asm.c" +# include "./enc_loop_asm.c" #else # include "../ssse3/enc_translate.c" # include "../ssse3/enc_reshuffle.c" diff --git a/mypyc/lib-rt/base64/arch/avx2/codec.c b/mypyc/lib-rt/base64/arch/avx2/codec.c index fe9200296914f..a54385bf89bea 100644 --- a/mypyc/lib-rt/base64/arch/avx2/codec.c +++ b/mypyc/lib-rt/base64/arch/avx2/codec.c @@ -20,15 +20,15 @@ # endif #endif -#include "dec_reshuffle.c" -#include "dec_loop.c" +#include "./dec_reshuffle.c" +#include "./dec_loop.c" #if BASE64_AVX2_USE_ASM -# include "enc_loop_asm.c" +# include "./enc_loop_asm.c" #else -# include "enc_translate.c" -# include "enc_reshuffle.c" -# include "enc_loop.c" +# include "./enc_translate.c" +# include "./enc_reshuffle.c" +# include "./enc_loop.c" #endif #endif // HAVE_AVX2 diff --git a/mypyc/lib-rt/base64/config.h b/mypyc/lib-rt/base64/config.h index b5e47fb04e756..467a722c2f117 100644 --- a/mypyc/lib-rt/base64/config.h +++ b/mypyc/lib-rt/base64/config.h @@ -1,29 +1,15 @@ #ifndef BASE64_CONFIG_H #define BASE64_CONFIG_H -#define BASE64_WITH_SSSE3 0 -#define HAVE_SSSE3 BASE64_WITH_SSSE3 - -#define BASE64_WITH_SSE41 0 -#define HAVE_SSE41 BASE64_WITH_SSE41 - -#if defined(__x86_64__) || defined(_M_X64) -#define BASE64_WITH_SSE42 1 -#else -#define BASE64_WITH_SSE42 0 +#if !defined(__APPLE__) && ((defined(__x86_64__) && defined(__LP64__)) || defined(_M_X64)) + #define HAVE_SSSE3 1 + #define HAVE_SSE41 1 + #define HAVE_SSE42 1 + #define HAVE_AVX 1 + #define HAVE_AVX2 1 + #define HAVE_AVX512 0 #endif -#define HAVE_SSE42 BASE64_WITH_SSE42 - -#define BASE64_WITH_AVX 0 -#define HAVE_AVX BASE64_WITH_AVX - -#define BASE64_WITH_AVX2 0 -#define HAVE_AVX2 BASE64_WITH_AVX2 - -#define BASE64_WITH_AVX512 0 -#define HAVE_AVX512 BASE64_WITH_AVX512 - #define BASE64_WITH_NEON32 0 #define HAVE_NEON32 BASE64_WITH_NEON32 diff --git a/mypyc/lib-rt/setup.py b/mypyc/lib-rt/setup.py index 6a56c65306aeb..72dfc15d8588d 100644 --- a/mypyc/lib-rt/setup.py +++ b/mypyc/lib-rt/setup.py @@ -25,9 +25,55 @@ "pythonsupport.c", ] +EXTRA_FLAGS_PER_COMPILER_TYPE_PER_PATH_COMPONENT = { + "unix": { + "base64/arch/ssse3": ["-mssse3"], + "base64/arch/sse41": ["-msse4.1"], + "base64/arch/sse42": ["-msse4.2"], + "base64/arch/avx2": ["-mavx2"], + "base64/arch/avx": ["-mavx"], + }, + "msvc": { + "base64/arch/sse42": ["/arch:SSE4.2"], + "base64/arch/avx2": ["/arch:AVX2"], + "base64/arch/avx": ["/arch:AVX"], + }, +} + +ccompiler.CCompiler.__spawn = ccompiler.CCompiler.spawn # type: ignore[attr-defined] X86_64 = platform.machine() in ("x86_64", "AMD64", "amd64") +def spawn(self, cmd, **kwargs) -> None: # type: ignore[no-untyped-def] + compiler_type: str = self.compiler_type + extra_options = EXTRA_FLAGS_PER_COMPILER_TYPE_PER_PATH_COMPONENT[compiler_type] + new_cmd = list(cmd) + if X86_64 and extra_options is not None: + # filenames are closer to the end of command line + for argument in reversed(new_cmd): + # Check if the matching argument contains a source filename. + if not str(argument).endswith(".c"): + continue + + for path in extra_options.keys(): + if path in str(argument): + if compiler_type == "bcpp": + compiler = new_cmd.pop() + # Borland accepts a source file name at the end, + # insert the options before it + new_cmd.extend(extra_options[path]) + new_cmd.append(compiler) + else: + new_cmd.extend(extra_options[path]) + + # path component is found, no need to search any further + break + self.__spawn(new_cmd, **kwargs) + + +ccompiler.CCompiler.spawn = spawn # type: ignore[method-assign] + + class BuildExtGtest(build_ext): def get_library_names(self) -> list[str]: return ["gtest"] @@ -80,14 +126,10 @@ def run(self) -> None: compiler = ccompiler.new_compiler() sysconfig.customize_compiler(compiler) cflags: list[str] = [] - if compiler.compiler_type == "unix": + if compiler.compiler_type == "unix": # type: ignore[attr-defined] cflags += ["-O3"] - if X86_64: - cflags.append("-msse4.2") # Enable SIMD (see also mypyc/build.py) - elif compiler.compiler_type == "msvc": + elif compiler.compiler_type == "msvc": # type: ignore[attr-defined] cflags += ["/O2"] - if X86_64: - cflags.append("/arch:SSE4.2") # Enable SIMD (see also mypyc/build.py) setup( ext_modules=[ diff --git a/setup.py b/setup.py index 0037624f9bbc2..f20c1db5d0450 100644 --- a/setup.py +++ b/setup.py @@ -99,6 +99,7 @@ def run(self) -> None: os.path.join("mypyc", "lib-rt", "setup.py"), # Uses __file__ at top level https://github.com/mypyc/mypyc/issues/700 os.path.join("mypyc", "__main__.py"), + os.path.join("mypyc", "build_setup.py"), # for monkeypatching ) everything = [os.path.join("mypy", x) for x in find_package_data("mypy", ["*.py"])] + [ From 3c813083b27c87cf3a32e7422191b02bf59fab6e Mon Sep 17 00:00:00 2001 From: Piotr Sawicki Date: Thu, 27 Nov 2025 11:50:14 +0100 Subject: [PATCH 410/424] Add draft version of 1.19 release notes (#20296) Added a draft version with the commits filtered to remove internal changes and grouped into sections. --------- Co-authored-by: Jacopo Abramo --- CHANGELOG.md | 150 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 134d251d90b14..ec3f0cbb59bf2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,156 @@ ## Next Release +## Mypy 1.19 (Unreleased) + +We’ve just uploaded mypy 1.19.0 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). +Mypy is a static type checker for Python. This release includes new features, performance +improvements and bug fixes. You can install it as follows: + + python3 -m pip install -U mypy + +You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). + +### Performance improvements +- Switch to a more dynamic SCC processing logic (Ivan Levkivskyi, PR [20053](https://github.com/python/mypy/pull/20053)) +- Try some aliases speed-up (Ivan Levkivskyi, PR [19810](https://github.com/python/mypy/pull/19810)) + +### Fixed‑Format Cache +- Force-discard cache if cache format changed (Ivan Levkivskyi, PR [20152](https://github.com/python/mypy/pull/20152)) +- Use more efficient serialization format for long integers in cache files (Jukka Lehtosalo, PR [20151](https://github.com/python/mypy/pull/20151)) +- More robust packing of flats in FF cache (Ivan Levkivskyi, PR [20150](https://github.com/python/mypy/pull/20150)) +- Use self-descriptive cache with type tags (Ivan Levkivskyi, PR [20137](https://github.com/python/mypy/pull/20137)) +- Use fixed format for cache metas (Ivan Levkivskyi, PR [20088](https://github.com/python/mypy/pull/20088)) +- Make metas more compact; fix indirect suppression (Ivan Levkivskyi, PR [20075](https://github.com/python/mypy/pull/20075)) +- Add tool to convert binary cache files to JSON (Jukka Lehtosalo, PR [20071](https://github.com/python/mypy/pull/20071)) +- Use dedicated tags for most common instances (Ivan Levkivskyi, PR [19762](https://github.com/python/mypy/pull/19762)) + +### PEP 747 - Annotating Type Forms +- [PEP 747] Recognize `TypeForm[T]` type and values (#9773) (David Foster, PR [19596](https://github.com/python/mypy/pull/19596)) + +### Fixes to crashes +- Do not push partial types to the binder (Stanislav Terliakov, PR [20202](https://github.com/python/mypy/pull/20202)) +- Fix crash on recursive tuple with Hashable (Ivan Levkivskyi, PR [20232](https://github.com/python/mypy/pull/20232)) +- Do not assume that args of decorated functions can be cleanly mapped to their nodes (Stanislav Terliakov, PR [20203](https://github.com/python/mypy/pull/20203)) +- Do not abort constructing TypeAlias if only type parameters hold us back (Stanislav Terliakov, PR [20162](https://github.com/python/mypy/pull/20162)) +- Use the fallback for `ModuleSpec` early if it can never be resolved (Stanislav Terliakov, PR [20167](https://github.com/python/mypy/pull/20167)) +- Do not store deferred NamedTuple fields as redefinitions (Stanislav Terliakov, PR [20147](https://github.com/python/mypy/pull/20147)) +- Discard partials remaining after inference failure (Stanislav Terliakov, PR [20126](https://github.com/python/mypy/pull/20126)) +- Remember the pair in `is_overlapping_types` if at least one of them is an alias (Stanislav Terliakov, PR [20127](https://github.com/python/mypy/pull/20127)) +- Fix IsADirectoryError for namespace packages when using --linecoverage-report (wyattscarpenter, PR [20109](https://github.com/python/mypy/pull/20109)) +- Fix an INTERNAL ERROR when creating cobertura output for namespace package (wyattscarpenter, PR [20112](https://github.com/python/mypy/pull/20112)) +- Allow type parameters reusing the name missing from current module (Stanislav Terliakov, PR [20081](https://github.com/python/mypy/pull/20081)) +- Prevent TypeGuardedType leak from `narrow_declared_type` as part of typevar bound (Stanislav Terliakov, PR [20046](https://github.com/python/mypy/pull/20046)) +- Fix crash on invalid unpack in base class (Ivan Levkivskyi, PR [19962](https://github.com/python/mypy/pull/19962)) +- Traverse ParamSpec prefix where we should (Ivan Levkivskyi, PR [19800](https://github.com/python/mypy/pull/19800)) + +### Mypyc: Support for `__getattr__`, `__setattr__`, and `__delattr__` +- Support deleting attributes in `__setattr__` wrapper (Piotr Sawicki, PR [19997](https://github.com/python/mypy/pull/19997)) +- Generate `__setattr__` wrapper (Piotr Sawicki, PR [19937](https://github.com/python/mypy/pull/19937)) +- Generate `__getattr__` wrapper (Piotr Sawicki, PR [19909](https://github.com/python/mypy/pull/19909)) + +### Miscellaneous Mypyc Improvements +- Fix crash on `super` in generator (Ivan Levkivskyi, PR [20291](https://github.com/python/mypy/pull/20291)) +- Fix calling base class async method using `super()` (Jukka Lehtosalo, PR [20254](https://github.com/python/mypy/pull/20254)) +- Fix async or generator methods in traits (Jukka Lehtosalo, PR [20246](https://github.com/python/mypy/pull/20246)) +- Optimize equality check with string literals [1/1] (BobTheBuidler, PR [19883](https://github.com/python/mypy/pull/19883)) +- Fix inheritance of async defs (Jukka Lehtosalo, PR [20044](https://github.com/python/mypy/pull/20044)) +- Reject invalid `mypyc_attr` args [1/1] (BobTheBuidler, PR [19963](https://github.com/python/mypy/pull/19963)) +- Optimize `isinstance` with tuple of primitive types (BobTheBuidler, PR [19949](https://github.com/python/mypy/pull/19949)) +- Optimize away first index check in for loops if length > 1 (BobTheBuidler, PR [19933](https://github.com/python/mypy/pull/19933)) +- Fix broken exception/cancellation handling in async def (Jukka Lehtosalo, PR [19951](https://github.com/python/mypy/pull/19951)) +- Transform `object.__new__` inside `__new__` (Piotr Sawicki, PR [19866](https://github.com/python/mypy/pull/19866)) +- Fix crash with NewType and other non-class types in incremental builds (Jukka Lehtosalo, PR [19837](https://github.com/python/mypy/pull/19837)) +- Optimize container creation from expressions with length known at compile time (BobTheBuidler, PR [19503](https://github.com/python/mypy/pull/19503)) +- Allow per-class free list to be used with inheritance (Jukka Lehtosalo, PR [19790](https://github.com/python/mypy/pull/19790)) +- Fix object finalization (Marc Mueller, PR [19749](https://github.com/python/mypy/pull/19749)) +- Allow defining a single-item free "list" for a native class (Jukka Lehtosalo, PR [19785](https://github.com/python/mypy/pull/19785)) +- Speed up unary "not" (Jukka Lehtosalo, PR [19774](https://github.com/python/mypy/pull/19774)) + +### Stubtest Improvements +- Check `_value_` for ellipsis-valued stub enum members (Stanislav Terliakov, PR [19760](https://github.com/python/mypy/pull/19760)) +- Include function name in overload assertion messages (Joren Hammudoglu, PR [20063](https://github.com/python/mypy/pull/20063)) +- Small fix in get_default_function_sig (iap, PR [19822](https://github.com/python/mypy/pull/19822)) +- Adjust stubtest test stubs for PEP 728 (Python 3.15) (Marc Mueller, PR [20009](https://github.com/python/mypy/pull/20009)) +- Improve `allowlist` docs with better example (sobolevn, PR [20007](https://github.com/python/mypy/pull/20007)) + +### Documentation Updates +- Update duck_type_compatibility.rst: mention strict-bytes & mypy 2.0 (wyattscarpenter, PR [20121](https://github.com/python/mypy/pull/20121)) +- document --enable-incomplete-feature TypeForm, minimally but sufficiently (wyattscarpenter, PR [20173](https://github.com/python/mypy/pull/20173)) +- Change the InlineTypedDict example (wyattscarpenter, PR [20172](https://github.com/python/mypy/pull/20172)) +- Update kinds_of_types.rst: keep old anchor for #no-strict-optional (wyattscarpenter, PR [19828](https://github.com/python/mypy/pull/19828)) +- Replace `List` with built‑in `list` (PEP 585) (Thiago J. Barbalho, PR [20000](https://github.com/python/mypy/pull/20000)) +- main.py: junit documentation elaboration (wyattscarpenter, PR [19867](https://github.com/python/mypy/pull/19867)) + +### Other Notable Fixes and Improvements +- Update import map when new modules added (Ivan Levkivskyi, PR [20271](https://github.com/python/mypy/pull/20271)) +- Fix annotated with function as type keyword list parameter (KarelKenens, PR [20094](https://github.com/python/mypy/pull/20094)) +- Fix errors for raise NotImplemented (Shantanu, PR [20168](https://github.com/python/mypy/pull/20168)) +- Don't let help formatter line-wrap URLs (Frank Dana, PR [19825](https://github.com/python/mypy/pull/19825)) +- Do not cache fast container types inside lambdas (Stanislav Terliakov, PR [20166](https://github.com/python/mypy/pull/20166)) +- Respect force-union-syntax flag in error hint (Marc Mueller, PR [20165](https://github.com/python/mypy/pull/20165)) +- Fix type checking of dict type aliases (Shantanu, PR [20170](https://github.com/python/mypy/pull/20170)) +- Use pretty_callable more often for callable expressions (Theodore Ando, PR [20128](https://github.com/python/mypy/pull/20128)) +- Use dummy concrete type instead of `Any` when checking protocol variance (bzoracler, PR [20110](https://github.com/python/mypy/pull/20110)) +- [PEP 696] Fix swapping TypeVars with defaults (Randolf Scholz, PR [19449](https://github.com/python/mypy/pull/19449)) +- Fix narrowing of class pattern with union-argument (Randolf Scholz, PR [19517](https://github.com/python/mypy/pull/19517)) +- Do not emit unreachable warnings for lines that return `NotImplemented` (Christoph Tyralla, PR [20083](https://github.com/python/mypy/pull/20083)) +- fix matching against `typing.Callable` and `Protocol` types (Randolf Scholz, PR [19471](https://github.com/python/mypy/pull/19471)) +- Make --pretty work better on multi-line issues (A5rocks, PR [20056](https://github.com/python/mypy/pull/20056)) +- More precise return types for `TypedDict.get` (Randolf Scholz, PR [19897](https://github.com/python/mypy/pull/19897)) +- prevent false unreachable warnings for @final instances that occur when strict optional checking is disabled (Christoph Tyralla, PR [20045](https://github.com/python/mypy/pull/20045)) +- Check class references to catch non-existent classes in match cases (A5rocks, PR [20042](https://github.com/python/mypy/pull/20042)) +- Do not sort unused error codes in unused error codes warning (wyattscarpenter, PR [20036](https://github.com/python/mypy/pull/20036)) +- Fix `[name-defined]` false-positive in `class A[X, Y=X]:` case (sobolevn, PR [20021](https://github.com/python/mypy/pull/20021)) +- Filter SyntaxWarnings during AST parsing (Marc Mueller, PR [20023](https://github.com/python/mypy/pull/20023)) +- Make untyped decorator its own code (wyattscarpenter, PR [19911](https://github.com/python/mypy/pull/19911)) +- Support error codes from plugins in options (Sigve Sebastian Farstad, PR [19719](https://github.com/python/mypy/pull/19719)) +- Allow returning Literals in `__new__` (James Hilton-Balfe, PR [15687](https://github.com/python/mypy/pull/15687)) +- Inverse interface freshness logic (Ivan Levkivskyi, PR [19809](https://github.com/python/mypy/pull/19809)) +- Do not report exhaustive-match after deferral (Stanislav Terliakov, PR [19804](https://github.com/python/mypy/pull/19804)) +- Make untyped_calls_exclude invalidate cache (Ivan Levkivskyi, PR [19801](https://github.com/python/mypy/pull/19801)) +- Add await to empty context hack (Stanislav Terliakov, PR [19777](https://github.com/python/mypy/pull/19777)) +- Consider non-empty enums assignable to Self (Stanislav Terliakov, PR [19779](https://github.com/python/mypy/pull/19779)) + +### Typeshed updates + +Please see [git log](https://github.com/python/typeshed/commits/main?after=ebce8d766b41fbf4d83cf47c1297563a9508ff60+0&branch=main&path=stdlib) for full list of standard library typeshed stub changes. + +### Acknowledgements + +Thanks to all mypy contributors who contributed to this release: +- A5rocks +- BobTheBuidler +- bzoracler +- Chainfire +- Christoph Tyralla +- David Foster +- Frank Dana +- Guo Ci +- iap +- Ivan Levkivskyi +- James Hilton-Balfe +- jhance +- Joren Hammudoglu +- Jukka Lehtosalo +- KarelKenens +- Kevin Kannammalil +- Marc Mueller +- Michael Carlstrom +- Michael J. Sullivan +- Piotr Sawicki +- Randolf Scholz +- Shantanu +- Sigve Sebastian Farstad +- sobolevn +- Stanislav Terliakov +- Stephen Morton +- Theodore Ando +- Thiago J. Barbalho +- wyattscarpenter + +I’d also like to thank my employer, Dropbox, for supporting mypy development. + ## Mypy 1.18.1 We’ve just uploaded mypy 1.18.1 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). From 6d5cf52e67da306b62455cdce4ce9a9ccec35d02 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 28 Nov 2025 11:53:21 +0000 Subject: [PATCH 411/424] Various updates to 1.19 changelog (#20304) The first draft was added in #20296. --- CHANGELOG.md | 155 +++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 120 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec3f0cbb59bf2..0be81310c6e1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Next Release -## Mypy 1.19 (Unreleased) +## Mypy 1.19 We’ve just uploaded mypy 1.19.0 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). Mypy is a static type checker for Python. This release includes new features, performance @@ -12,51 +12,139 @@ improvements and bug fixes. You can install it as follows: You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). -### Performance improvements +### Performance Improvements - Switch to a more dynamic SCC processing logic (Ivan Levkivskyi, PR [20053](https://github.com/python/mypy/pull/20053)) -- Try some aliases speed-up (Ivan Levkivskyi, PR [19810](https://github.com/python/mypy/pull/19810)) +- Speed up type aliases (Ivan Levkivskyi, PR [19810](https://github.com/python/mypy/pull/19810)) + +### Fixed‑Format Cache Improvements + +Mypy uses a cache by default to speed up incremental runs by reusing partial results +from earlier runs. Mypy 1.18 added a new binary fixed-format cache representation as +an experimental feature. The feature is no longer experimental, and we are planning +to enable it by default in a future mypy release (possibly 1.20), since it's faster +and uses less space than the original, JSON-based cache format. Use +`--fixed-format-cache` to enable the fixed-format cache. + +Mypy now has an extra dependency on the `librt` PyPI package, as it's needed for +cache serialization and deserialization. + +Mypy ships with a tool to convert fixed-format cache files to the old JSON format. +Example of how to use this: +``` +$ python -m mypy.exportjson .mypy_cache/.../my_module.data.ff +``` + +This way existing use cases that parse JSON cache files can be supported when using +the new format, though an extra conversion step is needed. + +This release includes these improvements: -### Fixed‑Format Cache - Force-discard cache if cache format changed (Ivan Levkivskyi, PR [20152](https://github.com/python/mypy/pull/20152)) +- Add tool to convert binary cache files to JSON (Jukka Lehtosalo, PR [20071](https://github.com/python/mypy/pull/20071)) - Use more efficient serialization format for long integers in cache files (Jukka Lehtosalo, PR [20151](https://github.com/python/mypy/pull/20151)) -- More robust packing of flats in FF cache (Ivan Levkivskyi, PR [20150](https://github.com/python/mypy/pull/20150)) +- More robust packing of floats in fixed-format cache (Ivan Levkivskyi, PR [20150](https://github.com/python/mypy/pull/20150)) - Use self-descriptive cache with type tags (Ivan Levkivskyi, PR [20137](https://github.com/python/mypy/pull/20137)) - Use fixed format for cache metas (Ivan Levkivskyi, PR [20088](https://github.com/python/mypy/pull/20088)) - Make metas more compact; fix indirect suppression (Ivan Levkivskyi, PR [20075](https://github.com/python/mypy/pull/20075)) -- Add tool to convert binary cache files to JSON (Jukka Lehtosalo, PR [20071](https://github.com/python/mypy/pull/20071)) -- Use dedicated tags for most common instances (Ivan Levkivskyi, PR [19762](https://github.com/python/mypy/pull/19762)) +- Use dedicated tags for most common cached instances (Ivan Levkivskyi, PR [19762](https://github.com/python/mypy/pull/19762)) + +### PEP 747: Annotating Type Forms + +Mypy now recognizes `TypeForm[T]` as a type and implements +[PEP 747](https://peps.python.org/pep-0747/). The feature is still experimental, +and it's disabled by default. Use `--enable-incomplete-feature=TypeForm` to +enable type forms. A type form object captures the type information provided by a +runtime type expression. Example: + +```python +from typing_extensions import TypeForm -### PEP 747 - Annotating Type Forms -- [PEP 747] Recognize `TypeForm[T]` type and values (#9773) (David Foster, PR [19596](https://github.com/python/mypy/pull/19596)) +def trycast[T](typx: TypeForm[T], value: object) -> T | None: ... -### Fixes to crashes +def example(o: object) -> None: + # 'int | str' below is an expression that represents a type. + # Unlike type[T], TypeForm[T] can be used with all kinds of types, + # including union types. + x = trycast(int | str, o) + if x is not None: + # Type of 'x' is 'int | str' here + ... +``` + +This feature was contributed by David Foster (PR [19596](https://github.com/python/mypy/pull/19596)). + +### Fixes to Crashes - Do not push partial types to the binder (Stanislav Terliakov, PR [20202](https://github.com/python/mypy/pull/20202)) - Fix crash on recursive tuple with Hashable (Ivan Levkivskyi, PR [20232](https://github.com/python/mypy/pull/20232)) -- Do not assume that args of decorated functions can be cleanly mapped to their nodes (Stanislav Terliakov, PR [20203](https://github.com/python/mypy/pull/20203)) +- Fix crash related to decorated functions (Stanislav Terliakov, PR [20203](https://github.com/python/mypy/pull/20203)) - Do not abort constructing TypeAlias if only type parameters hold us back (Stanislav Terliakov, PR [20162](https://github.com/python/mypy/pull/20162)) - Use the fallback for `ModuleSpec` early if it can never be resolved (Stanislav Terliakov, PR [20167](https://github.com/python/mypy/pull/20167)) - Do not store deferred NamedTuple fields as redefinitions (Stanislav Terliakov, PR [20147](https://github.com/python/mypy/pull/20147)) -- Discard partials remaining after inference failure (Stanislav Terliakov, PR [20126](https://github.com/python/mypy/pull/20126)) -- Remember the pair in `is_overlapping_types` if at least one of them is an alias (Stanislav Terliakov, PR [20127](https://github.com/python/mypy/pull/20127)) +- Discard partial types remaining after inference failure (Stanislav Terliakov, PR [20126](https://github.com/python/mypy/pull/20126)) +- Fix an infinite recursion bug (Stanislav Terliakov, PR [20127](https://github.com/python/mypy/pull/20127)) - Fix IsADirectoryError for namespace packages when using --linecoverage-report (wyattscarpenter, PR [20109](https://github.com/python/mypy/pull/20109)) -- Fix an INTERNAL ERROR when creating cobertura output for namespace package (wyattscarpenter, PR [20112](https://github.com/python/mypy/pull/20112)) +- Fix an internal error when creating cobertura output for namespace package (wyattscarpenter, PR [20112](https://github.com/python/mypy/pull/20112)) - Allow type parameters reusing the name missing from current module (Stanislav Terliakov, PR [20081](https://github.com/python/mypy/pull/20081)) -- Prevent TypeGuardedType leak from `narrow_declared_type` as part of typevar bound (Stanislav Terliakov, PR [20046](https://github.com/python/mypy/pull/20046)) +- Prevent TypeGuardedType leak from narrowing declared type as part of type variable bound (Stanislav Terliakov, PR [20046](https://github.com/python/mypy/pull/20046)) - Fix crash on invalid unpack in base class (Ivan Levkivskyi, PR [19962](https://github.com/python/mypy/pull/19962)) - Traverse ParamSpec prefix where we should (Ivan Levkivskyi, PR [19800](https://github.com/python/mypy/pull/19800)) +- Fix daemon crash related to imports (Ivan Levkivskyi, PR [20271](https://github.com/python/mypy/pull/20271)) ### Mypyc: Support for `__getattr__`, `__setattr__`, and `__delattr__` -- Support deleting attributes in `__setattr__` wrapper (Piotr Sawicki, PR [19997](https://github.com/python/mypy/pull/19997)) + +Mypyc now has partial support for `__getattr__`, `__setattr__` and +`__delattr__` methods in native classes. + +Note that native attributes are not stored using `__dict__`. Setting attributes +directly while bypassing `__setattr__` is possible by using +`super().__setattr__(...)` or `object.__setattr__(...)`, but not via `__dict__`. + +Example: +```python +class Demo: + _data: dict[str, str] + + def __init__(self) -> None: + # Initialize data dict without calling our __setattr__ + super().__setattr__("_data", {}) + + def __setattr__(self, name: str, value: str) -> None: + print(f"Setting {name} = {value!r}") + + if name == "_data": + raise AttributeError("'_data' cannot be set") + + self._data[name] = value + + def __getattr__(self, name: str) -> str: + print(f"Getting {name}") + + try: + return self._data[name] + except KeyError: + raise AttributeError(name) + +d = Demo() +d.x = "hello" +d.y = "world" + +print(d.x) +print(d.y) +``` + +Related PRs: - Generate `__setattr__` wrapper (Piotr Sawicki, PR [19937](https://github.com/python/mypy/pull/19937)) - Generate `__getattr__` wrapper (Piotr Sawicki, PR [19909](https://github.com/python/mypy/pull/19909)) +- Support deleting attributes in `__setattr__` wrapper (Piotr Sawicki, PR [19997](https://github.com/python/mypy/pull/19997)) ### Miscellaneous Mypyc Improvements +- Fix `__new__` in native classes with inheritance (Piotr Sawicki, PR [20302](https://github.com/python/mypy/pull/20302)) - Fix crash on `super` in generator (Ivan Levkivskyi, PR [20291](https://github.com/python/mypy/pull/20291)) - Fix calling base class async method using `super()` (Jukka Lehtosalo, PR [20254](https://github.com/python/mypy/pull/20254)) - Fix async or generator methods in traits (Jukka Lehtosalo, PR [20246](https://github.com/python/mypy/pull/20246)) -- Optimize equality check with string literals [1/1] (BobTheBuidler, PR [19883](https://github.com/python/mypy/pull/19883)) +- Optimize equality check with string literals (BobTheBuidler, PR [19883](https://github.com/python/mypy/pull/19883)) - Fix inheritance of async defs (Jukka Lehtosalo, PR [20044](https://github.com/python/mypy/pull/20044)) -- Reject invalid `mypyc_attr` args [1/1] (BobTheBuidler, PR [19963](https://github.com/python/mypy/pull/19963)) +- Reject invalid `mypyc_attr` args (BobTheBuidler, PR [19963](https://github.com/python/mypy/pull/19963)) - Optimize `isinstance` with tuple of primitive types (BobTheBuidler, PR [19949](https://github.com/python/mypy/pull/19949)) - Optimize away first index check in for loops if length > 1 (BobTheBuidler, PR [19933](https://github.com/python/mypy/pull/19933)) - Fix broken exception/cancellation handling in async def (Jukka Lehtosalo, PR [19951](https://github.com/python/mypy/pull/19951)) @@ -71,45 +159,42 @@ You can read the full documentation for this release on [Read the Docs](http://m ### Stubtest Improvements - Check `_value_` for ellipsis-valued stub enum members (Stanislav Terliakov, PR [19760](https://github.com/python/mypy/pull/19760)) - Include function name in overload assertion messages (Joren Hammudoglu, PR [20063](https://github.com/python/mypy/pull/20063)) -- Small fix in get_default_function_sig (iap, PR [19822](https://github.com/python/mypy/pull/19822)) -- Adjust stubtest test stubs for PEP 728 (Python 3.15) (Marc Mueller, PR [20009](https://github.com/python/mypy/pull/20009)) +- Fix special case in analyzing function signature (iap, PR [19822](https://github.com/python/mypy/pull/19822)) - Improve `allowlist` docs with better example (sobolevn, PR [20007](https://github.com/python/mypy/pull/20007)) ### Documentation Updates -- Update duck_type_compatibility.rst: mention strict-bytes & mypy 2.0 (wyattscarpenter, PR [20121](https://github.com/python/mypy/pull/20121)) -- document --enable-incomplete-feature TypeForm, minimally but sufficiently (wyattscarpenter, PR [20173](https://github.com/python/mypy/pull/20173)) -- Change the InlineTypedDict example (wyattscarpenter, PR [20172](https://github.com/python/mypy/pull/20172)) -- Update kinds_of_types.rst: keep old anchor for #no-strict-optional (wyattscarpenter, PR [19828](https://github.com/python/mypy/pull/19828)) +- Update duck type compatibility: mention strict-bytes and mypy 2.0 (wyattscarpenter, PR [20121](https://github.com/python/mypy/pull/20121)) +- Document `--enable-incomplete-feature TypeForm` (wyattscarpenter, PR [20173](https://github.com/python/mypy/pull/20173)) +- Change the inline TypedDict example (wyattscarpenter, PR [20172](https://github.com/python/mypy/pull/20172)) - Replace `List` with built‑in `list` (PEP 585) (Thiago J. Barbalho, PR [20000](https://github.com/python/mypy/pull/20000)) -- main.py: junit documentation elaboration (wyattscarpenter, PR [19867](https://github.com/python/mypy/pull/19867)) +- Improve junit documentation (wyattscarpenter, PR [19867](https://github.com/python/mypy/pull/19867)) ### Other Notable Fixes and Improvements -- Update import map when new modules added (Ivan Levkivskyi, PR [20271](https://github.com/python/mypy/pull/20271)) - Fix annotated with function as type keyword list parameter (KarelKenens, PR [20094](https://github.com/python/mypy/pull/20094)) - Fix errors for raise NotImplemented (Shantanu, PR [20168](https://github.com/python/mypy/pull/20168)) - Don't let help formatter line-wrap URLs (Frank Dana, PR [19825](https://github.com/python/mypy/pull/19825)) - Do not cache fast container types inside lambdas (Stanislav Terliakov, PR [20166](https://github.com/python/mypy/pull/20166)) - Respect force-union-syntax flag in error hint (Marc Mueller, PR [20165](https://github.com/python/mypy/pull/20165)) - Fix type checking of dict type aliases (Shantanu, PR [20170](https://github.com/python/mypy/pull/20170)) -- Use pretty_callable more often for callable expressions (Theodore Ando, PR [20128](https://github.com/python/mypy/pull/20128)) +- Use pretty callable formatting more often for callable expressions (Theodore Ando, PR [20128](https://github.com/python/mypy/pull/20128)) - Use dummy concrete type instead of `Any` when checking protocol variance (bzoracler, PR [20110](https://github.com/python/mypy/pull/20110)) -- [PEP 696] Fix swapping TypeVars with defaults (Randolf Scholz, PR [19449](https://github.com/python/mypy/pull/19449)) -- Fix narrowing of class pattern with union-argument (Randolf Scholz, PR [19517](https://github.com/python/mypy/pull/19517)) +- PEP 696: Fix swapping TypeVars with defaults (Randolf Scholz, PR [19449](https://github.com/python/mypy/pull/19449)) +- Fix narrowing of class pattern with union type (Randolf Scholz, PR [19517](https://github.com/python/mypy/pull/19517)) - Do not emit unreachable warnings for lines that return `NotImplemented` (Christoph Tyralla, PR [20083](https://github.com/python/mypy/pull/20083)) -- fix matching against `typing.Callable` and `Protocol` types (Randolf Scholz, PR [19471](https://github.com/python/mypy/pull/19471)) -- Make --pretty work better on multi-line issues (A5rocks, PR [20056](https://github.com/python/mypy/pull/20056)) +- Fix matching against `typing.Callable` and `Protocol` types (Randolf Scholz, PR [19471](https://github.com/python/mypy/pull/19471)) +- Make `--pretty` work better on multi-line issues (A5rocks, PR [20056](https://github.com/python/mypy/pull/20056)) - More precise return types for `TypedDict.get` (Randolf Scholz, PR [19897](https://github.com/python/mypy/pull/19897)) -- prevent false unreachable warnings for @final instances that occur when strict optional checking is disabled (Christoph Tyralla, PR [20045](https://github.com/python/mypy/pull/20045)) +- Prevent false unreachable warnings for `@final` instances that occur when strict optional checking is disabled (Christoph Tyralla, PR [20045](https://github.com/python/mypy/pull/20045)) - Check class references to catch non-existent classes in match cases (A5rocks, PR [20042](https://github.com/python/mypy/pull/20042)) - Do not sort unused error codes in unused error codes warning (wyattscarpenter, PR [20036](https://github.com/python/mypy/pull/20036)) -- Fix `[name-defined]` false-positive in `class A[X, Y=X]:` case (sobolevn, PR [20021](https://github.com/python/mypy/pull/20021)) +- Fix `[name-defined]` false positive in `class A[X, Y=X]:` case (sobolevn, PR [20021](https://github.com/python/mypy/pull/20021)) - Filter SyntaxWarnings during AST parsing (Marc Mueller, PR [20023](https://github.com/python/mypy/pull/20023)) -- Make untyped decorator its own code (wyattscarpenter, PR [19911](https://github.com/python/mypy/pull/19911)) +- Make untyped decorator its own error code (wyattscarpenter, PR [19911](https://github.com/python/mypy/pull/19911)) - Support error codes from plugins in options (Sigve Sebastian Farstad, PR [19719](https://github.com/python/mypy/pull/19719)) - Allow returning Literals in `__new__` (James Hilton-Balfe, PR [15687](https://github.com/python/mypy/pull/15687)) - Inverse interface freshness logic (Ivan Levkivskyi, PR [19809](https://github.com/python/mypy/pull/19809)) - Do not report exhaustive-match after deferral (Stanislav Terliakov, PR [19804](https://github.com/python/mypy/pull/19804)) -- Make untyped_calls_exclude invalidate cache (Ivan Levkivskyi, PR [19801](https://github.com/python/mypy/pull/19801)) +- Make `untyped_calls_exclude` invalidate cache (Ivan Levkivskyi, PR [19801](https://github.com/python/mypy/pull/19801)) - Add await to empty context hack (Stanislav Terliakov, PR [19777](https://github.com/python/mypy/pull/19777)) - Consider non-empty enums assignable to Self (Stanislav Terliakov, PR [19779](https://github.com/python/mypy/pull/19779)) From 0f068c9ec604daa09e69c92545b059f4b44f566e Mon Sep 17 00:00:00 2001 From: Piotr Sawicki Date: Fri, 28 Nov 2025 12:57:08 +0100 Subject: [PATCH 412/424] Remove +dev --- mypy/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/version.py b/mypy/version.py index af216bddded1a..f550cd929eb5e 100644 --- a/mypy/version.py +++ b/mypy/version.py @@ -8,7 +8,7 @@ # - Release versions have the form "1.2.3". # - Dev versions have the form "1.2.3+dev" (PLUS sign to conform to PEP 440). # - Before 1.0 we had the form "0.NNN". -__version__ = "1.19.0+dev" +__version__ = "1.19.0" base_version = __version__ mypy_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) From dbf97df271f3e69f0f52c9fc99e38a03ecadde79 Mon Sep 17 00:00:00 2001 From: Shantanu Jain Date: Sat, 13 Dec 2025 14:26:04 -0800 Subject: [PATCH 413/424] Bump version to 1.19.1+dev --- mypy/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/version.py b/mypy/version.py index f550cd929eb5e..eef99f7203cf7 100644 --- a/mypy/version.py +++ b/mypy/version.py @@ -8,7 +8,7 @@ # - Release versions have the form "1.2.3". # - Dev versions have the form "1.2.3+dev" (PLUS sign to conform to PEP 440). # - Before 1.0 we had the form "0.NNN". -__version__ = "1.19.0" +__version__ = "1.19.1+dev" base_version = __version__ mypy_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) From d503cf87a130449a053fd9ac098be7c9482ea540 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 1 Dec 2025 14:06:09 +0000 Subject: [PATCH 414/424] Fix crash on typevar with forward ref used in other module (#20334) Fixes https://github.com/python/mypy/issues/20326 Type variables with forward references in upper bound are known to be problematic. Existing mechanisms to work with them implicitly assumed that they are used in the same module where they are defined, which is not necessarily the case for "old-style" type variables that can be imported. Note that the simplification I made in `semanal_typeargs.py` would be probably sufficient to fix this, but that would be papering over the real issue, so I am making a bit more principled fix. --- mypy/plugins/proper_plugin.py | 1 + mypy/semanal.py | 2 +- mypy/semanal_typeargs.py | 12 ++-- mypy/typeanal.py | 9 +++ test-data/unit/check-type-aliases.test | 90 ++++++++++++++++++++++++++ 5 files changed, 105 insertions(+), 9 deletions(-) diff --git a/mypy/plugins/proper_plugin.py b/mypy/plugins/proper_plugin.py index 0189bfbd22fcd..872903ea6b47d 100644 --- a/mypy/plugins/proper_plugin.py +++ b/mypy/plugins/proper_plugin.py @@ -108,6 +108,7 @@ def is_special_target(right: ProperType) -> bool: "mypy.types.RequiredType", "mypy.types.ReadOnlyType", "mypy.types.TypeGuardedType", + "mypy.types.PlaceholderType", ): # Special case: these are not valid targets for a type alias and thus safe. # TODO: introduce a SyntheticType base to simplify this? diff --git a/mypy/semanal.py b/mypy/semanal.py index 973a28db0588b..f9f0e4d710986 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -4935,7 +4935,7 @@ def get_typevarlike_argument( ) if analyzed is None: # Type variables are special: we need to place them in the symbol table - # soon, even if upper bound is not ready yet. Otherwise avoiding + # soon, even if upper bound is not ready yet. Otherwise, avoiding # a "deadlock" in this common pattern would be tricky: # T = TypeVar('T', bound=Custom[Any]) # class Custom(Generic[T]): diff --git a/mypy/semanal_typeargs.py b/mypy/semanal_typeargs.py index 86f8a8700def6..9d1ce1fd60806 100644 --- a/mypy/semanal_typeargs.py +++ b/mypy/semanal_typeargs.py @@ -176,12 +176,12 @@ def validate_args( code=codes.VALID_TYPE, ) continue + if self.in_type_alias_expr and isinstance(arg, TypeVarType): + # Type aliases are allowed to use unconstrained type variables + # error will be checked at substitution point. + continue if tvar.values: if isinstance(arg, TypeVarType): - if self.in_type_alias_expr: - # Type aliases are allowed to use unconstrained type variables - # error will be checked at substitution point. - continue arg_values = arg.values if not arg_values: is_error = True @@ -205,10 +205,6 @@ def validate_args( and upper_bound.type.fullname == "builtins.object" ) if not object_upper_bound and not is_subtype(arg, upper_bound): - if self.in_type_alias_expr and isinstance(arg, TypeVarType): - # Type aliases are allowed to use unconstrained type variables - # error will be checked at substitution point. - continue is_error = True self.fail( message_registry.INVALID_TYPEVAR_ARG_BOUND.format( diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 06fa847c54345..3e5f522f39076 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -346,6 +346,15 @@ def visit_unbound_type_nonoptional(self, t: UnboundType, defining_literal: bool) if hook is not None: return hook(AnalyzeTypeContext(t, t, self)) tvar_def = self.tvar_scope.get_binding(sym) + if tvar_def is not None: + # We need to cover special-case explained in get_typevarlike_argument() here, + # since otherwise the deferral will not be triggered if the type variable is + # used in a different module. Using isinstance() should be safe for this purpose. + tvar_params = [tvar_def.upper_bound, tvar_def.default] + if isinstance(tvar_def, TypeVarType): + tvar_params += tvar_def.values + if any(isinstance(tp, PlaceholderType) for tp in tvar_params): + self.api.defer() if isinstance(sym.node, ParamSpecExpr): if tvar_def is None: if self.allow_unbound_tvars: diff --git a/test-data/unit/check-type-aliases.test b/test-data/unit/check-type-aliases.test index 6923b0d8f0064..1fb2b038a1a18 100644 --- a/test-data/unit/check-type-aliases.test +++ b/test-data/unit/check-type-aliases.test @@ -1351,3 +1351,93 @@ reveal_type(D(x="asdf")) # E: No overload variant of "dict" matches argument ty # N: def __init__(self, arg: Iterable[tuple[str, int]], **kwargs: int) -> dict[str, int] \ # N: Revealed type is "Any" [builtins fixtures/dict.pyi] + +[case testTypeAliasesInCyclicImport1] +import p.aliases + +[file p/__init__.py] +[file p/aliases.py] +from typing_extensions import TypeAlias +from .defs import C, Alias1 + +Alias2: TypeAlias = Alias1[C] + +[file p/defs.py] +from typing import TypeVar +from typing_extensions import TypeAlias +import p.aliases + +C = TypeVar("C", bound="SomeClass") +Alias1: TypeAlias = C + +class SomeClass: + pass +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] + +[case testTypeAliasesInCyclicImport2] +import p.aliases + +[file p/__init__.py] +[file p/aliases.py] +from typing_extensions import TypeAlias +from .defs import C, Alias1 + +Alias2: TypeAlias = Alias1[C] + +[file p/defs.py] +from typing import TypeVar, Union +from typing_extensions import TypeAlias +import p.aliases + +C = TypeVar("C", bound="SomeClass") +Alias1: TypeAlias = Union[C, int] + +class SomeClass: + pass +[builtins fixtures/tuple.pyi] + +[case testTypeAliasesInCyclicImport3] +import p.aliases + +[file p/__init__.py] +[file p/aliases.py] +from typing_extensions import TypeAlias +from .defs import C, Alias1 + +Alias2: TypeAlias = Alias1[C] + +[file p/defs.py] +from typing import TypeVar +from typing_extensions import TypeAlias +import p.aliases + +C = TypeVar("C", bound="list[SomeClass]") +Alias1: TypeAlias = C + +class SomeClass: + pass +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] + +[case testTypeAliasesInCyclicImport4] +import p.aliases + +[file p/__init__.py] +[file p/aliases.py] +from typing_extensions import TypeAlias +from .defs import C, Alias1 + +Alias2: TypeAlias = Alias1[C] + +[file p/defs.py] +from typing import TypeVar, Union +from typing_extensions import TypeAlias +import p.aliases + +C = TypeVar("C", bound="list[SomeClass]") +Alias1: TypeAlias = Union[C, int] + +class SomeClass: + pass +[builtins fixtures/tuple.pyi] From c93d917a86993e06dcc88e508f28f4f5199ce1c8 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 30 Nov 2025 16:30:01 +0000 Subject: [PATCH 415/424] Fix crash on star import of redefinition (#20333) Fixes https://github.com/python/mypy/issues/20327 Fix is trivial, do not grab various internal/temporary symbols with star imports. This may create an invalid cross-reference (and is generally dangerous). Likely, this worked previously because we processed all fresh modules in queue, not just the dependencies of current SCC. --- mypy/semanal.py | 4 ++++ test-data/unit/check-incremental.test | 30 +++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/mypy/semanal.py b/mypy/semanal.py index f9f0e4d710986..1035efb29061c 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -3195,6 +3195,10 @@ def visit_import_all(self, i: ImportAll) -> None: # namespace is incomplete. self.mark_incomplete("*", i) for name, node in m.names.items(): + if node.no_serialize: + # This is either internal or generated symbol, skip it to avoid problems + # like accidental name conflicts or invalid cross-references. + continue fullname = i_id + "." + name self.set_future_import_flags(fullname) # if '__all__' exists, all nodes not included have had module_public set to diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 56c9cef80f342..170a883ce25da 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -7596,3 +7596,33 @@ X = 0 tmp/a.py:6: error: "object" has no attribute "dtypes" [out2] tmp/a.py:2: error: "object" has no attribute "dtypes" + +[case testStarImportCycleRedefinition] +import m + +[file m.py] +import a + +[file m.py.2] +import a +reveal_type(a.C) + +[file a/__init__.py] +from a.b import * +from a.c import * +x = 1 + +[file a/b.py] +from other import C +from a.c import y +class C: ... # type: ignore + +[file a/c.py] +from other import C +from a import x +y = 1 + +[file other.py] +class C: ... +[out2] +tmp/m.py:2: note: Revealed type is "def () -> other.C" From 3890fc49bf7cc02db04b1e63eb2540aaacdeecc0 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sat, 29 Nov 2025 06:42:03 -0800 Subject: [PATCH 416/424] Fix crash involving Unpack-ed TypeVarTuple (#20323) Fixes #20093 This fixes the crash, but not the false positive (the false positive existed prior to the regression that introduced the crash) --- mypy/typeops.py | 10 ++++++++-- test-data/unit/check-overloading.test | 13 +++++++++++++ test-data/unit/check-typevar-tuple.test | 23 +++++++++++++++++++++++ 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/mypy/typeops.py b/mypy/typeops.py index 050252eb62050..f6646740031d0 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -508,7 +508,7 @@ def erase_to_bound(t: Type) -> Type: def callable_corresponding_argument( typ: NormalizedCallableType | Parameters, model: FormalArgument ) -> FormalArgument | None: - """Return the argument a function that corresponds to `model`""" + """Return the argument of a function that corresponds to `model`""" by_name = typ.argument_by_name(model.name) by_pos = typ.argument_by_position(model.pos) @@ -522,17 +522,23 @@ def callable_corresponding_argument( # taking both *args and **args, or a pair of functions like so: # def right(a: int = ...) -> None: ... - # def left(__a: int = ..., *, a: int = ...) -> None: ... + # def left(x: int = ..., /, *, a: int = ...) -> None: ... from mypy.meet import meet_types if ( not (by_name.required or by_pos.required) and by_pos.name is None and by_name.pos is None + # This is not principled, but prevents a crash. It's weird to have a FormalArgument + # that has an UnpackType. + and not isinstance(by_name.typ, UnpackType) + and not isinstance(by_pos.typ, UnpackType) ): return FormalArgument( by_name.name, by_pos.pos, meet_types(by_name.typ, by_pos.typ), False ) + return by_name + return by_name if by_name is not None else by_pos diff --git a/test-data/unit/check-overloading.test b/test-data/unit/check-overloading.test index be55a182b87bb..1830a0c5ce3c0 100644 --- a/test-data/unit/check-overloading.test +++ b/test-data/unit/check-overloading.test @@ -263,6 +263,19 @@ def foo(*args: int | str, **kw: int | Foo) -> None: pass [builtins fixtures/tuple.pyi] + +[case testTypeCheckOverloadImplOverlapVarArgsAndKwargsNever] +from __future__ import annotations +from typing import overload + +@overload # E: Single overload definition, multiple required +def foo(x: int) -> None: ... + +def foo(*args: int, **kw: str) -> None: # E: Overloaded function implementation does not accept all possible arguments of signature 1 + pass +[builtins fixtures/tuple.pyi] + + [case testTypeCheckOverloadWithImplTooSpecificRetType] from typing import overload, Any diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index cb5029ee4e6d2..c60d0aec0835a 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -2716,3 +2716,26 @@ class MyTuple(tuple[Unpack[Union[int, str]]], Generic[Unpack[Ts]]): # E: "Union x: MyTuple[int, str] reveal_type(x[0]) # N: Revealed type is "Any" [builtins fixtures/tuple.pyi] + +[case testHigherOrderFunctionUnpackTypeVarTupleViaParamSpec] +from typing import Callable, ParamSpec, TypeVar, TypeVarTuple, Unpack + +P = ParamSpec("P") +T = TypeVar("T") +Ts = TypeVarTuple("Ts") + +def call(func: Callable[P, T], *args: P.args, **kwargs: P.kwargs) -> T: + return func(*args, **kwargs) + + +def run(func: Callable[[Unpack[Ts]], T], *args: Unpack[Ts], some_kwarg: str = "asdf") -> T: + raise + + +def foo() -> str: + return "hello" + + +# this is a false positive, but it no longer crashes +call(run, foo, some_kwarg="a") # E: Argument 1 to "call" has incompatible type "def [Ts`-1, T] run(func: def (*Unpack[Ts]) -> T, *args: Unpack[Ts], some_kwarg: str = ...) -> T"; expected "Callable[[Callable[[], str], str], str]" +[builtins fixtures/tuple.pyi] From 70eceea682c041c0d8e8462dffef9c7bb252e014 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sat, 13 Dec 2025 14:24:20 -0800 Subject: [PATCH 417/424] Fix noncommutative joins with bounded TypeVars (#20345) Fixes #20344 --- mypy/join.py | 13 +++++++++---- mypy/test/testtypes.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/mypy/join.py b/mypy/join.py index 0822ddbfd89aa..a074fa522588c 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -297,10 +297,15 @@ def visit_erased_type(self, t: ErasedType) -> ProperType: return self.s def visit_type_var(self, t: TypeVarType) -> ProperType: - if isinstance(self.s, TypeVarType) and self.s.id == t.id: - if self.s.upper_bound == t.upper_bound: - return self.s - return self.s.copy_modified(upper_bound=join_types(self.s.upper_bound, t.upper_bound)) + if isinstance(self.s, TypeVarType): + if self.s.id == t.id: + if self.s.upper_bound == t.upper_bound: + return self.s + return self.s.copy_modified( + upper_bound=join_types(self.s.upper_bound, t.upper_bound) + ) + # Fix non-commutative joins + return get_proper_type(join_types(self.s.upper_bound, t.upper_bound)) else: return self.default(self.s) diff --git a/mypy/test/testtypes.py b/mypy/test/testtypes.py index fc68d9aa6eac2..f5f4c6797db2d 100644 --- a/mypy/test/testtypes.py +++ b/mypy/test/testtypes.py @@ -1051,6 +1051,35 @@ def test_join_type_type_type_var(self) -> None: self.assert_join(self.fx.type_a, self.fx.t, self.fx.o) self.assert_join(self.fx.t, self.fx.type_a, self.fx.o) + def test_join_type_var_bounds(self) -> None: + tvar1 = TypeVarType( + "tvar1", + "tvar1", + TypeVarId(-100), + [], + self.fx.o, + AnyType(TypeOfAny.from_omitted_generics), + INVARIANT, + ) + any_type = AnyType(TypeOfAny.special_form) + tvar2 = TypeVarType( + "tvar2", + "tvar2", + TypeVarId(-101), + [], + upper_bound=UnionType( + [ + TupleType([any_type], self.fx.std_tuple), + TupleType([any_type, any_type], self.fx.std_tuple), + ] + ), + default=AnyType(TypeOfAny.from_omitted_generics), + variance=INVARIANT, + ) + + self.assert_join(tvar1, tvar2, self.fx.o) + self.assert_join(tvar2, tvar1, self.fx.o) + # There are additional test cases in check-inference.test. # TODO: Function types + varargs and default args. From 8a6eff478416cd3ed3931a6ed77ce61c88ab69e9 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Tue, 9 Dec 2025 02:27:25 -0500 Subject: [PATCH 418/424] [mypyc] fix generator regression with empty tuple (#20371) This PR fixes #20341 --- mypyc/irbuild/builder.py | 4 ++- mypyc/irbuild/for_helpers.py | 42 +++++++++++++++++++++-------- mypyc/test-data/run-generators.test | 8 ++++++ mypyc/test-data/run-loops.test | 9 +++++-- 4 files changed, 49 insertions(+), 14 deletions(-) diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index 63930123135fe..51a02ed5446d3 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -990,8 +990,10 @@ def get_sequence_type_from_type(self, target_type: Type) -> RType: elif isinstance(target_type, TypeVarLikeType): return self.get_sequence_type_from_type(target_type.upper_bound) elif isinstance(target_type, TupleType): + items = target_type.items + assert items, "This function does not support empty tuples" # Tuple might have elements of different types. - rtypes = {self.mapper.type_to_rtype(item) for item in target_type.items} + rtypes = set(map(self.mapper.type_to_rtype, items)) if len(rtypes) == 1: return rtypes.pop() else: diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index 715f5432cd133..33e4429356414 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -7,7 +7,7 @@ from __future__ import annotations -from typing import Callable, ClassVar +from typing import Callable, ClassVar, cast from mypy.nodes import ( ARG_POS, @@ -241,25 +241,45 @@ def sequence_from_generator_preallocate_helper( rtype = builder.node_type(sequence_expr) if not (is_sequence_rprimitive(rtype) or isinstance(rtype, RTuple)): return None - sequence = builder.accept(sequence_expr) - length = get_expr_length_value(builder, sequence_expr, sequence, line, use_pyssize_t=True) + if isinstance(rtype, RTuple): # If input is RTuple, box it to tuple_rprimitive for generic iteration # TODO: this can be optimized a bit better with an unrolled ForRTuple helper proper_type = get_proper_type(builder.types[sequence_expr]) assert isinstance(proper_type, TupleType), proper_type - get_item_ops = [ - ( - LoadLiteral(typ.value, object_rprimitive) - if isinstance(typ, LiteralType) - else TupleGet(sequence, i, line) - ) - for i, typ in enumerate(get_proper_types(proper_type.items)) - ] + # the for_loop_helper_with_index crashes for empty tuples, bail out + if not proper_type.items: + return None + + proper_types = get_proper_types(proper_type.items) + + get_item_ops: list[LoadLiteral | TupleGet] + if all(isinstance(typ, LiteralType) for typ in proper_types): + get_item_ops = [ + LoadLiteral(cast(LiteralType, typ).value, object_rprimitive) + for typ in proper_types + ] + + else: + sequence = builder.accept(sequence_expr) + get_item_ops = [ + ( + LoadLiteral(typ.value, object_rprimitive) + if isinstance(typ, LiteralType) + else TupleGet(sequence, i, line) + ) + for i, typ in enumerate(proper_types) + ] + items = list(map(builder.add, get_item_ops)) sequence = builder.new_tuple(items, line) + else: + sequence = builder.accept(sequence_expr) + + length = get_expr_length_value(builder, sequence_expr, sequence, line, use_pyssize_t=True) + target_op = empty_op_llbuilder(length, line) def set_item(item_index: Value) -> None: diff --git a/mypyc/test-data/run-generators.test b/mypyc/test-data/run-generators.test index c8e83173474de..cf1dac7c57333 100644 --- a/mypyc/test-data/run-generators.test +++ b/mypyc/test-data/run-generators.test @@ -936,3 +936,11 @@ def test_generator_override() -> None: assert base1_foo(Base1()) == [1] assert base1_foo(Derived1()) == [2, 3] assert derived1_foo(Derived1()) == [2, 3] + +[case testGeneratorEmptyTuple] +from collections.abc import Generator +from typing import Optional, Union + +def test_compiledGeneratorEmptyTuple() -> None: + jobs: Generator[Optional[str], None, None] = (_ for _ in ()) + assert list(jobs) == [] diff --git a/mypyc/test-data/run-loops.test b/mypyc/test-data/run-loops.test index 3cbb07297e6e1..106c2271d3264 100644 --- a/mypyc/test-data/run-loops.test +++ b/mypyc/test-data/run-loops.test @@ -1,7 +1,7 @@ # Test cases for "range" objects, "for" and "while" loops (compile and run) [case testFor] -from typing import List, Tuple +from typing import Any, List, Tuple def count(n: int) -> None: for i in range(n): print(i) @@ -21,6 +21,10 @@ def list_iter(l: List[int]) -> None: def tuple_iter(l: Tuple[int, ...]) -> None: for i in l: print(i) +def empty_tuple_iter(l: Tuple[()]) -> None: + i: Any + for i in l: + print(i) def str_iter(l: str) -> None: for i in l: print(i) @@ -39,7 +43,7 @@ def count_down_short() -> None: [file driver.py] from native import ( count, list_iter, list_rev_iter, list_rev_iter_lol, count_between, count_down, count_double, - count_down_short, tuple_iter, str_iter, + count_down_short, tuple_iter, empty_tuple_iter, str_iter, ) count(5) list_iter(list(reversed(range(5)))) @@ -52,6 +56,7 @@ count_down_short() print('==') list_rev_iter_lol(list(reversed(range(5)))) tuple_iter((1, 2, 3)) +empty_tuple_iter(()) str_iter("abc") [out] 0 From a4b31a26788b70c4a2a19adbafa2bbda43dc2e8b Mon Sep 17 00:00:00 2001 From: A5rocks Date: Tue, 9 Dec 2025 07:03:09 -0500 Subject: [PATCH 419/424] Allow `types.NoneType` in match cases (#20383) Fixes https://github.com/python/mypy/issues/20367 --- mypy/checkpattern.py | 3 +++ test-data/unit/check-python310.test | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/mypy/checkpattern.py b/mypy/checkpattern.py index 3c51c41069097..cafc69490e09d 100644 --- a/mypy/checkpattern.py +++ b/mypy/checkpattern.py @@ -46,6 +46,7 @@ Type, TypedDictType, TypeOfAny, + TypeType, TypeVarTupleType, TypeVarType, UninhabitedType, @@ -556,6 +557,8 @@ def visit_class_pattern(self, o: ClassPattern) -> PatternType: fallback = self.chk.named_type("builtins.function") any_type = AnyType(TypeOfAny.unannotated) typ = callable_with_ellipsis(any_type, ret_type=any_type, fallback=fallback) + elif isinstance(p_typ, TypeType) and isinstance(p_typ.item, NoneType): + typ = p_typ.item elif not isinstance(p_typ, AnyType): self.msg.fail( message_registry.CLASS_PATTERN_TYPE_REQUIRED.format( diff --git a/test-data/unit/check-python310.test b/test-data/unit/check-python310.test index 1e27e30d4b041..8bc781d091c3e 100644 --- a/test-data/unit/check-python310.test +++ b/test-data/unit/check-python310.test @@ -3178,3 +3178,19 @@ match 5: reveal_type(b) # N: Revealed type is "Any" case BlahBlah(c=c): # E: Name "BlahBlah" is not defined reveal_type(c) # N: Revealed type is "Any" + +[case testMatchAllowsNoneTypeAsClass] +import types + +class V: + X = types.NoneType + +def fun(val: str | None): + match val: + case V.X(): + reveal_type(val) # N: Revealed type is "None" + + match val: + case types.NoneType(): + reveal_type(val) # N: Revealed type is "None" +[builtins fixtures/tuple.pyi] From 58d485b4ea4776e0b9d4045b306cb0818ecc2aa6 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 9 Dec 2025 14:08:13 +0000 Subject: [PATCH 420/424] Fail with an explicit error on PyPy (#20384) Fixes https://github.com/mypyc/librt/issues/21 Fail with an explicit user-friendly error on PyPy. --- mypy-requirements.txt | 2 +- pyproject.toml | 4 ++-- setup.py | 9 +++++++++ test-requirements.txt | 2 +- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index b0c632dddac56..6984d9a5d070c 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -4,4 +4,4 @@ typing_extensions>=4.6.0 mypy_extensions>=1.0.0 pathspec>=0.9.0 tomli>=1.1.0; python_version<'3.11' -librt>=0.6.2 +librt>=0.6.2; platform_python_implementation != 'PyPy' diff --git a/pyproject.toml b/pyproject.toml index bb41c82b1a3ce..fa56caeaa4bc9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ requires = [ "mypy_extensions>=1.0.0", "pathspec>=0.9.0", "tomli>=1.1.0; python_version<'3.11'", - "librt>=0.6.2", + "librt>=0.6.2; platform_python_implementation != 'PyPy'", # the following is from build-requirements.txt "types-psutil", "types-setuptools", @@ -54,7 +54,7 @@ dependencies = [ "mypy_extensions>=1.0.0", "pathspec>=0.9.0", "tomli>=1.1.0; python_version<'3.11'", - "librt>=0.6.2", + "librt>=0.6.2; platform_python_implementation != 'PyPy'", ] dynamic = ["version"] diff --git a/setup.py b/setup.py index f20c1db5d0450..8cba27ae0f85c 100644 --- a/setup.py +++ b/setup.py @@ -5,6 +5,7 @@ import glob import os import os.path +import platform import sys from typing import TYPE_CHECKING, Any @@ -12,6 +13,14 @@ sys.stderr.write("ERROR: You need Python 3.9 or later to use mypy.\n") exit(1) +if platform.python_implementation() == "PyPy": + sys.stderr.write( + "ERROR: Running mypy on PyPy is not supported yet.\n" + "To type-check a PyPy library please use an equivalent CPython version,\n" + "see https://github.com/mypyc/librt/issues/16 for possible workarounds.\n" + ) + exit(1) + # we'll import stuff from the source tree, let's ensure is on the sys path sys.path.insert(0, os.path.dirname(os.path.realpath(__file__))) diff --git a/test-requirements.txt b/test-requirements.txt index 953e7a750c755..d8334108fc1d0 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -22,7 +22,7 @@ identify==2.6.15 # via pre-commit iniconfig==2.1.0 # via pytest -librt==0.6.2 +librt==0.7.3 ; platform_python_implementation != 'PyPy' # via -r mypy-requirements.txt lxml==6.0.2 ; python_version < "3.15" # via -r test-requirements.in From f60f90fb8872bf722e32aefd548daaf6d8560e05 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 10 Dec 2025 02:02:26 +0000 Subject: [PATCH 421/424] Fail on PyPy in main instead of setup.py (#20389) Follow-up for https://github.com/python/mypy/pull/20384 --- mypy/main.py | 8 ++++++++ setup.py | 9 --------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index 7d5721851c3d6..5b8f8b5a54765 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -4,6 +4,7 @@ import argparse import os +import platform import subprocess import sys import time @@ -39,6 +40,13 @@ if TYPE_CHECKING: from _typeshed import SupportsWrite +if platform.python_implementation() == "PyPy": + sys.stderr.write( + "ERROR: Running mypy on PyPy is not supported yet.\n" + "To type-check a PyPy library please use an equivalent CPython version,\n" + "see https://github.com/mypyc/librt/issues/16 for possible workarounds.\n" + ) + sys.exit(2) orig_stat: Final = os.stat MEM_PROFILE: Final = False # If True, dump memory profile diff --git a/setup.py b/setup.py index 8cba27ae0f85c..f20c1db5d0450 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,6 @@ import glob import os import os.path -import platform import sys from typing import TYPE_CHECKING, Any @@ -13,14 +12,6 @@ sys.stderr.write("ERROR: You need Python 3.9 or later to use mypy.\n") exit(1) -if platform.python_implementation() == "PyPy": - sys.stderr.write( - "ERROR: Running mypy on PyPy is not supported yet.\n" - "To type-check a PyPy library please use an equivalent CPython version,\n" - "see https://github.com/mypyc/librt/issues/16 for possible workarounds.\n" - ) - exit(1) - # we'll import stuff from the source tree, let's ensure is on the sys path sys.path.insert(0, os.path.dirname(os.path.realpath(__file__))) From 2b23b507524bf1bd7513eea6f2a16fb91e072cb6 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 10 Dec 2025 00:51:04 +0000 Subject: [PATCH 422/424] Serialize raw errors in cache metas (#20372) Fixes https://github.com/python/mypy/issues/20353 This makes us respect e.g. `--output json` for cached files without re-checking the files (which is the desired behavior for users, see issue). This is also a first step towards resolving the "foo defined here" conundrum for parallel checking. The fix is straightforward. The only question was whether to continue using `ErrorTuple`s or switch to a proper class. I decided to keep the tuples for now to minimize the scope of change. Note I am also adjusting generic "JSON" fixed-format helpers to natively support tuples (unlike real JSON). We already use tuples in few other places, so it makes sense to just make it "official" (this format is still internal to mypy obviously). --- mypy/build.py | 69 +++++++++++++++++++++---- mypy/cache.py | 72 +++++++++++++++++++++++---- mypy/errors.py | 25 ++++++---- test-data/unit/check-incremental.test | 10 ++++ 4 files changed, 146 insertions(+), 30 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 853e54e445ac6..aee099fed316a 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -31,10 +31,17 @@ from librt.internal import cache_version import mypy.semanal_main -from mypy.cache import CACHE_VERSION, CacheMeta, ReadBuffer, WriteBuffer +from mypy.cache import ( + CACHE_VERSION, + CacheMeta, + ReadBuffer, + SerializedError, + WriteBuffer, + write_json, +) from mypy.checker import TypeChecker from mypy.error_formatter import OUTPUT_CHOICES, ErrorFormatter -from mypy.errors import CompileError, ErrorInfo, Errors, report_internal_error +from mypy.errors import CompileError, ErrorInfo, Errors, ErrorTuple, report_internal_error from mypy.graph_utils import prepare_sccs, strongly_connected_components, topsort from mypy.indirection import TypeIndirectionVisitor from mypy.messages import MessageBuilder @@ -1869,7 +1876,7 @@ class State: dep_hashes: dict[str, bytes] = {} # List of errors reported for this file last time. - error_lines: list[str] = [] + error_lines: list[SerializedError] = [] # Parent package, its parent, etc. ancestors: list[str] | None = None @@ -3286,9 +3293,13 @@ def find_stale_sccs( scc = order_ascc_ex(graph, ascc) for id in scc: if graph[id].error_lines: - manager.flush_errors( - manager.errors.simplify_path(graph[id].xpath), graph[id].error_lines, False + path = manager.errors.simplify_path(graph[id].xpath) + formatted = manager.errors.format_messages( + path, + deserialize_codes(graph[id].error_lines), + formatter=manager.error_formatter, ) + manager.flush_errors(path, formatted, False) fresh_sccs.append(ascc) else: size = len(ascc.mod_ids) @@ -3492,13 +3503,16 @@ def process_stale_scc(graph: Graph, ascc: SCC, manager: BuildManager) -> None: # Flush errors, and write cache in two phases: first data files, then meta files. meta_tuples = {} errors_by_id = {} + formatted_by_id = {} for id in stale: if graph[id].xpath not in manager.errors.ignored_files: - errors = manager.errors.file_messages( - graph[id].xpath, formatter=manager.error_formatter + errors = manager.errors.file_messages(graph[id].xpath) + formatted = manager.errors.format_messages( + graph[id].xpath, errors, formatter=manager.error_formatter ) - manager.flush_errors(manager.errors.simplify_path(graph[id].xpath), errors, False) + manager.flush_errors(manager.errors.simplify_path(graph[id].xpath), formatted, False) errors_by_id[id] = errors + formatted_by_id[id] = formatted meta_tuples[id] = graph[id].write_cache() graph[id].mark_as_rechecked() for id in stale: @@ -3507,7 +3521,7 @@ def process_stale_scc(graph: Graph, ascc: SCC, manager: BuildManager) -> None: continue meta, meta_file = meta_tuple meta.dep_hashes = [graph[dep].interface_hash for dep in graph[id].dependencies] - meta.error_lines = errors_by_id.get(id, []) + meta.error_lines = serialize_codes(errors_by_id.get(id, [])) write_cache_meta(meta, manager, meta_file) manager.done_sccs.add(ascc.id) @@ -3640,3 +3654,40 @@ def write_undocumented_ref_info( deps_json = get_undocumented_ref_info_json(state.tree, type_map) metastore.write(ref_info_file, json_dumps(deps_json)) + + +def sources_to_bytes(sources: list[BuildSource]) -> bytes: + source_tuples = [(s.path, s.module, s.text, s.base_dir, s.followed) for s in sources] + buf = WriteBuffer() + write_json(buf, {"sources": source_tuples}) + return buf.getvalue() + + +def sccs_to_bytes(sccs: list[SCC]) -> bytes: + scc_tuples = [(list(scc.mod_ids), scc.id, list(scc.deps)) for scc in sccs] + buf = WriteBuffer() + write_json(buf, {"sccs": scc_tuples}) + return buf.getvalue() + + +def serialize_codes(errs: list[ErrorTuple]) -> list[SerializedError]: + return [ + (path, line, column, end_line, end_column, severity, message, code.code if code else None) + for path, line, column, end_line, end_column, severity, message, code in errs + ] + + +def deserialize_codes(errs: list[SerializedError]) -> list[ErrorTuple]: + return [ + ( + path, + line, + column, + end_line, + end_column, + severity, + message, + codes.error_codes.get(code) if code else None, + ) + for path, line, column, end_line, end_column, severity, message, code in errs + ] diff --git a/mypy/cache.py b/mypy/cache.py index ad12fd96f1fa4..7755755898c0d 100644 --- a/mypy/cache.py +++ b/mypy/cache.py @@ -48,7 +48,7 @@ from __future__ import annotations from collections.abc import Sequence -from typing import Any, Final, Union +from typing import Any, Final, Optional, Union from typing_extensions import TypeAlias as _TypeAlias from librt.internal import ( @@ -70,7 +70,9 @@ from mypy_extensions import u8 # High-level cache layout format -CACHE_VERSION: Final = 0 +CACHE_VERSION: Final = 1 + +SerializedError: _TypeAlias = tuple[Optional[str], int, int, int, int, str, str, Optional[str]] class CacheMeta: @@ -93,7 +95,7 @@ def __init__( dep_lines: list[int], dep_hashes: list[bytes], interface_hash: bytes, - error_lines: list[str], + error_lines: list[SerializedError], version_id: str, ignore_all: bool, plugin_data: Any, @@ -158,7 +160,7 @@ def deserialize(cls, meta: dict[str, Any], data_file: str) -> CacheMeta | None: dep_lines=meta["dep_lines"], dep_hashes=[bytes.fromhex(dep) for dep in meta["dep_hashes"]], interface_hash=bytes.fromhex(meta["interface_hash"]), - error_lines=meta["error_lines"], + error_lines=[tuple(err) for err in meta["error_lines"]], version_id=meta["version_id"], ignore_all=meta["ignore_all"], plugin_data=meta["plugin_data"], @@ -180,7 +182,7 @@ def write(self, data: WriteBuffer) -> None: write_int_list(data, self.dep_lines) write_bytes_list(data, self.dep_hashes) write_bytes(data, self.interface_hash) - write_str_list(data, self.error_lines) + write_errors(data, self.error_lines) write_str(data, self.version_id) write_bool(data, self.ignore_all) # Plugin data may be not a dictionary, so we use @@ -205,7 +207,7 @@ def read(cls, data: ReadBuffer, data_file: str) -> CacheMeta | None: dep_lines=read_int_list(data), dep_hashes=read_bytes_list(data), interface_hash=read_bytes(data), - error_lines=read_str_list(data), + error_lines=read_errors(data), version_id=read_str(data), ignore_all=read_bool(data), plugin_data=read_json_value(data), @@ -232,6 +234,7 @@ def read(cls, data: ReadBuffer, data_file: str) -> CacheMeta | None: LIST_INT: Final[Tag] = 21 LIST_STR: Final[Tag] = 22 LIST_BYTES: Final[Tag] = 23 +TUPLE_GEN: Final[Tag] = 24 DICT_STR_GEN: Final[Tag] = 30 # Misc classes. @@ -391,7 +394,13 @@ def write_str_opt_list(data: WriteBuffer, value: list[str | None]) -> None: write_str_opt(data, item) -JsonValue: _TypeAlias = Union[None, int, str, bool, list["JsonValue"], dict[str, "JsonValue"]] +Value: _TypeAlias = Union[None, int, str, bool] + +# Our JSON format is somewhat non-standard as we distinguish lists and tuples. +# This is convenient for some internal things, like mypyc plugin and error serialization. +JsonValue: _TypeAlias = Union[ + Value, list["JsonValue"], dict[str, "JsonValue"], tuple["JsonValue", ...] +] def read_json_value(data: ReadBuffer) -> JsonValue: @@ -409,15 +418,16 @@ def read_json_value(data: ReadBuffer) -> JsonValue: if tag == LIST_GEN: size = read_int_bare(data) return [read_json_value(data) for _ in range(size)] + if tag == TUPLE_GEN: + size = read_int_bare(data) + return tuple(read_json_value(data) for _ in range(size)) if tag == DICT_STR_GEN: size = read_int_bare(data) return {read_str_bare(data): read_json_value(data) for _ in range(size)} assert False, f"Invalid JSON tag: {tag}" -# Currently tuples are used by mypyc plugin. They will be normalized to -# JSON lists after a roundtrip. -def write_json_value(data: WriteBuffer, value: JsonValue | tuple[JsonValue, ...]) -> None: +def write_json_value(data: WriteBuffer, value: JsonValue) -> None: if value is None: write_tag(data, LITERAL_NONE) elif isinstance(value, bool): @@ -428,11 +438,16 @@ def write_json_value(data: WriteBuffer, value: JsonValue | tuple[JsonValue, ...] elif isinstance(value, str): write_tag(data, LITERAL_STR) write_str_bare(data, value) - elif isinstance(value, (list, tuple)): + elif isinstance(value, list): write_tag(data, LIST_GEN) write_int_bare(data, len(value)) for val in value: write_json_value(data, val) + elif isinstance(value, tuple): + write_tag(data, TUPLE_GEN) + write_int_bare(data, len(value)) + for val in value: + write_json_value(data, val) elif isinstance(value, dict): write_tag(data, DICT_STR_GEN) write_int_bare(data, len(value)) @@ -457,3 +472,38 @@ def write_json(data: WriteBuffer, value: dict[str, Any]) -> None: for key in sorted(value): write_str_bare(data, key) write_json_value(data, value[key]) + + +def write_errors(data: WriteBuffer, errs: list[SerializedError]) -> None: + write_tag(data, LIST_GEN) + write_int_bare(data, len(errs)) + for path, line, column, end_line, end_column, severity, message, code in errs: + write_tag(data, TUPLE_GEN) + write_str_opt(data, path) + write_int(data, line) + write_int(data, column) + write_int(data, end_line) + write_int(data, end_column) + write_str(data, severity) + write_str(data, message) + write_str_opt(data, code) + + +def read_errors(data: ReadBuffer) -> list[SerializedError]: + assert read_tag(data) == LIST_GEN + result = [] + for _ in range(read_int_bare(data)): + assert read_tag(data) == TUPLE_GEN + result.append( + ( + read_str_opt(data), + read_int(data), + read_int(data), + read_int(data), + read_int(data), + read_str(data), + read_str(data), + read_str_opt(data), + ) + ) + return result diff --git a/mypy/errors.py b/mypy/errors.py index 69e4fb4cf0654..ce5c6cc8215fb 100644 --- a/mypy/errors.py +++ b/mypy/errors.py @@ -951,7 +951,7 @@ def raise_error(self, use_stdout: bool = True) -> NoReturn: self.new_messages(), use_stdout=use_stdout, module_with_blocker=self.blocker_module() ) - def format_messages( + def format_messages_default( self, error_tuples: list[ErrorTuple], source_lines: list[str] | None ) -> list[str]: """Return a string list that represents the error messages. @@ -1009,24 +1009,28 @@ def format_messages( a.append(" " * (DEFAULT_SOURCE_OFFSET + column) + marker) return a - def file_messages(self, path: str, formatter: ErrorFormatter | None = None) -> list[str]: - """Return a string list of new error messages from a given file. - - Use a form suitable for displaying to the user. - """ + def file_messages(self, path: str) -> list[ErrorTuple]: + """Return an error tuple list of new error messages from a given file.""" if path not in self.error_info_map: return [] error_info = self.error_info_map[path] error_info = [info for info in error_info if not info.hidden] error_info = self.remove_duplicates(self.sort_messages(error_info)) - error_tuples = self.render_messages(error_info) + return self.render_messages(error_info) + def format_messages( + self, path: str, error_tuples: list[ErrorTuple], formatter: ErrorFormatter | None = None + ) -> list[str]: + """Return a string list of new error messages from a given file. + + Use a form suitable for displaying to the user. + """ + self.flushed_files.add(path) if formatter is not None: errors = create_errors(error_tuples) return [formatter.report_error(err) for err in errors] - self.flushed_files.add(path) source_lines = None if self.options.pretty and self.read_source: # Find shadow file mapping and read source lines if a shadow file exists for the given path. @@ -1036,7 +1040,7 @@ def file_messages(self, path: str, formatter: ErrorFormatter | None = None) -> l source_lines = self.read_source(mapped_path) else: source_lines = self.read_source(path) - return self.format_messages(error_tuples, source_lines) + return self.format_messages_default(error_tuples, source_lines) def find_shadow_file_mapping(self, path: str) -> str | None: """Return the shadow file path for a given source file path or None.""" @@ -1058,7 +1062,8 @@ def new_messages(self) -> list[str]: msgs = [] for path in self.error_info_map.keys(): if path not in self.flushed_files: - msgs.extend(self.file_messages(path)) + error_tuples = self.file_messages(path) + msgs.extend(self.format_messages(path, error_tuples)) return msgs def targets(self) -> set[str]: diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 170a883ce25da..fdda5f64284de 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -7626,3 +7626,13 @@ y = 1 class C: ... [out2] tmp/m.py:2: note: Revealed type is "def () -> other.C" + +[case testOutputFormatterIncremental] +# flags2: --output json +def wrong() -> int: + if wrong(): + return 0 +[out] +main:2: error: Missing return statement +[out2] +{"file": "main", "line": 2, "column": 0, "message": "Missing return statement", "hint": null, "code": "return", "severity": "error"} From 20aea0a6ca0710f5427239bdd2fd8e8bf1caf634 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sun, 14 Dec 2025 15:44:59 -0800 Subject: [PATCH 423/424] Update changelog for 1.19.1 (#20414) Also change the header for 1.18 because of https://github.com/python/mypy/issues/19910 --- CHANGELOG.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0be81310c6e1c..ed5d947cb8292 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -202,6 +202,17 @@ Related PRs: Please see [git log](https://github.com/python/typeshed/commits/main?after=ebce8d766b41fbf4d83cf47c1297563a9508ff60+0&branch=main&path=stdlib) for full list of standard library typeshed stub changes. +### Mypy 1.19.1 + +- Fix noncommutative joins with bounded TypeVars (Shantanu, PR [20345](https://github.com/python/mypy/pull/20345)) +- Respect output format for cached runs by serializing raw errors in cache metas (Ivan Levkivskyi, PR [20372](https://github.com/python/mypy/pull/20372)) +- Allow `types.NoneType` in match cases (A5rocks, PR [20383](https://github.com/python/mypy/pull/20383)) +- Fix mypyc generator regression with empty tuple (BobTheBuidler, PR [20371](https://github.com/python/mypy/pull/20371)) +- Fix crash involving Unpack-ed TypeVarTuple (Shantanu, PR [20323](https://github.com/python/mypy/pull/20323)) +- Fix crash on star import of redefinition (Ivan Levkivskyi, PR [20333](https://github.com/python/mypy/pull/20333)) +- Fix crash on typevar with forward ref used in other module (Ivan Levkivskyi, PR [20334](https://github.com/python/mypy/pull/20334)) +- Fail with an explicit error on PyPy (Ivan Levkivskyi, PR [20389](https://github.com/python/mypy/pull/20389)) + ### Acknowledgements Thanks to all mypy contributors who contributed to this release: @@ -237,7 +248,7 @@ Thanks to all mypy contributors who contributed to this release: I’d also like to thank my employer, Dropbox, for supporting mypy development. -## Mypy 1.18.1 +## Mypy 1.18 We’ve just uploaded mypy 1.18.1 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). Mypy is a static type checker for Python. This release includes new features, performance From 412c19a6bde31e7afa7f41afdf8356664689ae80 Mon Sep 17 00:00:00 2001 From: Shantanu Jain Date: Sun, 14 Dec 2025 15:46:31 -0800 Subject: [PATCH 424/424] Bump version to 1.19.1 --- mypy/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/version.py b/mypy/version.py index eef99f7203cf7..eded284e79415 100644 --- a/mypy/version.py +++ b/mypy/version.py @@ -8,7 +8,7 @@ # - Release versions have the form "1.2.3". # - Dev versions have the form "1.2.3+dev" (PLUS sign to conform to PEP 440). # - Before 1.0 we had the form "0.NNN". -__version__ = "1.19.1+dev" +__version__ = "1.19.1" base_version = __version__ mypy_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))