[ty] Fix stack overflow for self-referential TypeOf in annotations (#23407)

charliermarsh · web-flow · commit 97acaaea5f99 · 2026-02-19T16:03:15.000Z
## Summary When a function references itself via `TypeOf` in a deferred annotation (`def foo(x: "TypeOf[foo]")`), the resulting `FunctionType` contains itself as a parameter type, forming a cycle. Closes astral-sh/ty#2800.
diff --git a/crates/ty_python_semantic/resources/mdtest/ty_extensions.md b/crates/ty_python_semantic/resources/mdtest/ty_extensions.md
@@ -427,6 +427,18 @@ def f(x: TypeOf) -> None:
     reveal_type(x)  # revealed: Unknown
 ```
 
+## Self-referential `TypeOf` in annotations
+
+A function can reference itself via `TypeOf` in a deferred annotation. This should not cause a stack
+overflow:
+
+```py
+from ty_extensions import TypeOf
+
+def foo(x: "TypeOf[foo]"):
+    reveal_type(x)  # revealed: def foo(x: def foo(...)) -> Unknown
+```
+
 ## `CallableTypeOf`
 
 The `CallableTypeOf` special form can be used to extract the `Callable` structural type inhabited by
diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs
@@ -6408,25 +6408,27 @@ impl<'db> Type<'db> {
                 },
             }
 
-            Type::FunctionLiteral(function) => match type_mapping {
-                // Promote the types within the signature before promoting the signature to its
-                // callable form.
-                TypeMapping::PromoteLiterals(PromoteLiteralsMode::On) => {
-                    Type::FunctionLiteral(function.apply_type_mapping_impl(
+            Type::FunctionLiteral(function) => visitor.visit(self, || {
+                match type_mapping {
+                    // Promote the types within the signature before promoting the signature to its
+                    // callable form.
+                    TypeMapping::PromoteLiterals(PromoteLiteralsMode::On) => {
+                        Type::FunctionLiteral(function.apply_type_mapping_impl(
+                            db,
+                            type_mapping,
+                            tcx,
+                            visitor,
+                        ))
+                        .promote_literals_impl(db)
+                    }
+                    _ => Type::FunctionLiteral(function.apply_type_mapping_impl(
                         db,
                         type_mapping,
                         tcx,
                         visitor,
-                    ))
-                    .promote_literals_impl(db)
+                    )),
                 }
-                _ => Type::FunctionLiteral(function.apply_type_mapping_impl(
-                    db,
-                    type_mapping,
-                    tcx,
-                    visitor,
-                )),
-            },
+            }),
 
             Type::BoundMethod(method) => Type::BoundMethod(BoundMethodType::new(
                 db,
@@ -6692,7 +6694,9 @@ impl<'db> Type<'db> {
             }
 
             Type::FunctionLiteral(function) => {
-                function.find_legacy_typevars_impl(db, binding_context, typevars, visitor);
+                visitor.visit(self, || {
+                    function.find_legacy_typevars_impl(db, binding_context, typevars, visitor);
+                });
             }
 
             Type::BoundMethod(method) => {
diff --git a/crates/ty_python_semantic/src/types/display.rs b/crates/ty_python_semantic/src/types/display.rs
@@ -87,6 +87,9 @@ pub struct DisplaySettings<'db> {
     /// whose type parameters are currently being displayed).
     /// Used to suppress redundant `@{scope}` suffixes for type variables.
     pub active_scopes: Rc<FxHashSet<Definition<'db>>>,
+    /// Function types that are currently being displayed.
+    /// Used to prevent infinite recursion when displaying self-referential function types.
+    pub visited_function_types: Rc<FxHashSet<FunctionType<'db>>>,
 }
 
 impl<'db> DisplaySettings<'db> {
@@ -1466,6 +1469,19 @@ pub(crate) struct DisplayFunctionType<'db> {
 
 impl<'db> FmtDetailed<'db> for DisplayFunctionType<'db> {
     fn fmt_detailed(&self, f: &mut TypeWriter<'_, '_, 'db>) -> fmt::Result {
+        // Detect self-referential function types to prevent infinite recursion.
+        if self.settings.visited_function_types.contains(&self.ty) {
+            f.set_invalid_type_annotation();
+            f.write_str("def ")?;
+            write!(f, "{}", self.ty.name(self.db))?;
+            return f.write_str("(...)");
+        }
+
+        let mut settings = self.settings.clone();
+        let mut visited = (*settings.visited_function_types).clone();
+        visited.insert(self.ty);
+        settings.visited_function_types = Rc::new(visited);
+
         let signature = self.ty.signature(self.db);
 
         match signature.overloads.as_slice() {
@@ -1475,32 +1491,32 @@ impl<'db> FmtDetailed<'db> for DisplayFunctionType<'db> {
                 let type_parameters = DisplayOptionalGenericContext {
                     generic_context: signature.generic_context.as_ref(),
                     db: self.db,
-                    settings: self.settings.clone(),
+                    settings: settings.clone(),
                     hide_unused_self,
                 };
                 f.set_invalid_type_annotation();
                 f.write_str("def ")?;
                 write!(f, "{}", self.ty.name(self.db))?;
                 type_parameters.fmt_detailed(f)?;
                 signature
-                    .display_with(self.db, self.settings.disallow_signature_name())
+                    .display_with(self.db, settings.disallow_signature_name())
                     .fmt_detailed(f)
             }
             signatures => {
                 // TODO: How to display overloads?
-                if !self.settings.multiline {
+                if !settings.multiline {
                     // TODO: This should ideally have a TypeDetail but we actually
                     // don't have a type for @overload (we just detect the decorator)
                     f.write_str("Overload")?;
                     f.write_char('[')?;
                 }
-                let separator = if self.settings.multiline { "\n" } else { ", " };
+                let separator = if settings.multiline { "\n" } else { ", " };
                 let mut join = f.join(separator);
                 for signature in signatures {
-                    join.entry(&signature.display_with(self.db, self.settings.clone()));
+                    join.entry(&signature.display_with(self.db, settings.clone()));
                 }
                 join.finish()?;
-                if !self.settings.multiline {
+                if !settings.multiline {
                     f.write_str("]")?;
                 }
                 Ok(())
