cleaner

scijava · tlambert03 · Apr 23, 2025 · Apr 23, 2025 · Apr 23, 2025 · Apr 23, 2025
commit 93e4e51ea184d2e9888b2a1bb71a1f1aff3a8f06
diff --git a/src/scyjava/_stubs/__init__.py b/src/scyjava/_stubs/__init__.py
@@ -1,4 +1,4 @@
 from ._dynamic_import import setup_java_imports
 from ._genstubs import generate_stubs
 
-__all__ = ["setup_java_imports", "generate_stubs"]
+__all__ = ["generate_stubs", "setup_java_imports"]
diff --git a/src/scyjava/_stubs/_genstubs.py b/src/scyjava/_stubs/_genstubs.py
@@ -65,8 +65,8 @@ def generate_stubs(
         called with the `convertStrings` argument set to True or False.  By setting
         this `convert_strings` argument to true, the type stubs will be generated as if
         `convertStrings` is set to True: that is, all string types will be listed as
-        `str` rather than `java.lang.String | str`.  This is a safer default (as `str`)
-        is a subtype of `java.lang.String`), but may lead to type errors in some cases.
+        `str` rather than `java.lang.String | str`.  This is a safer default (as `str`
+        is a base of `java.lang.String`), but may lead to type errors in some cases.
     include_javadoc : bool, optional
         Whether to include Javadoc in the generated stubs. Defaults to True.
     add_runtime_imports : bool, optional
@@ -90,9 +90,13 @@ def generate_stubs(
             "stubgenj is not installed, but is required to generate java stubs. "
             "Please install it with `pip/conda install stubgenj`."
         ) from e
-    print("GENERATE")
     import jpype
 
+    # if jpype.isJVMStarted():
+    # raise RuntimeError(
+    # "Generating type stubs after the JVM has started is not supported."
+    # )
+
     startJVM = jpype.startJVM
 
     scyjava.config.endpoints.extend(endpoints)
@@ -117,23 +121,24 @@ def _patched_start(*args: Any, **kwargs: Any) -> None:
     logger.info(f"Generating stubs for: {prefixes}")
     logger.info(f"Writing stubs to: {output_dir}")
 
-    metapath = sys.meta_path
+    metapath = sys.meta_path.copy()
     try:
         import jpype.imports
 
         jmodules = [import_module(prefix) for prefix in prefixes]
+
+        stubgenj.generateJavaStubs(
+            jmodules,
+            useStubsSuffix=False,
+            outputDir=str(output_dir),
+            jpypeJPackageStubs=False,
+            includeJavadoc=include_javadoc,
+        )
+
     finally:
-        # remove the jpype.imports magic from the import system
-        # if it wasn't there to begin with
-        sys.meta_path = metapath
-
-    stubgenj.generateJavaStubs(
-        jmodules,
-        useStubsSuffix=False,
-        outputDir=str(output_dir),
-        jpypeJPackageStubs=False,
-        includeJavadoc=include_javadoc,
-    )
+        # restore sys.metapath
+        # (remove the jpype.imports magic if it wasn't there to begin with)
+        sys.meta_path[:] = metapath
 
     output_dir = Path(output_dir)
     if add_runtime_imports:

diff --git a/src/scyjava/py.typed b/src/scyjava/py.typed
diff --git a/src/scyjava/types/__init__.py b/src/scyjava/types/__init__.py
@@ -9,136 +9,96 @@
 
 from __future__ import annotations
 
-import importlib.util
+import os
 import sys
 import threading
-import types
-from ast import mod
-from importlib.abc import Loader, MetaPathFinder
-from importlib.machinery import SourceFileLoader
 from pathlib import Path
 from typing import TYPE_CHECKING
 
-import scyjava
-from scyjava._stubs import generate_stubs
-
 if TYPE_CHECKING:
+    import types
     from collections.abc import Sequence
     from importlib.machinery import ModuleSpec
 
 
+# where generated stubs should land (defaults to this dir: `scyjava.types`)
+STUBS_DIR = os.getenv("SCYJAVA_STUBS_DIR", str(Path(__file__).parent))
+# namespace under which generated stubs will be placed
+STUBS_NAMESPACE = __name__
+# module lock to prevent concurrent stub generation
 _STUBS_LOCK = threading.Lock()
-TYPES_DIR = Path(__file__).parent
 
 
-class ScyJavaTypesMetaFinder(MetaPathFinder):
-    """Meta path finder for scyjava.types that delegates to our loader."""
+class ScyJavaTypesMetaFinder:
+    """Meta path finder for scyjava.types that generates stubs on demand."""
 
     def find_spec(
         self,
         fullname: str,
         path: Sequence[str] | None,
         target: types.ModuleType | None = None,
-        /,
     ) -> ModuleSpec | None:
         """Return a spec for names under scyjava.types (except the base)."""
-        base_package = __name__
-
-        if not fullname.startswith(base_package + ".") or fullname == base_package:
-            return None
-
-        return importlib.util.spec_from_loader(
-            fullname,
-            ScyJavaTypesLoader(fullname),
-            origin="dynamic",
-        )
-
-
-class ScyJavaTypesLoader(Loader):
-    """Loader that lazily generates stubs and loads/synthesizes modules."""
-
-    def __init__(self, fullname: str) -> None:
-        self.fullname = fullname
-
-    def create_module(self, spec: ModuleSpec) -> types.ModuleType | None:
-        """Load an existing module/package or lazily generate stubs then load."""
-        pkg_dir, pkg_init, mod_file = _paths_for(spec.name, TYPES_DIR)
-
-        def _load_module() -> types.ModuleType | None:
-            # Fast paths: concrete module file or package present
-            if pkg_init.exists() or mod_file.exists():
-                return _load_generated_module(spec.name, TYPES_DIR)
-            if pkg_dir.is_dir():
-                return _namespace_package(spec, pkg_dir)
-            return None
-
-        if module := _load_module():
-            return module
-
-        # Nothing exists for this name: generate once under a lock
-        with _STUBS_LOCK:
-            # Re-check under the lock to avoid duplicate work
-            if not (pkg_init.exists() or mod_file.exists() or pkg_dir.exists()):
-                endpoints = ["org.scijava:parsington:3.1.0"]  # TODO
-                generate_stubs(endpoints, output_dir=TYPES_DIR)
-
-        # Retry after generation (or if another thread created it)
-        if module := _load_module():
-            return module
+        # if this is an import from this module ('scyjava.types.<name>')
+        # check if the module exists, and if not, call generation routines
+        if fullname.startswith(f"{__name__}."):
+            with _STUBS_LOCK:
+                # check if the spec already exists
+                # under the module lock to avoid duplicate work
+                if not _find_spec(fullname, path, target, skip=self):
+                    _generate_stubs()
 
-        raise ImportError(f"Generated module not found: {spec.name} under {pkg_dir}")
+        return None
 
-    def exec_module(self, module: types.ModuleType) -> None:
-        pass
 
+def _generate_stubs() -> None:
+    """Install stubs for all endpoints detected in `scyjava.config`.
 
-def _paths_for(fullname: str, out_dir: Path) -> tuple[Path, Path, Path]:
-    """Return (pkg_dir, pkg_init, mod_file) for a scyjava.types.* fullname."""
-    rel = fullname.split("scyjava.types.", 1)[1]
-    pkg_dir = out_dir / rel.replace(".", "/")
-    pkg_init = pkg_dir / "__init__.py"
-    mod_file = out_dir / (rel.replace(".", "/") + ".py")
-    return pkg_dir, pkg_init, mod_file
-
-
-def _namespace_package(spec: ModuleSpec, pkg_dir: Path) -> types.ModuleType:
-    """Create a simple package module pointing at pkg_dir.
-
-    This fills the role of a namespace package, (a folder with no __init__.py).
+    This could be expanded to include additional endpoints detected in, for example,
+    python entry-points discovered in packages in the environment.
     """
-    module = types.ModuleType(spec.name)
-    module.__package__ = spec.name
-    module.__path__ = [str(pkg_dir)]
-    module.__spec__ = spec
-    return module
-
-
-def _load_generated_module(fullname: str, out_dir: Path) -> types.ModuleType:
-    """Load a just-generated module/package from disk and return it."""
-    _, pkg_init, mod_file = _paths_for(fullname, out_dir)
-    path = pkg_init if pkg_init.exists() else mod_file
-    if not path.exists():
-        raise ImportError(f"Generated module not found: {fullname} at {path}")
-
-    loader = SourceFileLoader(fullname, str(path))
-    spec = importlib.util.spec_from_loader(fullname, loader, origin=str(path))
-    if spec is None or spec.loader is None:
-        raise ImportError(f"Failed to build spec for: {fullname}")
+    from scyjava import config
+    from scyjava._stubs import generate_stubs
+
+    generate_stubs(
+        config.endpoints,
+        output_dir=STUBS_DIR,
+        add_runtime_imports=True,
+        remove_namespace_only_stubs=True,
+    )
+
+
+def _find_spec(
+    fullname: str,
+    path: Sequence[str] | None,
+    target: types.ModuleType | None = None,
+    skip: object | None = None,
+) -> ModuleSpec | None:
+    """Find a module spec, skipping finder `skip` to avoid recursion."""
+    # if the module is already loaded and has a spec, return it
+    if module := sys.modules.get(fullname):
+        try:
+            if module.__spec__ is not None:
+                return module.__spec__
+        except AttributeError:
+            pass
 
-    spec.has_location = True  # populate __file__
-    sys.modules[fullname] = module = importlib.util.module_from_spec(spec)
-    spec.loader.exec_module(module)
-    return module
-
-
-# -----------------------------------------------------------
+    for finder in sys.meta_path:
+        if finder is not skip:
+            try:
+                spec = finder.find_spec(fullname, path, target)
+            except AttributeError:
+                continue
+            else:
+                if spec is not None:
+                    return spec
+    return None
 
 
 def _install_meta_finder() -> None:
-    for finder in sys.meta_path:
-        if isinstance(finder, ScyJavaTypesMetaFinder):
-            return
-
+    """Install the ScyJavaTypesMetaFinder into sys.meta_path if not already there."""
+    if any(isinstance(finder, ScyJavaTypesMetaFinder) for finder in sys.meta_path):
+        return
     sys.meta_path.insert(0, ScyJavaTypesMetaFinder())
 
 

diff --git a/tests/test_stubgen.py b/tests/test_stubgen.py
@@ -14,10 +14,11 @@
     from pathlib import Path
 
 
-@pytest.mark.skipif(
-    scyjava.config.mode != scyjava.config.Mode.JPYPE,
-    reason="Stubgen not supported in JEP",
-)
+JEP_MODE = scyjava.config.mode != scyjava.config.Mode.JPYPE
+skip_if_jep = pytest.mark.skipif(JEP_MODE, reason="Stubgen not supported in JEP")
+
+
+@skip_if_jep
 def test_stubgen(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
     # run the stubgen command as if it was run from the command line
     monkeypatch.setattr(
@@ -32,19 +33,15 @@ def test_stubgen(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
     )
     _cli.main()
 
-    # remove the `jpype.imports` magic from the import system if present
-    mp = [x for x in sys.meta_path if not isinstance(x, jpype.imports._JImportLoader)]
-    monkeypatch.setattr(sys, "meta_path", mp)
-
     # add tmp_path to the import path
     monkeypatch.setattr(sys, "path", [str(tmp_path)])
 
     # first cleanup to make sure we are not importing from the cache
     sys.modules.pop("org", None)
     sys.modules.pop("org.scijava", None)
     sys.modules.pop("org.scijava.parsington", None)
-    # make sure the stubgen command works and that we can now impmort stuff
 
+    # make sure the stubgen command works and that we can now import stuff
     with patch.object(scyjava._jvm, "start_jvm") as mock_start_jvm:
         from org.scijava.parsington import Function