soft deps tree

RustPython · youknowone · Jan 20, 2026 · Jan 20, 2026 · Jan 20, 2026 · Jan 20, 2026
commit 3178783d2630dbb7f5ecd7c0a76ec0a349442ce9
diff --git a/scripts/update_lib/auto_mark.py b/scripts/update_lib/auto_mark.py
@@ -455,15 +455,13 @@ def extract_test_methods(contents: str) -> set[tuple[str, str]]:
     Returns:
         Set of (class_name, method_name) tuples
     """
-    import ast
+    from update_lib.io_utils import safe_parse_ast
+    from update_lib.patch_spec import iter_tests
 
-    try:
-        tree = ast.parse(contents)
-    except SyntaxError:
+    tree = safe_parse_ast(contents)
+    if tree is None:
         return set()
 
-    from update_lib.patch_spec import iter_tests
-
     return {(cls_node.name, fn_node.name) for cls_node, fn_node in iter_tests(tree)}
 
 

diff --git a/scripts/update_lib/copy_lib.py b/scripts/update_lib/copy_lib.py
@@ -64,17 +64,14 @@ def copy_lib(
 
     # Extract module name and cpython prefix from path
     path_str = str(src_path).replace("\\", "/")
-    if "/Lib/" in path_str:
-        cpython_prefix, after_lib = path_str.split("/Lib/", 1)
-        # Get module name (first component, without .py)
-        name = after_lib.split("/")[0]
-        if name.endswith(".py"):
-            name = name[:-3]
-    else:
-        # Fallback: just copy the single file
-        lib_path = parse_lib_path(src_path)
-        _copy_single(src_path, lib_path, verbose)
-        return
+    if "/Lib/" not in path_str:
+        raise ValueError(f"Path must contain '/Lib/' (got: {src_path})")
+
+    cpython_prefix, after_lib = path_str.split("/Lib/", 1)
+    # Get module name (first component, without .py)
+    name = after_lib.split("/")[0]
+    if name.endswith(".py"):
+        name = name[:-3]
 
     # Get all paths to copy from DEPENDENCIES table
     all_src_paths = get_lib_paths(name, cpython_prefix)

diff --git a/scripts/update_lib/deps.py b/scripts/update_lib/deps.py
@@ -7,9 +7,11 @@
 - Test dependencies (auto-detected from 'from test import ...')
 """
 
-import ast
 import pathlib
 
+from update_lib.io_utils import read_python_files, safe_parse_ast, safe_read_text
+from update_lib.path import construct_lib_path, resolve_module_path
+
 # Manual dependency table for irregular cases
 # Format: "name" -> {"lib": [...], "test": [...], "data": [...], "hard_deps": [...]}
 # - lib: override default path (default: name.py or name/)
@@ -153,28 +155,18 @@ def get_lib_paths(name: str, cpython_prefix: str = "cpython") -> list[pathlib.Pa
     Returns:
         List of paths to copy
     """
-    paths = []
     dep_info = DEPENDENCIES.get(name, {})
 
     # Get main lib path (override or default)
     if "lib" in dep_info:
-        paths = [pathlib.Path(f"{cpython_prefix}/Lib/{p}") for p in dep_info["lib"]]
+        paths = [construct_lib_path(cpython_prefix, p) for p in dep_info["lib"]]
     else:
         # Default: try file first, then directory
-        file_path = pathlib.Path(f"{cpython_prefix}/Lib/{name}.py")
-        if file_path.exists():
-            paths = [file_path]
-        else:
-            dir_path = pathlib.Path(f"{cpython_prefix}/Lib/{name}")
-            if dir_path.exists():
-                paths = [dir_path]
-            else:
-                paths = [file_path]  # Default to file path
+        paths = [resolve_module_path(name, cpython_prefix, prefer="file")]
 
     # Add hard_deps
-    if "hard_deps" in dep_info:
-        for dep in dep_info["hard_deps"]:
-            paths.append(pathlib.Path(f"{cpython_prefix}/Lib/{dep}"))
+    for dep in dep_info.get("hard_deps", []):
+        paths.append(construct_lib_path(cpython_prefix, dep))
 
     return paths
 
@@ -191,18 +183,11 @@ def get_test_paths(name: str, cpython_prefix: str = "cpython") -> list[pathlib.P
     """
     if name in DEPENDENCIES and "test" in DEPENDENCIES[name]:
         return [
-            pathlib.Path(f"{cpython_prefix}/Lib/{p}")
-            for p in DEPENDENCIES[name]["test"]
+            construct_lib_path(cpython_prefix, p) for p in DEPENDENCIES[name]["test"]
         ]
 
     # Default: try directory first, then file
-    dir_path = pathlib.Path(f"{cpython_prefix}/Lib/test/test_{name}")
-    if dir_path.exists():
-        return [dir_path]
-    file_path = pathlib.Path(f"{cpython_prefix}/Lib/test/test_{name}.py")
-    if file_path.exists():
-        return [file_path]
-    return [dir_path]  # Default to directory path
+    return [resolve_module_path(f"test/test_{name}", cpython_prefix, prefer="dir")]
 
 
 def get_data_paths(name: str, cpython_prefix: str = "cpython") -> list[pathlib.Path]:
@@ -217,8 +202,7 @@ def get_data_paths(name: str, cpython_prefix: str = "cpython") -> list[pathlib.P
     """
     if name in DEPENDENCIES and "data" in DEPENDENCIES[name]:
         return [
-            pathlib.Path(f"{cpython_prefix}/Lib/{p}")
-            for p in DEPENDENCIES[name]["data"]
+            construct_lib_path(cpython_prefix, p) for p in DEPENDENCIES[name]["data"]
         ]
     return []
 
@@ -232,9 +216,10 @@ def parse_test_imports(content: str) -> set[str]:
     Returns:
         Set of module names imported from test package
     """
-    try:
-        tree = ast.parse(content)
-    except SyntaxError:
+    import ast
+
+    tree = safe_parse_ast(content)
+    if tree is None:
         return set()
 
     imports = set()
@@ -267,9 +252,10 @@ def parse_lib_imports(content: str) -> set[str]:
     Returns:
         Set of imported module names (top-level only)
     """
-    try:
-        tree = ast.parse(content)
-    except SyntaxError:
+    import ast
+
+    tree = safe_parse_ast(content)
+    if tree is None:
         return set()
 
     imports = set()
@@ -286,49 +272,125 @@ def parse_lib_imports(content: str) -> set[str]:
     return imports
 
 
-def get_soft_deps(name: str, cpython_prefix: str = "cpython") -> set[str]:
-    """Get soft dependencies by parsing imports from library file.
+def get_all_imports(name: str, cpython_prefix: str = "cpython") -> set[str]:
+    """Get all imports from a library file.
 
     Args:
         name: Module name
         cpython_prefix: CPython directory prefix
 
     Returns:
-        Set of imported stdlib module names
+        Set of all imported module names
     """
-    lib_paths = get_lib_paths(name, cpython_prefix)
-
     all_imports = set()
-    for lib_path in lib_paths:
+    for lib_path in get_lib_paths(name, cpython_prefix):
         if lib_path.exists():
-            if lib_path.is_file():
-                try:
-                    content = lib_path.read_text(encoding="utf-8")
-                    all_imports.update(parse_lib_imports(content))
-                except (OSError, UnicodeDecodeError):
-                    continue
-            else:
-                # Directory - parse all .py files
-                for py_file in lib_path.glob("**/*.py"):
-                    try:
-                        content = py_file.read_text(encoding="utf-8")
-                        all_imports.update(parse_lib_imports(content))
-                    except (OSError, UnicodeDecodeError):
-                        continue
+            for _, content in read_python_files(lib_path):
+                all_imports.update(parse_lib_imports(content))
+
+    # Remove self
+    all_imports.discard(name)
+    return all_imports
+
+
+def get_soft_deps(name: str, cpython_prefix: str = "cpython") -> set[str]:
+    """Get soft dependencies by parsing imports from library file.
+
+    Args:
+        name: Module name
+        cpython_prefix: CPython directory prefix
+
+    Returns:
+        Set of imported stdlib module names (those that exist in cpython/Lib/)
+    """
+    all_imports = get_all_imports(name, cpython_prefix)
 
     # Filter: only include modules that exist in cpython/Lib/
     stdlib_deps = set()
     for imp in all_imports:
-        if imp == name:
-            continue  # Skip self
-        file_path = pathlib.Path(f"{cpython_prefix}/Lib/{imp}.py")
-        dir_path = pathlib.Path(f"{cpython_prefix}/Lib/{imp}")
-        if file_path.exists() or dir_path.exists():
+        module_path = resolve_module_path(imp, cpython_prefix)
+        if module_path.exists():
             stdlib_deps.add(imp)
 
     return stdlib_deps
 
 
+def get_rust_deps(name: str, cpython_prefix: str = "cpython") -> set[str]:
+    """Get Rust/C dependencies (imports that don't exist in cpython/Lib/).
+
+    Args:
+        name: Module name
+        cpython_prefix: CPython directory prefix
+
+    Returns:
+        Set of imported module names that are built-in or C extensions
+    """
+    all_imports = get_all_imports(name, cpython_prefix)
+    soft_deps = get_soft_deps(name, cpython_prefix)
+    return all_imports - soft_deps
+
+
+def _dircmp_is_same(dcmp) -> bool:
+    """Recursively check if two directories are identical.
+
+    Args:
+        dcmp: filecmp.dircmp object
+
+    Returns:
+        True if directories are identical (including subdirectories)
+    """
+    if dcmp.diff_files or dcmp.left_only or dcmp.right_only:
+        return False
+
+    # Recursively check subdirectories
+    for subdir in dcmp.subdirs.values():
+        if not _dircmp_is_same(subdir):
+            return False
+
+    return True
+
+
+def is_up_to_date(
+    name: str, cpython_prefix: str = "cpython", lib_prefix: str = "Lib"
+) -> bool:
+    """Check if a module is up-to-date by comparing files.
+
+    Args:
+        name: Module name
+        cpython_prefix: CPython directory prefix
+        lib_prefix: Local Lib directory prefix
+
+    Returns:
+        True if all files match, False otherwise
+    """
+    import filecmp
+
+    lib_paths = get_lib_paths(name, cpython_prefix)
+
+    for cpython_path in lib_paths:
+        if not cpython_path.exists():
+            continue
+
+        # Convert cpython path to local path
+        # cpython/Lib/foo.py -> Lib/foo.py
+        rel_path = cpython_path.relative_to(cpython_prefix)
+        local_path = pathlib.Path(lib_prefix) / rel_path.relative_to("Lib")
+
+        if not local_path.exists():
+            return False
+
+        if cpython_path.is_file():
+            if not filecmp.cmp(cpython_path, local_path, shallow=False):
+                return False
+        else:
+            # Directory comparison (recursive)
+            dcmp = filecmp.dircmp(cpython_path, local_path)
+            if not _dircmp_is_same(dcmp):
+                return False
+
+    return True
+
+
 def get_test_dependencies(
     test_path: pathlib.Path,
 ) -> dict[str, list[pathlib.Path]]:
@@ -345,20 +407,10 @@ def get_test_dependencies(
     if not test_path.exists():
         return result
 
-    # Collect all test files
-    if test_path.is_file():
-        files = [test_path]
-    else:
-        files = list(test_path.glob("**/*.py"))
-
     # Parse all files for imports (auto-detect deps)
     all_imports = set()
-    for f in files:
-        try:
-            content = f.read_text(encoding="utf-8")
-            all_imports.update(parse_test_imports(content))
-        except (OSError, UnicodeDecodeError):
-            continue
+    for _, content in read_python_files(test_path):
+        all_imports.update(parse_test_imports(content))
 
     # Also add manual dependencies from TEST_DEPENDENCIES
     test_name = test_path.stem if test_path.is_file() else test_path.name
@@ -430,8 +482,11 @@ def resolve_all_paths(
         # Auto-detect test dependencies
         for test_path in result["test"]:
             deps = get_test_dependencies(test_path)
-            for dep in deps:
-                if dep not in result["test_deps"]:
-                    result["test_deps"].append(dep)
+            for dep_path in deps["hard_deps"]:
+                if dep_path not in result["test_deps"]:
+                    result["test_deps"].append(dep_path)
+            for data_path in deps["data"]:
+                if data_path not in result["data"]:
+                    result["data"].append(data_path)
 
     return result