Skip to content

fix(profiler): prevent filtering target package name in module discovery #18072

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
hebaalazzeh wants to merge 3 commits into
base: main
Choose a base branch
from
+91 −13
Open

fix(profiler): prevent filtering target package name in module discovery #18072

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f24e53d
refactor(profiler): clarify module resolution filtering and variable …
hebaalazzeh Aug 11, 2026
baadb13
test(profiler): update assertion in test_find_module_from_package_tes…
hebaalazzeh Aug 11, 2026
6b41730
refactor(profiler): remove PARENT_NAMESPACES constant and rely on ini…
hebaalazzeh Aug 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 52 additions & 11 deletions scripts/import_profiler/profiler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -377,6 +377,46 @@ def validate_module_name(module_name):
raise argparse.ArgumentTypeError(f"'{module_name}' is not a valid Python module identifier.")
return module_name

IGNORED_TOP_LEVEL_NAMES = {
"tests", "samples", "examples", "benchmark", "benchmarks", "third_party",
"testing", "test_utils", "docs", "build", "dist", "bin", "ci", "scripts",
"cloudbuild", "notebooks", "assets", "scratch", "specs"
}
IGNORED_NAME_PREFIXES = (
"test_", "tests_", "sample_", "samples_", "bench_", "benchmarks_",
"example_", "examples_", "doc_", "docs_", "notebook_", "notebooks_"
)


def _should_ignore_namespace_package(top_level, target_pkg):
"""Determines if a discovered top-level directory should be excluded from module selection.

Args:
top_level: Top-level directory component of the package (e.g. 'tests' or 'google').
target_pkg: The distribution package being profiled (e.g. 'google-cloud-storage' or 'google-cloud-testutils').

Returns:
True if the top-level directory represents a non-library folder, False otherwise.
"""
is_non_library_dir = (
top_level in IGNORED_TOP_LEVEL_NAMES
or top_level.startswith(IGNORED_NAME_PREFIXES)
)

if not is_non_library_dir:
return False

# Exception: If the top-level folder name is part of the target package name
# (e.g. top_level='test_utils' when target_pkg='google-cloud-testutils'), do not ignore it.
normalized_target_pkg = target_pkg.replace("-", "").replace("_", "").lower()
normalized_top_level = top_level.replace("-", "").replace("_", "").lower()

if normalized_top_level in normalized_target_pkg:
return False

return True


def find_module_from_package(pkg):
import importlib.metadata
import importlib.util
Expand All @@ -385,10 +425,17 @@ def find_module_from_package(pkg):
try:
files = importlib.metadata.files(pkg)
if files:
ignored_parts = {'tests', 'testing', 'samples', 'examples', 'benchmark', 'benchmarks', 'third_party', 'test_utils', 'docs', 'build', 'dist', 'bin', 'ci', 'scripts', 'cloudbuild', 'notebooks', 'assets', 'scratch', 'specs'}
if pkg == "google-cloud-testutils":
ignored_parts.discard('test_utils')
init_files = [str(f) for f in files if str(f).endswith('__init__.py') and '__pycache__' not in str(f) and not any(part in ignored_parts for part in str(f).replace('\\', '/').split('/'))]
pkg_norm = pkg.replace("-", "").replace("_", "").lower()
init_files = [
str(f) for f in files
if str(f).endswith('__init__.py')
and '__pycache__' not in str(f)
and not any(
part in IGNORED_TOP_LEVEL_NAMES
and part.replace("-", "").replace("_", "").lower() not in pkg_norm
for part in str(f).replace('\\', '/').split('/')
)
]
if init_files:
from pathlib import Path
shortest_init = min(init_files, key=lambda p: len(Path(p).parts))
Expand All @@ -412,16 +459,10 @@ def find_module_from_package(pkg):
if abs_where_dir not in sys.path:
sys.path.insert(0, abs_where_dir)
pkgs = setuptools.find_namespace_packages(where=where_dir)
ignored_prefixes = ("tests", "samples", "examples", "benchmark", "benchmarks", "third_party", "testing", "test_utils", "docs", "build", "dist", "bin", "ci", "scripts", "cloudbuild", "notebooks", "assets", "scratch", "specs")
ignored_starts = ("test_", "tests_", "sample_", "samples_", "bench_", "benchmarks_", "example_", "examples_", "doc_", "docs_", "notebook_", "notebooks_")

filtered = []
for p in pkgs:
top = p.split(".")[0]
is_ignored_top = top in ignored_prefixes or top.startswith(ignored_starts)
if is_ignored_top and pkg == "google-cloud-testutils" and top == "test_utils":
is_ignored_top = False
if is_ignored_top or p in ("google", "google.cloud"):
if _should_ignore_namespace_package(top, pkg):
continue
Comment thread
hebaalazzeh marked this conversation as resolved.
filtered.append(p)

Expand Down
41 changes: 39 additions & 2 deletions scripts/import_profiler/test_profiler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -650,10 +650,13 @@ def test_find_module_from_package_metadata_test_utils():

def test_find_module_from_package_setuptools():
sys.modules.setdefault("setuptools", MagicMock())
def mock_isfile(path):
return "my_pkg" in path
with patch("importlib.metadata.files", side_effect=Exception), \
patch("os.path.exists", return_value=True), \
patch("profiler.os.path.exists", return_value=True), \
patch("profiler.os.path.isdir", return_value=True), \
patch("setuptools.find_namespace_packages", return_value=["google", "google.cloud", "tests.dummy", "my_pkg"]), \
patch("os.path.isfile", return_value=True), \
patch("profiler.os.path.isfile", side_effect=mock_isfile), \
patch("importlib.util.find_spec", return_value=True):
res = find_module_from_package("my-pkg")
assert res == "my_pkg"
Expand Down Expand Up @@ -830,4 +833,38 @@ def test_cli_main_options():
runpy.run_path(profiler_path, run_name="__main__")


def test_should_ignore_namespace_package():
from profiler import _should_ignore_namespace_package

# Standard non-library top-level directories should be ignored
assert _should_ignore_namespace_package("tests", "google-cloud-storage") is True
assert _should_ignore_namespace_package("samples", "google-cloud-storage") is True
assert _should_ignore_namespace_package("test_utils", "google-cloud-storage") is True
assert _should_ignore_namespace_package("test_helpers", "google-cloud-storage") is True

# Exception: Target package explicitly contains the top-level directory name (e.g. google-cloud-testutils -> test_utils)
assert _should_ignore_namespace_package("test_utils", "google-cloud-testutils") is False

# Valid library package top-level should not be ignored
assert _should_ignore_namespace_package("google", "google-cloud-storage") is False
assert _should_ignore_namespace_package("my_library", "my-library") is False


def test_find_module_from_package_testutils():
"""Verifies that google-cloud-testutils correctly resolves to test_utils namespace package."""
sys.modules.setdefault("setuptools", MagicMock())
def mock_isfile(path):
return "test_utils" in path
with patch("importlib.metadata.files", side_effect=Exception), \
patch("profiler.os.path.exists", return_value=True), \
patch("profiler.os.path.isdir", return_value=True), \
patch("setuptools.find_namespace_packages", return_value=["google", "google.cloud", "tests", "test_utils"]) as mock_find, \
patch("profiler.os.path.isfile", side_effect=mock_isfile), \
patch("importlib.util.find_spec", return_value=True):
res = find_module_from_package("google-cloud-testutils")
assert res == "test_utils"
mock_find.assert_called_once_with(where="src")




Loading