From 1ca19533e0ab8e74cb9e8b2f57f389ab2b79d1ad Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Thu, 25 Aug 2022 14:48:29 +0200 Subject: [PATCH 1/2] Python: Add import problem test from the wild --- .../dataflow/typetracking_imports/README.md | 5 +++ .../highlight_problem.expected | 5 +++ .../typetracking_imports/highlight_problem.ql | 15 +++++++++ .../dataflow/typetracking_imports/options | 1 + .../typetracking_imports/pkg/__init__.py | 0 .../pkg/alias_only_direct.py | 1 + .../typetracking_imports/pkg/alias_problem.py | 2 ++ .../pkg/alias_problem_fixed.py | 3 ++ .../typetracking_imports/pkg/alias_star.py | 2 ++ .../typetracking_imports/pkg/foo_def.py | 5 +++ .../typetracking_imports/pkg/other.py | 1 + .../dataflow/typetracking_imports/pkg/use.py | 33 +++++++++++++++++++ .../typetracking_imports/tracked.expected | 0 .../typetracking_imports/tracked.qlref | 1 + 14 files changed, 74 insertions(+) create mode 100644 python/ql/test/experimental/dataflow/typetracking_imports/README.md create mode 100644 python/ql/test/experimental/dataflow/typetracking_imports/highlight_problem.expected create mode 100644 python/ql/test/experimental/dataflow/typetracking_imports/highlight_problem.ql create mode 100644 python/ql/test/experimental/dataflow/typetracking_imports/options create mode 100644 python/ql/test/experimental/dataflow/typetracking_imports/pkg/__init__.py create mode 100644 python/ql/test/experimental/dataflow/typetracking_imports/pkg/alias_only_direct.py create mode 100644 python/ql/test/experimental/dataflow/typetracking_imports/pkg/alias_problem.py create mode 100644 python/ql/test/experimental/dataflow/typetracking_imports/pkg/alias_problem_fixed.py create mode 100644 python/ql/test/experimental/dataflow/typetracking_imports/pkg/alias_star.py create mode 100644 python/ql/test/experimental/dataflow/typetracking_imports/pkg/foo_def.py create mode 100644 python/ql/test/experimental/dataflow/typetracking_imports/pkg/other.py create mode 100644 python/ql/test/experimental/dataflow/typetracking_imports/pkg/use.py create mode 100644 python/ql/test/experimental/dataflow/typetracking_imports/tracked.expected create mode 100644 python/ql/test/experimental/dataflow/typetracking_imports/tracked.qlref diff --git a/python/ql/test/experimental/dataflow/typetracking_imports/README.md b/python/ql/test/experimental/dataflow/typetracking_imports/README.md new file mode 100644 index 000000000000..4063a2851f91 --- /dev/null +++ b/python/ql/test/experimental/dataflow/typetracking_imports/README.md @@ -0,0 +1,5 @@ +A testcase observed in real code, where mixing `from .this import that` with `from .other import *` (in that order) causes import resolution to not work properly. + +This needs to be in a separate folder, since using relative imports requires a valid top-level package. We emulate real extractor behavior using `-R` extractor option. + +From this directory, you can run the code with `python -m pkg.use`. diff --git a/python/ql/test/experimental/dataflow/typetracking_imports/highlight_problem.expected b/python/ql/test/experimental/dataflow/typetracking_imports/highlight_problem.expected new file mode 100644 index 000000000000..4336dcfafcd3 --- /dev/null +++ b/python/ql/test/experimental/dataflow/typetracking_imports/highlight_problem.expected @@ -0,0 +1,5 @@ +| pkg/alias_only_direct.py:0:0:0:0 | Module pkg.alias_only_direct | pkg/alias_only_direct.py:1:22:1:24 | GSSA Variable foo | use to normal exit | +| pkg/alias_problem.py:0:0:0:0 | Module pkg.alias_problem | pkg/alias_problem.py:1:22:1:24 | GSSA Variable foo | no use to normal exit | +| pkg/alias_problem.py:0:0:0:0 | Module pkg.alias_problem | pkg/alias_problem.py:2:1:2:20 | GSSA Variable foo | use to normal exit | +| pkg/alias_problem_fixed.py:0:0:0:0 | Module pkg.alias_problem_fixed | pkg/alias_problem_fixed.py:0:0:0:0 | GSSA Variable foo | no use to normal exit | +| pkg/alias_problem_fixed.py:0:0:0:0 | Module pkg.alias_problem_fixed | pkg/alias_problem_fixed.py:3:22:3:24 | GSSA Variable foo | use to normal exit | diff --git a/python/ql/test/experimental/dataflow/typetracking_imports/highlight_problem.ql b/python/ql/test/experimental/dataflow/typetracking_imports/highlight_problem.ql new file mode 100644 index 000000000000..fc1441be4797 --- /dev/null +++ b/python/ql/test/experimental/dataflow/typetracking_imports/highlight_problem.ql @@ -0,0 +1,15 @@ +import python + +// looking at `module_export` predicate in DataFlowPrivate, the core of the problem is +// that in alias_problem.py, the direct import of `foo` does not flow to a normal exit of +// the module. Instead there is a second variable foo coming from `from .other import*` that +// goes to the normal exit of the module. +from Module m, EssaVariable v, string useToNormalExit +where + m = v.getScope().getEnclosingModule() and + not m.getName() in ["pkg.use", "pkg.foo_def"] and + v.getName() = "foo" and + if v.getAUse() = m.getANormalExit() + then useToNormalExit = "use to normal exit" + else useToNormalExit = "no use to normal exit" +select m, v, useToNormalExit diff --git a/python/ql/test/experimental/dataflow/typetracking_imports/options b/python/ql/test/experimental/dataflow/typetracking_imports/options new file mode 100644 index 000000000000..1bbc489d1536 --- /dev/null +++ b/python/ql/test/experimental/dataflow/typetracking_imports/options @@ -0,0 +1 @@ +semmle-extractor-options: --max-import-depth=1 -R ./pkg/ diff --git a/python/ql/test/experimental/dataflow/typetracking_imports/pkg/__init__.py b/python/ql/test/experimental/dataflow/typetracking_imports/pkg/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/ql/test/experimental/dataflow/typetracking_imports/pkg/alias_only_direct.py b/python/ql/test/experimental/dataflow/typetracking_imports/pkg/alias_only_direct.py new file mode 100644 index 000000000000..f609b0fc1aab --- /dev/null +++ b/python/ql/test/experimental/dataflow/typetracking_imports/pkg/alias_only_direct.py @@ -0,0 +1 @@ +from .foo_def import foo # $ tracked diff --git a/python/ql/test/experimental/dataflow/typetracking_imports/pkg/alias_problem.py b/python/ql/test/experimental/dataflow/typetracking_imports/pkg/alias_problem.py new file mode 100644 index 000000000000..0f0d11e47ae5 --- /dev/null +++ b/python/ql/test/experimental/dataflow/typetracking_imports/pkg/alias_problem.py @@ -0,0 +1,2 @@ +from .foo_def import foo # $ tracked +from .other import * diff --git a/python/ql/test/experimental/dataflow/typetracking_imports/pkg/alias_problem_fixed.py b/python/ql/test/experimental/dataflow/typetracking_imports/pkg/alias_problem_fixed.py new file mode 100644 index 000000000000..19eb63bfc305 --- /dev/null +++ b/python/ql/test/experimental/dataflow/typetracking_imports/pkg/alias_problem_fixed.py @@ -0,0 +1,3 @@ +# this ordering makes the problem go away +from .other import * +from .foo_def import foo # $ tracked diff --git a/python/ql/test/experimental/dataflow/typetracking_imports/pkg/alias_star.py b/python/ql/test/experimental/dataflow/typetracking_imports/pkg/alias_star.py new file mode 100644 index 000000000000..45a80b5b57f2 --- /dev/null +++ b/python/ql/test/experimental/dataflow/typetracking_imports/pkg/alias_star.py @@ -0,0 +1,2 @@ +from .foo_def import * +from .other import * diff --git a/python/ql/test/experimental/dataflow/typetracking_imports/pkg/foo_def.py b/python/ql/test/experimental/dataflow/typetracking_imports/pkg/foo_def.py new file mode 100644 index 000000000000..515c77e5e576 --- /dev/null +++ b/python/ql/test/experimental/dataflow/typetracking_imports/pkg/foo_def.py @@ -0,0 +1,5 @@ +# apparently adding the assignment makes type-tracker unhappy, so we add this eval so +# it's possible to run the example and see that everything works +exec("tracked = 'tracked'") +foo = tracked # $ tracked +print(foo) # $ tracked diff --git a/python/ql/test/experimental/dataflow/typetracking_imports/pkg/other.py b/python/ql/test/experimental/dataflow/typetracking_imports/pkg/other.py new file mode 100644 index 000000000000..6d2e01a45d87 --- /dev/null +++ b/python/ql/test/experimental/dataflow/typetracking_imports/pkg/other.py @@ -0,0 +1 @@ +bar = "bar-text" diff --git a/python/ql/test/experimental/dataflow/typetracking_imports/pkg/use.py b/python/ql/test/experimental/dataflow/typetracking_imports/pkg/use.py new file mode 100644 index 000000000000..8e6bb099d6d9 --- /dev/null +++ b/python/ql/test/experimental/dataflow/typetracking_imports/pkg/use.py @@ -0,0 +1,33 @@ +def test_direct_import(): + from .foo_def import foo # $ tracked + print(foo) # $ tracked + +test_direct_import() + + +def test_alias_problem(): + from .alias_problem import foo # $ MISSING: tracked + print(foo) # $ MISSING: tracked + +test_alias_problem() + + +def test_alias_problem_fixed(): + from .alias_problem_fixed import foo # $ tracked + print(foo) # $ tracked + +test_alias_problem_fixed() + + +def test_alias_star(): + from .alias_star import foo # $ tracked + print(foo) # $ tracked + +test_alias_star() + + +def test_alias_only_direct(): + from .alias_only_direct import foo # $ tracked + print(foo) # $ tracked + +test_alias_only_direct() diff --git a/python/ql/test/experimental/dataflow/typetracking_imports/tracked.expected b/python/ql/test/experimental/dataflow/typetracking_imports/tracked.expected new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/ql/test/experimental/dataflow/typetracking_imports/tracked.qlref b/python/ql/test/experimental/dataflow/typetracking_imports/tracked.qlref new file mode 100644 index 000000000000..2c005903470f --- /dev/null +++ b/python/ql/test/experimental/dataflow/typetracking_imports/tracked.qlref @@ -0,0 +1 @@ +../typetracking/tracked.ql From 0728ecebbbe13ddc0f77d439d8286f9c9823ab4c Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Thu, 25 Aug 2022 15:51:26 +0200 Subject: [PATCH 2/2] Python: Highlight that import problem is not just a relative problem --- .../typetracking_imports/highlight_problem.expected | 4 ++++ .../pkg/problem_absolute_import.py | 2 ++ .../dataflow/typetracking_imports/pkg/use.py | 13 +++++++++++++ .../pkg/works_absolute_import.py | 2 ++ 4 files changed, 21 insertions(+) create mode 100644 python/ql/test/experimental/dataflow/typetracking_imports/pkg/problem_absolute_import.py create mode 100644 python/ql/test/experimental/dataflow/typetracking_imports/pkg/works_absolute_import.py diff --git a/python/ql/test/experimental/dataflow/typetracking_imports/highlight_problem.expected b/python/ql/test/experimental/dataflow/typetracking_imports/highlight_problem.expected index 4336dcfafcd3..9aa55c97f1cb 100644 --- a/python/ql/test/experimental/dataflow/typetracking_imports/highlight_problem.expected +++ b/python/ql/test/experimental/dataflow/typetracking_imports/highlight_problem.expected @@ -3,3 +3,7 @@ | pkg/alias_problem.py:0:0:0:0 | Module pkg.alias_problem | pkg/alias_problem.py:2:1:2:20 | GSSA Variable foo | use to normal exit | | pkg/alias_problem_fixed.py:0:0:0:0 | Module pkg.alias_problem_fixed | pkg/alias_problem_fixed.py:0:0:0:0 | GSSA Variable foo | no use to normal exit | | pkg/alias_problem_fixed.py:0:0:0:0 | Module pkg.alias_problem_fixed | pkg/alias_problem_fixed.py:3:22:3:24 | GSSA Variable foo | use to normal exit | +| pkg/problem_absolute_import.py:0:0:0:0 | Module pkg.problem_absolute_import | pkg/problem_absolute_import.py:1:25:1:27 | GSSA Variable foo | no use to normal exit | +| pkg/problem_absolute_import.py:0:0:0:0 | Module pkg.problem_absolute_import | pkg/problem_absolute_import.py:2:1:2:23 | GSSA Variable foo | use to normal exit | +| pkg/works_absolute_import.py:0:0:0:0 | Module pkg.works_absolute_import | pkg/works_absolute_import.py:0:0:0:0 | GSSA Variable foo | no use to normal exit | +| pkg/works_absolute_import.py:0:0:0:0 | Module pkg.works_absolute_import | pkg/works_absolute_import.py:2:25:2:27 | GSSA Variable foo | use to normal exit | diff --git a/python/ql/test/experimental/dataflow/typetracking_imports/pkg/problem_absolute_import.py b/python/ql/test/experimental/dataflow/typetracking_imports/pkg/problem_absolute_import.py new file mode 100644 index 000000000000..289f108ea967 --- /dev/null +++ b/python/ql/test/experimental/dataflow/typetracking_imports/pkg/problem_absolute_import.py @@ -0,0 +1,2 @@ +from pkg.foo_def import foo # $ tracked +from pkg.other import * diff --git a/python/ql/test/experimental/dataflow/typetracking_imports/pkg/use.py b/python/ql/test/experimental/dataflow/typetracking_imports/pkg/use.py index 8e6bb099d6d9..023af8684f21 100644 --- a/python/ql/test/experimental/dataflow/typetracking_imports/pkg/use.py +++ b/python/ql/test/experimental/dataflow/typetracking_imports/pkg/use.py @@ -31,3 +31,16 @@ def test_alias_only_direct(): print(foo) # $ tracked test_alias_only_direct() + + +def test_problem_absolute_import(): + from pkg.problem_absolute_import import foo # $ MISSING: tracked + print(foo) # $ MISSING: tracked + +test_problem_absolute_import() + +def test_works_absolute_import(): + from pkg.works_absolute_import import foo # $ tracked + print(foo) # $ tracked + +test_works_absolute_import() diff --git a/python/ql/test/experimental/dataflow/typetracking_imports/pkg/works_absolute_import.py b/python/ql/test/experimental/dataflow/typetracking_imports/pkg/works_absolute_import.py new file mode 100644 index 000000000000..3199bb24a155 --- /dev/null +++ b/python/ql/test/experimental/dataflow/typetracking_imports/pkg/works_absolute_import.py @@ -0,0 +1,2 @@ +from pkg.other import * +from pkg.foo_def import foo # $ tracked