Skip to content

Commit 4d51d5d

Browse files
grouvillegrouville
committed
feat: make Python module discovery cwd-aware
1 parent a66d16c commit 4d51d5d

10 files changed

Lines changed: 156 additions & 74 deletions

File tree

‎.dagger/modules/e2e/dagger.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "e2e",
3-
"engineVersion": "v0.21.4",
3+
"engineVersion": "v1.0.0-0",
44
"sdk": {
55
"source": "dang"
66
},

‎.dagger/modules/e2e/fixtures/toml/.dagger-python-sdk-skip-generate‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+

‎.dagger/modules/e2e/fixtures/toml/app/dagger-module.toml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
engineVersion = "v1.0.0-0"
2+
3+
[runtime]
4+
source = "python"

‎.dagger/modules/e2e/fixtures/toml/app/nested/.keep‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+

‎.dagger/modules/e2e/main.dang‎

Lines changed: 32 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ type E2e {
1313
let depsModulePath: String! = fixtureRoot + "/deps/app"
1414
let configModulePath: String! = fixtureRoot + "/config/app"
1515
let configuredModulePath: String! = fixtureRoot + "/config/configured"
16+
let tomlModulePath: String! = fixtureRoot + "/toml/app"
1617

1718
let generatedMarkerPath: String! = "sdk/src/dagger/client/gen.py"
1819
let generatedMarkerContents: String! = "Code generated by dagger."
@@ -73,12 +74,41 @@ type E2e {
7374
let nested = pythonSdk.mod(ws, path: lookupNestedPath)
7475
let exact = pythonSdk.mod(ws, path: lookupModulePath, findUp: false)
7576

76-
assert(nested.path == lookupModulePath, "find-up lookup returned the wrong module path")
77-
assert(exact.path == lookupModulePath, "exact lookup returned the wrong module path")
77+
assert(nested.rootPath == lookupModulePath, "find-up lookup returned the wrong module path")
78+
assert(exact.rootPath == lookupModulePath, "exact lookup returned the wrong module path")
7879

7980
null
8081
}
8182

83+
"""
84+
Lookup must choose the nearest module config even when its filename differs
85+
from an ancestor config, and modern TOML modules must be supported.
86+
"""
87+
pub mixedConfigLookupCheck(ws: Workspace!): Void @check {
88+
let mod = pythonSdk.mod(ws, path: tomlModulePath + "/nested")
89+
assert(mod.rootPath == tomlModulePath, "lookup chose an outer dagger.json instead of the nearer dagger-module.toml")
90+
null
91+
}
92+
93+
"""
94+
Managed-module discovery is cwd-aware and supports both config formats.
95+
"""
96+
pub moduleDiscoveryCheck(ws: Workspace!): Void @check {
97+
let roots = pythonSdk.modules(ws).{{rootPath}}.map { mod => mod.rootPath }
98+
assert(contains(roots, generateModulePath), "root discovery missed a legacy Python module")
99+
assert(contains(roots, tomlModulePath), "root discovery missed a TOML Python module")
100+
assert(contains(roots, nonPythonModulePath) == false, "discovery included a non-Python module")
101+
102+
let nestedWs = ws.directory("/").asWorkspace(cwd: lookupNestedPath)
103+
let scoped = pythonSdk.modules(nestedWs).{{rootPath, path}}
104+
let scopedRoots = scoped.map { mod => mod.rootPath }
105+
let scopedPaths = scoped.map { mod => mod.path }
106+
assert(scoped.length == 1, "nested cwd should see only its nearest managed module")
107+
assert(contains(scopedRoots, lookupModulePath), "nested cwd resolved the wrong managed module")
108+
assert(contains(scopedPaths, ".."), "module path should be relative to the nested cwd")
109+
null
110+
}
111+
82112
"""
83113
initModule should seed SDK-owned template files without running codegen and
84114
without writing engine-owned files (dagger.json / dagger-module.toml).
@@ -153,18 +183,6 @@ type E2e {
153183
null
154184
}
155185

156-
"""
157-
initClient should not materialize files itself. The engine records the client
158-
and owns the generated context changeset.
159-
"""
160-
pub initClientCheck(ws: Workspace!): Void @check {
161-
let changes = pythonSdk.initClient(ws, path: outputRoot + "/client", module: generateModulePath, dev: true)
162-
163-
assert(changes.isEmpty, "initClient should return an empty SDK changeset")
164-
165-
null
166-
}
167-
168186
"""
169187
config.get should reflect pyproject.toml and report unset values as null
170188
rather than guessing, and config.set should edit only pyproject.toml.

‎dagger.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@
77
"dependencies": [
88
{
99
"name": "polyfill",
10-
"source": "github.com/dagger/sdk-sdk/polyfill@main",
11-
"pin": "a16466390cacd68bd72f7ae9910d990966fb2226"
10+
"source": "github.com/dagger/polyfill@main",
11+
"pin": "b30eb839000fe179e5d0544a9b667b21817b8120"
1212
}
1313
]
1414
}

‎dagger.toml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,3 +27,6 @@ path = ".dagger/modules/e2e/fixtures/deps/app"
2727

2828
[[modules.python-sdk.as-sdk.modules]]
2929
path = ".dagger/modules/e2e/fixtures/skip/app"
30+
31+
[[modules.python-sdk.as-sdk.modules]]
32+
path = ".dagger/modules/e2e/fixtures/toml/app"

‎docs/cwd-aware-discovery.md‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
# CWD-aware module discovery
2+
3+
The Python SDK delegates config discovery to `github.com/dagger/polyfill`, then
4+
intersects the discovered directories with `currentModule.asSDK.modules`. The
5+
engine's managed-module list remains authoritative while the caller's current
6+
directory determines scope.
7+
8+
Discovery returns modules at or below the cwd and, when the cwd has no module
9+
config, its nearest enclosing module. Both `dagger-module.toml` and legacy
10+
`dagger.json` are considered together, so the nearest config wins regardless of
11+
filename. Virtual environments and installed packages are excluded.
12+
13+
```console
14+
dagger check -l
15+
dagger call e-2-e mixed-config-lookup-check
16+
dagger call e-2-e module-discovery-check
17+
```
18+
19+
The fixtures cover mixed nested config formats, modern and legacy configs,
20+
non-Python exclusion, root discovery, and discovery from inside a module.

‎mod.dang‎

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,9 @@ A Dagger module that uses the Python SDK.
33
"""
44
type Mod {
55
"""
6-
Workspace-relative path of this module root.
6+
Workspace-root-relative path of this module root.
77
"""
8-
pub path: String!
8+
pub rootPath: String!
99

1010
"""
1111
The workspace this module belongs to.
@@ -17,18 +17,37 @@ type Mod {
1717
"""
1818
let skipGenerateFilename: String!
1919

20+
"""
21+
Module root relative to the client's cwd.
22+
"""
23+
pub path: String! {
24+
let cwd = ws.cwd.trimPrefix("/").trimSuffix("/")
25+
if (rootPath == cwd) {
26+
"."
27+
} else if (cwd == "") {
28+
rootPath
29+
} else if (rootPath.trimPrefix(cwd + "/") != rootPath) {
30+
rootPath.trimPrefix(cwd + "/")
31+
} else if (rootPath == "." or cwd.trimPrefix(rootPath + "/") != cwd) {
32+
let depth = if (rootPath == ".") { 0 } else { rootPath.split("/").length }
33+
cwd.split("/").dropFirst(depth).map { segment => ".." }.join("/")
34+
} else {
35+
rootPath
36+
}
37+
}
38+
2039
"""
2140
Whether this module or an ancestor contains the configured generate skip marker.
2241
"""
2342
pub skipGenerate: Boolean! {
24-
ws.findUp(name: skipGenerateFilename, from: path) != null
43+
ws.findUp(name: skipGenerateFilename, from: rootPath) != null
2544
}
2645

2746
"""
2847
Manage this module's Python build configuration (pyproject.toml).
2948
"""
3049
pub config: ModConfig! {
31-
ModConfig(path: path, ws: ws)
50+
ModConfig(path: rootPath, ws: ws)
3251
}
3352

3453
"""
@@ -39,7 +58,7 @@ type Mod {
3958
if (skipGenerate) {
4059
polyfill.workspace(ws).fork.changes
4160
} else {
42-
polyfill.workspace(ws).moduleSource("/" + path).generate.changes
61+
polyfill.workspace(ws).moduleSource("/" + rootPath).generate.changes
4362
}
4463
}
4564
}

‎python-sdk.dang‎

Lines changed: 68 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,10 @@
22
Manage Dagger modules that use the Python SDK.
33
"""
44
type PythonSdk {
5+
let legacyConfigPattern = "\"sdk\"\\s*:\\s*\\{[^}]*\"source\"\\s*:\\s*\"python\""
6+
let tomlConfigPattern = "\\[runtime\\][^\\[]*source\\s*=\\s*\"python\""
7+
let moduleConfigFilenames: [String!]! = ["dagger-module.toml", "dagger.json"]
8+
59
"""
610
Marker filename that skips generate when found at or above a Python SDK module root.
711
"""
@@ -13,29 +17,46 @@ type PythonSdk {
1317
pub targetRuntime: String! { "python" }
1418

1519
"""
16-
Return every legacy dagger.json Dagger module whose sdk.source is "python".
17-
18-
This discovery path is obsolete for workspace-managed modules; the engine
19-
owns the modules.<sdk>.as-sdk.modules source of truth.
20+
Return every managed Python SDK module visible from the client's cwd: the
21+
nearest enclosing module plus modules at or below the cwd. Discovery uses the
22+
shared polyfill and intersects its results with the engine-owned as-SDK list.
2023
"""
2124
pub modules(ws: Workspace!): [Mod!]! {
22-
ws
23-
.directory("/", include: ["**/dagger.json"])
24-
.search(
25-
pattern: "\"sdk\"\\s*:\\s*\\{[^}]*\"source\"\\s*:\\s*\"python\"",
26-
globs: ["**/dagger.json"],
27-
filesOnly: true,
28-
multiline: true,
29-
dotall: true,
30-
)
31-
.{{filePath}}
32-
.map { configPath =>
33-
mod(
34-
ws,
35-
if (configPath.filePath == "dagger.json") { "." } else { configPath.filePath.trimSuffix("/dagger.json") },
36-
findUp: false,
37-
)
25+
let managed = currentModule.asSDK.modules.{{path}}
26+
let cwd = normalizePath(ws.cwd)
27+
polyfill.workspace(ws)
28+
.findConfigDirs(moduleConfigFilenames, exclude: ["**/.venv/**", "**/site-packages/**"])
29+
.map { dir => workspacePath(cwd, dir) }
30+
.uniq
31+
.filter { path => managed.filter { m => normalizePath(m.path) == path }.length > 0 }
32+
.map { path => Mod(rootPath: path, ws: ws, skipGenerateFilename: skipGenerateFilename) }
33+
}
34+
35+
let workspacePath(cwd: String!, path: String!): String! {
36+
let base = if (cwd == ".") { [] } else { cwd.split("/") }
37+
let segments = path.split("/").reduce(base) { acc, segment =>
38+
if (segment == "..") {
39+
acc.dropLast(1)
40+
} else if (segment == "." or segment == "") {
41+
acc
42+
} else {
43+
acc + [segment]
3844
}
45+
}
46+
if (segments.length == 0) { "." } else { segments.join("/") }
47+
}
48+
49+
let normalizePath(path: String!): String! {
50+
let normalized = path.trimPrefix("./").trimPrefix("/").trimSuffix("/")
51+
if (normalized == "") { "." } else { normalized }
52+
}
53+
54+
let pathDepth(path: String): Int! {
55+
if (path == null) { -1 } else if (path == ".") { 0 } else { path.split("/").length }
56+
}
57+
58+
let configDir(path: String!, filename: String!): String! {
59+
normalizePath(path.trimPrefix("/").trimSuffix(filename))
3960
}
4061

4162
"""
@@ -71,42 +92,42 @@ type PythonSdk {
7192
"""
7293
pub mod(ws: Workspace!, path: String! = ".", findUp: Boolean! = true): Mod! {
7394
let modPath = if (findUp) {
74-
let foundConfigPath = ws.findUp("dagger.json", path)
75-
if (foundConfigPath == null) {
95+
let foundToml = ws.findUp("dagger-module.toml", path)
96+
let foundJson = ws.findUp("dagger.json", path)
97+
let tomlBase = if (foundToml == null) { null } else { configDir(foundToml, "dagger-module.toml") }
98+
let jsonBase = if (foundJson == null) { null } else { configDir(foundJson, "dagger.json") }
99+
100+
if (tomlBase == null and jsonBase == null) {
76101
raise "no Dagger module found containing path: " + path
102+
} else if (tomlBase != null and pathDepth(tomlBase) > pathDepth(jsonBase)) {
103+
validateConfig(ws, tomlBase, "dagger-module.toml", tomlConfigPattern, path)
104+
} else if (jsonBase != null) {
105+
validateConfig(ws, jsonBase, "dagger.json", legacyConfigPattern, path)
77106
} else {
78-
let configPath = foundConfigPath.trimPrefix("/")
79-
if (
80-
ws
81-
.directory("/", include: [configPath])
82-
.file(configPath)
83-
.search(
84-
pattern: "\"sdk\"\\s*:\\s*\\{[^}]*\"source\"\\s*:\\s*\"python\"",
85-
multiline: true,
86-
dotall: true,
87-
limit: 1,
88-
)
89-
.{{id}}
90-
.length == 0
91-
) {
92-
raise "Dagger module does not use the Python SDK: " + path
93-
} else if (configPath == "dagger.json") {
94-
"."
95-
} else {
96-
configPath.trimSuffix("/dagger.json")
97-
}
107+
raise "no Dagger module found containing path: " + path
98108
}
99109
} else {
100-
path.trimPrefix("/")
110+
normalizePath(path)
101111
}
102112

103113
Mod(
104-
path: modPath,
114+
rootPath: modPath,
105115
ws: ws,
106116
skipGenerateFilename: skipGenerateFilename,
107117
)
108118
}
109119

120+
let validateConfig(ws: Workspace!, base: String!, filename: String!, pattern: String!, requestedPath: String!): String! {
121+
let configPath = if (base == ".") { filename } else { base + "/" + filename }
122+
let matches = ws
123+
.directory("/", include: [configPath])
124+
.file(configPath)
125+
.search(pattern: pattern, multiline: true, dotall: true, limit: 1)
126+
.{{id}}
127+
.length > 0
128+
if (matches) { base } else { raise "Dagger module does not use the Python SDK: " + requestedPath }
129+
}
130+
110131
"""
111132
Initialize Python-owned files for a new Dagger module.
112133

@@ -191,22 +212,17 @@ type PythonSdk {
191212
}
192213

193214
"""
194-
Generate all discovered legacy dagger.json Python SDK modules.
195-
196-
This discovery path is obsolete for workspace-managed modules; the engine
197-
owns the modules.<sdk>.as-sdk.modules source of truth.
215+
Generate every managed Python SDK module visible from the client's cwd.
198216

199217
Modules with the generate skip marker are skipped.
200218
"""
201219
pub generateAll(ws: Workspace!): Changeset! @generate {
202220
let pws = polyfill.workspace(ws)
203221

204-
currentModule.asSDK.modules
205-
.{{path}}
206-
.map { m => Mod(path: m.path, ws: ws, skipGenerateFilename: skipGenerateFilename) }
222+
modules(ws)
207223
.filter { mod => mod.skipGenerate == false }
208224
.reduce(pws.fork) { fork, mod =>
209-
fork.merge(pws.moduleSource("/" + mod.path).generate)
225+
fork.merge(pws.moduleSource("/" + mod.rootPath).generate)
210226
}
211227
.changes
212228
}

0 commit comments

Comments
 (0)