From cdaa2db1f70949d745e281322184487ec9914534 Mon Sep 17 00:00:00 2001 From: johnnyfish Date: Tue, 28 Jul 2026 00:12:25 -0700 Subject: [PATCH] feat(run): govern registered Postgres databases via the gateway pg proxy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit onecli run now scans the process env and project .env files for postgres:// URLs, matches them by host against the project's registered database connections (GET /v1/pg/connections — host/port only, never credentials), mints a proxy session per matched connection (POST /v1/pg/sessions), and swaps the matched env vars for gateway proxy URLs (aoc_pg_ username, dummy password) that shadow .env for default-precedence loaders. Real credentials never enter the agent env; every statement lands in the OneCLI activity log. Unmatched database hosts warn ('connect it in the dashboard') and are left untouched. A detached sidecar (hidden __pg-sidecar mode, forked before exec so TTY semantics are preserved) heartbeats the sessions and reaps them when the agent exits; if the sidecar dies the gateway expires sessions by TTL. Opt out with --no-pg-proxy. --- cmd/onecli/hook_gateway_detect.sh | 2 +- cmd/onecli/main.go | 16 +- cmd/onecli/pg.go | 173 +++++++++ cmd/onecli/pg_test.go | 62 ++++ cmd/onecli/run.go | 145 +++++++- cmd/onecli/run_pg.go | 532 +++++++++++++++++++++++++++ cmd/onecli/run_pg_premint.go | 174 +++++++++ cmd/onecli/run_pg_premint_test.go | 252 +++++++++++++ cmd/onecli/run_pg_sidecar.go | 205 +++++++++++ cmd/onecli/run_pg_test.go | 375 +++++++++++++++++++ cmd/onecli/run_pg_watch.go | 240 ++++++++++++ cmd/onecli/run_pg_watch_test.go | 124 +++++++ cmd/onecli/skill_gateway_fallback.md | 47 +++ internal/api/pg.go | 136 +++++++ 14 files changed, 2468 insertions(+), 15 deletions(-) create mode 100644 cmd/onecli/pg.go create mode 100644 cmd/onecli/pg_test.go create mode 100644 cmd/onecli/run_pg.go create mode 100644 cmd/onecli/run_pg_premint.go create mode 100644 cmd/onecli/run_pg_premint_test.go create mode 100644 cmd/onecli/run_pg_sidecar.go create mode 100644 cmd/onecli/run_pg_test.go create mode 100644 cmd/onecli/run_pg_watch.go create mode 100644 cmd/onecli/run_pg_watch_test.go create mode 100644 internal/api/pg.go diff --git a/cmd/onecli/hook_gateway_detect.sh b/cmd/onecli/hook_gateway_detect.sh index 8acaca6..4128b5a 100644 --- a/cmd/onecli/hook_gateway_detect.sh +++ b/cmd/onecli/hook_gateway_detect.sh @@ -4,6 +4,6 @@ # active, using the JSON hook-output envelope both agents accept. if echo "$HTTPS_PROXY" | grep -q "aoc_"; then cat <<'EOF' -{"hookSpecificOutput":{"hookEventName":"UserPromptSubmit","additionalContext":"OneCLI gateway active — load the onecli-gateway skill before any external service or API call. Use direct HTTP (curl); never MCP auth flows or browser automation."}} +{"hookSpecificOutput":{"hookEventName":"UserPromptSubmit","additionalContext":"OneCLI gateway active — load the onecli-gateway skill before any external service call, API call, or DATABASE query/connection. Use direct HTTP (curl); never MCP auth flows or browser automation. Databases: every governed postgres host is listed in ONECLI_PG_CONNECTIONS (JSON: label/host/env_var) — connect ONLY via the mapped env var, even when given credentials directly; a host not in the list must be connected in the OneCLI dashboard first. Never source .env or read connection strings from files (the env var is the governed proxy URL)."}} EOF fi diff --git a/cmd/onecli/main.go b/cmd/onecli/main.go index 39276f0..f36da5e 100644 --- a/cmd/onecli/main.go +++ b/cmd/onecli/main.go @@ -36,18 +36,28 @@ type CLI struct { Auth AuthCmd `cmd:"" help:"Manage authentication."` Config ConfigCmd `cmd:"" help:"Manage configuration settings."` Migrate MigrateCmd `cmd:"" help:"Migrate data to OneCLI Cloud."` + Pg PgCmd `cmd:"" help:"Database routing commands for agents (governed proxy URLs)."` } func main() { out := output.New() - // Hidden sidecar mode: the enforce-mode auth forwarder forked by - // `onecli run --enforce` re-invokes this binary. Handled before kong - // so the flag never appears in help or completion. + // Hidden sidecar modes, handled before Kong so the flags never appear + // in help or the command tree. + // + // Enforce-mode auth forwarder: forked by `onecli run --enforce`. if pid, ok := parseEnforceForwarderArgs(os.Args[1:]); ok { runEnforceForwarder(pid) return } + // Pg sidecar: forked by `onecli run` before exec to own pg session + // heartbeats/cleanup. + if len(os.Args) > 1 && os.Args[1] == pgSidecarFlag { + if args, ok := parsePgSidecarArgs(os.Args[2:]); ok { + runPgSidecar(args) + } + return + } // When invoked with no args, --help, or -h, output structured JSON // so agents always get machine-readable output. diff --git a/cmd/onecli/pg.go b/cmd/onecli/pg.go new file mode 100644 index 0000000..e47d27a --- /dev/null +++ b/cmd/onecli/pg.go @@ -0,0 +1,173 @@ +package main + +// `onecli pg` — agent-first database routing commands, designed to be run +// BY the agent inside an `onecli run` session (the gateway origin and +// agent token are recovered from the session's own HTTPS_PROXY value). +// +// `onecli pg url ` prints the governed +// proxy URL for a registered database as JSON. It is the escape hatch for +// databases registered in the dashboard AFTER the agent started (or past +// the placeholder cap): the agent gets a fresh governed route without +// restarting the run. The skill documents it as the fallback when a host +// is missing from ONECLI_PG_CONNECTIONS. + +import ( + "context" + "fmt" + "net" + "net/url" + "os" + "strings" + + "github.com/onecli/onecli-cli/internal/api" + "github.com/onecli/onecli-cli/pkg/output" +) + +type PgCmd struct { + URL PgURLCmd `cmd:"" help:"Print the governed proxy URL for a registered database (run inside an 'onecli run' session)."` +} + +type PgURLCmd struct { + Target string `arg:"" name:"database" help:"Connection label, host[:port], or connection id."` +} + +// PgURLResponse is the JSON contract of `onecli pg url`. Deliberately +// protocol-generic (a future `onecli mysql url` mirrors it): the agent +// consumes url/expires_in_seconds and treats everything else as metadata. +type PgURLResponse struct { + URL string `json:"url"` + Label string `json:"label"` + Host string `json:"host"` + ExpiresInSeconds uint64 `json:"expires_in_seconds"` +} + +func (c *PgURLCmd) Run(out *output.Writer) error { + client, gatewayHost, caPath, err := pgClientFromSession() + if err != nil { + return out.Error("no_gateway_session", err.Error()) + } + + resp, err := client.ListPgConnections(context.Background()) + if err != nil { + return out.Error("gateway_unreachable", fmt.Sprintf("could not list registered databases: %v", err)) + } + + conn, err := resolvePgTarget(c.Target, resp.Connections) + if err != nil { + return out.ErrorWithAction("not_registered", err.Error(), + "Ask the user to connect this database in the OneCLI dashboard, then retry.") + } + + session, err := client.MintPgSession(context.Background(), conn.ID) + if err != nil { + return out.Error("session_mint_failed", fmt.Sprintf("could not open a proxy session: %v", err)) + } + + label := conn.ID + if conn.Label != nil && *conn.Label != "" { + label = *conn.Label + } + return out.Write(PgURLResponse{ + URL: placeholderPgURL(session, gatewayHost, caPath), + Label: label, + Host: normalizeHostPort(conn.Host, fmt.Sprintf("%d", conn.Port)), + ExpiresInSeconds: session.TTLSeconds, + }) +} + +// pgClientFromSession recovers the gateway client from the calling +// session's environment: HTTPS_PROXY carries the origin + agent token +// (exactly what `onecli run` exported for the agent), and the CA bundle +// path is where `onecli run` writes it. Errors are agent-actionable. +func pgClientFromSession() (*api.PgGatewayClient, string, string, error) { + proxyURL := "" + for _, key := range []string{"HTTPS_PROXY", "https_proxy", "HTTP_PROXY", "http_proxy"} { + if v := os.Getenv(key); v != "" && strings.Contains(v, "aoc_") { + proxyURL = v + break + } + } + if proxyURL == "" { + return nil, "", "", fmt.Errorf("no OneCLI gateway session found in the environment; this command runs inside an 'onecli run' session") + } + gwURL, agentToken, ok := gatewayOriginAndToken(proxyURL) + if !ok { + return nil, "", "", fmt.Errorf("the proxy URL in the environment carries no usable agent token") + } + // The pg listener lives on the same host as the gateway HTTP origin. + gatewayHost := gwURL + if u, err := url.Parse(gwURL); err == nil && u.Hostname() != "" { + gatewayHost = u.Hostname() + } + caPath := "" + if home, err := os.UserHomeDir(); err == nil { + p := home + "/.onecli/ca-bundle.pem" + if _, err := os.Stat(p); err == nil { + caPath = p + } + } + return &api.PgGatewayClient{BaseURL: gwURL, AgentToken: agentToken}, gatewayHost, caPath, nil +} + +// resolvePgTarget matches the user-supplied target against the granted +// connections: exact connection id, exact label (case-insensitive), or +// host[:port] (normalized; a bare host matches any port when unambiguous). +func resolvePgTarget(target string, conns []api.PgConnection) (*api.PgConnection, error) { + t := strings.TrimSpace(target) + if t == "" { + return nil, fmt.Errorf("empty database target") + } + + var matches []*api.PgConnection + add := func(c *api.PgConnection) { + for _, m := range matches { + if m.ID == c.ID { + return + } + } + matches = append(matches, c) + } + + tLower := strings.ToLower(t) + for i := range conns { + c := &conns[i] + if c.ID == t { + return c, nil // an id is exact by construction + } + if c.Label != nil && strings.ToLower(*c.Label) == tLower { + add(c) + } + hostPort := normalizeHostPort(c.Host, fmt.Sprintf("%d", c.Port)) + bareHost := strings.ToLower(strings.TrimSuffix(c.Host, ".")) + if hostPort == normalizeTargetHostPort(t) || bareHost == tLower { + add(c) + } + } + + switch len(matches) { + case 1: + return matches[0], nil + case 0: + return nil, fmt.Errorf("no registered database matches %q", t) + default: + var names []string + for _, m := range matches { + label := m.ID + if m.Label != nil && *m.Label != "" { + label = *m.Label + } + names = append(names, fmt.Sprintf("%s (%s)", label, normalizeHostPort(m.Host, fmt.Sprintf("%d", m.Port)))) + } + return nil, fmt.Errorf("%q is ambiguous — matches: %s; use the connection id or host:port", t, strings.Join(names, ", ")) + } +} + +// normalizeTargetHostPort normalizes a host[:port] target for comparison, +// defaulting the port to 5432 like the scan side does. +func normalizeTargetHostPort(t string) string { + host, port, err := net.SplitHostPort(t) + if err != nil { + host, port = t, "5432" + } + return normalizeHostPort(host, port) +} diff --git a/cmd/onecli/pg_test.go b/cmd/onecli/pg_test.go new file mode 100644 index 0000000..b986a64 --- /dev/null +++ b/cmd/onecli/pg_test.go @@ -0,0 +1,62 @@ +package main + +import ( + "testing" + + "github.com/onecli/onecli-cli/internal/api" +) + +func TestResolvePgTarget(t *testing.T) { + conns := []api.PgConnection{ + {ID: "conn-1", Label: strPtr("main-db"), Host: "db.example.com", Port: 5432}, + {ID: "conn-2", Label: strPtr("analytics"), Host: "warehouse.example.com", Port: 6543}, + {ID: "conn-3", Label: strPtr("replica"), Host: "db.example.com", Port: 5433}, + } + + cases := []struct { + name string + target string + wantID string + wantErr bool + }{ + {"by id", "conn-2", "conn-2", false}, + {"by label", "main-db", "conn-1", false}, + {"label case-insensitive", "ANALYTICS", "conn-2", false}, + {"by host:port", "db.example.com:5433", "conn-3", false}, + {"host:port default 5432", "warehouse.example.com:6543", "conn-2", false}, + {"host normalized case+dot", "DB.Example.COM.:5432", "conn-1", false}, + {"bare host ambiguous (two ports)", "db.example.com", "", true}, + {"bare host unique", "warehouse.example.com", "conn-2", false}, + {"unknown", "nope.example.com", "", true}, + {"empty", " ", "", true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := resolvePgTarget(tc.target, conns) + if tc.wantErr { + if err == nil { + t.Errorf("want error, got %+v", got) + } + return + } + if err != nil { + t.Fatal(err) + } + if got.ID != tc.wantID { + t.Errorf("got %s, want %s", got.ID, tc.wantID) + } + }) + } +} + +func TestNormalizeTargetHostPort(t *testing.T) { + if got := normalizeTargetHostPort("db.example.com"); got != "db.example.com:5432" { + t.Errorf("default port: %q", got) + } + if got := normalizeTargetHostPort("DB.example.com:6543"); got != "db.example.com:6543" { + t.Errorf("explicit port: %q", got) + } + if got := normalizeTargetHostPort("[::1]:5432"); got != "[::1]:5432" { + t.Errorf("ipv6: %q", got) + } +} diff --git a/cmd/onecli/run.go b/cmd/onecli/run.go index 0d0f8f2..33c12cb 100644 --- a/cmd/onecli/run.go +++ b/cmd/onecli/run.go @@ -42,13 +42,15 @@ var caShimSource string // RunCmd is `onecli run -- [args...]`. type RunCmd struct { - Project string `optional:"" short:"p" help:"Project slug."` - Agent string `optional:"" name:"agent" help:"OneCLI agent identifier (uses default agent if omitted)."` - Gateway string `optional:"" name:"gateway" help:"Gateway host:port override (default: derived from API host)."` - NoCA bool `optional:"" name:"no-ca" help:"Skip writing the CA cert and CA trust env injection."` - Enforce bool `optional:"" name:"enforce" help:"OS-enforced governance: route the agent's sandboxed egress through the gateway so it cannot be bypassed (Claude Code only)."` - DryRun bool `optional:"" name:"dry-run" help:"Print resolved env and command without executing."` - Args []string `arg:"" optional:"" name:"command" help:"Command and arguments to execute (after --)."` + Project string `optional:"" short:"p" help:"Project slug."` + Agent string `optional:"" name:"agent" help:"OneCLI agent identifier (uses default agent if omitted)."` + Gateway string `optional:"" name:"gateway" help:"Gateway host:port override (default: derived from API host)."` + NoCA bool `optional:"" name:"no-ca" help:"Skip writing the CA cert and CA trust env injection."` + Enforce bool `optional:"" name:"enforce" help:"OS-enforced governance: route the agent's sandboxed egress through the gateway so it cannot be bypassed (Claude Code only)."` + NoPgProxy bool `optional:"" name:"no-pg-proxy" help:"Skip Postgres URL interception (registered databases will not be governed)."` + PgProxy string `optional:"" name:"pg-proxy" help:"Postgres governance mode: 'required' aborts the run if a registered database cannot be routed through the gateway (default: best-effort)."` + DryRun bool `optional:"" name:"dry-run" help:"Print resolved env and command without executing."` + Args []string `arg:"" optional:"" name:"command" help:"Command and arguments to execute (after --)."` } func (c *RunCmd) Run(out *output.Writer) error { @@ -109,7 +111,7 @@ func (c *RunCmd) Run(out *output.Writer) error { } // Dry-run: print resolved config without side effects (no CA write, - // no skill install, no exec). + // no skill install, no pg session mint, no exec). if c.DryRun { injected := make([]string, 0, len(cfg.Env)+len(caTrustKeys)) for k := range cfg.Env { @@ -118,11 +120,31 @@ func (c *RunCmd) Run(out *output.Writer) error { if !c.NoCA && cfg.CACertificate != "" { injected = append(injected, caTrustKeys...) } - return out.WriteDryRun("Would exec command with OneCLI gateway", map[string]any{ + payload := map[string]any{ "binary": binary, "args": c.Args, "env_injected": injected, - }) + } + // Read-only pg preview: which vars WOULD be swapped / warned on. + // No sessions are minted (that is a side effect). + if !c.NoPgProxy { + if proxyURL := firstProxyURL(cfg.Env); proxyURL != "" { + if gwURL, agentToken, ok := gatewayOriginAndToken(proxyURL); ok { + cwd, _ := os.Getwd() + swap, unmatched := previewPgSwap( + &api.PgGatewayClient{BaseURL: gwURL, AgentToken: agentToken}, + os.Environ(), cwd, + ) + if len(swap) > 0 { + payload["pg_env_swapped"] = swap + } + if len(unmatched) > 0 { + payload["pg_unregistered_hosts"] = unmatched + } + } + } + } + return out.WriteDryRun("Would exec command with OneCLI gateway", payload) } // Write CA cert to disk (unless --no-ca). @@ -198,10 +220,76 @@ func (c *RunCmd) Run(out *output.Writer) error { out.Stderr(fmt.Sprintf("onecli: warning: %s", w)) } + // Postgres governance (design: pg-interception): scan env + .env for + // postgres URLs (and the libpq PG* group), swap registered ones for + // gateway proxy sessions, and fork the heartbeat/cleanup sidecar. + // Best-effort by default; with --pg-proxy=required a failure to govern a + // matched database aborts the run. (Unreachable in --dry-run: that path + // returns above, so required is intentionally a no-op there.) + if !c.NoPgProxy { + pgRequired := strings.EqualFold(c.PgProxy, "required") + sandbox := false + if spec, ok := agentSkillDir(c.Args[0]); ok { + sandbox = spec.dockerSandbox + } + if sandbox { + // Docker-sandbox agents (e.g. Hermes) run tools in a container + // that does not inherit this env, so a swapped DATABASE_URL never + // reaches the tool making the DB call. Routing that path is a + // phase-3 concern; surface the gap (only when the agent actually + // carries a database in its env) rather than pretend the host-env + // swap covers it. + cwd, _ := os.Getwd() + if len(scanPgURLs(env, cwd)) > 0 { + out.Stderr("onecli: note: database access from sandboxed tools is not yet routed through OneCLI (phase 3); direct connections bypass governance.") + } + } else { + gwURL, agentToken, ok := "", "", false + if proxyURL := firstProxyURL(cfg.Env); proxyURL != "" { + gwURL, agentToken, ok = gatewayOriginAndToken(proxyURL) + } + if !ok { + if pgRequired { + return fmt.Errorf("--pg-proxy=required but the gateway proxy is unavailable (no usable proxy URL); cannot govern database access") + } + } else { + pgClient := &api.PgGatewayClient{BaseURL: gwURL, AgentToken: agentToken} + cwd, _ := os.Getwd() + outcome, err := setupPgProxy(out, pgClient, gatewayHost, caPath, env, cwd, pgRequired) + if err != nil { + return err + } + if outcome != nil { + for name, value := range outcome.Swapped { + // Append AFTER the inherited env: POSIX getenv returns + // the first match, but Go child processes and libc both + // honor later duplicates via exec env ordering — so + // strip the original first, matching buildChildEnv. + env = removeEnvKey(env, name) + env = append(env, name+"="+value) + } + // Placeholder vars for granted databases the scan did + // not cover, plus the index the skill points the agent + // at (ONECLI_PG_CONNECTIONS). + for name, value := range outcome.Placeholders { + env = removeEnvKey(env, name) + env = append(env, name+"="+value) + } + if outcome.IndexJSON != "" { + env = removeEnvKey(env, pgIndexVar) + env = append(env, pgIndexVar+"="+outcome.IndexJSON) + } + spawnPgSidecar(gwURL, agentToken, outcome.SessionIDs, outcome.TTLSeconds, outcome.WatchHosts, outcome.GatewayPgAddr) + } + } + } + } + // Enforce mode: fork the loopback auth forwarder, write the sandbox // settings, and extend the agent argv. Fails closed — a broken // forwarder would leave the sandbox with no route to the gateway, - // which is worse than an explicit error. + // which is worse than an explicit error. Runs AFTER the pg swap so + // sandboxed DB clients read the already-governed env. args := c.Args if c.Enforce { a, ok := agentSkillDir(c.Args[0]) @@ -979,6 +1067,41 @@ func firstProxyURL(env map[string]string) string { return "" } +// gatewayOriginAndToken splits a proxy URL (http://x:aoc_...@host:port) +// into the gateway HTTP origin and the embedded agent token. The pg +// session surface lives on the same listener as the HTTP proxy. +func gatewayOriginAndToken(proxyURL string) (origin, token string, ok bool) { + u, err := url.Parse(proxyURL) + if err != nil || u.Host == "" || u.User == nil { + return "", "", false + } + password, _ := u.User.Password() + // The token rides either the username or the password position + // depending on the server's URL shape; take whichever has the prefix. + for _, cand := range []string{password, u.User.Username()} { + if strings.HasPrefix(cand, "aoc_") { + token = cand + break + } + } + if token == "" { + return "", "", false + } + return "http://" + u.Host, token, true +} + +// removeEnvKey strips every entry for key from an environ-style slice. +func removeEnvKey(env []string, key string) []string { + out := env[:0] + prefix := key + "=" + for _, kv := range env { + if !strings.HasPrefix(kv, prefix) { + out = append(out, kv) + } + } + return out +} + // proxyURLWithHost rewrites the host of a proxy URL, preserving scheme, // credentials, and port. Returns "" for empty input. func proxyURLWithHost(raw, host string) string { diff --git a/cmd/onecli/run_pg.go b/cmd/onecli/run_pg.go new file mode 100644 index 0000000..e8eaa3e --- /dev/null +++ b/cmd/onecli/run_pg.go @@ -0,0 +1,532 @@ +package main + +// Postgres URL interception for `onecli run` — phase 1 of +// docs/design/pg-interception.md (cloud repo): scan the environment and +// project .env* files for postgres:// URLs, match them BY HOST against the +// dashboard-registered connections available to this agent, mint a proxy +// session per matched connection, and swap the matched env values for +// proxy URLs (aoc_pg_ as the username, dummy password). Unmatched +// database URLs are left untouched and produce a warning naming the fix. +// +// The files are never modified: swapped values are injected into the +// CHILD env under the same variable names, which shadows .env for +// default-precedence loaders (dotenv, python-dotenv, Prisma). + +import ( + "bufio" + "context" + "fmt" + "net" + "net/url" + "os" + "path/filepath" + "strings" + + "github.com/onecli/onecli-cli/internal/api" + "github.com/onecli/onecli-cli/pkg/output" +) + +// pgURLPrefixes match postgres connection strings in env values. +var pgURLPrefixes = []string{"postgres://", "postgresql://"} + +// dotenvFiles are the project files scanned (read-only) for database URLs. +var dotenvFiles = []string{".env", ".env.local", ".env.development", ".env.development.local"} + +// pgScanResult is one discovered postgres URL and where it came from. +type pgScanResult struct { + // VarName is the environment variable carrying the URL. For a libpq + // group scan this is "PGHOST" (the group's anchor). + VarName string + // URL is the parsed original connection URL. For a libpq group scan + // it is a synthetic URL assembled from the PG* vars (host/port/db). + URL *url.URL + // FromEnvFile is true when the value came from a .env file rather + // than the process environment (env wins on conflicts). + FromEnvFile bool + // Libpq is true when this scan represents the libpq PG* variable group + // (PGHOST/PGPORT/...) rather than a single URL-valued variable; the + // rewrite then emits the whole group, not one URL. + Libpq bool +} + +// libpqVars are the connection vars of the libpq environment group. PGHOST +// anchors the group; the rest refine it. +var libpqVars = []string{"PGHOST", "PGPORT", "PGUSER", "PGPASSWORD", "PGDATABASE"} + +// pgSwapOutcome reports what setupPgProxy did, for messaging + tests. +type pgSwapOutcome struct { + // Swapped maps env var name → proxy URL. + Swapped map[string]string + // Placeholders maps ONECLI_PG_