From 91c079a5b891068aa4c94a1e8f8a8018b08d324b Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Mon, 4 May 2026 16:51:46 -0400 Subject: [PATCH 1/9] Add offline GitHub proxy for E2E tests Route SDK E2E test subprocess traffic through a shared CONNECT proxy so GitHub and MCP requests are handled locally while CAPI traffic continues to use replay snapshots. Wire the proxy metadata into the Node, Python, Go, and .NET harnesses and add coverage for the proxy and GitHub CLI auth normalization. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- dotnet/test/Harness/CapiProxy.cs | 21 +- dotnet/test/Harness/E2ETestContext.cs | 25 ++ go/internal/e2e/testharness/context.go | 8 + go/internal/e2e/testharness/proxy.go | 81 ++++- nodejs/test/e2e/harness/CapiProxy.ts | 77 ++++- nodejs/test/e2e/harness/sdkTestContext.ts | 6 + python/e2e/testharness/context.py | 6 + python/e2e/testharness/proxy.py | 55 +++- test/harness/certUtils.ts | 81 +++++ test/harness/connectProxy.test.ts | 205 +++++++++++++ test/harness/connectProxy.ts | 353 ++++++++++++++++++++++ test/harness/mockHandlers.ts | 174 +++++++++++ test/harness/package-lock.json | 22 ++ test/harness/package.json | 2 + test/harness/replayingCapiProxy.test.ts | 54 ++++ test/harness/replayingCapiProxy.ts | 109 +++++-- test/harness/server.ts | 48 ++- test/harness/util.ts | 8 +- 18 files changed, 1272 insertions(+), 63 deletions(-) create mode 100644 test/harness/certUtils.ts create mode 100644 test/harness/connectProxy.test.ts create mode 100644 test/harness/connectProxy.ts create mode 100644 test/harness/mockHandlers.ts diff --git a/dotnet/test/Harness/CapiProxy.cs b/dotnet/test/Harness/CapiProxy.cs index f863b651c..780264095 100644 --- a/dotnet/test/Harness/CapiProxy.cs +++ b/dotnet/test/Harness/CapiProxy.cs @@ -17,6 +17,9 @@ public sealed partial class CapiProxy : IAsyncDisposable private Process? _process; private Task? _startupTask; + public string? ConnectProxyUrl { get; private set; } + public string? CaFilePath { get; private set; } + public Task StartAsync() { return _startupTask ??= StartCoreAsync(); @@ -57,8 +60,19 @@ async Task StartCoreAsync() _process.OutputDataReceived += (_, e) => { if (e.Data == null) return; - var match = Regex.Match(e.Data, @"Listening: (http://[^\s]+)"); - if (match.Success) tcs.TrySetResult(match.Groups[1].Value); + var match = Regex.Match(e.Data, @"Listening: (?http://[^\s]+)(?:\s+(?\{.*\}))?"); + if (match.Success) + { + if (match.Groups["metadata"].Success) + { + var metadata = JsonSerializer.Deserialize( + match.Groups["metadata"].Value, + CapiProxyJsonContext.Default.ProxyStartupMetadata); + ConnectProxyUrl = metadata?.ConnectProxyUrl; + CaFilePath = metadata?.CaFilePath; + } + tcs.TrySetResult(match.Groups["url"].Value); + } }; _process.ErrorDataReceived += (_, e) => @@ -124,6 +138,8 @@ public async Task ConfigureAsync(string filePath, string workDir) private record ConfigureRequest(string FilePath, string WorkDir); + private record ProxyStartupMetadata(string? ConnectProxyUrl, string? CaFilePath); + public async Task> GetExchangesAsync() { var url = await (_startupTask ?? throw new InvalidOperationException("Proxy not started")); @@ -165,6 +181,7 @@ private static string FindRepoRoot() [JsonSerializable(typeof(List))] [JsonSerializable(typeof(CopilotUserByTokenRequest))] [JsonSerializable(typeof(Dictionary))] + [JsonSerializable(typeof(ProxyStartupMetadata))] private partial class CapiProxyJsonContext : JsonSerializerContext; } diff --git a/dotnet/test/Harness/E2ETestContext.cs b/dotnet/test/Harness/E2ETestContext.cs index 88627ba6d..19777e09b 100644 --- a/dotnet/test/Harness/E2ETestContext.cs +++ b/dotnet/test/Harness/E2ETestContext.cs @@ -166,8 +166,33 @@ public IReadOnlyDictionary GetEnvironment() env["COPILOT_API_URL"] = ProxyUrl; env["COPILOT_HOME"] = HomeDir; + env["GH_CONFIG_DIR"] = HomeDir; env["XDG_CONFIG_HOME"] = HomeDir; env["XDG_STATE_HOME"] = HomeDir; + if (!string.IsNullOrEmpty(_proxy.ConnectProxyUrl) && !string.IsNullOrEmpty(_proxy.CaFilePath)) + { + const string noProxy = "127.0.0.1,localhost,::1"; + env["HTTP_PROXY"] = _proxy.ConnectProxyUrl; + env["HTTPS_PROXY"] = _proxy.ConnectProxyUrl; + env["http_proxy"] = _proxy.ConnectProxyUrl; + env["https_proxy"] = _proxy.ConnectProxyUrl; + env["NO_PROXY"] = noProxy; + env["no_proxy"] = noProxy; + env["NODE_EXTRA_CA_CERTS"] = _proxy.CaFilePath; + env["SSL_CERT_FILE"] = _proxy.CaFilePath; + env["REQUESTS_CA_BUNDLE"] = _proxy.CaFilePath; + env["CURL_CA_BUNDLE"] = _proxy.CaFilePath; + env["GIT_SSL_CAINFO"] = _proxy.CaFilePath; + env["GH_TOKEN"] = ""; + env["GITHUB_TOKEN"] = ""; + env["GH_ENTERPRISE_TOKEN"] = ""; + env["GITHUB_ENTERPRISE_TOKEN"] = ""; + } + if (Environment.GetEnvironmentVariable("GITHUB_ACTIONS") == "true") + { + env["GH_TOKEN"] = "fake-token-for-e2e-tests"; + env["GITHUB_TOKEN"] = "fake-token-for-e2e-tests"; + } return env!; } diff --git a/go/internal/e2e/testharness/context.go b/go/internal/e2e/testharness/context.go index bf6f160df..e8efda82f 100644 --- a/go/internal/e2e/testharness/context.go +++ b/go/internal/e2e/testharness/context.go @@ -163,12 +163,20 @@ func (c *TestContext) Env() []string { env := os.Environ() // Add overrides (later values take precedence in most systems) + env = append(env, c.proxy.ProxyEnv()...) env = append(env, "COPILOT_API_URL="+c.ProxyURL, "COPILOT_HOME="+c.HomeDir, + "GH_CONFIG_DIR="+c.HomeDir, "XDG_CONFIG_HOME="+c.HomeDir, "XDG_STATE_HOME="+c.HomeDir, ) + if os.Getenv("GITHUB_ACTIONS") == "true" { + env = append(env, + "GH_TOKEN=fake-token-for-e2e-tests", + "GITHUB_TOKEN=fake-token-for-e2e-tests", + ) + } return env } diff --git a/go/internal/e2e/testharness/proxy.go b/go/internal/e2e/testharness/proxy.go index 4fb98e98d..03ddecfbb 100644 --- a/go/internal/e2e/testharness/proxy.go +++ b/go/internal/e2e/testharness/proxy.go @@ -17,9 +17,11 @@ import ( // CapiProxy manages a child process that acts as a replaying proxy to AI endpoints. // It spawns the shared test harness server from test/harness/server.ts. type CapiProxy struct { - cmd *exec.Cmd - proxyURL string - mu sync.Mutex + cmd *exec.Cmd + proxyURL string + connectProxyURL string + caFilePath string + mu sync.Mutex } // NewCapiProxy creates a new proxy instance. @@ -54,23 +56,42 @@ func (p *CapiProxy) Start() (string, error) { return "", fmt.Errorf("failed to start proxy server: %w", err) } - // Read the first line to get the listening URL + // Read until the server prints "Listening: http://..."; npm/npx may emit + // wrapper output first on some platforms. reader := bufio.NewReader(stdout) - line, err := reader.ReadString('\n') - if err != nil && err != io.EOF { - p.cmd.Process.Kill() - return "", fmt.Errorf("failed to read proxy URL: %w", err) - } - - // Parse "Listening: http://..." from output - re := regexp.MustCompile(`Listening: (http://[^\s]+)`) - matches := re.FindStringSubmatch(strings.TrimSpace(line)) - if len(matches) < 2 { - p.cmd.Process.Kill() - return "", fmt.Errorf("unexpected proxy output: %s", line) + re := regexp.MustCompile(`Listening: (http://[^\s]+)(?:\s+(\{.*\}))?`) + var matches []string + var line string + for { + nextLine, err := reader.ReadString('\n') + if err != nil && err != io.EOF { + p.cmd.Process.Kill() + return "", fmt.Errorf("failed to read proxy URL: %w", err) + } + line = strings.TrimSpace(nextLine) + matches = re.FindStringSubmatch(line) + if len(matches) >= 2 { + break + } + if err == io.EOF { + p.cmd.Process.Kill() + return "", fmt.Errorf("proxy exited before startup; last output: %s", line) + } } p.proxyURL = matches[1] + if len(matches) >= 3 && matches[2] != "" { + var metadata struct { + ConnectProxyURL string `json:"connectProxyUrl"` + CAFilePath string `json:"caFilePath"` + } + if err := json.Unmarshal([]byte(matches[2]), &metadata); err != nil { + p.cmd.Process.Kill() + return "", fmt.Errorf("failed to parse proxy startup metadata: %w", err) + } + p.connectProxyURL = metadata.ConnectProxyURL + p.caFilePath = metadata.CAFilePath + } return p.proxyURL, nil } @@ -254,6 +275,34 @@ func (p *CapiProxy) URL() string { return p.proxyURL } +// ProxyEnv returns environment variables that route HTTPS traffic through the CONNECT proxy. +func (p *CapiProxy) ProxyEnv() []string { + p.mu.Lock() + defer p.mu.Unlock() + if p.connectProxyURL == "" || p.caFilePath == "" { + return nil + } + + noProxy := "127.0.0.1,localhost,::1" + return []string{ + "HTTP_PROXY=" + p.connectProxyURL, + "HTTPS_PROXY=" + p.connectProxyURL, + "http_proxy=" + p.connectProxyURL, + "https_proxy=" + p.connectProxyURL, + "NO_PROXY=" + noProxy, + "no_proxy=" + noProxy, + "NODE_EXTRA_CA_CERTS=" + p.caFilePath, + "SSL_CERT_FILE=" + p.caFilePath, + "REQUESTS_CA_BUNDLE=" + p.caFilePath, + "CURL_CA_BUNDLE=" + p.caFilePath, + "GIT_SSL_CAINFO=" + p.caFilePath, + "GH_TOKEN=", + "GITHUB_TOKEN=", + "GH_ENTERPRISE_TOKEN=", + "GITHUB_ENTERPRISE_TOKEN=", + } +} + // SetCopilotUserByToken registers a per-token user configuration on the proxy. func (p *CapiProxy) SetCopilotUserByToken(token string, response map[string]interface{}) error { p.mu.Lock() diff --git a/nodejs/test/e2e/harness/CapiProxy.ts b/nodejs/test/e2e/harness/CapiProxy.ts index eace18739..d44433ea6 100644 --- a/nodejs/test/e2e/harness/CapiProxy.ts +++ b/nodejs/test/e2e/harness/CapiProxy.ts @@ -7,10 +7,18 @@ import { } from "../../../../test/harness/replayingCapiProxy"; const HARNESS_SERVER_PATH = resolve(__dirname, "../../../../test/harness/server.ts"); +const NO_PROXY = "127.0.0.1,localhost,::1"; + +interface ProxyStartupInfo { + capiProxyUrl: string; + connectProxyUrl?: string; + caFilePath?: string; +} // Manages a child process that acts as a replaying proxy to the underlying AI endpoints export class CapiProxy { private proxyUrl: string | undefined; + private startupInfo: ProxyStartupInfo | undefined; /** * Returns the URL of the running proxy. Throws if the proxy has not been started. @@ -28,16 +36,56 @@ export class CapiProxy { shell: true, }); - this.proxyUrl = await new Promise((resolve) => { - serverProcess.stdout!.once("data", (chunk: Buffer) => { - const match = chunk.toString().match(/Listening: (http:\/\/[^\s]+)/); - resolve(match![1]); - }); + this.startupInfo = await new Promise((resolve, reject) => { + let output = ""; + const cleanup = () => { + serverProcess.stdout!.off("data", onData); + serverProcess.off("exit", onExit); + }; + const onData = (chunk: Buffer) => { + output += chunk.toString(); + const info = tryParseStartupInfo(output); + if (info) { + cleanup(); + resolve(info); + } + }; + const onExit = (code: number | null) => { + cleanup(); + reject(new Error(`Proxy exited before startup with code ${code}: ${output}`)); + }; + serverProcess.stdout!.on("data", onData); + serverProcess.once("exit", onExit); }); + this.proxyUrl = this.startupInfo.capiProxyUrl; return this.proxyUrl; } + getProxyEnv(): Record { + if (!this.startupInfo?.connectProxyUrl || !this.startupInfo.caFilePath) { + return {}; + } + + return { + HTTP_PROXY: this.startupInfo.connectProxyUrl, + HTTPS_PROXY: this.startupInfo.connectProxyUrl, + http_proxy: this.startupInfo.connectProxyUrl, + https_proxy: this.startupInfo.connectProxyUrl, + NO_PROXY, + no_proxy: NO_PROXY, + NODE_EXTRA_CA_CERTS: this.startupInfo.caFilePath, + SSL_CERT_FILE: this.startupInfo.caFilePath, + REQUESTS_CA_BUNDLE: this.startupInfo.caFilePath, + CURL_CA_BUNDLE: this.startupInfo.caFilePath, + GIT_SSL_CAINFO: this.startupInfo.caFilePath, + GH_TOKEN: "", + GITHUB_TOKEN: "", + GH_ENTERPRISE_TOKEN: "", + GITHUB_ENTERPRISE_TOKEN: "", + }; + } + async updateConfig(config: { filePath: string; workDir: string; @@ -78,3 +126,22 @@ export class CapiProxy { expect(res.ok).toBe(true); } } + +function tryParseStartupInfo(output: string): ProxyStartupInfo | undefined { + const line = output.split(/\r?\n/).find((candidate) => candidate.includes("Listening: ")); + if (!line) { + return undefined; + } + + const match = line.match(/Listening: (http:\/\/[^\s]+)(?:\s+(\{.*\}))?/); + if (!match) { + throw new Error(`Unexpected proxy output: ${line}`); + } + + const metadata = match[2] ? (JSON.parse(match[2]) as Partial) : {}; + return { + capiProxyUrl: match[1], + connectProxyUrl: metadata.connectProxyUrl, + caFilePath: metadata.caFilePath, + }; +} diff --git a/nodejs/test/e2e/harness/sdkTestContext.ts b/nodejs/test/e2e/harness/sdkTestContext.ts index c68bc6f86..7fe2b9cc7 100644 --- a/nodejs/test/e2e/harness/sdkTestContext.ts +++ b/nodejs/test/e2e/harness/sdkTestContext.ts @@ -37,8 +37,10 @@ export async function createSdkTestContext({ const proxyUrl = await openAiEndpoint.start(); const env = { ...process.env, + ...openAiEndpoint.getProxyEnv(), COPILOT_API_URL: proxyUrl, COPILOT_HOME: copilotHomeDir, + GH_CONFIG_DIR: homeDir, // TODO: I'm not convinced the SDK should default to using whatever config you happen to have in your homedir. // The SDK config should be independent of the regular CLI app. Likewise it shouldn't mix sessions from the @@ -46,6 +48,10 @@ export async function createSdkTestContext({ XDG_CONFIG_HOME: homeDir, XDG_STATE_HOME: homeDir, }; + if (isCI) { + env.GH_TOKEN = "fake-token-for-e2e-tests"; + env.GITHUB_TOKEN = "fake-token-for-e2e-tests"; + } const copilotClient = new CopilotClient({ cwd: workDir, diff --git a/python/e2e/testharness/context.py b/python/e2e/testharness/context.py index 5bb7b326b..dae71d99e 100644 --- a/python/e2e/testharness/context.py +++ b/python/e2e/testharness/context.py @@ -128,15 +128,21 @@ async def configure_for_test(self, test_file: str, test_name: str): def get_env(self) -> dict: """Return environment variables configured for isolated testing.""" env = os.environ.copy() + if self._proxy: + env.update(self._proxy.get_proxy_env()) env.update( { "COPILOT_API_URL": self.proxy_url, "COPILOT_HOME": self.home_dir, + "GH_CONFIG_DIR": self.home_dir, "XDG_CONFIG_HOME": self.home_dir, "XDG_STATE_HOME": self.home_dir, } ) + if os.environ.get("GITHUB_ACTIONS") == "true": + env["GH_TOKEN"] = "fake-token-for-e2e-tests" + env["GITHUB_TOKEN"] = "fake-token-for-e2e-tests" return env @property diff --git a/python/e2e/testharness/proxy.py b/python/e2e/testharness/proxy.py index e125375e0..7d4444629 100644 --- a/python/e2e/testharness/proxy.py +++ b/python/e2e/testharness/proxy.py @@ -9,6 +9,7 @@ import platform import re import subprocess +import json from typing import Any import httpx @@ -20,6 +21,8 @@ class CapiProxy: def __init__(self): self._process: subprocess.Popen | None = None self._proxy_url: str | None = None + self._connect_proxy_url: str | None = None + self._ca_file_path: str | None = None async def start(self) -> str: """Launch the proxy server and return its URL.""" @@ -44,19 +47,25 @@ async def start(self) -> str: shell=use_shell, ) - # Read the first line to get the listening URL - line = self._process.stdout.readline() - if not line: - self._process.kill() - raise RuntimeError("Failed to read proxy URL") - - # Parse "Listening: http://..." from output - match = re.search(r"Listening: (http://[^\s]+)", line.strip()) - if not match: - self._process.kill() - raise RuntimeError(f"Unexpected proxy output: {line}") + # Read until the server prints "Listening: http://..."; npm/npx may emit + # wrapper output first on some platforms. + line = "" + match = None + while True: + line = self._process.stdout.readline() + if not line: + self._process.kill() + raise RuntimeError("Failed to read proxy URL") + match = re.search(r"Listening: (http://[^\s]+)", line.strip()) + if match: + break self._proxy_url = match.group(1) + metadata_match = re.search(r"(\{.*\})\s*$", line.strip()) + if metadata_match: + metadata = json.loads(metadata_match.group(1)) + self._connect_proxy_url = metadata.get("connectProxyUrl") + self._ca_file_path = metadata.get("caFilePath") return self._proxy_url async def stop(self, skip_writing_cache: bool = False): @@ -122,3 +131,27 @@ async def set_copilot_user_by_token(self, token: str, response: dict[str, Any]) def url(self) -> str | None: """Return the proxy URL, or None if not started.""" return self._proxy_url + + def get_proxy_env(self) -> dict[str, str]: + """Return environment variables that route HTTPS traffic through the CONNECT proxy.""" + if not self._connect_proxy_url or not self._ca_file_path: + return {} + + no_proxy = "127.0.0.1,localhost,::1" + return { + "HTTP_PROXY": self._connect_proxy_url, + "HTTPS_PROXY": self._connect_proxy_url, + "http_proxy": self._connect_proxy_url, + "https_proxy": self._connect_proxy_url, + "NO_PROXY": no_proxy, + "no_proxy": no_proxy, + "NODE_EXTRA_CA_CERTS": self._ca_file_path, + "SSL_CERT_FILE": self._ca_file_path, + "REQUESTS_CA_BUNDLE": self._ca_file_path, + "CURL_CA_BUNDLE": self._ca_file_path, + "GIT_SSL_CAINFO": self._ca_file_path, + "GH_TOKEN": "", + "GITHUB_TOKEN": "", + "GH_ENTERPRISE_TOKEN": "", + "GITHUB_ENTERPRISE_TOKEN": "", + } diff --git a/test/harness/certUtils.ts b/test/harness/certUtils.ts new file mode 100644 index 000000000..ed1754547 --- /dev/null +++ b/test/harness/certUtils.ts @@ -0,0 +1,81 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------------------------------------------*/ + +import tls from "tls"; + +import forge from "node-forge"; + +export interface CaData { + certPem: string; + keyPem: string; + caCert: forge.pki.Certificate; + caKey: forge.pki.rsa.PrivateKey; +} + +export function generateCA(): CaData { + const keys = forge.pki.rsa.generateKeyPair(2048); + const cert = forge.pki.createCertificate(); + cert.publicKey = keys.publicKey; + cert.serialNumber = "01"; + + const now = new Date(); + const oneYearLater = new Date(); + oneYearLater.setFullYear(oneYearLater.getFullYear() + 1); + cert.validity.notBefore = now; + cert.validity.notAfter = oneYearLater; + + const attrs: forge.pki.CertificateField[] = [ + { name: "commonName", value: "SDK E2E Test CA" }, + { name: "organizationName", value: "Copilot SDK Tests" }, + ]; + cert.setSubject(attrs); + cert.setIssuer(attrs); + + cert.setExtensions([ + { name: "basicConstraints", cA: true, critical: true }, + { name: "keyUsage", keyCertSign: true, cRLSign: true, critical: true }, + ]); + + cert.sign(keys.privateKey, forge.md.sha256.create()); + + return { + certPem: forge.pki.certificateToPem(cert), + keyPem: forge.pki.privateKeyToPem(keys.privateKey), + caCert: cert, + caKey: keys.privateKey, + }; +} + +export function createSecureContextForHost( + hostname: string, + ca: CaData, +): tls.SecureContext { + const keys = forge.pki.rsa.generateKeyPair(2048); + const cert = forge.pki.createCertificate(); + cert.publicKey = keys.publicKey; + cert.serialNumber = String(Date.now()); + + const now = new Date(); + const oneYearLater = new Date(); + oneYearLater.setFullYear(oneYearLater.getFullYear() + 1); + cert.validity.notBefore = now; + cert.validity.notAfter = oneYearLater; + + cert.setSubject([{ name: "commonName", value: hostname }]); + cert.setIssuer(ca.caCert.subject.attributes); + cert.setExtensions([ + { + name: "subjectAltName", + altNames: [{ type: 2, value: hostname }], + }, + ]); + + cert.sign(ca.caKey, forge.md.sha256.create()); + + return tls.createSecureContext({ + key: forge.pki.privateKeyToPem(keys.privateKey), + cert: forge.pki.certificateToPem(cert), + ca: ca.certPem, + }); +} diff --git a/test/harness/connectProxy.test.ts b/test/harness/connectProxy.test.ts new file mode 100644 index 000000000..202e403f5 --- /dev/null +++ b/test/harness/connectProxy.test.ts @@ -0,0 +1,205 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------------------------------------------*/ + +import fs from "fs"; +import http from "http"; +import https from "https"; +import tls from "tls"; +import { describe, expect, test } from "vitest"; +import { + ConnectProxy, + parseConnectTarget, + type RequestHandler, +} from "./connectProxy"; +import { createE2eRequestHandler } from "./mockHandlers"; + +describe("parseConnectTarget", () => { + test("parses host:port", () => { + expect(parseConnectTarget("example.com:443")).toEqual({ + host: "example.com", + port: "443", + }); + }); + + test("defaults missing port to 443", () => { + expect(parseConnectTarget("example.com")).toEqual({ + host: "example.com", + port: "443", + }); + }); + + test("parses IPv6 bracket form", () => { + expect(parseConnectTarget("[::1]:8443")).toEqual({ + host: "::1", + port: "8443", + }); + }); + + test("rejects malformed IPv6 authority", () => { + expect(parseConnectTarget("[::1:443")).toEqual({ host: "", port: "" }); + expect(parseConnectTarget("[::1]443")).toEqual({ host: "", port: "" }); + }); +}); + +describe("ConnectProxy", () => { + test("starts and stops cleanly", async () => { + const proxy = new ConnectProxy( + (_req, res) => { + res.writeHead(200); + res.end("ok"); + return true; + }, + { interceptDomains: ["example.com"] }, + ); + await proxy.start(); + + expect(proxy.proxyUrl).toMatch(/^http:\/\/127\.0\.0\.1:\d+$/); + expect(proxy.caFilePath).toMatch(/test-ca\.pem$/); + + await proxy.stop(); + }); + + test("intercepts HTTPS requests to configured domains", async () => { + const requests: Array<{ host: string; url: string }> = []; + const handler: RequestHandler = (req, res, targetHost) => { + requests.push({ host: targetHost, url: req.url ?? "/" }); + res.writeHead(200, { "content-type": "text/plain" }); + res.end("mocked"); + return true; + }; + + const proxy = new ConnectProxy(handler, { + interceptDomains: ["test.example.com"], + }); + await proxy.start(); + + try { + const response = await makeHttpsRequest( + proxy.proxyUrl, + proxy.caFilePath, + "test.example.com", + "/api/test", + ); + expect(response.statusCode).toBe(200); + expect(response.body).toBe("mocked"); + expect(requests).toEqual([ + { host: "test.example.com", url: "/api/test" }, + ]); + expect(proxy.connectLog[0].host).toBe("test.example.com"); + } finally { + await proxy.stop(); + } + }); + + test("rejects CONNECT to non-intercepted domains", async () => { + const blocked: string[] = []; + const proxy = new ConnectProxy( + (_req, res) => { + res.writeHead(200); + res.end("ok"); + return true; + }, + { + interceptDomains: ["allowed.example.com"], + onBlockedConnection: (host) => blocked.push(host), + }, + ); + await proxy.start(); + + try { + await expect( + makeHttpsRequest( + proxy.proxyUrl, + proxy.caFilePath, + "blocked.example.com", + "/", + ), + ).rejects.toThrow(); + expect(blocked).toEqual(["blocked.example.com"]); + } finally { + await proxy.stop(); + } + }); + + test("mocks GitHub HTTPS requests without reaching the network", async () => { + const proxy = new ConnectProxy( + createE2eRequestHandler({ capiProxyUrl: "http://127.0.0.1:1" }), + { interceptDomains: ["github.com", "api.github.com"] }, + ); + await proxy.start(); + + try { + const githubResponse = await makeHttpsRequest( + proxy.proxyUrl, + proxy.caFilePath, + "github.com", + "/github/copilot-sdk/issues/1234", + ); + expect(githubResponse.statusCode).toBe(404); + expect(githubResponse.body).toContain("Not Found (e2e mock)"); + + const apiResponse = await makeHttpsRequest( + proxy.proxyUrl, + proxy.caFilePath, + "api.github.com", + "/user", + ); + expect(apiResponse.statusCode).toBe(200); + expect(JSON.parse(apiResponse.body)).toMatchObject({ + login: "sdk-e2e-user", + }); + } finally { + await proxy.stop(); + } + }); +}); + +function makeHttpsRequest( + proxyUrl: string, + caFilePath: string, + hostname: string, + path: string, +): Promise<{ statusCode: number; body: string }> { + return new Promise((resolve, reject) => { + const proxy = new URL(proxyUrl); + const connectReq = http.request({ + host: proxy.hostname, + port: Number(proxy.port), + method: "CONNECT", + path: `${hostname}:443`, + }); + + connectReq.on("connect", (_res, socket) => { + const ca = fs.readFileSync(caFilePath); + const req = https.request( + { + hostname, + path, + method: "GET", + createConnection: () => + tls.connect({ socket, servername: hostname, ca }), + }, + (res) => { + let body = ""; + res.on("data", (chunk: Buffer) => { + body += chunk.toString(); + }); + res.on("end", () => + resolve({ statusCode: res.statusCode ?? 0, body }), + ); + }, + ); + req.on("error", reject); + req.end(); + }); + + connectReq.on("response", (res) => { + res.resume(); + reject(new Error(`CONNECT failed with status ${res.statusCode}`)); + }); + + connectReq.on("error", reject); + connectReq.end(); + }); +} diff --git a/test/harness/connectProxy.ts b/test/harness/connectProxy.ts new file mode 100644 index 000000000..0e475075d --- /dev/null +++ b/test/harness/connectProxy.ts @@ -0,0 +1,353 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------------------------------------------*/ + +import fs from "fs"; +import http from "http"; +import net from "net"; +import os from "os"; +import path from "path"; +import tls from "tls"; +import { + type CaData, + createSecureContextForHost, + generateCA, +} from "./certUtils"; + +const debugLogPath = process.env.E2E_PROXY_DEBUG + ? path.join(os.tmpdir(), `e2e-proxy-debug-${process.pid}.log`) + : undefined; + +function debugLog(msg: string): void { + if (debugLogPath) { + fs.appendFileSync( + debugLogPath, + `[${new Date().toISOString()}] [connect] ${msg}\n`, + ); + } +} + +export type RequestHandler = ( + req: http.IncomingMessage, + res: http.ServerResponse, + targetHost: string, +) => boolean | Promise; + +export class ConnectProxy { + private proxyServer?: http.Server; + private internalServer?: http.Server; + private ca?: CaData; + private certCache = new Map(); + private _caFilePath?: string; + private _proxyUrl?: string; + private _connectLog: Array<{ + host: string; + port: string; + timestamp: number; + }> = []; + private interceptDomains: Set; + private passthroughDomains: Set; + private onBlockedConnection?: (host: string, port: string) => void; + private openSockets = new Set(); + + constructor( + private handler: RequestHandler, + options?: { + interceptDomains?: string[]; + passthroughDomains?: string[]; + onBlockedConnection?: (host: string, port: string) => void; + }, + ) { + this.interceptDomains = new Set(options?.interceptDomains ?? []); + this.passthroughDomains = new Set(options?.passthroughDomains ?? []); + this.onBlockedConnection = options?.onBlockedConnection; + } + + get proxyUrl(): string { + if (!this._proxyUrl) { + throw new Error("ConnectProxy not started"); + } + return this._proxyUrl; + } + + get caFilePath(): string { + if (!this._caFilePath) { + throw new Error("ConnectProxy not started"); + } + return this._caFilePath; + } + + get connectLog(): ReadonlyArray<{ + host: string; + port: string; + timestamp: number; + }> { + return this._connectLog; + } + + async start(): Promise { + this.ca = generateCA(); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-proxy-ca-")); + this._caFilePath = path.join(tmpDir, "test-ca.pem"); + fs.writeFileSync(this._caFilePath, this.ca.certPem); + + this.internalServer = http.createServer((req, res) => { + const socket = req.socket as tls.TLSSocket & { _connectTarget?: string }; + const targetHost = socket._connectTarget ?? req.headers.host ?? "unknown"; + + void Promise.resolve(this.handler(req, res, targetHost)) + .then((handled) => { + if (!handled && !res.headersSent) { + res.writeHead(502, { "content-type": "text/plain" }); + res.end( + `E2E proxy: no handler for ${req.method} ${targetHost}${req.url}`, + ); + } + }) + .catch((err) => { + console.warn( + `[E2E proxy] handler error for ${req.method} ${targetHost}${req.url}: ${err}`, + ); + if (!res.headersSent) { + res.writeHead(502, { "content-type": "text/plain" }); + res.end("E2E proxy: handler error"); + } + }); + }); + + this.proxyServer = http.createServer((req, res) => { + this.handleForwardProxy(req, res); + }); + + this.proxyServer.on("connect", (req, clientSocket, head) => { + this.handleConnect(req, clientSocket as net.Socket, head); + }); + + await new Promise((resolve, reject) => { + this.proxyServer!.on("error", reject); + this.proxyServer!.listen(0, "127.0.0.1", () => resolve()); + }); + + const addr = this.proxyServer.address() as net.AddressInfo; + this._proxyUrl = `http://${addr.address}:${addr.port}`; + } + + async stop(): Promise { + for (const socket of this.openSockets) { + socket.destroy(); + } + this.openSockets.clear(); + + const closeServer = (server?: http.Server) => + new Promise((resolve) => { + if (!server) { + resolve(); + return; + } + server.close(() => resolve()); + }); + + await Promise.all([ + closeServer(this.proxyServer), + closeServer(this.internalServer), + ]); + + if (this._caFilePath) { + try { + fs.rmSync(path.dirname(this._caFilePath), { + recursive: true, + force: true, + }); + } catch { + // Best-effort cleanup. + } + } + } + + private handleConnect( + req: http.IncomingMessage, + clientSocket: net.Socket, + head: Buffer, + ) { + const { host, port } = parseConnectTarget(req.url ?? ""); + debugLog(`CONNECT ${host}:${port}`); + if (!host) { + clientSocket.write("HTTP/1.1 400 Bad Request\r\n\r\n"); + clientSocket.destroy(); + return; + } + + this._connectLog.push({ host, port, timestamp: Date.now() }); + + if (this.passthroughDomains.has(host)) { + this.pipeToRealTarget(clientSocket, head, host, port); + return; + } + + if (!this.interceptDomains.has(host)) { + this.onBlockedConnection?.(host, port); + clientSocket.write("HTTP/1.1 502 Blocked by E2E proxy\r\n\r\n"); + clientSocket.destroy(); + return; + } + + clientSocket.write("HTTP/1.1 200 Connection Established\r\n\r\n"); + + const tlsSocket = new tls.TLSSocket(clientSocket, { + isServer: true, + secureContext: this.getOrCreateSecureContext(host), + ALPNProtocols: ["http/1.1"], + }); + + this.openSockets.add(clientSocket); + this.openSockets.add(tlsSocket); + let cleaned = false; + const cleanup = () => { + if (cleaned) { + return; + } + cleaned = true; + tlsSocket.off("close", cleanup); + clientSocket.off("close", cleanup); + tlsSocket.off("error", onTlsError); + clientSocket.off("error", onClientError); + this.openSockets.delete(clientSocket); + this.openSockets.delete(tlsSocket); + }; + const onTlsError = (err: Error) => { + debugLog(`TLS error for ${host}: ${err.message}`); + cleanup(); + clientSocket.destroy(); + }; + const onClientError = () => { + cleanup(); + tlsSocket.destroy(); + }; + tlsSocket.on("close", cleanup); + clientSocket.on("close", cleanup); + tlsSocket.on("error", onTlsError); + clientSocket.on("error", onClientError); + + (tlsSocket as tls.TLSSocket & { _connectTarget?: string })._connectTarget = + host; + if (head.length > 0) { + tlsSocket.unshift(head); + } + this.internalServer!.emit("connection", tlsSocket); + } + + private handleForwardProxy( + req: http.IncomingMessage, + res: http.ServerResponse, + ) { + let targetHost: string; + try { + const url = new URL(req.url ?? ""); + targetHost = url.hostname; + req.url = url.pathname + url.search; + } catch { + targetHost = req.headers.host ?? "unknown"; + } + + void Promise.resolve(this.handler(req, res, targetHost)) + .then((handled) => { + if (!handled && !res.headersSent) { + res.writeHead(502, { "content-type": "text/plain" }); + res.end( + `E2E proxy: no handler for HTTP ${req.method} ${targetHost}${req.url}`, + ); + } + }) + .catch(() => { + if (!res.headersSent) { + res.writeHead(502, { "content-type": "text/plain" }); + res.end("E2E proxy: handler error"); + } + }); + } + + private getOrCreateSecureContext(hostname: string): tls.SecureContext { + let context = this.certCache.get(hostname); + if (!context) { + context = createSecureContextForHost(hostname, this.ca!); + this.certCache.set(hostname, context); + } + return context; + } + + private pipeToRealTarget( + clientSocket: net.Socket, + head: Buffer, + host: string, + port: string, + ) { + const targetSocket = net.connect(Number.parseInt(port, 10), host, () => { + if (clientSocket.destroyed || targetSocket.destroyed) { + return; + } + clientSocket.write("HTTP/1.1 200 Connection Established\r\n\r\n"); + if (head.length > 0) { + targetSocket.write(head); + } + clientSocket.pipe(targetSocket); + targetSocket.pipe(clientSocket); + }); + + this.openSockets.add(clientSocket); + this.openSockets.add(targetSocket); + + let cleaned = false; + const cleanup = () => { + if (cleaned) { + return; + } + cleaned = true; + clientSocket.off("error", cleanup); + clientSocket.off("close", cleanup); + targetSocket.off("error", cleanup); + targetSocket.off("close", cleanup); + clientSocket.destroy(); + targetSocket.destroy(); + this.openSockets.delete(clientSocket); + this.openSockets.delete(targetSocket); + }; + clientSocket.on("error", cleanup); + clientSocket.on("close", cleanup); + targetSocket.on("error", cleanup); + targetSocket.on("close", cleanup); + } +} + +export function parseConnectTarget(authority: string): { + host: string; + port: string; +} { + if (!authority) { + return { host: "", port: "" }; + } + + if (authority.startsWith("[")) { + const closeBracket = authority.indexOf("]"); + if (closeBracket === -1) { + return { host: "", port: "" }; + } + const host = authority.slice(1, closeBracket); + const afterBracket = authority.slice(closeBracket + 1); + if (afterBracket === "" || afterBracket === ":") { + return { host, port: "443" }; + } + if (afterBracket[0] !== ":") { + return { host: "", port: "" }; + } + return { host, port: afterBracket.slice(1) || "443" }; + } + + const lastColon = authority.lastIndexOf(":"); + if (lastColon === -1) { + return { host: authority, port: "443" }; + } + + const host = authority.slice(0, lastColon); + const port = authority.slice(lastColon + 1) || "443"; + return { host, port }; +} diff --git a/test/harness/mockHandlers.ts b/test/harness/mockHandlers.ts new file mode 100644 index 000000000..9f75d6819 --- /dev/null +++ b/test/harness/mockHandlers.ts @@ -0,0 +1,174 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------------------------------------------*/ + +import http from "http"; +import type { RequestHandler } from "./connectProxy"; + +export function createE2eRequestHandler(options: { + capiProxyUrl: string; + onUnhandled?: (host: string, method: string, path: string) => void; +}): RequestHandler { + return async (req, res, targetHost) => { + if (targetHost === "api.githubcopilot.com") { + return forwardToCapiProxy(req, res, options.capiProxyUrl); + } + + if (targetHost === "api.github.com") { + return handleGitHubApi(req, res, options); + } + + if (targetHost === "github.com") { + respondJson(res, 404, { message: "Not Found (e2e mock)" }); + return true; + } + + if (targetHost === "api.mcp.github.com") { + return handleMcpRegistry(req, res); + } + + options.onUnhandled?.(targetHost, req.method ?? "GET", req.url ?? "/"); + return false; + }; +} + +function handleGitHubApi( + req: http.IncomingMessage, + res: http.ServerResponse, + options: { capiProxyUrl: string }, +): boolean { + const url = req.url ?? "/"; + + if (req.method === "GET" && url === "/user") { + respondJson(res, 200, { + login: "sdk-e2e-user", + id: 12345, + type: "User", + name: "SDK E2E User", + }); + return true; + } + + if (req.method === "GET" && url.startsWith("/user/copilot_billing")) { + respondJson(res, 200, { + seat: { plan: { plan_type: "business" } }, + }); + return true; + } + + if (req.method === "GET" && url.startsWith("/copilot_internal/user")) { + respondJson(res, 200, { + login: "sdk-e2e-user", + analytics_tracking_id: "sdk-e2e-tracking-id", + organization_list: [], + copilot_plan: "individual_pro", + is_mcp_enabled: true, + endpoints: { + api: options.capiProxyUrl, + telemetry: "https://localhost:1/telemetry", + }, + }); + return true; + } + + if (req.method === "POST" && url === "/graphql") { + respondJson(res, 401, { + message: "Requires authentication", + documentation_url: "https://docs.github.com/graphql", + }); + return true; + } + + respondJson(res, 404, { message: "Not Found (e2e mock)" }); + return true; +} + +function handleMcpRegistry( + req: http.IncomingMessage, + res: http.ServerResponse, +): boolean { + const url = new URL(req.url ?? "/", "https://api.mcp.github.com"); + + if (req.method === "GET" && url.pathname.startsWith("/v0.1/servers")) { + respondJson(res, 200, { servers: [], metadata: {} }); + return true; + } + + respondJson(res, 404, { error: "Not Found (e2e mock)" }); + return true; +} + +function respondJson( + res: http.ServerResponse, + statusCode: number, + body: unknown, +): void { + const data = JSON.stringify(body); + res.writeHead(statusCode, { + "content-type": "application/json", + "content-length": Buffer.byteLength(data), + }); + res.end(data); +} + +function forwardToCapiProxy( + clientReq: http.IncomingMessage, + clientRes: http.ServerResponse, + capiProxyUrl: string, +): Promise { + return new Promise((resolve) => { + const target = new URL(capiProxyUrl); + const chunks: Buffer[] = []; + clientReq.on("data", (chunk: Buffer) => chunks.push(chunk)); + clientReq.on("error", (err) => { + if (!clientRes.headersSent) { + clientRes.writeHead(502, { "content-type": "text/plain" }); + clientRes.end(`E2E proxy: client request error: ${err.message}`); + } else { + clientRes.destroy(err); + } + resolve(true); + }); + clientReq.on("end", () => { + const proxyReq = http.request( + { + hostname: target.hostname, + port: target.port, + path: clientReq.url, + method: clientReq.method, + headers: { + ...clientReq.headers, + host: target.host, + }, + }, + (proxyRes) => { + clientRes.writeHead(proxyRes.statusCode ?? 502, proxyRes.headers); + proxyRes.pipe(clientRes); + proxyRes.on("end", () => resolve(true)); + proxyRes.on("error", (err) => { + clientRes.destroy(err); + resolve(true); + }); + }, + ); + proxyReq.on("error", (err) => { + if (!clientRes.headersSent) { + clientRes.writeHead(502, { + "content-type": "application/json", + "x-github-request-id": "e2e-proxy-error", + }); + clientRes.end( + JSON.stringify({ + error: `E2E proxy: CAPI forward error: ${err.message}`, + }), + ); + } + resolve(true); + }); + if (chunks.length > 0) { + proxyReq.write(Buffer.concat(chunks)); + } + proxyReq.end(); + }); + }); +} diff --git a/test/harness/package-lock.json b/test/harness/package-lock.json index 02cfeb2cc..9ece434e9 100644 --- a/test/harness/package-lock.json +++ b/test/harness/package-lock.json @@ -12,6 +12,8 @@ "@github/copilot": "^1.0.41-0", "@modelcontextprotocol/sdk": "^1.26.0", "@types/node": "^25.3.3", + "@types/node-forge": "^1.3.14", + "node-forge": "^1.4.0", "openai": "^6.17.0", "tsx": "^4.21.0", "typescript": "^5.9.3", @@ -1034,6 +1036,16 @@ "undici-types": "~7.18.0" } }, + "node_modules/@types/node-forge": { + "version": "1.3.14", + "resolved": "https://registry.npmjs.org/@types/node-forge/-/node-forge-1.3.14.tgz", + "integrity": "sha512-mhVF2BnD4BO+jtOp7z1CdzaK4mbuK0LLQYAvdOLqHTavxFNq4zA1EmYkpnFjP8HOUzedfQkRnp0E2ulSAYSzAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@vitest/expect": { "version": "4.0.18", "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.0.18.tgz", @@ -2041,6 +2053,16 @@ "node": ">= 0.6" } }, + "node_modules/node-forge": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.4.0.tgz", + "integrity": "sha512-LarFH0+6VfriEhqMMcLX2F7SwSXeWwnEAJEsYm5QKWchiVYVvJyV9v7UDvUv+w5HO23ZpQTXDv/GxdDdMyOuoQ==", + "dev": true, + "license": "(BSD-3-Clause OR GPL-2.0)", + "engines": { + "node": ">= 6.13.0" + } + }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", diff --git a/test/harness/package.json b/test/harness/package.json index 6600204ab..a0d9e5469 100644 --- a/test/harness/package.json +++ b/test/harness/package.json @@ -14,6 +14,8 @@ "@github/copilot": "^1.0.41-0", "@modelcontextprotocol/sdk": "^1.26.0", "@types/node": "^25.3.3", + "@types/node-forge": "^1.3.14", + "node-forge": "^1.4.0", "openai": "^6.17.0", "tsx": "^4.21.0", "typescript": "^5.9.3", diff --git a/test/harness/replayingCapiProxy.test.ts b/test/harness/replayingCapiProxy.test.ts index 0a57c5e01..57861dba3 100644 --- a/test/harness/replayingCapiProxy.test.ts +++ b/test/harness/replayingCapiProxy.test.ts @@ -415,6 +415,60 @@ describe("ReplayingCapiProxy", () => { expect(toolMessages[1].content).toBe("[beta result]"); }); + test("normalizes GitHub CLI proxy auth failures", async () => { + const requestBody = JSON.stringify({ + messages: [ + { role: "user", content: "Summarize this issue" }, + { + role: "assistant", + tool_calls: [ + { + id: "tc1", + type: "function", + function: { name: "web_fetch", arguments: "{}" }, + }, + ], + }, + { + role: "tool", + tool_call_id: "tc1", + content: + 'Post "https://api.github.com/graphql": tls: failed to verify certificate: x509: certificate signed by unknown authority\n', + }, + { + role: "tool", + tool_call_id: "tc1", + content: + "HTTP: 401 Requires authentication (https://api.github.com/graphql)\n", + }, + ], + }); + const responseBody = JSON.stringify({ + choices: [{ message: { role: "assistant", content: "Done" } }], + }); + + const outputPath = await createProxy([ + { url: "/chat/completions", requestBody, responseBody }, + ]); + + const result = await readYamlOutput(outputPath); + const toolMessages = result.conversations[0].messages.filter( + (m) => m.role === "tool", + ); + expect(toolMessages).toEqual([ + { + role: "tool", + tool_call_id: "toolcall_0", + content: "${gh_auth_required}\n", + }, + { + role: "tool", + tool_call_id: "toolcall_0", + content: "${gh_auth_required}\n", + }, + ]); + }); + test("ignores non-chat-completion endpoints", async () => { const outputPath = await createProxy([ { url: "/models", requestBody: "{}", responseBody: "{}" }, diff --git a/test/harness/replayingCapiProxy.ts b/test/harness/replayingCapiProxy.ts index faff7de00..534881d3b 100644 --- a/test/harness/replayingCapiProxy.ts +++ b/test/harness/replayingCapiProxy.ts @@ -25,7 +25,7 @@ export const workingDirPlaceholder = "${workdir}"; const chatCompletionEndpoint = "/chat/completions"; const shellConfig = process.platform === "win32" ? ShellConfig.powerShell : ShellConfig.bash; -const normalizedToolNames = { +const normalizedToolNames: Record = { [shellConfig.shellToolName]: "${shell}", [shellConfig.readShellToolName]: "${read_shell}", [shellConfig.writeShellToolName]: "${write_shell}", @@ -69,6 +69,7 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { * If true, cached responses are played back slowly (~ 2KiB/sec). Otherwise streaming responses are sent as fast as possible. */ slowStreaming = false; + onStopRequested?: (skipWritingCache: boolean) => Promise | void; constructor( targetUrl: string, @@ -176,7 +177,10 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { options.requestOptions.path === "/copilot-user-config" && options.requestOptions.method === "POST" ) { - const config = JSON.parse(options.body!) as { token: string; response: CopilotUserResponse }; + const config = JSON.parse(options.body!) as { + token: string; + response: CopilotUserResponse; + }; this.copilotUserByToken.set(config.token, config.response); options.onResponseStart(200, {}); options.onResponseEnd(); @@ -204,6 +208,7 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { ); options.onResponseStart(200, {}); options.onResponseEnd(); + await this.onStopRequested?.(skipWritingCache); await this.stop(skipWritingCache); process.exit(0); } @@ -218,7 +223,11 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { ); const parsedExchanges = await Promise.all( chatCompletionExchanges.map((e) => - parseHttpExchange(e.request.body, e.response?.body, e.request.headers), + parseHttpExchange( + e.request.body, + e.response?.body, + e.request.headers, + ), ), ); options.onResponseStart(200, {}); @@ -256,9 +265,22 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { // Handle /copilot_internal/user endpoint for per-session auth if (options.requestOptions.path === "/copilot_internal/user") { - const authHeader = options.requestOptions.headers?.["authorization"] as string | undefined; + const headers = options.requestOptions.headers; + const headerMap = headers as + | Record + | undefined; + const rawAuthHeader = Array.isArray(headers) + ? undefined + : (headerMap?.authorization ?? headerMap?.Authorization); + const authHeader = Array.isArray(rawAuthHeader) + ? rawAuthHeader[0] + : typeof rawAuthHeader === "string" + ? rawAuthHeader + : undefined; const token = authHeader?.replace("Bearer ", ""); - const userResponse = token ? this.copilotUserByToken.get(token) : undefined; + const userResponse = token + ? this.copilotUserByToken.get(token) + : undefined; if (userResponse) { const headers = { "content-type": "application/json", @@ -269,7 +291,9 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { options.onResponseEnd(); } else { options.onResponseStart(401, commonResponseHeaders); - options.onData(Buffer.from(JSON.stringify({ message: "Bad credentials" }))); + options.onData( + Buffer.from(JSON.stringify({ message: "Bad credentials" })), + ); options.onResponseEnd(); } return; @@ -445,7 +469,9 @@ function diagnoseMatchFailure( storedData: NormalizedData | undefined, ): string { const lines: string[] = []; - lines.push(`Request has ${requestMessages.length} normalized messages (${rawMessages.length} raw).`); + lines.push( + `Request has ${requestMessages.length} normalized messages (${rawMessages.length} raw).`, + ); if (!storedData || storedData.conversations.length === 0) { lines.push("No stored conversations to match against."); @@ -459,7 +485,7 @@ function diagnoseMatchFailure( if (requestMessages.length >= saved.length) { lines.push( `Conversation ${c} (${saved.length} messages): ` + - `skipped — request has ${requestMessages.length} messages, need fewer than ${saved.length}.`, + `skipped — request has ${requestMessages.length} messages, need fewer than ${saved.length}.`, ); continue; } @@ -474,9 +500,10 @@ function diagnoseMatchFailure( } if (mismatchIndex >= 0) { - const raw = mismatchIndex < rawMessages.length - ? JSON.stringify(rawMessages[mismatchIndex]).slice(0, 300) - : "(no raw message)"; + const raw = + mismatchIndex < rawMessages.length + ? JSON.stringify(rawMessages[mismatchIndex]).slice(0, 300) + : "(no raw message)"; lines.push( `Conversation ${c} (${saved.length} messages): mismatch at message ${mismatchIndex}:`, ` request: ${JSON.stringify(requestMessages[mismatchIndex]).slice(0, 200)}`, @@ -485,10 +512,11 @@ function diagnoseMatchFailure( ); } else { // Prefix matched, but the next saved message isn't an assistant turn - const nextRole = saved[requestMessages.length]?.role ?? "(end of conversation)"; + const nextRole = + saved[requestMessages.length]?.role ?? "(end of conversation)"; lines.push( `Conversation ${c} (${saved.length} messages): ` + - `prefix matched, but next saved message is "${nextRole}" (need "assistant").`, + `prefix matched, but next saved message is "${nextRole}" (need "assistant").`, ); } } @@ -505,28 +533,43 @@ async function exitWithNoMatchingRequestError( ) { let diagnostics: string; try { - const normalized = await parseAndNormalizeRequest(options.body, workDir, toolResultNormalizers); + const normalized = await parseAndNormalizeRequest( + options.body, + workDir, + toolResultNormalizers, + ); const requestMessages = normalized.conversations[0]?.messages ?? []; let rawMessages: unknown[] = []; try { - rawMessages = (JSON.parse(options.body ?? "{}") as { messages?: unknown[] }).messages ?? []; - } catch { /* non-JSON body */ } + rawMessages = + (JSON.parse(options.body ?? "{}") as { messages?: unknown[] }) + .messages ?? []; + } catch { + /* non-JSON body */ + } - diagnostics = diagnoseMatchFailure(requestMessages, rawMessages, storedData); + diagnostics = diagnoseMatchFailure( + requestMessages, + rawMessages, + storedData, + ); } catch (e) { diagnostics = `(unable to parse request for diagnostics: ${e})`; } - const errorMessage = - `No cached response found for ${options.requestOptions.method} ${options.requestOptions.path}.\n${diagnostics}`; + const errorMessage = `No cached response found for ${options.requestOptions.method} ${options.requestOptions.path}.\n${diagnostics}`; // Format as GitHub Actions annotation when test location is available const annotation = [ testInfo?.file ? `file=${testInfo.file}` : "", typeof testInfo?.line === "number" ? `line=${testInfo.line}` : "", - ].filter(Boolean).join(","); - process.stderr.write(`::error${annotation ? ` ${annotation}` : ""}::${errorMessage}\n`); + ] + .filter(Boolean) + .join(","); + process.stderr.write( + `::error${annotation ? ` ${annotation}` : ""}::${errorMessage}\n`, + ); options.onError(new Error(errorMessage)); } @@ -817,7 +860,11 @@ function transformOpenAIRequestMessage( // Extract and normalize text parts; represent image_url parts as a stable marker. const parts: string[] = []; for (const part of m.content) { - if (typeof part === "object" && part.type === "text" && typeof part.text === "string") { + if ( + typeof part === "object" && + part.type === "text" && + typeof part.text === "string" + ) { parts.push(normalizeUserMessage(part.text)); } else if (typeof part === "object" && part.type === "image_url") { parts.push("[image]"); @@ -836,9 +883,10 @@ function transformOpenAIRequestMessage( parsed.resultType === "success" && "textResultForLlm" in parsed ) { - content = typeof parsed.textResultForLlm === "string" - ? parsed.textResultForLlm - : JSON.stringify(sortJsonKeys(parsed.textResultForLlm)); + content = + typeof parsed.textResultForLlm === "string" + ? parsed.textResultForLlm + : JSON.stringify(sortJsonKeys(parsed.textResultForLlm)); } else { content = JSON.stringify(sortJsonKeys(parsed)); } @@ -901,6 +949,17 @@ function normalizeGhAuthMessages(result: string): string { /To get started with GitHub CLI, please run:\s*gh auth login\s*\n\s*Alternatively, populate the GH_TOKEN environment variable with a GitHub API authentication token\./g, "${gh_auth_required}", ); + // When the GitHub CLI is run under the local CONNECT proxy on Windows, it + // can try its auth probe before trusting the generated CA. This is still the + // same unauthenticated-GitHub condition from the snapshot's perspective. + normalized = normalized.replace( + /[^\n]*Post "https:\/\/api\.github\.com\/graphql": tls: failed to verify certificate: x509: certificate signed by unknown authority\s*\n/g, + "${gh_auth_required}\n", + ); + normalized = normalized.replace( + /(?:HTTP|GraphQL): 401[^\n]*Requires authentication[^\n]*\n/g, + "${gh_auth_required}\n", + ); return normalized; } diff --git a/test/harness/server.ts b/test/harness/server.ts index e6a9e4dc8..887a57178 100644 --- a/test/harness/server.ts +++ b/test/harness/server.ts @@ -3,11 +3,57 @@ *--------------------------------------------------------------------------------------------*/ import { ReplayingCapiProxy } from "./replayingCapiProxy"; +import { ConnectProxy } from "./connectProxy"; +import { createE2eRequestHandler } from "./mockHandlers"; // Starts up an instance of the ReplayingCapiProxy server // The intention is for this to be usable in E2E tests across all languages const proxy = new ReplayingCapiProxy("https://api.githubcopilot.com"); const proxyUrl = await proxy.start(); +const blockedHosts: string[] = []; +const unhandledRequests: string[] = []; -console.log(`Listening: ${proxyUrl}`); +const connectProxy = new ConnectProxy( + createE2eRequestHandler({ + capiProxyUrl: proxyUrl, + onUnhandled: (host, method, requestPath) => { + const entry = `${method} ${host}${requestPath}`; + unhandledRequests.push(entry); + console.error(`[E2E proxy] Unhandled intercepted request: ${entry}`); + }, + }), + { + interceptDomains: [ + "api.githubcopilot.com", + "api.github.com", + "github.com", + "api.mcp.github.com", + ], + passthroughDomains: ["registry.npmjs.org"], + onBlockedConnection: (host, port) => { + const entry = `${host}:${port}`; + blockedHosts.push(entry); + console.error(`[E2E proxy] Blocked connection to: ${entry}`); + }, + }, +); +await connectProxy.start(); + +proxy.onStopRequested = async () => { + if (blockedHosts.length || unhandledRequests.length) { + const details = [ + ...blockedHosts.map((host) => `blocked ${host}`), + ...unhandledRequests.map((request) => `unhandled ${request}`), + ].join(", "); + console.error(`[E2E proxy] Unexpected network activity: ${details}`); + } + await connectProxy.stop(); +}; + +console.log( + `Listening: ${proxyUrl} ${JSON.stringify({ + connectProxyUrl: connectProxy.proxyUrl, + caFilePath: connectProxy.caFilePath, + })}`, +); diff --git a/test/harness/util.ts b/test/harness/util.ts index b696e06c5..020e07658 100644 --- a/test/harness/util.ts +++ b/test/harness/util.ts @@ -2,8 +2,6 @@ * Copyright (c) Microsoft Corporation. All rights reserved. *--------------------------------------------------------------------------------------------*/ -import type { SessionOptions } from "@github/copilot/sdk"; - export function iife(fn: () => Promise): Promise { return fn(); } @@ -12,7 +10,11 @@ export function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } -type ShellConfigType = NonNullable; +type ShellConfigType = { + shellToolName: string; + readShellToolName: string; + writeShellToolName: string; +}; /** * Shell configuration for platform-specific tool names. From 9b34335dae39156f68a5a8446ceb736b38720cd4 Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Mon, 4 May 2026 16:54:29 -0400 Subject: [PATCH 2/9] Fix Python harness import order Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- python/e2e/testharness/proxy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/e2e/testharness/proxy.py b/python/e2e/testharness/proxy.py index 7d4444629..be4ef0689 100644 --- a/python/e2e/testharness/proxy.py +++ b/python/e2e/testharness/proxy.py @@ -5,11 +5,11 @@ It spawns the shared test harness server from test/harness/server.ts. """ +import json import os import platform import re import subprocess -import json from typing import Any import httpx From 85f9465e78bb37a4cc2a80a10f10e97d9cc338d5 Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Mon, 4 May 2026 17:04:56 -0400 Subject: [PATCH 3/9] Address E2E proxy review feedback Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- dotnet/test/Harness/CapiProxy.cs | 18 ++++++++--- nodejs/test/e2e/harness/CapiProxy.ts | 41 +++++++++++++++++-------- test/harness/connectProxy.test.ts | 2 +- test/harness/connectProxy.ts | 8 +++-- test/harness/replayingCapiProxy.test.ts | 2 +- test/harness/replayingCapiProxy.ts | 2 +- 6 files changed, 51 insertions(+), 22 deletions(-) diff --git a/dotnet/test/Harness/CapiProxy.cs b/dotnet/test/Harness/CapiProxy.cs index 780264095..1778c7a6a 100644 --- a/dotnet/test/Harness/CapiProxy.cs +++ b/dotnet/test/Harness/CapiProxy.cs @@ -65,11 +65,19 @@ async Task StartCoreAsync() { if (match.Groups["metadata"].Success) { - var metadata = JsonSerializer.Deserialize( - match.Groups["metadata"].Value, - CapiProxyJsonContext.Default.ProxyStartupMetadata); - ConnectProxyUrl = metadata?.ConnectProxyUrl; - CaFilePath = metadata?.CaFilePath; + try + { + var metadata = JsonSerializer.Deserialize( + match.Groups["metadata"].Value, + CapiProxyJsonContext.Default.ProxyStartupMetadata); + ConnectProxyUrl = metadata?.ConnectProxyUrl; + CaFilePath = metadata?.CaFilePath; + } + catch (Exception ex) when (ex is JsonException or NotSupportedException) + { + tcs.TrySetException(new InvalidOperationException($"Failed to parse proxy startup metadata: {match.Groups["metadata"].Value}", ex)); + return; + } } tcs.TrySetResult(match.Groups["url"].Value); } diff --git a/nodejs/test/e2e/harness/CapiProxy.ts b/nodejs/test/e2e/harness/CapiProxy.ts index d44433ea6..a5fffc37a 100644 --- a/nodejs/test/e2e/harness/CapiProxy.ts +++ b/nodejs/test/e2e/harness/CapiProxy.ts @@ -1,5 +1,6 @@ import { spawn } from "child_process"; import { resolve } from "path"; +import { createInterface } from "readline"; import { expect } from "vitest"; import { CopilotUserResponse, @@ -37,24 +38,35 @@ export class CapiProxy { }); this.startupInfo = await new Promise((resolve, reject) => { - let output = ""; + const stdout = serverProcess.stdout!; + const lines: string[] = []; + const lineReader = createInterface({ input: stdout }); const cleanup = () => { - serverProcess.stdout!.off("data", onData); + lineReader.off("line", onLine); serverProcess.off("exit", onExit); + lineReader.close(); }; - const onData = (chunk: Buffer) => { - output += chunk.toString(); - const info = tryParseStartupInfo(output); - if (info) { + const onLine = (line: string) => { + lines.push(line); + try { + const info = tryParseStartupInfo(line); + if (!info) { + return; + } cleanup(); resolve(info); + } catch (error) { + cleanup(); + reject(error); } }; const onExit = (code: number | null) => { cleanup(); - reject(new Error(`Proxy exited before startup with code ${code}: ${output}`)); + reject( + new Error(`Proxy exited before startup with code ${code}: ${lines.join("\n")}`) + ); }; - serverProcess.stdout!.on("data", onData); + lineReader.on("line", onLine); serverProcess.once("exit", onExit); }); this.proxyUrl = this.startupInfo.capiProxyUrl; @@ -127,18 +139,23 @@ export class CapiProxy { } } -function tryParseStartupInfo(output: string): ProxyStartupInfo | undefined { - const line = output.split(/\r?\n/).find((candidate) => candidate.includes("Listening: ")); +function tryParseStartupInfo(line: string): ProxyStartupInfo | undefined { if (!line) { return undefined; } - const match = line.match(/Listening: (http:\/\/[^\s]+)(?:\s+(\{.*\}))?/); + const match = line.match(/Listening: (http:\/\/[^\s]+)\s+(\{.*\})$/); if (!match) { + if (!line.includes("Listening: ")) { + return undefined; + } throw new Error(`Unexpected proxy output: ${line}`); } - const metadata = match[2] ? (JSON.parse(match[2]) as Partial) : {}; + const metadata = JSON.parse(match[2]) as Partial; + if (!metadata.connectProxyUrl || !metadata.caFilePath) { + throw new Error(`Proxy startup metadata missing CONNECT proxy details: ${line}`); + } return { capiProxyUrl: match[1], connectProxyUrl: metadata.connectProxyUrl, diff --git a/test/harness/connectProxy.test.ts b/test/harness/connectProxy.test.ts index 202e403f5..86d205dd3 100644 --- a/test/harness/connectProxy.test.ts +++ b/test/harness/connectProxy.test.ts @@ -55,7 +55,7 @@ describe("ConnectProxy", () => { await proxy.start(); expect(proxy.proxyUrl).toMatch(/^http:\/\/127\.0\.0\.1:\d+$/); - expect(proxy.caFilePath).toMatch(/test-ca\.pem$/); + expect(proxy.caFilePath).toMatch(/test-ca-bundle\.pem$/); await proxy.stop(); }); diff --git a/test/harness/connectProxy.ts b/test/harness/connectProxy.ts index 0e475075d..d5aade087 100644 --- a/test/harness/connectProxy.ts +++ b/test/harness/connectProxy.ts @@ -88,8 +88,12 @@ export class ConnectProxy { async start(): Promise { this.ca = generateCA(); const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-proxy-ca-")); - this._caFilePath = path.join(tmpDir, "test-ca.pem"); - fs.writeFileSync(this._caFilePath, this.ca.certPem); + fs.writeFileSync(path.join(tmpDir, "test-ca.pem"), this.ca.certPem); + this._caFilePath = path.join(tmpDir, "test-ca-bundle.pem"); + fs.writeFileSync( + this._caFilePath, + [...tls.rootCertificates, this.ca.certPem].join("\n"), + ); this.internalServer = http.createServer((req, res) => { const socket = req.socket as tls.TLSSocket & { _connectTarget?: string }; diff --git a/test/harness/replayingCapiProxy.test.ts b/test/harness/replayingCapiProxy.test.ts index 57861dba3..bef17d4ac 100644 --- a/test/harness/replayingCapiProxy.test.ts +++ b/test/harness/replayingCapiProxy.test.ts @@ -439,7 +439,7 @@ describe("ReplayingCapiProxy", () => { role: "tool", tool_call_id: "tc1", content: - "HTTP: 401 Requires authentication (https://api.github.com/graphql)\n", + "\u28fe\u28fdHTTP 401: Requires authentication (https://api.github.com/graphql)\nTry authenticating with: gh auth login\n", }, ], }); diff --git a/test/harness/replayingCapiProxy.ts b/test/harness/replayingCapiProxy.ts index 534881d3b..430f07a1a 100644 --- a/test/harness/replayingCapiProxy.ts +++ b/test/harness/replayingCapiProxy.ts @@ -957,7 +957,7 @@ function normalizeGhAuthMessages(result: string): string { "${gh_auth_required}\n", ); normalized = normalized.replace( - /(?:HTTP|GraphQL): 401[^\n]*Requires authentication[^\n]*\n/g, + /[^\n]*(?:HTTP|GraphQL)[\s:]+401[^\n]*Requires authentication(?:[^\n]*\r?\n)*?/g, "${gh_auth_required}\n", ); return normalized; From c3c50a498f74bdb965beb61e863326aacdd50f05 Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Mon, 4 May 2026 17:16:54 -0400 Subject: [PATCH 4/9] Tighten proxy startup metadata handling Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- dotnet/test/Harness/CapiProxy.cs | 48 ++++++++++++++++++---------- go/internal/e2e/testharness/proxy.go | 32 +++++++++++-------- python/e2e/testharness/proxy.py | 15 +++++++-- test/harness/replayingCapiProxy.ts | 37 ++++++++++++++++++--- 4 files changed, 94 insertions(+), 38 deletions(-) diff --git a/dotnet/test/Harness/CapiProxy.cs b/dotnet/test/Harness/CapiProxy.cs index 1778c7a6a..274055540 100644 --- a/dotnet/test/Harness/CapiProxy.cs +++ b/dotnet/test/Harness/CapiProxy.cs @@ -60,27 +60,41 @@ async Task StartCoreAsync() _process.OutputDataReceived += (_, e) => { if (e.Data == null) return; - var match = Regex.Match(e.Data, @"Listening: (?http://[^\s]+)(?:\s+(?\{.*\}))?"); - if (match.Success) + var match = Regex.Match(e.Data, @"Listening: (?http://[^\s]+)\s+(?\{.*\})$"); + if (!match.Success) { - if (match.Groups["metadata"].Success) + if (e.Data.Contains("Listening: ", StringComparison.Ordinal)) { - try - { - var metadata = JsonSerializer.Deserialize( - match.Groups["metadata"].Value, - CapiProxyJsonContext.Default.ProxyStartupMetadata); - ConnectProxyUrl = metadata?.ConnectProxyUrl; - CaFilePath = metadata?.CaFilePath; - } - catch (Exception ex) when (ex is JsonException or NotSupportedException) - { - tcs.TrySetException(new InvalidOperationException($"Failed to parse proxy startup metadata: {match.Groups["metadata"].Value}", ex)); - return; - } + tcs.TrySetException( + new InvalidOperationException( + $"Proxy startup line missing CONNECT proxy metadata: {e.Data}")); } - tcs.TrySetResult(match.Groups["url"].Value); + return; } + try + { + var metadata = JsonSerializer.Deserialize( + match.Groups["metadata"].Value, + CapiProxyJsonContext.Default.ProxyStartupMetadata); + ConnectProxyUrl = metadata?.ConnectProxyUrl; + CaFilePath = metadata?.CaFilePath; + } + catch (Exception ex) when (ex is JsonException or NotSupportedException) + { + tcs.TrySetException( + new InvalidOperationException( + $"Failed to parse proxy startup metadata: {match.Groups["metadata"].Value}", + ex)); + return; + } + if (string.IsNullOrEmpty(ConnectProxyUrl) || string.IsNullOrEmpty(CaFilePath)) + { + tcs.TrySetException( + new InvalidOperationException( + $"Proxy startup metadata missing CONNECT proxy details: {e.Data}")); + return; + } + tcs.TrySetResult(match.Groups["url"].Value); }; _process.ErrorDataReceived += (_, e) => diff --git a/go/internal/e2e/testharness/proxy.go b/go/internal/e2e/testharness/proxy.go index 03ddecfbb..e407f13e0 100644 --- a/go/internal/e2e/testharness/proxy.go +++ b/go/internal/e2e/testharness/proxy.go @@ -59,7 +59,7 @@ func (p *CapiProxy) Start() (string, error) { // Read until the server prints "Listening: http://..."; npm/npx may emit // wrapper output first on some platforms. reader := bufio.NewReader(stdout) - re := regexp.MustCompile(`Listening: (http://[^\s]+)(?:\s+(\{.*\}))?`) + re := regexp.MustCompile(`Listening: (http://[^\s]+)\s+(\{.*\})$`) var matches []string var line string for { @@ -70,9 +70,13 @@ func (p *CapiProxy) Start() (string, error) { } line = strings.TrimSpace(nextLine) matches = re.FindStringSubmatch(line) - if len(matches) >= 2 { + if len(matches) >= 3 { break } + if strings.Contains(line, "Listening: ") { + p.cmd.Process.Kill() + return "", fmt.Errorf("proxy startup line missing CONNECT proxy metadata: %s", line) + } if err == io.EOF { p.cmd.Process.Kill() return "", fmt.Errorf("proxy exited before startup; last output: %s", line) @@ -80,17 +84,19 @@ func (p *CapiProxy) Start() (string, error) { } p.proxyURL = matches[1] - if len(matches) >= 3 && matches[2] != "" { - var metadata struct { - ConnectProxyURL string `json:"connectProxyUrl"` - CAFilePath string `json:"caFilePath"` - } - if err := json.Unmarshal([]byte(matches[2]), &metadata); err != nil { - p.cmd.Process.Kill() - return "", fmt.Errorf("failed to parse proxy startup metadata: %w", err) - } - p.connectProxyURL = metadata.ConnectProxyURL - p.caFilePath = metadata.CAFilePath + var metadata struct { + ConnectProxyURL string `json:"connectProxyUrl"` + CAFilePath string `json:"caFilePath"` + } + if err := json.Unmarshal([]byte(matches[2]), &metadata); err != nil { + p.cmd.Process.Kill() + return "", fmt.Errorf("failed to parse proxy startup metadata: %w", err) + } + p.connectProxyURL = metadata.ConnectProxyURL + p.caFilePath = metadata.CAFilePath + if p.connectProxyURL == "" || p.caFilePath == "" { + p.cmd.Process.Kill() + return "", fmt.Errorf("proxy startup metadata missing CONNECT proxy details: %s", line) } return p.proxyURL, nil } diff --git a/python/e2e/testharness/proxy.py b/python/e2e/testharness/proxy.py index be4ef0689..58584b831 100644 --- a/python/e2e/testharness/proxy.py +++ b/python/e2e/testharness/proxy.py @@ -62,10 +62,19 @@ async def start(self) -> str: self._proxy_url = match.group(1) metadata_match = re.search(r"(\{.*\})\s*$", line.strip()) - if metadata_match: + if not metadata_match: + self._process.kill() + raise RuntimeError(f"Proxy startup line missing CONNECT proxy metadata: {line}") + try: metadata = json.loads(metadata_match.group(1)) - self._connect_proxy_url = metadata.get("connectProxyUrl") - self._ca_file_path = metadata.get("caFilePath") + except json.JSONDecodeError as exc: + self._process.kill() + raise RuntimeError(f"Failed to parse proxy startup metadata: {line}") from exc + self._connect_proxy_url = metadata.get("connectProxyUrl") + self._ca_file_path = metadata.get("caFilePath") + if not self._connect_proxy_url or not self._ca_file_path: + self._process.kill() + raise RuntimeError(f"Proxy startup metadata missing CONNECT proxy details: {line}") return self._proxy_url async def stop(self, skip_writing_cache: bool = False): diff --git a/test/harness/replayingCapiProxy.ts b/test/harness/replayingCapiProxy.ts index 430f07a1a..4d5832535 100644 --- a/test/harness/replayingCapiProxy.ts +++ b/test/harness/replayingCapiProxy.ts @@ -956,11 +956,38 @@ function normalizeGhAuthMessages(result: string): string { /[^\n]*Post "https:\/\/api\.github\.com\/graphql": tls: failed to verify certificate: x509: certificate signed by unknown authority\s*\n/g, "${gh_auth_required}\n", ); - normalized = normalized.replace( - /[^\n]*(?:HTTP|GraphQL)[\s:]+401[^\n]*Requires authentication(?:[^\n]*\r?\n)*?/g, - "${gh_auth_required}\n", - ); - return normalized; + return normalizeGh401AuthMessages(normalized); +} + +function normalizeGh401AuthMessages(result: string): string { + const lines = result.split(/\r?\n/); + const normalizedLines: string[] = []; + let changed = false; + + for (let i = 0; i < lines.length; i++) { + if ( + /(?:HTTP|GraphQL)[ \t:]+401/.test(lines[i]) && + lines[i].includes("Requires authentication") + ) { + let replaced = false; + for (let j = i + 1; j < lines.length; j++) { + if (/^$/.test(lines[j].trim())) { + normalizedLines.push("${gh_auth_required}"); + normalizedLines.push(""); + i = j; + changed = true; + replaced = true; + break; + } + } + if (replaced) { + continue; + } + } + normalizedLines.push(lines[i]); + } + + return changed ? normalizedLines.join("\n") : result; } // Transforms a single OpenAI-style inbound response message into normalized form From 8ebc51a6ebdda12b5bd291f624bd0c689c2b6d18 Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Mon, 4 May 2026 18:32:44 -0400 Subject: [PATCH 5/9] Fix extension test proxy environment merge Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- dotnet/test/E2E/RpcExtensionsLoadedE2ETests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotnet/test/E2E/RpcExtensionsLoadedE2ETests.cs b/dotnet/test/E2E/RpcExtensionsLoadedE2ETests.cs index dce6d4f1e..a7c3cbf6c 100644 --- a/dotnet/test/E2E/RpcExtensionsLoadedE2ETests.cs +++ b/dotnet/test/E2E/RpcExtensionsLoadedE2ETests.cs @@ -40,7 +40,7 @@ public class RpcExtensionsLoadedE2ETests(E2ETestFixture fixture, ITestOutputHelp /// private Dictionary ExtensionsEnabledEnvironment() { - var env = new Dictionary(Ctx.GetEnvironment(), StringComparer.OrdinalIgnoreCase) + var env = new Dictionary(Ctx.GetEnvironment()) { ["COPILOT_CLI_ENABLED_FEATURE_FLAGS"] = "EXTENSIONS", }; From c00300bc5cfcde7d532d25d3bdd099609484237d Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Mon, 4 May 2026 18:49:16 -0400 Subject: [PATCH 6/9] Normalize parallel tool result order Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- test/harness/replayingCapiProxy.test.ts | 102 ++++++++++++++++++++++++ test/harness/replayingCapiProxy.ts | 46 +++++++++++ 2 files changed, 148 insertions(+) diff --git a/test/harness/replayingCapiProxy.test.ts b/test/harness/replayingCapiProxy.test.ts index bef17d4ac..b5e3ff4cc 100644 --- a/test/harness/replayingCapiProxy.test.ts +++ b/test/harness/replayingCapiProxy.test.ts @@ -730,6 +730,108 @@ describe("ReplayingCapiProxy", () => { } }); + test("matches parallel tool results regardless of arrival order", async () => { + const cachePath = path.join(tempDir, "cache.yaml"); + const cacheContent = yaml.stringify({ + models: ["test-model"], + conversations: [ + { + messages: [ + { role: "system", content: "${system}" }, + { role: "user", content: "Lookup city and country" }, + { + role: "assistant", + tool_calls: [ + { + id: "toolcall_0", + type: "function", + function: { + name: "lookup_city", + arguments: '{"city":"Paris"}', + }, + }, + { + id: "toolcall_1", + type: "function", + function: { + name: "lookup_country", + arguments: '{"country":"France"}', + }, + }, + ], + }, + { + role: "tool", + tool_call_id: "toolcall_0", + content: "CITY_PARIS", + }, + { + role: "tool", + tool_call_id: "toolcall_1", + content: "COUNTRY_FRANCE", + }, + { role: "assistant", content: "Paris is in France." }, + ], + }, + ], + } satisfies NormalizedData); + await writeFile(cachePath, cacheContent); + + const proxy = new ReplayingCapiProxy( + "http://localhost:9999", + cachePath, + workDir, + ); + const proxyUrl = await proxy.start(); + + try { + const response = await makeRequest(proxyUrl, "/chat/completions", { + body: { + model: "test-model", + messages: [ + { role: "system", content: "Be helpful" }, + { role: "user", content: "Lookup city and country" }, + { + role: "assistant", + tool_calls: [ + { + id: "city-id", + type: "function", + function: { + name: "lookup_city", + arguments: '{"city":"Paris"}', + }, + }, + { + id: "country-id", + type: "function", + function: { + name: "lookup_country", + arguments: '{"country":"France"}', + }, + }, + ], + }, + { + role: "tool", + tool_call_id: "country-id", + content: "COUNTRY_FRANCE", + }, + { role: "tool", tool_call_id: "city-id", content: "CITY_PARIS" }, + ], + }, + }); + + expect(response.status).toBe(200); + expect( + (JSON.parse(response.body) as ChatCompletion).choices[0].message + .content, + ).toBe("Paris is in France."); + } finally { + await proxy.stop(); + } + }); + test("returns streaming response when stream: true", async () => { const cachePath = path.join(tempDir, "cache.yaml"); const cacheContent = yaml.stringify({ diff --git a/test/harness/replayingCapiProxy.ts b/test/harness/replayingCapiProxy.ts index 4d5832535..1cfaf89e7 100644 --- a/test/harness/replayingCapiProxy.ts +++ b/test/harness/replayingCapiProxy.ts @@ -677,6 +677,7 @@ async function transformHttpExchanges( ); normalizeToolCalls(dedupedExchanges, toolResultNormalizers); + normalizeToolResultOrder(dedupedExchanges); normalizeFilenames(dedupedExchanges, workDir); return { models: Array.from(dedupedModels), conversations: dedupedExchanges }; } @@ -782,6 +783,51 @@ function normalizeToolCalls( } } +function normalizeToolResultOrder(conversations: NormalizedConversation[]) { + for (const conv of conversations) { + for (let start = 0; start < conv.messages.length; ) { + if (conv.messages[start].role !== "tool") { + start++; + continue; + } + + let end = start + 1; + while (end < conv.messages.length && conv.messages[end].role === "tool") { + end++; + } + + conv.messages + .slice(start, end) + .sort(compareToolResultMessages) + .forEach((message, index) => { + conv.messages[start + index] = message; + }); + start = end; + } + } +} + +function compareToolResultMessages( + left: NormalizedMessage, + right: NormalizedMessage, +) { + return compareToolCallIds(left.tool_call_id, right.tool_call_id); +} + +function compareToolCallIds(left?: string, right?: string) { + const leftNumber = parseNormalizedToolCallId(left); + const rightNumber = parseNormalizedToolCallId(right); + if (leftNumber !== undefined && rightNumber !== undefined) { + return leftNumber - rightNumber; + } + return (left ?? "").localeCompare(right ?? ""); +} + +function parseNormalizedToolCallId(id?: string) { + const match = id?.match(/^toolcall_(\d+)$/); + return match ? Number(match[1]) : undefined; +} + // As we capture LLM calls, we see: // - Request A, response AB // - Request ABC, response ABCD From 3781929783bf4c5c1820c6321831a778b9ac1741 Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Mon, 4 May 2026 18:57:17 -0400 Subject: [PATCH 7/9] Normalize loaded tool result snapshots Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- test/harness/replayingCapiProxy.test.ts | 8 ++++---- test/harness/replayingCapiProxy.ts | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/test/harness/replayingCapiProxy.test.ts b/test/harness/replayingCapiProxy.test.ts index b5e3ff4cc..c7abf01f2 100644 --- a/test/harness/replayingCapiProxy.test.ts +++ b/test/harness/replayingCapiProxy.test.ts @@ -762,13 +762,13 @@ describe("ReplayingCapiProxy", () => { }, { role: "tool", - tool_call_id: "toolcall_0", - content: "CITY_PARIS", + tool_call_id: "toolcall_1", + content: "COUNTRY_FRANCE", }, { role: "tool", - tool_call_id: "toolcall_1", - content: "COUNTRY_FRANCE", + tool_call_id: "toolcall_0", + content: "CITY_PARIS", }, { role: "assistant", content: "Paris is in France." }, ], diff --git a/test/harness/replayingCapiProxy.ts b/test/harness/replayingCapiProxy.ts index 1cfaf89e7..cd6399922 100644 --- a/test/harness/replayingCapiProxy.ts +++ b/test/harness/replayingCapiProxy.ts @@ -128,6 +128,7 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { if (this.state && existsSync(this.state.filePath)) { const content = await readFile(this.state.filePath, "utf-8"); this.state.storedData = yaml.parse(content) as NormalizedData; + normalizeToolResultOrder(this.state.storedData.conversations); } } From dcb7c563167980a9f09abbd058a8e6ef7050f77a Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Mon, 4 May 2026 19:10:58 -0400 Subject: [PATCH 8/9] Add background agent continuation snapshot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...tart_background_agent_and_report_task_details.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/snapshots/rpc_tasks_and_handlers/should_start_background_agent_and_report_task_details.yaml b/test/snapshots/rpc_tasks_and_handlers/should_start_background_agent_and_report_task_details.yaml index 01c49201f..85b0c767e 100644 --- a/test/snapshots/rpc_tasks_and_handlers/should_start_background_agent_and_report_task_details.yaml +++ b/test/snapshots/rpc_tasks_and_handlers/should_start_background_agent_and_report_task_details.yaml @@ -15,3 +15,14 @@ conversations: content: Reply with TASK_AGENT_DONE exactly. - role: assistant content: TASK_AGENT_DONE + - messages: + - role: system + content: ${system} + - role: user + content: Reply with TASK_AGENT_READY exactly. + - role: assistant + content: TASK_AGENT_READY + - role: user + content: Reply with TASK_AGENT_DONE exactly. + - role: assistant + content: TASK_AGENT_DONE From e057d6e9a1204ad6e8e02fca7828a9e9ba4c581a Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Mon, 4 May 2026 19:23:57 -0400 Subject: [PATCH 9/9] Add background agent notification snapshot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...t_background_agent_and_report_task_details.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/snapshots/rpc_tasks_and_handlers/should_start_background_agent_and_report_task_details.yaml b/test/snapshots/rpc_tasks_and_handlers/should_start_background_agent_and_report_task_details.yaml index 85b0c767e..41bbe583d 100644 --- a/test/snapshots/rpc_tasks_and_handlers/should_start_background_agent_and_report_task_details.yaml +++ b/test/snapshots/rpc_tasks_and_handlers/should_start_background_agent_and_report_task_details.yaml @@ -15,6 +15,20 @@ conversations: content: Reply with TASK_AGENT_DONE exactly. - role: assistant content: TASK_AGENT_DONE + - messages: + - role: system + content: ${system} + - role: user + content: Reply with TASK_AGENT_READY exactly. + - role: assistant + content: TASK_AGENT_READY + - role: user + content: |- + + Agent "sdk-background-agent" (general-purpose) has completed successfully. Use read_agent with agent_id "sdk-background-agent" to retrieve the full results. + + - role: assistant + content: TASK_AGENT_DONE - messages: - role: system content: ${system}