diff --git a/dotnet/test/E2E/RpcExtensionsLoadedE2ETests.cs b/dotnet/test/E2E/RpcExtensionsLoadedE2ETests.cs index dce6d4f1e..a7c3cbf6c 100644 --- a/dotnet/test/E2E/RpcExtensionsLoadedE2ETests.cs +++ b/dotnet/test/E2E/RpcExtensionsLoadedE2ETests.cs @@ -40,7 +40,7 @@ public class RpcExtensionsLoadedE2ETests(E2ETestFixture fixture, ITestOutputHelp /// private Dictionary ExtensionsEnabledEnvironment() { - var env = new Dictionary(Ctx.GetEnvironment(), StringComparer.OrdinalIgnoreCase) + var env = new Dictionary(Ctx.GetEnvironment()) { ["COPILOT_CLI_ENABLED_FEATURE_FLAGS"] = "EXTENSIONS", }; diff --git a/dotnet/test/Harness/CapiProxy.cs b/dotnet/test/Harness/CapiProxy.cs index f863b651c..274055540 100644 --- a/dotnet/test/Harness/CapiProxy.cs +++ b/dotnet/test/Harness/CapiProxy.cs @@ -17,6 +17,9 @@ public sealed partial class CapiProxy : IAsyncDisposable private Process? _process; private Task? _startupTask; + public string? ConnectProxyUrl { get; private set; } + public string? CaFilePath { get; private set; } + public Task StartAsync() { return _startupTask ??= StartCoreAsync(); @@ -57,8 +60,41 @@ async Task StartCoreAsync() _process.OutputDataReceived += (_, e) => { if (e.Data == null) return; - var match = Regex.Match(e.Data, @"Listening: (http://[^\s]+)"); - if (match.Success) tcs.TrySetResult(match.Groups[1].Value); + var match = Regex.Match(e.Data, @"Listening: (?http://[^\s]+)\s+(?\{.*\})$"); + if (!match.Success) + { + if (e.Data.Contains("Listening: ", StringComparison.Ordinal)) + { + tcs.TrySetException( + new InvalidOperationException( + $"Proxy startup line missing CONNECT proxy metadata: {e.Data}")); + } + return; + } + try + { + var metadata = JsonSerializer.Deserialize( + match.Groups["metadata"].Value, + CapiProxyJsonContext.Default.ProxyStartupMetadata); + ConnectProxyUrl = metadata?.ConnectProxyUrl; + CaFilePath = metadata?.CaFilePath; + } + catch (Exception ex) when (ex is JsonException or NotSupportedException) + { + tcs.TrySetException( + new InvalidOperationException( + $"Failed to parse proxy startup metadata: {match.Groups["metadata"].Value}", + ex)); + return; + } + if (string.IsNullOrEmpty(ConnectProxyUrl) || string.IsNullOrEmpty(CaFilePath)) + { + tcs.TrySetException( + new InvalidOperationException( + $"Proxy startup metadata missing CONNECT proxy details: {e.Data}")); + return; + } + tcs.TrySetResult(match.Groups["url"].Value); }; _process.ErrorDataReceived += (_, e) => @@ -124,6 +160,8 @@ public async Task ConfigureAsync(string filePath, string workDir) private record ConfigureRequest(string FilePath, string WorkDir); + private record ProxyStartupMetadata(string? ConnectProxyUrl, string? CaFilePath); + public async Task> GetExchangesAsync() { var url = await (_startupTask ?? throw new InvalidOperationException("Proxy not started")); @@ -165,6 +203,7 @@ private static string FindRepoRoot() [JsonSerializable(typeof(List))] [JsonSerializable(typeof(CopilotUserByTokenRequest))] [JsonSerializable(typeof(Dictionary))] + [JsonSerializable(typeof(ProxyStartupMetadata))] private partial class CapiProxyJsonContext : JsonSerializerContext; } diff --git a/dotnet/test/Harness/E2ETestContext.cs b/dotnet/test/Harness/E2ETestContext.cs index 88627ba6d..19777e09b 100644 --- a/dotnet/test/Harness/E2ETestContext.cs +++ b/dotnet/test/Harness/E2ETestContext.cs @@ -166,8 +166,33 @@ public IReadOnlyDictionary GetEnvironment() env["COPILOT_API_URL"] = ProxyUrl; env["COPILOT_HOME"] = HomeDir; + env["GH_CONFIG_DIR"] = HomeDir; env["XDG_CONFIG_HOME"] = HomeDir; env["XDG_STATE_HOME"] = HomeDir; + if (!string.IsNullOrEmpty(_proxy.ConnectProxyUrl) && !string.IsNullOrEmpty(_proxy.CaFilePath)) + { + const string noProxy = "127.0.0.1,localhost,::1"; + env["HTTP_PROXY"] = _proxy.ConnectProxyUrl; + env["HTTPS_PROXY"] = _proxy.ConnectProxyUrl; + env["http_proxy"] = _proxy.ConnectProxyUrl; + env["https_proxy"] = _proxy.ConnectProxyUrl; + env["NO_PROXY"] = noProxy; + env["no_proxy"] = noProxy; + env["NODE_EXTRA_CA_CERTS"] = _proxy.CaFilePath; + env["SSL_CERT_FILE"] = _proxy.CaFilePath; + env["REQUESTS_CA_BUNDLE"] = _proxy.CaFilePath; + env["CURL_CA_BUNDLE"] = _proxy.CaFilePath; + env["GIT_SSL_CAINFO"] = _proxy.CaFilePath; + env["GH_TOKEN"] = ""; + env["GITHUB_TOKEN"] = ""; + env["GH_ENTERPRISE_TOKEN"] = ""; + env["GITHUB_ENTERPRISE_TOKEN"] = ""; + } + if (Environment.GetEnvironmentVariable("GITHUB_ACTIONS") == "true") + { + env["GH_TOKEN"] = "fake-token-for-e2e-tests"; + env["GITHUB_TOKEN"] = "fake-token-for-e2e-tests"; + } return env!; } diff --git a/go/internal/e2e/testharness/context.go b/go/internal/e2e/testharness/context.go index bf6f160df..e8efda82f 100644 --- a/go/internal/e2e/testharness/context.go +++ b/go/internal/e2e/testharness/context.go @@ -163,12 +163,20 @@ func (c *TestContext) Env() []string { env := os.Environ() // Add overrides (later values take precedence in most systems) + env = append(env, c.proxy.ProxyEnv()...) env = append(env, "COPILOT_API_URL="+c.ProxyURL, "COPILOT_HOME="+c.HomeDir, + "GH_CONFIG_DIR="+c.HomeDir, "XDG_CONFIG_HOME="+c.HomeDir, "XDG_STATE_HOME="+c.HomeDir, ) + if os.Getenv("GITHUB_ACTIONS") == "true" { + env = append(env, + "GH_TOKEN=fake-token-for-e2e-tests", + "GITHUB_TOKEN=fake-token-for-e2e-tests", + ) + } return env } diff --git a/go/internal/e2e/testharness/proxy.go b/go/internal/e2e/testharness/proxy.go index 4fb98e98d..e407f13e0 100644 --- a/go/internal/e2e/testharness/proxy.go +++ b/go/internal/e2e/testharness/proxy.go @@ -17,9 +17,11 @@ import ( // CapiProxy manages a child process that acts as a replaying proxy to AI endpoints. // It spawns the shared test harness server from test/harness/server.ts. type CapiProxy struct { - cmd *exec.Cmd - proxyURL string - mu sync.Mutex + cmd *exec.Cmd + proxyURL string + connectProxyURL string + caFilePath string + mu sync.Mutex } // NewCapiProxy creates a new proxy instance. @@ -54,23 +56,48 @@ func (p *CapiProxy) Start() (string, error) { return "", fmt.Errorf("failed to start proxy server: %w", err) } - // Read the first line to get the listening URL + // Read until the server prints "Listening: http://..."; npm/npx may emit + // wrapper output first on some platforms. reader := bufio.NewReader(stdout) - line, err := reader.ReadString('\n') - if err != nil && err != io.EOF { - p.cmd.Process.Kill() - return "", fmt.Errorf("failed to read proxy URL: %w", err) + re := regexp.MustCompile(`Listening: (http://[^\s]+)\s+(\{.*\})$`) + var matches []string + var line string + for { + nextLine, err := reader.ReadString('\n') + if err != nil && err != io.EOF { + p.cmd.Process.Kill() + return "", fmt.Errorf("failed to read proxy URL: %w", err) + } + line = strings.TrimSpace(nextLine) + matches = re.FindStringSubmatch(line) + if len(matches) >= 3 { + break + } + if strings.Contains(line, "Listening: ") { + p.cmd.Process.Kill() + return "", fmt.Errorf("proxy startup line missing CONNECT proxy metadata: %s", line) + } + if err == io.EOF { + p.cmd.Process.Kill() + return "", fmt.Errorf("proxy exited before startup; last output: %s", line) + } } - // Parse "Listening: http://..." from output - re := regexp.MustCompile(`Listening: (http://[^\s]+)`) - matches := re.FindStringSubmatch(strings.TrimSpace(line)) - if len(matches) < 2 { + p.proxyURL = matches[1] + var metadata struct { + ConnectProxyURL string `json:"connectProxyUrl"` + CAFilePath string `json:"caFilePath"` + } + if err := json.Unmarshal([]byte(matches[2]), &metadata); err != nil { p.cmd.Process.Kill() - return "", fmt.Errorf("unexpected proxy output: %s", line) + return "", fmt.Errorf("failed to parse proxy startup metadata: %w", err) + } + p.connectProxyURL = metadata.ConnectProxyURL + p.caFilePath = metadata.CAFilePath + if p.connectProxyURL == "" || p.caFilePath == "" { + p.cmd.Process.Kill() + return "", fmt.Errorf("proxy startup metadata missing CONNECT proxy details: %s", line) } - - p.proxyURL = matches[1] return p.proxyURL, nil } @@ -254,6 +281,34 @@ func (p *CapiProxy) URL() string { return p.proxyURL } +// ProxyEnv returns environment variables that route HTTPS traffic through the CONNECT proxy. +func (p *CapiProxy) ProxyEnv() []string { + p.mu.Lock() + defer p.mu.Unlock() + if p.connectProxyURL == "" || p.caFilePath == "" { + return nil + } + + noProxy := "127.0.0.1,localhost,::1" + return []string{ + "HTTP_PROXY=" + p.connectProxyURL, + "HTTPS_PROXY=" + p.connectProxyURL, + "http_proxy=" + p.connectProxyURL, + "https_proxy=" + p.connectProxyURL, + "NO_PROXY=" + noProxy, + "no_proxy=" + noProxy, + "NODE_EXTRA_CA_CERTS=" + p.caFilePath, + "SSL_CERT_FILE=" + p.caFilePath, + "REQUESTS_CA_BUNDLE=" + p.caFilePath, + "CURL_CA_BUNDLE=" + p.caFilePath, + "GIT_SSL_CAINFO=" + p.caFilePath, + "GH_TOKEN=", + "GITHUB_TOKEN=", + "GH_ENTERPRISE_TOKEN=", + "GITHUB_ENTERPRISE_TOKEN=", + } +} + // SetCopilotUserByToken registers a per-token user configuration on the proxy. func (p *CapiProxy) SetCopilotUserByToken(token string, response map[string]interface{}) error { p.mu.Lock() diff --git a/nodejs/test/e2e/harness/CapiProxy.ts b/nodejs/test/e2e/harness/CapiProxy.ts index eace18739..a5fffc37a 100644 --- a/nodejs/test/e2e/harness/CapiProxy.ts +++ b/nodejs/test/e2e/harness/CapiProxy.ts @@ -1,5 +1,6 @@ import { spawn } from "child_process"; import { resolve } from "path"; +import { createInterface } from "readline"; import { expect } from "vitest"; import { CopilotUserResponse, @@ -7,10 +8,18 @@ import { } from "../../../../test/harness/replayingCapiProxy"; const HARNESS_SERVER_PATH = resolve(__dirname, "../../../../test/harness/server.ts"); +const NO_PROXY = "127.0.0.1,localhost,::1"; + +interface ProxyStartupInfo { + capiProxyUrl: string; + connectProxyUrl?: string; + caFilePath?: string; +} // Manages a child process that acts as a replaying proxy to the underlying AI endpoints export class CapiProxy { private proxyUrl: string | undefined; + private startupInfo: ProxyStartupInfo | undefined; /** * Returns the URL of the running proxy. Throws if the proxy has not been started. @@ -28,16 +37,67 @@ export class CapiProxy { shell: true, }); - this.proxyUrl = await new Promise((resolve) => { - serverProcess.stdout!.once("data", (chunk: Buffer) => { - const match = chunk.toString().match(/Listening: (http:\/\/[^\s]+)/); - resolve(match![1]); - }); + this.startupInfo = await new Promise((resolve, reject) => { + const stdout = serverProcess.stdout!; + const lines: string[] = []; + const lineReader = createInterface({ input: stdout }); + const cleanup = () => { + lineReader.off("line", onLine); + serverProcess.off("exit", onExit); + lineReader.close(); + }; + const onLine = (line: string) => { + lines.push(line); + try { + const info = tryParseStartupInfo(line); + if (!info) { + return; + } + cleanup(); + resolve(info); + } catch (error) { + cleanup(); + reject(error); + } + }; + const onExit = (code: number | null) => { + cleanup(); + reject( + new Error(`Proxy exited before startup with code ${code}: ${lines.join("\n")}`) + ); + }; + lineReader.on("line", onLine); + serverProcess.once("exit", onExit); }); + this.proxyUrl = this.startupInfo.capiProxyUrl; return this.proxyUrl; } + getProxyEnv(): Record { + if (!this.startupInfo?.connectProxyUrl || !this.startupInfo.caFilePath) { + return {}; + } + + return { + HTTP_PROXY: this.startupInfo.connectProxyUrl, + HTTPS_PROXY: this.startupInfo.connectProxyUrl, + http_proxy: this.startupInfo.connectProxyUrl, + https_proxy: this.startupInfo.connectProxyUrl, + NO_PROXY, + no_proxy: NO_PROXY, + NODE_EXTRA_CA_CERTS: this.startupInfo.caFilePath, + SSL_CERT_FILE: this.startupInfo.caFilePath, + REQUESTS_CA_BUNDLE: this.startupInfo.caFilePath, + CURL_CA_BUNDLE: this.startupInfo.caFilePath, + GIT_SSL_CAINFO: this.startupInfo.caFilePath, + GH_TOKEN: "", + GITHUB_TOKEN: "", + GH_ENTERPRISE_TOKEN: "", + GITHUB_ENTERPRISE_TOKEN: "", + }; + } + async updateConfig(config: { filePath: string; workDir: string; @@ -78,3 +138,27 @@ export class CapiProxy { expect(res.ok).toBe(true); } } + +function tryParseStartupInfo(line: string): ProxyStartupInfo | undefined { + if (!line) { + return undefined; + } + + const match = line.match(/Listening: (http:\/\/[^\s]+)\s+(\{.*\})$/); + if (!match) { + if (!line.includes("Listening: ")) { + return undefined; + } + throw new Error(`Unexpected proxy output: ${line}`); + } + + const metadata = JSON.parse(match[2]) as Partial; + if (!metadata.connectProxyUrl || !metadata.caFilePath) { + throw new Error(`Proxy startup metadata missing CONNECT proxy details: ${line}`); + } + return { + capiProxyUrl: match[1], + connectProxyUrl: metadata.connectProxyUrl, + caFilePath: metadata.caFilePath, + }; +} diff --git a/nodejs/test/e2e/harness/sdkTestContext.ts b/nodejs/test/e2e/harness/sdkTestContext.ts index c68bc6f86..7fe2b9cc7 100644 --- a/nodejs/test/e2e/harness/sdkTestContext.ts +++ b/nodejs/test/e2e/harness/sdkTestContext.ts @@ -37,8 +37,10 @@ export async function createSdkTestContext({ const proxyUrl = await openAiEndpoint.start(); const env = { ...process.env, + ...openAiEndpoint.getProxyEnv(), COPILOT_API_URL: proxyUrl, COPILOT_HOME: copilotHomeDir, + GH_CONFIG_DIR: homeDir, // TODO: I'm not convinced the SDK should default to using whatever config you happen to have in your homedir. // The SDK config should be independent of the regular CLI app. Likewise it shouldn't mix sessions from the @@ -46,6 +48,10 @@ export async function createSdkTestContext({ XDG_CONFIG_HOME: homeDir, XDG_STATE_HOME: homeDir, }; + if (isCI) { + env.GH_TOKEN = "fake-token-for-e2e-tests"; + env.GITHUB_TOKEN = "fake-token-for-e2e-tests"; + } const copilotClient = new CopilotClient({ cwd: workDir, diff --git a/python/e2e/testharness/context.py b/python/e2e/testharness/context.py index 5bb7b326b..dae71d99e 100644 --- a/python/e2e/testharness/context.py +++ b/python/e2e/testharness/context.py @@ -128,15 +128,21 @@ async def configure_for_test(self, test_file: str, test_name: str): def get_env(self) -> dict: """Return environment variables configured for isolated testing.""" env = os.environ.copy() + if self._proxy: + env.update(self._proxy.get_proxy_env()) env.update( { "COPILOT_API_URL": self.proxy_url, "COPILOT_HOME": self.home_dir, + "GH_CONFIG_DIR": self.home_dir, "XDG_CONFIG_HOME": self.home_dir, "XDG_STATE_HOME": self.home_dir, } ) + if os.environ.get("GITHUB_ACTIONS") == "true": + env["GH_TOKEN"] = "fake-token-for-e2e-tests" + env["GITHUB_TOKEN"] = "fake-token-for-e2e-tests" return env @property diff --git a/python/e2e/testharness/proxy.py b/python/e2e/testharness/proxy.py index e125375e0..58584b831 100644 --- a/python/e2e/testharness/proxy.py +++ b/python/e2e/testharness/proxy.py @@ -5,6 +5,7 @@ It spawns the shared test harness server from test/harness/server.ts. """ +import json import os import platform import re @@ -20,6 +21,8 @@ class CapiProxy: def __init__(self): self._process: subprocess.Popen | None = None self._proxy_url: str | None = None + self._connect_proxy_url: str | None = None + self._ca_file_path: str | None = None async def start(self) -> str: """Launch the proxy server and return its URL.""" @@ -44,19 +47,34 @@ async def start(self) -> str: shell=use_shell, ) - # Read the first line to get the listening URL - line = self._process.stdout.readline() - if not line: - self._process.kill() - raise RuntimeError("Failed to read proxy URL") - - # Parse "Listening: http://..." from output - match = re.search(r"Listening: (http://[^\s]+)", line.strip()) - if not match: - self._process.kill() - raise RuntimeError(f"Unexpected proxy output: {line}") + # Read until the server prints "Listening: http://..."; npm/npx may emit + # wrapper output first on some platforms. + line = "" + match = None + while True: + line = self._process.stdout.readline() + if not line: + self._process.kill() + raise RuntimeError("Failed to read proxy URL") + match = re.search(r"Listening: (http://[^\s]+)", line.strip()) + if match: + break self._proxy_url = match.group(1) + metadata_match = re.search(r"(\{.*\})\s*$", line.strip()) + if not metadata_match: + self._process.kill() + raise RuntimeError(f"Proxy startup line missing CONNECT proxy metadata: {line}") + try: + metadata = json.loads(metadata_match.group(1)) + except json.JSONDecodeError as exc: + self._process.kill() + raise RuntimeError(f"Failed to parse proxy startup metadata: {line}") from exc + self._connect_proxy_url = metadata.get("connectProxyUrl") + self._ca_file_path = metadata.get("caFilePath") + if not self._connect_proxy_url or not self._ca_file_path: + self._process.kill() + raise RuntimeError(f"Proxy startup metadata missing CONNECT proxy details: {line}") return self._proxy_url async def stop(self, skip_writing_cache: bool = False): @@ -122,3 +140,27 @@ async def set_copilot_user_by_token(self, token: str, response: dict[str, Any]) def url(self) -> str | None: """Return the proxy URL, or None if not started.""" return self._proxy_url + + def get_proxy_env(self) -> dict[str, str]: + """Return environment variables that route HTTPS traffic through the CONNECT proxy.""" + if not self._connect_proxy_url or not self._ca_file_path: + return {} + + no_proxy = "127.0.0.1,localhost,::1" + return { + "HTTP_PROXY": self._connect_proxy_url, + "HTTPS_PROXY": self._connect_proxy_url, + "http_proxy": self._connect_proxy_url, + "https_proxy": self._connect_proxy_url, + "NO_PROXY": no_proxy, + "no_proxy": no_proxy, + "NODE_EXTRA_CA_CERTS": self._ca_file_path, + "SSL_CERT_FILE": self._ca_file_path, + "REQUESTS_CA_BUNDLE": self._ca_file_path, + "CURL_CA_BUNDLE": self._ca_file_path, + "GIT_SSL_CAINFO": self._ca_file_path, + "GH_TOKEN": "", + "GITHUB_TOKEN": "", + "GH_ENTERPRISE_TOKEN": "", + "GITHUB_ENTERPRISE_TOKEN": "", + } diff --git a/test/harness/certUtils.ts b/test/harness/certUtils.ts new file mode 100644 index 000000000..ed1754547 --- /dev/null +++ b/test/harness/certUtils.ts @@ -0,0 +1,81 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------------------------------------------*/ + +import tls from "tls"; + +import forge from "node-forge"; + +export interface CaData { + certPem: string; + keyPem: string; + caCert: forge.pki.Certificate; + caKey: forge.pki.rsa.PrivateKey; +} + +export function generateCA(): CaData { + const keys = forge.pki.rsa.generateKeyPair(2048); + const cert = forge.pki.createCertificate(); + cert.publicKey = keys.publicKey; + cert.serialNumber = "01"; + + const now = new Date(); + const oneYearLater = new Date(); + oneYearLater.setFullYear(oneYearLater.getFullYear() + 1); + cert.validity.notBefore = now; + cert.validity.notAfter = oneYearLater; + + const attrs: forge.pki.CertificateField[] = [ + { name: "commonName", value: "SDK E2E Test CA" }, + { name: "organizationName", value: "Copilot SDK Tests" }, + ]; + cert.setSubject(attrs); + cert.setIssuer(attrs); + + cert.setExtensions([ + { name: "basicConstraints", cA: true, critical: true }, + { name: "keyUsage", keyCertSign: true, cRLSign: true, critical: true }, + ]); + + cert.sign(keys.privateKey, forge.md.sha256.create()); + + return { + certPem: forge.pki.certificateToPem(cert), + keyPem: forge.pki.privateKeyToPem(keys.privateKey), + caCert: cert, + caKey: keys.privateKey, + }; +} + +export function createSecureContextForHost( + hostname: string, + ca: CaData, +): tls.SecureContext { + const keys = forge.pki.rsa.generateKeyPair(2048); + const cert = forge.pki.createCertificate(); + cert.publicKey = keys.publicKey; + cert.serialNumber = String(Date.now()); + + const now = new Date(); + const oneYearLater = new Date(); + oneYearLater.setFullYear(oneYearLater.getFullYear() + 1); + cert.validity.notBefore = now; + cert.validity.notAfter = oneYearLater; + + cert.setSubject([{ name: "commonName", value: hostname }]); + cert.setIssuer(ca.caCert.subject.attributes); + cert.setExtensions([ + { + name: "subjectAltName", + altNames: [{ type: 2, value: hostname }], + }, + ]); + + cert.sign(ca.caKey, forge.md.sha256.create()); + + return tls.createSecureContext({ + key: forge.pki.privateKeyToPem(keys.privateKey), + cert: forge.pki.certificateToPem(cert), + ca: ca.certPem, + }); +} diff --git a/test/harness/connectProxy.test.ts b/test/harness/connectProxy.test.ts new file mode 100644 index 000000000..86d205dd3 --- /dev/null +++ b/test/harness/connectProxy.test.ts @@ -0,0 +1,205 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------------------------------------------*/ + +import fs from "fs"; +import http from "http"; +import https from "https"; +import tls from "tls"; +import { describe, expect, test } from "vitest"; +import { + ConnectProxy, + parseConnectTarget, + type RequestHandler, +} from "./connectProxy"; +import { createE2eRequestHandler } from "./mockHandlers"; + +describe("parseConnectTarget", () => { + test("parses host:port", () => { + expect(parseConnectTarget("example.com:443")).toEqual({ + host: "example.com", + port: "443", + }); + }); + + test("defaults missing port to 443", () => { + expect(parseConnectTarget("example.com")).toEqual({ + host: "example.com", + port: "443", + }); + }); + + test("parses IPv6 bracket form", () => { + expect(parseConnectTarget("[::1]:8443")).toEqual({ + host: "::1", + port: "8443", + }); + }); + + test("rejects malformed IPv6 authority", () => { + expect(parseConnectTarget("[::1:443")).toEqual({ host: "", port: "" }); + expect(parseConnectTarget("[::1]443")).toEqual({ host: "", port: "" }); + }); +}); + +describe("ConnectProxy", () => { + test("starts and stops cleanly", async () => { + const proxy = new ConnectProxy( + (_req, res) => { + res.writeHead(200); + res.end("ok"); + return true; + }, + { interceptDomains: ["example.com"] }, + ); + await proxy.start(); + + expect(proxy.proxyUrl).toMatch(/^http:\/\/127\.0\.0\.1:\d+$/); + expect(proxy.caFilePath).toMatch(/test-ca-bundle\.pem$/); + + await proxy.stop(); + }); + + test("intercepts HTTPS requests to configured domains", async () => { + const requests: Array<{ host: string; url: string }> = []; + const handler: RequestHandler = (req, res, targetHost) => { + requests.push({ host: targetHost, url: req.url ?? "/" }); + res.writeHead(200, { "content-type": "text/plain" }); + res.end("mocked"); + return true; + }; + + const proxy = new ConnectProxy(handler, { + interceptDomains: ["test.example.com"], + }); + await proxy.start(); + + try { + const response = await makeHttpsRequest( + proxy.proxyUrl, + proxy.caFilePath, + "test.example.com", + "/api/test", + ); + expect(response.statusCode).toBe(200); + expect(response.body).toBe("mocked"); + expect(requests).toEqual([ + { host: "test.example.com", url: "/api/test" }, + ]); + expect(proxy.connectLog[0].host).toBe("test.example.com"); + } finally { + await proxy.stop(); + } + }); + + test("rejects CONNECT to non-intercepted domains", async () => { + const blocked: string[] = []; + const proxy = new ConnectProxy( + (_req, res) => { + res.writeHead(200); + res.end("ok"); + return true; + }, + { + interceptDomains: ["allowed.example.com"], + onBlockedConnection: (host) => blocked.push(host), + }, + ); + await proxy.start(); + + try { + await expect( + makeHttpsRequest( + proxy.proxyUrl, + proxy.caFilePath, + "blocked.example.com", + "/", + ), + ).rejects.toThrow(); + expect(blocked).toEqual(["blocked.example.com"]); + } finally { + await proxy.stop(); + } + }); + + test("mocks GitHub HTTPS requests without reaching the network", async () => { + const proxy = new ConnectProxy( + createE2eRequestHandler({ capiProxyUrl: "http://127.0.0.1:1" }), + { interceptDomains: ["github.com", "api.github.com"] }, + ); + await proxy.start(); + + try { + const githubResponse = await makeHttpsRequest( + proxy.proxyUrl, + proxy.caFilePath, + "github.com", + "/github/copilot-sdk/issues/1234", + ); + expect(githubResponse.statusCode).toBe(404); + expect(githubResponse.body).toContain("Not Found (e2e mock)"); + + const apiResponse = await makeHttpsRequest( + proxy.proxyUrl, + proxy.caFilePath, + "api.github.com", + "/user", + ); + expect(apiResponse.statusCode).toBe(200); + expect(JSON.parse(apiResponse.body)).toMatchObject({ + login: "sdk-e2e-user", + }); + } finally { + await proxy.stop(); + } + }); +}); + +function makeHttpsRequest( + proxyUrl: string, + caFilePath: string, + hostname: string, + path: string, +): Promise<{ statusCode: number; body: string }> { + return new Promise((resolve, reject) => { + const proxy = new URL(proxyUrl); + const connectReq = http.request({ + host: proxy.hostname, + port: Number(proxy.port), + method: "CONNECT", + path: `${hostname}:443`, + }); + + connectReq.on("connect", (_res, socket) => { + const ca = fs.readFileSync(caFilePath); + const req = https.request( + { + hostname, + path, + method: "GET", + createConnection: () => + tls.connect({ socket, servername: hostname, ca }), + }, + (res) => { + let body = ""; + res.on("data", (chunk: Buffer) => { + body += chunk.toString(); + }); + res.on("end", () => + resolve({ statusCode: res.statusCode ?? 0, body }), + ); + }, + ); + req.on("error", reject); + req.end(); + }); + + connectReq.on("response", (res) => { + res.resume(); + reject(new Error(`CONNECT failed with status ${res.statusCode}`)); + }); + + connectReq.on("error", reject); + connectReq.end(); + }); +} diff --git a/test/harness/connectProxy.ts b/test/harness/connectProxy.ts new file mode 100644 index 000000000..d5aade087 --- /dev/null +++ b/test/harness/connectProxy.ts @@ -0,0 +1,357 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------------------------------------------*/ + +import fs from "fs"; +import http from "http"; +import net from "net"; +import os from "os"; +import path from "path"; +import tls from "tls"; +import { + type CaData, + createSecureContextForHost, + generateCA, +} from "./certUtils"; + +const debugLogPath = process.env.E2E_PROXY_DEBUG + ? path.join(os.tmpdir(), `e2e-proxy-debug-${process.pid}.log`) + : undefined; + +function debugLog(msg: string): void { + if (debugLogPath) { + fs.appendFileSync( + debugLogPath, + `[${new Date().toISOString()}] [connect] ${msg}\n`, + ); + } +} + +export type RequestHandler = ( + req: http.IncomingMessage, + res: http.ServerResponse, + targetHost: string, +) => boolean | Promise; + +export class ConnectProxy { + private proxyServer?: http.Server; + private internalServer?: http.Server; + private ca?: CaData; + private certCache = new Map(); + private _caFilePath?: string; + private _proxyUrl?: string; + private _connectLog: Array<{ + host: string; + port: string; + timestamp: number; + }> = []; + private interceptDomains: Set; + private passthroughDomains: Set; + private onBlockedConnection?: (host: string, port: string) => void; + private openSockets = new Set(); + + constructor( + private handler: RequestHandler, + options?: { + interceptDomains?: string[]; + passthroughDomains?: string[]; + onBlockedConnection?: (host: string, port: string) => void; + }, + ) { + this.interceptDomains = new Set(options?.interceptDomains ?? []); + this.passthroughDomains = new Set(options?.passthroughDomains ?? []); + this.onBlockedConnection = options?.onBlockedConnection; + } + + get proxyUrl(): string { + if (!this._proxyUrl) { + throw new Error("ConnectProxy not started"); + } + return this._proxyUrl; + } + + get caFilePath(): string { + if (!this._caFilePath) { + throw new Error("ConnectProxy not started"); + } + return this._caFilePath; + } + + get connectLog(): ReadonlyArray<{ + host: string; + port: string; + timestamp: number; + }> { + return this._connectLog; + } + + async start(): Promise { + this.ca = generateCA(); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-proxy-ca-")); + fs.writeFileSync(path.join(tmpDir, "test-ca.pem"), this.ca.certPem); + this._caFilePath = path.join(tmpDir, "test-ca-bundle.pem"); + fs.writeFileSync( + this._caFilePath, + [...tls.rootCertificates, this.ca.certPem].join("\n"), + ); + + this.internalServer = http.createServer((req, res) => { + const socket = req.socket as tls.TLSSocket & { _connectTarget?: string }; + const targetHost = socket._connectTarget ?? req.headers.host ?? "unknown"; + + void Promise.resolve(this.handler(req, res, targetHost)) + .then((handled) => { + if (!handled && !res.headersSent) { + res.writeHead(502, { "content-type": "text/plain" }); + res.end( + `E2E proxy: no handler for ${req.method} ${targetHost}${req.url}`, + ); + } + }) + .catch((err) => { + console.warn( + `[E2E proxy] handler error for ${req.method} ${targetHost}${req.url}: ${err}`, + ); + if (!res.headersSent) { + res.writeHead(502, { "content-type": "text/plain" }); + res.end("E2E proxy: handler error"); + } + }); + }); + + this.proxyServer = http.createServer((req, res) => { + this.handleForwardProxy(req, res); + }); + + this.proxyServer.on("connect", (req, clientSocket, head) => { + this.handleConnect(req, clientSocket as net.Socket, head); + }); + + await new Promise((resolve, reject) => { + this.proxyServer!.on("error", reject); + this.proxyServer!.listen(0, "127.0.0.1", () => resolve()); + }); + + const addr = this.proxyServer.address() as net.AddressInfo; + this._proxyUrl = `http://${addr.address}:${addr.port}`; + } + + async stop(): Promise { + for (const socket of this.openSockets) { + socket.destroy(); + } + this.openSockets.clear(); + + const closeServer = (server?: http.Server) => + new Promise((resolve) => { + if (!server) { + resolve(); + return; + } + server.close(() => resolve()); + }); + + await Promise.all([ + closeServer(this.proxyServer), + closeServer(this.internalServer), + ]); + + if (this._caFilePath) { + try { + fs.rmSync(path.dirname(this._caFilePath), { + recursive: true, + force: true, + }); + } catch { + // Best-effort cleanup. + } + } + } + + private handleConnect( + req: http.IncomingMessage, + clientSocket: net.Socket, + head: Buffer, + ) { + const { host, port } = parseConnectTarget(req.url ?? ""); + debugLog(`CONNECT ${host}:${port}`); + if (!host) { + clientSocket.write("HTTP/1.1 400 Bad Request\r\n\r\n"); + clientSocket.destroy(); + return; + } + + this._connectLog.push({ host, port, timestamp: Date.now() }); + + if (this.passthroughDomains.has(host)) { + this.pipeToRealTarget(clientSocket, head, host, port); + return; + } + + if (!this.interceptDomains.has(host)) { + this.onBlockedConnection?.(host, port); + clientSocket.write("HTTP/1.1 502 Blocked by E2E proxy\r\n\r\n"); + clientSocket.destroy(); + return; + } + + clientSocket.write("HTTP/1.1 200 Connection Established\r\n\r\n"); + + const tlsSocket = new tls.TLSSocket(clientSocket, { + isServer: true, + secureContext: this.getOrCreateSecureContext(host), + ALPNProtocols: ["http/1.1"], + }); + + this.openSockets.add(clientSocket); + this.openSockets.add(tlsSocket); + let cleaned = false; + const cleanup = () => { + if (cleaned) { + return; + } + cleaned = true; + tlsSocket.off("close", cleanup); + clientSocket.off("close", cleanup); + tlsSocket.off("error", onTlsError); + clientSocket.off("error", onClientError); + this.openSockets.delete(clientSocket); + this.openSockets.delete(tlsSocket); + }; + const onTlsError = (err: Error) => { + debugLog(`TLS error for ${host}: ${err.message}`); + cleanup(); + clientSocket.destroy(); + }; + const onClientError = () => { + cleanup(); + tlsSocket.destroy(); + }; + tlsSocket.on("close", cleanup); + clientSocket.on("close", cleanup); + tlsSocket.on("error", onTlsError); + clientSocket.on("error", onClientError); + + (tlsSocket as tls.TLSSocket & { _connectTarget?: string })._connectTarget = + host; + if (head.length > 0) { + tlsSocket.unshift(head); + } + this.internalServer!.emit("connection", tlsSocket); + } + + private handleForwardProxy( + req: http.IncomingMessage, + res: http.ServerResponse, + ) { + let targetHost: string; + try { + const url = new URL(req.url ?? ""); + targetHost = url.hostname; + req.url = url.pathname + url.search; + } catch { + targetHost = req.headers.host ?? "unknown"; + } + + void Promise.resolve(this.handler(req, res, targetHost)) + .then((handled) => { + if (!handled && !res.headersSent) { + res.writeHead(502, { "content-type": "text/plain" }); + res.end( + `E2E proxy: no handler for HTTP ${req.method} ${targetHost}${req.url}`, + ); + } + }) + .catch(() => { + if (!res.headersSent) { + res.writeHead(502, { "content-type": "text/plain" }); + res.end("E2E proxy: handler error"); + } + }); + } + + private getOrCreateSecureContext(hostname: string): tls.SecureContext { + let context = this.certCache.get(hostname); + if (!context) { + context = createSecureContextForHost(hostname, this.ca!); + this.certCache.set(hostname, context); + } + return context; + } + + private pipeToRealTarget( + clientSocket: net.Socket, + head: Buffer, + host: string, + port: string, + ) { + const targetSocket = net.connect(Number.parseInt(port, 10), host, () => { + if (clientSocket.destroyed || targetSocket.destroyed) { + return; + } + clientSocket.write("HTTP/1.1 200 Connection Established\r\n\r\n"); + if (head.length > 0) { + targetSocket.write(head); + } + clientSocket.pipe(targetSocket); + targetSocket.pipe(clientSocket); + }); + + this.openSockets.add(clientSocket); + this.openSockets.add(targetSocket); + + let cleaned = false; + const cleanup = () => { + if (cleaned) { + return; + } + cleaned = true; + clientSocket.off("error", cleanup); + clientSocket.off("close", cleanup); + targetSocket.off("error", cleanup); + targetSocket.off("close", cleanup); + clientSocket.destroy(); + targetSocket.destroy(); + this.openSockets.delete(clientSocket); + this.openSockets.delete(targetSocket); + }; + clientSocket.on("error", cleanup); + clientSocket.on("close", cleanup); + targetSocket.on("error", cleanup); + targetSocket.on("close", cleanup); + } +} + +export function parseConnectTarget(authority: string): { + host: string; + port: string; +} { + if (!authority) { + return { host: "", port: "" }; + } + + if (authority.startsWith("[")) { + const closeBracket = authority.indexOf("]"); + if (closeBracket === -1) { + return { host: "", port: "" }; + } + const host = authority.slice(1, closeBracket); + const afterBracket = authority.slice(closeBracket + 1); + if (afterBracket === "" || afterBracket === ":") { + return { host, port: "443" }; + } + if (afterBracket[0] !== ":") { + return { host: "", port: "" }; + } + return { host, port: afterBracket.slice(1) || "443" }; + } + + const lastColon = authority.lastIndexOf(":"); + if (lastColon === -1) { + return { host: authority, port: "443" }; + } + + const host = authority.slice(0, lastColon); + const port = authority.slice(lastColon + 1) || "443"; + return { host, port }; +} diff --git a/test/harness/mockHandlers.ts b/test/harness/mockHandlers.ts new file mode 100644 index 000000000..9f75d6819 --- /dev/null +++ b/test/harness/mockHandlers.ts @@ -0,0 +1,174 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------------------------------------------*/ + +import http from "http"; +import type { RequestHandler } from "./connectProxy"; + +export function createE2eRequestHandler(options: { + capiProxyUrl: string; + onUnhandled?: (host: string, method: string, path: string) => void; +}): RequestHandler { + return async (req, res, targetHost) => { + if (targetHost === "api.githubcopilot.com") { + return forwardToCapiProxy(req, res, options.capiProxyUrl); + } + + if (targetHost === "api.github.com") { + return handleGitHubApi(req, res, options); + } + + if (targetHost === "github.com") { + respondJson(res, 404, { message: "Not Found (e2e mock)" }); + return true; + } + + if (targetHost === "api.mcp.github.com") { + return handleMcpRegistry(req, res); + } + + options.onUnhandled?.(targetHost, req.method ?? "GET", req.url ?? "/"); + return false; + }; +} + +function handleGitHubApi( + req: http.IncomingMessage, + res: http.ServerResponse, + options: { capiProxyUrl: string }, +): boolean { + const url = req.url ?? "/"; + + if (req.method === "GET" && url === "/user") { + respondJson(res, 200, { + login: "sdk-e2e-user", + id: 12345, + type: "User", + name: "SDK E2E User", + }); + return true; + } + + if (req.method === "GET" && url.startsWith("/user/copilot_billing")) { + respondJson(res, 200, { + seat: { plan: { plan_type: "business" } }, + }); + return true; + } + + if (req.method === "GET" && url.startsWith("/copilot_internal/user")) { + respondJson(res, 200, { + login: "sdk-e2e-user", + analytics_tracking_id: "sdk-e2e-tracking-id", + organization_list: [], + copilot_plan: "individual_pro", + is_mcp_enabled: true, + endpoints: { + api: options.capiProxyUrl, + telemetry: "https://localhost:1/telemetry", + }, + }); + return true; + } + + if (req.method === "POST" && url === "/graphql") { + respondJson(res, 401, { + message: "Requires authentication", + documentation_url: "https://docs.github.com/graphql", + }); + return true; + } + + respondJson(res, 404, { message: "Not Found (e2e mock)" }); + return true; +} + +function handleMcpRegistry( + req: http.IncomingMessage, + res: http.ServerResponse, +): boolean { + const url = new URL(req.url ?? "/", "https://api.mcp.github.com"); + + if (req.method === "GET" && url.pathname.startsWith("/v0.1/servers")) { + respondJson(res, 200, { servers: [], metadata: {} }); + return true; + } + + respondJson(res, 404, { error: "Not Found (e2e mock)" }); + return true; +} + +function respondJson( + res: http.ServerResponse, + statusCode: number, + body: unknown, +): void { + const data = JSON.stringify(body); + res.writeHead(statusCode, { + "content-type": "application/json", + "content-length": Buffer.byteLength(data), + }); + res.end(data); +} + +function forwardToCapiProxy( + clientReq: http.IncomingMessage, + clientRes: http.ServerResponse, + capiProxyUrl: string, +): Promise { + return new Promise((resolve) => { + const target = new URL(capiProxyUrl); + const chunks: Buffer[] = []; + clientReq.on("data", (chunk: Buffer) => chunks.push(chunk)); + clientReq.on("error", (err) => { + if (!clientRes.headersSent) { + clientRes.writeHead(502, { "content-type": "text/plain" }); + clientRes.end(`E2E proxy: client request error: ${err.message}`); + } else { + clientRes.destroy(err); + } + resolve(true); + }); + clientReq.on("end", () => { + const proxyReq = http.request( + { + hostname: target.hostname, + port: target.port, + path: clientReq.url, + method: clientReq.method, + headers: { + ...clientReq.headers, + host: target.host, + }, + }, + (proxyRes) => { + clientRes.writeHead(proxyRes.statusCode ?? 502, proxyRes.headers); + proxyRes.pipe(clientRes); + proxyRes.on("end", () => resolve(true)); + proxyRes.on("error", (err) => { + clientRes.destroy(err); + resolve(true); + }); + }, + ); + proxyReq.on("error", (err) => { + if (!clientRes.headersSent) { + clientRes.writeHead(502, { + "content-type": "application/json", + "x-github-request-id": "e2e-proxy-error", + }); + clientRes.end( + JSON.stringify({ + error: `E2E proxy: CAPI forward error: ${err.message}`, + }), + ); + } + resolve(true); + }); + if (chunks.length > 0) { + proxyReq.write(Buffer.concat(chunks)); + } + proxyReq.end(); + }); + }); +} diff --git a/test/harness/package-lock.json b/test/harness/package-lock.json index 02cfeb2cc..9ece434e9 100644 --- a/test/harness/package-lock.json +++ b/test/harness/package-lock.json @@ -12,6 +12,8 @@ "@github/copilot": "^1.0.41-0", "@modelcontextprotocol/sdk": "^1.26.0", "@types/node": "^25.3.3", + "@types/node-forge": "^1.3.14", + "node-forge": "^1.4.0", "openai": "^6.17.0", "tsx": "^4.21.0", "typescript": "^5.9.3", @@ -1034,6 +1036,16 @@ "undici-types": "~7.18.0" } }, + "node_modules/@types/node-forge": { + "version": "1.3.14", + "resolved": "https://registry.npmjs.org/@types/node-forge/-/node-forge-1.3.14.tgz", + "integrity": "sha512-mhVF2BnD4BO+jtOp7z1CdzaK4mbuK0LLQYAvdOLqHTavxFNq4zA1EmYkpnFjP8HOUzedfQkRnp0E2ulSAYSzAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@vitest/expect": { "version": "4.0.18", "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.0.18.tgz", @@ -2041,6 +2053,16 @@ "node": ">= 0.6" } }, + "node_modules/node-forge": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.4.0.tgz", + "integrity": "sha512-LarFH0+6VfriEhqMMcLX2F7SwSXeWwnEAJEsYm5QKWchiVYVvJyV9v7UDvUv+w5HO23ZpQTXDv/GxdDdMyOuoQ==", + "dev": true, + "license": "(BSD-3-Clause OR GPL-2.0)", + "engines": { + "node": ">= 6.13.0" + } + }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", diff --git a/test/harness/package.json b/test/harness/package.json index 6600204ab..a0d9e5469 100644 --- a/test/harness/package.json +++ b/test/harness/package.json @@ -14,6 +14,8 @@ "@github/copilot": "^1.0.41-0", "@modelcontextprotocol/sdk": "^1.26.0", "@types/node": "^25.3.3", + "@types/node-forge": "^1.3.14", + "node-forge": "^1.4.0", "openai": "^6.17.0", "tsx": "^4.21.0", "typescript": "^5.9.3", diff --git a/test/harness/replayingCapiProxy.test.ts b/test/harness/replayingCapiProxy.test.ts index 0a57c5e01..c7abf01f2 100644 --- a/test/harness/replayingCapiProxy.test.ts +++ b/test/harness/replayingCapiProxy.test.ts @@ -415,6 +415,60 @@ describe("ReplayingCapiProxy", () => { expect(toolMessages[1].content).toBe("[beta result]"); }); + test("normalizes GitHub CLI proxy auth failures", async () => { + const requestBody = JSON.stringify({ + messages: [ + { role: "user", content: "Summarize this issue" }, + { + role: "assistant", + tool_calls: [ + { + id: "tc1", + type: "function", + function: { name: "web_fetch", arguments: "{}" }, + }, + ], + }, + { + role: "tool", + tool_call_id: "tc1", + content: + 'Post "https://api.github.com/graphql": tls: failed to verify certificate: x509: certificate signed by unknown authority\n', + }, + { + role: "tool", + tool_call_id: "tc1", + content: + "\u28fe\u28fdHTTP 401: Requires authentication (https://api.github.com/graphql)\nTry authenticating with: gh auth login\n", + }, + ], + }); + const responseBody = JSON.stringify({ + choices: [{ message: { role: "assistant", content: "Done" } }], + }); + + const outputPath = await createProxy([ + { url: "/chat/completions", requestBody, responseBody }, + ]); + + const result = await readYamlOutput(outputPath); + const toolMessages = result.conversations[0].messages.filter( + (m) => m.role === "tool", + ); + expect(toolMessages).toEqual([ + { + role: "tool", + tool_call_id: "toolcall_0", + content: "${gh_auth_required}\n", + }, + { + role: "tool", + tool_call_id: "toolcall_0", + content: "${gh_auth_required}\n", + }, + ]); + }); + test("ignores non-chat-completion endpoints", async () => { const outputPath = await createProxy([ { url: "/models", requestBody: "{}", responseBody: "{}" }, @@ -676,6 +730,108 @@ describe("ReplayingCapiProxy", () => { } }); + test("matches parallel tool results regardless of arrival order", async () => { + const cachePath = path.join(tempDir, "cache.yaml"); + const cacheContent = yaml.stringify({ + models: ["test-model"], + conversations: [ + { + messages: [ + { role: "system", content: "${system}" }, + { role: "user", content: "Lookup city and country" }, + { + role: "assistant", + tool_calls: [ + { + id: "toolcall_0", + type: "function", + function: { + name: "lookup_city", + arguments: '{"city":"Paris"}', + }, + }, + { + id: "toolcall_1", + type: "function", + function: { + name: "lookup_country", + arguments: '{"country":"France"}', + }, + }, + ], + }, + { + role: "tool", + tool_call_id: "toolcall_1", + content: "COUNTRY_FRANCE", + }, + { + role: "tool", + tool_call_id: "toolcall_0", + content: "CITY_PARIS", + }, + { role: "assistant", content: "Paris is in France." }, + ], + }, + ], + } satisfies NormalizedData); + await writeFile(cachePath, cacheContent); + + const proxy = new ReplayingCapiProxy( + "http://localhost:9999", + cachePath, + workDir, + ); + const proxyUrl = await proxy.start(); + + try { + const response = await makeRequest(proxyUrl, "/chat/completions", { + body: { + model: "test-model", + messages: [ + { role: "system", content: "Be helpful" }, + { role: "user", content: "Lookup city and country" }, + { + role: "assistant", + tool_calls: [ + { + id: "city-id", + type: "function", + function: { + name: "lookup_city", + arguments: '{"city":"Paris"}', + }, + }, + { + id: "country-id", + type: "function", + function: { + name: "lookup_country", + arguments: '{"country":"France"}', + }, + }, + ], + }, + { + role: "tool", + tool_call_id: "country-id", + content: "COUNTRY_FRANCE", + }, + { role: "tool", tool_call_id: "city-id", content: "CITY_PARIS" }, + ], + }, + }); + + expect(response.status).toBe(200); + expect( + (JSON.parse(response.body) as ChatCompletion).choices[0].message + .content, + ).toBe("Paris is in France."); + } finally { + await proxy.stop(); + } + }); + test("returns streaming response when stream: true", async () => { const cachePath = path.join(tempDir, "cache.yaml"); const cacheContent = yaml.stringify({ diff --git a/test/harness/replayingCapiProxy.ts b/test/harness/replayingCapiProxy.ts index faff7de00..cd6399922 100644 --- a/test/harness/replayingCapiProxy.ts +++ b/test/harness/replayingCapiProxy.ts @@ -25,7 +25,7 @@ export const workingDirPlaceholder = "${workdir}"; const chatCompletionEndpoint = "/chat/completions"; const shellConfig = process.platform === "win32" ? ShellConfig.powerShell : ShellConfig.bash; -const normalizedToolNames = { +const normalizedToolNames: Record = { [shellConfig.shellToolName]: "${shell}", [shellConfig.readShellToolName]: "${read_shell}", [shellConfig.writeShellToolName]: "${write_shell}", @@ -69,6 +69,7 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { * If true, cached responses are played back slowly (~ 2KiB/sec). Otherwise streaming responses are sent as fast as possible. */ slowStreaming = false; + onStopRequested?: (skipWritingCache: boolean) => Promise | void; constructor( targetUrl: string, @@ -127,6 +128,7 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { if (this.state && existsSync(this.state.filePath)) { const content = await readFile(this.state.filePath, "utf-8"); this.state.storedData = yaml.parse(content) as NormalizedData; + normalizeToolResultOrder(this.state.storedData.conversations); } } @@ -176,7 +178,10 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { options.requestOptions.path === "/copilot-user-config" && options.requestOptions.method === "POST" ) { - const config = JSON.parse(options.body!) as { token: string; response: CopilotUserResponse }; + const config = JSON.parse(options.body!) as { + token: string; + response: CopilotUserResponse; + }; this.copilotUserByToken.set(config.token, config.response); options.onResponseStart(200, {}); options.onResponseEnd(); @@ -204,6 +209,7 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { ); options.onResponseStart(200, {}); options.onResponseEnd(); + await this.onStopRequested?.(skipWritingCache); await this.stop(skipWritingCache); process.exit(0); } @@ -218,7 +224,11 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { ); const parsedExchanges = await Promise.all( chatCompletionExchanges.map((e) => - parseHttpExchange(e.request.body, e.response?.body, e.request.headers), + parseHttpExchange( + e.request.body, + e.response?.body, + e.request.headers, + ), ), ); options.onResponseStart(200, {}); @@ -256,9 +266,22 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { // Handle /copilot_internal/user endpoint for per-session auth if (options.requestOptions.path === "/copilot_internal/user") { - const authHeader = options.requestOptions.headers?.["authorization"] as string | undefined; + const headers = options.requestOptions.headers; + const headerMap = headers as + | Record + | undefined; + const rawAuthHeader = Array.isArray(headers) + ? undefined + : (headerMap?.authorization ?? headerMap?.Authorization); + const authHeader = Array.isArray(rawAuthHeader) + ? rawAuthHeader[0] + : typeof rawAuthHeader === "string" + ? rawAuthHeader + : undefined; const token = authHeader?.replace("Bearer ", ""); - const userResponse = token ? this.copilotUserByToken.get(token) : undefined; + const userResponse = token + ? this.copilotUserByToken.get(token) + : undefined; if (userResponse) { const headers = { "content-type": "application/json", @@ -269,7 +292,9 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { options.onResponseEnd(); } else { options.onResponseStart(401, commonResponseHeaders); - options.onData(Buffer.from(JSON.stringify({ message: "Bad credentials" }))); + options.onData( + Buffer.from(JSON.stringify({ message: "Bad credentials" })), + ); options.onResponseEnd(); } return; @@ -445,7 +470,9 @@ function diagnoseMatchFailure( storedData: NormalizedData | undefined, ): string { const lines: string[] = []; - lines.push(`Request has ${requestMessages.length} normalized messages (${rawMessages.length} raw).`); + lines.push( + `Request has ${requestMessages.length} normalized messages (${rawMessages.length} raw).`, + ); if (!storedData || storedData.conversations.length === 0) { lines.push("No stored conversations to match against."); @@ -459,7 +486,7 @@ function diagnoseMatchFailure( if (requestMessages.length >= saved.length) { lines.push( `Conversation ${c} (${saved.length} messages): ` + - `skipped — request has ${requestMessages.length} messages, need fewer than ${saved.length}.`, + `skipped — request has ${requestMessages.length} messages, need fewer than ${saved.length}.`, ); continue; } @@ -474,9 +501,10 @@ function diagnoseMatchFailure( } if (mismatchIndex >= 0) { - const raw = mismatchIndex < rawMessages.length - ? JSON.stringify(rawMessages[mismatchIndex]).slice(0, 300) - : "(no raw message)"; + const raw = + mismatchIndex < rawMessages.length + ? JSON.stringify(rawMessages[mismatchIndex]).slice(0, 300) + : "(no raw message)"; lines.push( `Conversation ${c} (${saved.length} messages): mismatch at message ${mismatchIndex}:`, ` request: ${JSON.stringify(requestMessages[mismatchIndex]).slice(0, 200)}`, @@ -485,10 +513,11 @@ function diagnoseMatchFailure( ); } else { // Prefix matched, but the next saved message isn't an assistant turn - const nextRole = saved[requestMessages.length]?.role ?? "(end of conversation)"; + const nextRole = + saved[requestMessages.length]?.role ?? "(end of conversation)"; lines.push( `Conversation ${c} (${saved.length} messages): ` + - `prefix matched, but next saved message is "${nextRole}" (need "assistant").`, + `prefix matched, but next saved message is "${nextRole}" (need "assistant").`, ); } } @@ -505,28 +534,43 @@ async function exitWithNoMatchingRequestError( ) { let diagnostics: string; try { - const normalized = await parseAndNormalizeRequest(options.body, workDir, toolResultNormalizers); + const normalized = await parseAndNormalizeRequest( + options.body, + workDir, + toolResultNormalizers, + ); const requestMessages = normalized.conversations[0]?.messages ?? []; let rawMessages: unknown[] = []; try { - rawMessages = (JSON.parse(options.body ?? "{}") as { messages?: unknown[] }).messages ?? []; - } catch { /* non-JSON body */ } + rawMessages = + (JSON.parse(options.body ?? "{}") as { messages?: unknown[] }) + .messages ?? []; + } catch { + /* non-JSON body */ + } - diagnostics = diagnoseMatchFailure(requestMessages, rawMessages, storedData); + diagnostics = diagnoseMatchFailure( + requestMessages, + rawMessages, + storedData, + ); } catch (e) { diagnostics = `(unable to parse request for diagnostics: ${e})`; } - const errorMessage = - `No cached response found for ${options.requestOptions.method} ${options.requestOptions.path}.\n${diagnostics}`; + const errorMessage = `No cached response found for ${options.requestOptions.method} ${options.requestOptions.path}.\n${diagnostics}`; // Format as GitHub Actions annotation when test location is available const annotation = [ testInfo?.file ? `file=${testInfo.file}` : "", typeof testInfo?.line === "number" ? `line=${testInfo.line}` : "", - ].filter(Boolean).join(","); - process.stderr.write(`::error${annotation ? ` ${annotation}` : ""}::${errorMessage}\n`); + ] + .filter(Boolean) + .join(","); + process.stderr.write( + `::error${annotation ? ` ${annotation}` : ""}::${errorMessage}\n`, + ); options.onError(new Error(errorMessage)); } @@ -634,6 +678,7 @@ async function transformHttpExchanges( ); normalizeToolCalls(dedupedExchanges, toolResultNormalizers); + normalizeToolResultOrder(dedupedExchanges); normalizeFilenames(dedupedExchanges, workDir); return { models: Array.from(dedupedModels), conversations: dedupedExchanges }; } @@ -739,6 +784,51 @@ function normalizeToolCalls( } } +function normalizeToolResultOrder(conversations: NormalizedConversation[]) { + for (const conv of conversations) { + for (let start = 0; start < conv.messages.length; ) { + if (conv.messages[start].role !== "tool") { + start++; + continue; + } + + let end = start + 1; + while (end < conv.messages.length && conv.messages[end].role === "tool") { + end++; + } + + conv.messages + .slice(start, end) + .sort(compareToolResultMessages) + .forEach((message, index) => { + conv.messages[start + index] = message; + }); + start = end; + } + } +} + +function compareToolResultMessages( + left: NormalizedMessage, + right: NormalizedMessage, +) { + return compareToolCallIds(left.tool_call_id, right.tool_call_id); +} + +function compareToolCallIds(left?: string, right?: string) { + const leftNumber = parseNormalizedToolCallId(left); + const rightNumber = parseNormalizedToolCallId(right); + if (leftNumber !== undefined && rightNumber !== undefined) { + return leftNumber - rightNumber; + } + return (left ?? "").localeCompare(right ?? ""); +} + +function parseNormalizedToolCallId(id?: string) { + const match = id?.match(/^toolcall_(\d+)$/); + return match ? Number(match[1]) : undefined; +} + // As we capture LLM calls, we see: // - Request A, response AB // - Request ABC, response ABCD @@ -817,7 +907,11 @@ function transformOpenAIRequestMessage( // Extract and normalize text parts; represent image_url parts as a stable marker. const parts: string[] = []; for (const part of m.content) { - if (typeof part === "object" && part.type === "text" && typeof part.text === "string") { + if ( + typeof part === "object" && + part.type === "text" && + typeof part.text === "string" + ) { parts.push(normalizeUserMessage(part.text)); } else if (typeof part === "object" && part.type === "image_url") { parts.push("[image]"); @@ -836,9 +930,10 @@ function transformOpenAIRequestMessage( parsed.resultType === "success" && "textResultForLlm" in parsed ) { - content = typeof parsed.textResultForLlm === "string" - ? parsed.textResultForLlm - : JSON.stringify(sortJsonKeys(parsed.textResultForLlm)); + content = + typeof parsed.textResultForLlm === "string" + ? parsed.textResultForLlm + : JSON.stringify(sortJsonKeys(parsed.textResultForLlm)); } else { content = JSON.stringify(sortJsonKeys(parsed)); } @@ -901,7 +996,45 @@ function normalizeGhAuthMessages(result: string): string { /To get started with GitHub CLI, please run:\s*gh auth login\s*\n\s*Alternatively, populate the GH_TOKEN environment variable with a GitHub API authentication token\./g, "${gh_auth_required}", ); - return normalized; + // When the GitHub CLI is run under the local CONNECT proxy on Windows, it + // can try its auth probe before trusting the generated CA. This is still the + // same unauthenticated-GitHub condition from the snapshot's perspective. + normalized = normalized.replace( + /[^\n]*Post "https:\/\/api\.github\.com\/graphql": tls: failed to verify certificate: x509: certificate signed by unknown authority\s*\n/g, + "${gh_auth_required}\n", + ); + return normalizeGh401AuthMessages(normalized); +} + +function normalizeGh401AuthMessages(result: string): string { + const lines = result.split(/\r?\n/); + const normalizedLines: string[] = []; + let changed = false; + + for (let i = 0; i < lines.length; i++) { + if ( + /(?:HTTP|GraphQL)[ \t:]+401/.test(lines[i]) && + lines[i].includes("Requires authentication") + ) { + let replaced = false; + for (let j = i + 1; j < lines.length; j++) { + if (/^$/.test(lines[j].trim())) { + normalizedLines.push("${gh_auth_required}"); + normalizedLines.push(""); + i = j; + changed = true; + replaced = true; + break; + } + } + if (replaced) { + continue; + } + } + normalizedLines.push(lines[i]); + } + + return changed ? normalizedLines.join("\n") : result; } // Transforms a single OpenAI-style inbound response message into normalized form diff --git a/test/harness/server.ts b/test/harness/server.ts index e6a9e4dc8..887a57178 100644 --- a/test/harness/server.ts +++ b/test/harness/server.ts @@ -3,11 +3,57 @@ *--------------------------------------------------------------------------------------------*/ import { ReplayingCapiProxy } from "./replayingCapiProxy"; +import { ConnectProxy } from "./connectProxy"; +import { createE2eRequestHandler } from "./mockHandlers"; // Starts up an instance of the ReplayingCapiProxy server // The intention is for this to be usable in E2E tests across all languages const proxy = new ReplayingCapiProxy("https://api.githubcopilot.com"); const proxyUrl = await proxy.start(); +const blockedHosts: string[] = []; +const unhandledRequests: string[] = []; -console.log(`Listening: ${proxyUrl}`); +const connectProxy = new ConnectProxy( + createE2eRequestHandler({ + capiProxyUrl: proxyUrl, + onUnhandled: (host, method, requestPath) => { + const entry = `${method} ${host}${requestPath}`; + unhandledRequests.push(entry); + console.error(`[E2E proxy] Unhandled intercepted request: ${entry}`); + }, + }), + { + interceptDomains: [ + "api.githubcopilot.com", + "api.github.com", + "github.com", + "api.mcp.github.com", + ], + passthroughDomains: ["registry.npmjs.org"], + onBlockedConnection: (host, port) => { + const entry = `${host}:${port}`; + blockedHosts.push(entry); + console.error(`[E2E proxy] Blocked connection to: ${entry}`); + }, + }, +); +await connectProxy.start(); + +proxy.onStopRequested = async () => { + if (blockedHosts.length || unhandledRequests.length) { + const details = [ + ...blockedHosts.map((host) => `blocked ${host}`), + ...unhandledRequests.map((request) => `unhandled ${request}`), + ].join(", "); + console.error(`[E2E proxy] Unexpected network activity: ${details}`); + } + await connectProxy.stop(); +}; + +console.log( + `Listening: ${proxyUrl} ${JSON.stringify({ + connectProxyUrl: connectProxy.proxyUrl, + caFilePath: connectProxy.caFilePath, + })}`, +); diff --git a/test/harness/util.ts b/test/harness/util.ts index b696e06c5..020e07658 100644 --- a/test/harness/util.ts +++ b/test/harness/util.ts @@ -2,8 +2,6 @@ * Copyright (c) Microsoft Corporation. All rights reserved. *--------------------------------------------------------------------------------------------*/ -import type { SessionOptions } from "@github/copilot/sdk"; - export function iife(fn: () => Promise): Promise { return fn(); } @@ -12,7 +10,11 @@ export function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } -type ShellConfigType = NonNullable; +type ShellConfigType = { + shellToolName: string; + readShellToolName: string; + writeShellToolName: string; +}; /** * Shell configuration for platform-specific tool names. diff --git a/test/snapshots/rpc_tasks_and_handlers/should_start_background_agent_and_report_task_details.yaml b/test/snapshots/rpc_tasks_and_handlers/should_start_background_agent_and_report_task_details.yaml index 01c49201f..41bbe583d 100644 --- a/test/snapshots/rpc_tasks_and_handlers/should_start_background_agent_and_report_task_details.yaml +++ b/test/snapshots/rpc_tasks_and_handlers/should_start_background_agent_and_report_task_details.yaml @@ -15,3 +15,28 @@ conversations: content: Reply with TASK_AGENT_DONE exactly. - role: assistant content: TASK_AGENT_DONE + - messages: + - role: system + content: ${system} + - role: user + content: Reply with TASK_AGENT_READY exactly. + - role: assistant + content: TASK_AGENT_READY + - role: user + content: |- + + Agent "sdk-background-agent" (general-purpose) has completed successfully. Use read_agent with agent_id "sdk-background-agent" to retrieve the full results. + + - role: assistant + content: TASK_AGENT_DONE + - messages: + - role: system + content: ${system} + - role: user + content: Reply with TASK_AGENT_READY exactly. + - role: assistant + content: TASK_AGENT_READY + - role: user + content: Reply with TASK_AGENT_DONE exactly. + - role: assistant + content: TASK_AGENT_DONE