diff --git a/.github/scripts/automationPolicy.test.mjs b/.github/scripts/automationPolicy.test.mjs index fce6d3a0..8583a4ac 100644 --- a/.github/scripts/automationPolicy.test.mjs +++ b/.github/scripts/automationPolicy.test.mjs @@ -10,19 +10,30 @@ function read(relativePath) { return readFileSync(resolve(REPO_ROOT, relativePath), 'utf8'); } -test('runner attributes the tier whose reply becomes final and posts only commentBody', () => { +test('runner attributes the selected tier and posts only the trusted comment body', () => { const runner = read('.github/scripts/first-pass.mjs'); assert.match(runner, /const TRANSPORT = 'anthropic-messages'/); assert.match(runner, /resolveGeneratorAttribution\(env\.CCU_BOT_GENERATOR, model, TRANSPORT\)/); assert.match(runner, /resolveGeneratorAttribution\(env\.CCU_BOT_GENERATOR_PRO, modelPro, TRANSPORT\)/); assert.match(runner, /parseFirstPassResponse/); - assert.match(runner, /chooseFinalReply/); - assert.match(runner, /formatAutomatedComment\(reply, \{ kind, generator: finalGenerator \}\)/); + assert.match(runner, /resolveFirstPassCandidates/); + assert.match(runner, /formatAutomatedComment\(selected\.reply, \{/); + assert.match(runner, /generator: selected\.generator/); assert.match(runner, /JSON\.stringify\(\{\s*body:\s*commentBody\s*\}\)/); assert.doesNotMatch(runner, /JSON\.stringify\(\{\s*body:\s*reply\s*\}\)/); assert.doesNotMatch(runner, /CCU_BOT_TRANSPORT|prompt-injection safe|injection-safe/i); }); +test('issue and PR opened events both use the hardened shared runner', () => { + const issue = read('.github/workflows/issue-first-pass.yml'); + const pr = read('.github/workflows/pr-first-pass.yml'); + assert.match(issue, /issues:\s*\n\s+types: \[opened\]/); + assert.match(pr, /pull_request_target:\s*\n\s+types: \[opened\]/); + for (const workflow of [issue, pr]) { + assert.match(workflow, /run: node \.github\/scripts\/first-pass\.mjs/); + } +}); + test('runner uses one bounded reader for AGENTS grounding and requested source', () => { const runner = read('.github/scripts/first-pass.mjs'); assert.match(runner, /# AGENTS\.md/); diff --git a/.github/scripts/first-pass-lib.mjs b/.github/scripts/first-pass-lib.mjs index bd4a160e..b121bce1 100644 --- a/.github/scripts/first-pass-lib.mjs +++ b/.github/scripts/first-pass-lib.mjs @@ -340,6 +340,77 @@ export function parseFirstPassResponse(raw) { return { reply, ...control }; } +export function needsProFirstPass(parsed) { + return parsed?.answerable === false || String(parsed?.reply || '').trim() === ''; +} + +export function formatFirstPassFallback({ kind, isChinese = false }) { + if (!['reply', 'review'].includes(kind)) { + throw new Error(`Unsupported first-pass kind: ${kind}`); + } + const subject = kind === 'reply' ? 'issue' : 'pull request'; + const lines = [ + `🤖 Automated first-pass ${kind}`, + '', + `The configured analysis model did not return a usable analysis for this ${subject}. ` + + 'The item has still been received and remains available for maintainer review.', + '', + kind === 'reply' + ? 'To help with triage, please ensure the extension, VS Code, and operating-system versions plus anonymous diagnostic logs are included.' + : 'To help with review, please ensure the change summary, tests, and any user-facing documentation or localization updates are included.', + ]; + if (isChinese) { + lines.push( + '', + `自动分析服务没有为这个${kind === 'reply' ? '问题' : '拉取请求'}返回可用结果。` + + '该条目已经成功收到，维护者仍可继续审阅。', + '', + kind === 'reply' + ? '为便于排查，请确认已提供扩展、VS Code、操作系统版本以及匿名诊断日志。' + : '为便于审阅，请确认已提供改动摘要、测试结果，以及必要的文档和本地化更新。', + ); + } + lines.push( + '', + '---', + '🤖 Posted automatically by CCU Bot — not a maintainer decision.', + ); + return lines.join('\n'); +} + +export function hasTrustedFirstPass(comments, kind) { + if (!['reply', 'review'].includes(kind)) return false; + const marker = `🤖 Automated first-pass ${kind}`; + return Array.isArray(comments) && comments.some((comment) => + comment?.user?.login === 'github-actions[bot]' && + String(comment?.body || '').trimStart().startsWith(marker) + ); +} + +export async function resolveFirstPassCandidates({ cheap, pro }) { + let cheapCandidate; + try { + cheapCandidate = await cheap(); + } catch { + cheapCandidate = null; + } + + let proCandidate; + if (!cheapCandidate || needsProFirstPass(cheapCandidate)) { + try { + proCandidate = await pro(cheapCandidate); + } catch { + proCandidate = null; + } + } + + try { + return chooseFinalReply(cheapCandidate, proCandidate); + } catch { + return null; + } +} + export function chooseFinalReply(cheap, pro) { if (pro && typeof pro.reply === 'string' && pro.reply.trim() !== '') { return { reply: pro.reply.trim(), generator: pro.generator }; diff --git a/.github/scripts/first-pass.mjs b/.github/scripts/first-pass.mjs index cbdb92ff..9fddcf0d 100644 --- a/.github/scripts/first-pass.mjs +++ b/.github/scripts/first-pass.mjs @@ -12,10 +12,12 @@ import { } from 'node:fs'; import { resolve } from 'node:path'; import { - chooseFinalReply, createRepoReadSession, formatAutomatedComment, + formatFirstPassFallback, + hasTrustedFirstPass, parseFirstPassResponse, + resolveFirstPassCandidates, resolveGeneratorAttribution, validateFirstPassEnvironment, } from './first-pass-lib.mjs'; @@ -24,6 +26,7 @@ const TRANSPORT = 'anthropic-messages'; const REPO_ROOT = resolve(process.cwd()); const env = process.env; const base = (env.ANTHROPIC_BASE_URL || 'https://api.anthropic.com').replace(/\/$/, ''); +const githubApi = (env.GITHUB_API_URL || 'https://api.github.com').replace(/\/$/, ''); const model = env.CCU_BOT_MODEL || 'deepseek-v4-flash'; const modelPro = env.CCU_BOT_MODEL_PRO || 'deepseek-v4-pro'; @@ -162,43 +165,65 @@ async function askModel(useModel, system, userText, think = false) { .trim(); } -let selected; -try { - const first = parseFirstPassResponse(await askModel(model, buildSystem(false), buildUser(''))); - let proCandidate; +const commentsUrl = `${githubApi}/repos/${owner}/${repo}/issues/${num}/comments`; +const githubHeaders = { + authorization: `Bearer ${env.GH_TOKEN}`, + accept: 'application/vnd.github+json', + 'content-type': 'application/json', + 'user-agent': 'ccu-bot', +}; - if (!first.answerable) { - const extra = repoReader.read(first.want_files).text; - const second = parseFirstPassResponse( - await askModel(modelPro, buildSystem(true), buildUser(extra), true), - ); - proCandidate = { reply: second.reply, generator: proGenerator }; +try { + const response = await fetch(`${commentsUrl}?per_page=100`, { headers: githubHeaders }); + if (!response.ok) { + fail(`Comment lookup failed ${response.status}: ${(await response.text()).slice(0, 300)}`); + } + if (hasTrustedFirstPass(await response.json(), kind)) { + console.log(`First-pass ${kind} already exists on #${num}; skipping duplicate.`); + process.exit(0); } - selected = chooseFinalReply( - { reply: first.reply, generator: cheapGenerator }, - proCandidate, - ); } catch (error) { - fail(`Model call failed: ${error.message}`); + fail(`Comment lookup failed: ${error.message}`); } -const { reply, generator: finalGenerator } = selected; + +const selected = await resolveFirstPassCandidates({ + cheap: async () => ({ + ...parseFirstPassResponse(await askModel(model, buildSystem(false), buildUser(''))), + generator: cheapGenerator, + }), + pro: async (cheapCandidate) => { + const extra = repoReader.read(cheapCandidate?.want_files || []).text; + return { + ...parseFirstPassResponse( + await askModel(modelPro, buildSystem(true), buildUser(extra), true), + ), + generator: proGenerator, + }; + }, +}); let commentBody; -try { - commentBody = formatAutomatedComment(reply, { kind, generator: finalGenerator }); -} catch (error) { - fail(`First-pass formatting failed: ${error.message}`); +if (selected) { + try { + commentBody = formatAutomatedComment(selected.reply, { + kind, + generator: selected.generator, + }); + } catch (error) { + fail(`First-pass formatting failed: ${error.message}`); + } +} else { + const authorWroteInChinese = /[\u3400-\u9fff]/u.test( + `${env.ITEM_TITLE || ''}\n${env.ITEM_BODY || ''}`, + ); + commentBody = formatFirstPassFallback({ kind, isChinese: authorWroteInChinese }); + console.warn(`Both model tiers failed to return a usable first-pass ${kind}; posting fallback.`); } try { - const response = await fetch(`https://api.github.com/repos/${owner}/${repo}/issues/${num}/comments`, { + const response = await fetch(commentsUrl, { method: 'POST', - headers: { - authorization: `Bearer ${env.GH_TOKEN}`, - accept: 'application/vnd.github+json', - 'content-type': 'application/json', - 'user-agent': 'ccu-bot', - }, + headers: githubHeaders, body: JSON.stringify({ body: commentBody }), }); if (!response.ok) { diff --git a/.github/scripts/first-pass.test.mjs b/.github/scripts/first-pass.test.mjs index 55ded193..21ac6bac 100644 --- a/.github/scripts/first-pass.test.mjs +++ b/.github/scripts/first-pass.test.mjs @@ -1,5 +1,6 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; import { mkdtempSync, mkdirSync, @@ -7,19 +8,166 @@ import { symlinkSync, writeFileSync, } from 'node:fs'; +import { createServer } from 'node:http'; import { tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { join, resolve } from 'node:path'; import { createRepoReadSession, chooseFinalReply, + formatFirstPassFallback, formatAutomatedComment, + hasTrustedFirstPass, isAllowedRepoPath, + needsProFirstPass, parseFirstPassResponse, + resolveFirstPassCandidates, resolveGeneratorAttribution, validateFirstPassEnvironment, } from './first-pass-lib.mjs'; const ANTHROPIC_MESSAGES = 'anthropic-messages'; +const REPO_ROOT = resolve(import.meta.dirname, '..', '..'); + +async function runFallbackScenario(eventKind) { + const postedComments = []; + let modelCalls = 0; + const server = createServer(async (request, response) => { + if (request.method === 'POST' && request.url === '/v1/messages') { + modelCalls += 1; + response.writeHead(200, { 'content-type': 'application/json' }); + response.end(JSON.stringify({ content: [] })); + return; + } + if (request.method === 'GET' && request.url?.endsWith('/comments?per_page=100')) { + response.writeHead(200, { 'content-type': 'application/json' }); + response.end('[]'); + return; + } + if (request.method === 'POST' && request.url?.endsWith('/comments')) { + const chunks = []; + for await (const chunk of request) chunks.push(chunk); + postedComments.push(JSON.parse(Buffer.concat(chunks).toString('utf8')).body); + response.writeHead(201, { 'content-type': 'application/json' }); + response.end('{}'); + return; + } + response.writeHead(404); + response.end(); + }); + await new Promise((accept) => server.listen(0, '127.0.0.1', accept)); + const address = server.address(); + const apiUrl = `http://127.0.0.1:${address.port}`; + const fixtureRoot = mkdtempSync(join(tmpdir(), 'ccu-first-pass-')); + const diffFile = join(fixtureRoot, 'pr.diff'); + writeFileSync(diffFile, 'diff --git a/src/a.ts b/src/a.ts\n+export const value = 1;\n', 'utf8'); + + try { + const child = spawn(process.execPath, ['.github/scripts/first-pass.mjs'], { + cwd: REPO_ROOT, + env: { + ...process.env, + GH_TOKEN: 'test-github-token', + GITHUB_API_URL: apiUrl, + REPO: 'ClaudeCodeUsage/ClaudeCodeUsage', + EVENT_KIND: eventKind, + ITEM_NUMBER: eventKind === 'issue' ? '87' : '88', + ITEM_TITLE: eventKind === 'issue' ? 'High CPU usage' : 'Reduce background work', + ITEM_BODY: 'Reproduction details and tests are included.', + DIFF_FILE: eventKind === 'pr' ? diffFile : '', + ANTHROPIC_API_KEY: 'test-model-key', + ANTHROPIC_BASE_URL: apiUrl, + CCU_BOT_MODEL: 'deepseek-v4-flash', + CCU_BOT_MODEL_PRO: 'deepseek-v4-pro', + CCU_BOT_GENERATOR: 'deepseek', + CCU_BOT_GENERATOR_PRO: 'deepseek', + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stdout = ''; + let stderr = ''; + child.stdout.setEncoding('utf8'); + child.stderr.setEncoding('utf8'); + child.stdout.on('data', (chunk) => { stdout += chunk; }); + child.stderr.on('data', (chunk) => { stderr += chunk; }); + const exitCode = await new Promise((accept) => child.on('close', accept)); + return { exitCode, modelCalls, postedComments, stdout, stderr }; + } finally { + rmSync(fixtureRoot, { recursive: true, force: true }); + await new Promise((accept) => server.close(accept)); + } +} + +test('shared runner posts a deterministic fallback for issue and PR model failures', async () => { + for (const eventKind of ['issue', 'pr']) { + const result = await runFallbackScenario(eventKind); + assert.equal(result.exitCode, 0, result.stderr); + assert.equal(result.modelCalls, 2); + assert.equal(result.postedComments.length, 1); + assert.match( + result.postedComments[0], + new RegExp(`^🤖 Automated first-pass ${eventKind === 'issue' ? 'reply' : 'review'}`), + ); + assert.match(result.postedComments[0], /not a maintainer decision/); + } +}); + +test('empty cheap replies escalate even when the control says answerable', () => { + assert.equal(needsProFirstPass({ answerable: true, reply: '' }), true); + assert.equal(needsProFirstPass({ answerable: true, reply: 'usable' }), false); + assert.equal(needsProFirstPass({ answerable: false, reply: 'needs source' }), true); +}); + +test('static fallback is honest, English-first, and names no model provider', () => { + const english = formatFirstPassFallback({ kind: 'reply', isChinese: false }); + assert.match(english, /Automated first-pass reply/); + assert.match(english, /did not return a usable analysis/); + assert.match(english, /Posted automatically by CCU Bot/); + assert.doesNotMatch(english, /DeepSeek|Claude|OpenAI Codex|Generated (?:with|by)/); + + const bilingual = formatFirstPassFallback({ kind: 'reply', isChinese: true }); + assert.ok(bilingual.indexOf('did not return a usable analysis') < bilingual.indexOf('自动分析服务')); +}); + +test('only a trusted GitHub Actions author and wrapper marker suppress a duplicate', () => { + const trustedBody = '🤖 Automated first-pass reply\n\nUseful body'; + assert.equal(hasTrustedFirstPass([ + { user: { login: 'github-actions[bot]' }, body: trustedBody }, + ], 'reply'), true); + assert.equal(hasTrustedFirstPass([ + { user: { login: 'someone' }, body: trustedBody }, + ], 'reply'), false); + assert.equal(hasTrustedFirstPass([ + { user: { login: 'github-actions[bot]' }, body: 'ordinary comment' }, + ], 'reply'), false); +}); + +test('candidate resolution calls the pro tier when the cheap reply is empty', async () => { + const cheapGenerator = { id: 'cheap' }; + const proGenerator = { id: 'pro' }; + let proCalls = 0; + const selected = await resolveFirstPassCandidates({ + cheap: async () => ({ reply: '', answerable: true, generator: cheapGenerator }), + pro: async () => { + proCalls += 1; + return { reply: 'grounded pro reply', answerable: true, generator: proGenerator }; + }, + }); + assert.equal(proCalls, 1); + assert.deepEqual(selected, { reply: 'grounded pro reply', generator: proGenerator }); +}); + +test('candidate resolution returns null instead of throwing when both tiers fail', async () => { + let proCalls = 0; + const selected = await resolveFirstPassCandidates({ + cheap: async () => { throw new Error('temporary cheap failure'); }, + pro: async () => { + proCalls += 1; + return { reply: '', answerable: true, generator: { id: 'pro' } }; + }, + }); + assert.equal(proCalls, 1); + assert.equal(selected, null); +}); test('current transport truthfully supports DeepSeek and Claude per tier', () => { const cheap = resolveGeneratorAttribution(undefined, 'deepseek-v4-flash', ANTHROPIC_MESSAGES);