import { execSync } from 'child_process';

import * as fs from 'fs';

import * as path from 'path';

import { CodeGraph } from '../../src/index.js';

import { scoreSearchNodes, scoreFindRelevantContext } from './scoring.js';

import { testCases } from './test-cases.js';

import type { EvalReport, EvalResult } from './types.js';

const codebasePath = process.env.EVAL_CODEBASE || process.argv[2];

if (!codebasePath) {

console.error('Usage: EVAL_CODEBASE=/path/to/codebase npx tsx __tests__/evaluation/runner.ts');

console.error(' or: npx tsx __tests__/evaluation/runner.ts /path/to/codebase');

process.exit(1);

}

const resolvedPath = path.resolve(codebasePath);

if (!fs.existsSync(path.join(resolvedPath, '.codegraph', 'codegraph.db'))) {

console.error(`No .codegraph/codegraph.db found at ${resolvedPath}`);

process.exit(1);

}

let codegraphSha = 'unknown';

try {

codegraphSha = execSync('git rev-parse --short HEAD', { encoding: 'utf-8' }).trim();

} catch {}

console.log(`\nCodeGraph Eval — ${path.basename(resolvedPath)}`);

console.log(`Codebase: ${resolvedPath}`);

console.log(`Commit: ${codegraphSha}`);

console.log(`Cases: ${testCases.length}`);

console.log('');

async function run() {

const cg = CodeGraph.openSync(resolvedPath);

const results: EvalResult[] = [];

for (const tc of testCases) {

const start = performance.now();

if (tc.api === 'searchNodes') {

const searchResults = cg.searchNodes(tc.query, {

limit: 10,

kinds: tc.kinds,

...(tc.options as Record<string, unknown>),

});

const latency = performance.now() - start;

const result = scoreSearchNodes(tc.id, tc.expectedSymbols, searchResults, latency);

results.push(result);

} else {

const subgraph = await cg.findRelevantContext(tc.query, {

searchLimit: 8,

traversalDepth: 3,

maxNodes: 80,

minScore: 0.2,

...(tc.options as Record<string, unknown>),

});

const latency = performance.now() - start;

const result = scoreFindRelevantContext(tc.id, tc.expectedSymbols, subgraph, latency);

results.push(result);

}

}

cg.close();

// Print results table

const maxIdLen = Math.max(...results.map((r) => r.caseId.length));

for (const r of results) {

const status = r.pass ? '\x1b[32mPASS\x1b[0m' : '\x1b[31mFAIL\x1b[0m';

const id = r.caseId.padEnd(maxIdLen);

const recall = `recall=${r.recall.toFixed(2)}`;

const extra =

r.edgeDensity !== undefined

? `density=${r.edgeDensity.toFixed(2)}`

: `mrr=${r.mrr.toFixed(2)}`;

const latency = `${Math.round(r.latencyMs)}ms`;

console.log(` ${id} ${status} ${recall} ${extra} ${latency}`);

if (r.missedSymbols.length > 0) {

console.log(` ${' '.repeat(maxIdLen)} missed: ${r.missedSymbols.join(', ')}`);

}

}

// Summary

const passed = results.filter((r) => r.pass).length;

const failed = results.length - passed;

const meanRecall = results.reduce((s, r) => s + r.recall, 0) / results.length;

const mrrResults = results.filter((r) => r.mrr > 0 || r.caseId.startsWith('search-'));

const meanMRR =

mrrResults.length > 0 ? mrrResults.reduce((s, r) => s + r.mrr, 0) / mrrResults.length : 0;

console.log('');

const summaryColor = failed === 0 ? '\x1b[32m' : '\x1b[33m';

console.log(

`${summaryColor}SUMMARY: ${passed}/${results.length} passed | recall=${meanRecall.toFixed(2)} | mrr=${meanMRR.toFixed(2)}\x1b[0m`

);

// Save JSON report

const report: EvalReport = {

timestamp: new Date().toISOString(),

codebasePath: resolvedPath,

codegraphSha,

summary: { total: results.length, passed, failed, meanRecall, meanMRR },

results,

};

const resultsDir = path.join(__dirname, 'results');

fs.mkdirSync(resultsDir, { recursive: true });

const reportFile = path.join(

resultsDir,

`${new Date().toISOString().replace(/[:.]/g, '-')}.json`

);

fs.writeFileSync(reportFile, JSON.stringify(report, null, 2));

console.log(`\nReport saved: ${reportFile}`);

process.exit(failed > 0 ? 1 : 0);

}

run().catch((err) => {

console.error(err);

process.exit(1);

});