From ba4f6eb4206edc3621bc793ac97488efa5622f9f Mon Sep 17 00:00:00 2001 From: Zifan Zhu Date: Wed, 8 Oct 2025 15:25:11 -0400 Subject: [PATCH 1/5] feat: add evaluation script - supports take inference output file as input then genrate result files --- src/evaluate.py | 125 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 src/evaluate.py diff --git a/src/evaluate.py b/src/evaluate.py new file mode 100644 index 0000000..4db2de6 --- /dev/null +++ b/src/evaluate.py @@ -0,0 +1,125 @@ +import argparse +import asyncio +import logging +from pathlib import Path + +from sandbox_fusion import ( + SubmitRequest, + TestConfig, + set_endpoint, + submit_async, +) +from tqdm.asyncio import tqdm_asyncio + +from utils import read_jsonl, write_jsonl + +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s' +) +logger = logging.getLogger(__name__) + + +async def evaluate_sample(sample: dict) -> list[dict]: + completions: str | list[str] = sample.get('completion', []) + if isinstance(completions, str): + completions = [completions] + n = len(completions) + results = [] + pass_at_k = False + all_k_correct = True + for completion in completions: + result = {} + eval_result = await submit_async( + SubmitRequest( + dataset='FullStackBench', + id=sample['id'], + completion=completion, + config=TestConfig( + compile_timeout=50, + run_timeout=50, + dataset_type='AutoEvalDataset', + provided_data=sample + ) + ) + ) + passed = eval_result.accepted + result['accepted'] = passed + result['pass_at_1'] = 1 if passed else 0 + result['task_id']= sample['id'] + result['n' ] = n + if passed: + pass_at_k = True + else: + all_k_correct = False + if 'labels' in sample and 'programming_language' in sample['labels']: + result['programming_language'] = sample['labels']['programming_language'] + result['category'] = sample['labels'].get('category', '') + result['difficulty'] = sample['labels'].get('difficulty', '') + results.append(result) + for result in results: + result['pass_at_k'] = 1 if pass_at_k else 0 + result['all_k_correct'] = 1 if all_k_correct else 0 + return results + + +async def main(): + parser = argparse.ArgumentParser( + description='Evaluate FullStackBench predictions using SandboxFusion' + ) + parser.add_argument( + '--inference-file', + type=str, + required=True, + help='Path to inference output file (JSONL format)' + ) + parser.add_argument( + '--output-file', + type=str, + required=True, + help='Path to save evaluation results (JSONL format)' + ) + parser.add_argument( + '--sandbox-endpoint', + type=str, + default='http://localhost:8080', + help='SandboxFusion server endpoint' + ) + + args = parser.parse_args() + + inference_file = Path(args.inference_file) + if not inference_file.exists(): + logger.error(f"Inference file not found: {args.inference_file}") + return 1 + + set_endpoint(args.sandbox_endpoint) + logger.info(f"Using SandboxFusion endpoint: {args.sandbox_endpoint}") + + logger.info(f"Reading inference results from: {args.inference_file}") + samples = read_jsonl(args.inference_file) + logger.info(f"Found {len(samples)} samples to evaluate") + + logger.info("Starting evaluation...") + tasks = [evaluate_sample(sample) for sample in samples] + results = [] + + for result in await tqdm_asyncio.gather(*tasks, desc="Evaluating"): + for r in result: + results.append(r) + + pass_count = sum([r['pass_at_1'] for r in results]) + pass_rate = pass_count / len(results) if results else 0.0 + + logger.info("Evaluation complete!") + logger.info(f"Pass rate: {pass_rate:.4%} ({pass_count}/{len(results)})") + + logger.info(f"Writing results to: {args.output_file}") + write_jsonl(args.output_file, [r for r in results]) + + return 0 + + +if __name__ == '__main__': + exit_code = asyncio.run(main()) + exit(exit_code) From f8e9e49c722dc2df1fa61e78b22f5170b7a136ff Mon Sep 17 00:00:00 2001 From: Zifan Zhu Date: Wed, 15 Oct 2025 18:52:38 -0400 Subject: [PATCH 2/5] feat: add natural languge to result json file --- src/evaluate.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/evaluate.py b/src/evaluate.py index 4db2de6..08ec7a0 100644 --- a/src/evaluate.py +++ b/src/evaluate.py @@ -20,7 +20,7 @@ logger = logging.getLogger(__name__) -async def evaluate_sample(sample: dict) -> list[dict]: +async def evaluate_sample(sample: dict, natural_language: str) -> list[dict]: completions: str | list[str] = sample.get('completion', []) if isinstance(completions, str): completions = [completions] @@ -52,6 +52,7 @@ async def evaluate_sample(sample: dict) -> list[dict]: pass_at_k = True else: all_k_correct = False + result['natural_language'] = natural_language if 'labels' in sample and 'programming_language' in sample['labels']: result['programming_language'] = sample['labels']['programming_language'] result['category'] = sample['labels'].get('category', '') @@ -85,6 +86,13 @@ async def main(): default='http://localhost:8080', help='SandboxFusion server endpoint' ) + parser.add_argument( + '--language', + type=str, + required=True, + choices=['en', 'zh'], + help='Natural language of the dataset (en or zh)' + ) args = parser.parse_args() @@ -101,7 +109,7 @@ async def main(): logger.info(f"Found {len(samples)} samples to evaluate") logger.info("Starting evaluation...") - tasks = [evaluate_sample(sample) for sample in samples] + tasks = [evaluate_sample(sample, args.language) for sample in samples] results = [] for result in await tqdm_asyncio.gather(*tasks, desc="Evaluating"): From 2d696fca5a101d4ae902b0d490b5a9f23e7e41d3 Mon Sep 17 00:00:00 2001 From: Gustavo Ansaldi Oliva Date: Mon, 27 Oct 2025 21:38:30 -0400 Subject: [PATCH 3/5] Apply suggestion from @Copilot Simplify result generation Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/evaluate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/evaluate.py b/src/evaluate.py index 08ec7a0..53686ef 100644 --- a/src/evaluate.py +++ b/src/evaluate.py @@ -123,7 +123,7 @@ async def main(): logger.info(f"Pass rate: {pass_rate:.4%} ({pass_count}/{len(results)})") logger.info(f"Writing results to: {args.output_file}") - write_jsonl(args.output_file, [r for r in results]) + write_jsonl(args.output_file, results) return 0 From c889e390470767540159db05411b6aa31a8739a6 Mon Sep 17 00:00:00 2001 From: Zifan Zhu Date: Fri, 23 Jan 2026 14:03:39 -0500 Subject: [PATCH 4/5] fix: add num workers arg to evaluate command --- src/evaluate.py | 93 +++++++++++++++++++++++++++---------------------- 1 file changed, 52 insertions(+), 41 deletions(-) diff --git a/src/evaluate.py b/src/evaluate.py index 53686ef..9ea93ae 100644 --- a/src/evaluate.py +++ b/src/evaluate.py @@ -20,48 +20,51 @@ logger = logging.getLogger(__name__) -async def evaluate_sample(sample: dict, natural_language: str) -> list[dict]: - completions: str | list[str] = sample.get('completion', []) - if isinstance(completions, str): - completions = [completions] - n = len(completions) - results = [] - pass_at_k = False - all_k_correct = True - for completion in completions: - result = {} - eval_result = await submit_async( - SubmitRequest( - dataset='FullStackBench', - id=sample['id'], - completion=completion, - config=TestConfig( - compile_timeout=50, - run_timeout=50, - dataset_type='AutoEvalDataset', - provided_data=sample +async def evaluate_sample( + sample: dict, natural_language: str, semaphore: asyncio.Semaphore +) -> list[dict]: + async with semaphore: + completions: str | list[str] = sample.get('completion', []) + if isinstance(completions, str): + completions = [completions] + n = len(completions) + results = [] + pass_at_k = False + all_k_correct = True + for completion in completions: + result = {} + eval_result = await submit_async( + SubmitRequest( + dataset='FullStackBench', + id=sample['id'], + completion=completion, + config=TestConfig( + compile_timeout=50, + run_timeout=50, + dataset_type='AutoEvalDataset', + provided_data=sample + ) ) ) - ) - passed = eval_result.accepted - result['accepted'] = passed - result['pass_at_1'] = 1 if passed else 0 - result['task_id']= sample['id'] - result['n' ] = n - if passed: - pass_at_k = True - else: - all_k_correct = False - result['natural_language'] = natural_language - if 'labels' in sample and 'programming_language' in sample['labels']: - result['programming_language'] = sample['labels']['programming_language'] - result['category'] = sample['labels'].get('category', '') - result['difficulty'] = sample['labels'].get('difficulty', '') - results.append(result) - for result in results: - result['pass_at_k'] = 1 if pass_at_k else 0 - result['all_k_correct'] = 1 if all_k_correct else 0 - return results + passed = eval_result.accepted + result['accepted'] = passed + result['pass_at_1'] = 1 if passed else 0 + result['task_id'] = sample['id'] + result['n'] = n + if passed: + pass_at_k = True + else: + all_k_correct = False + result['natural_language'] = natural_language + if 'labels' in sample and 'programming_language' in sample['labels']: + result['programming_language'] = sample['labels']['programming_language'] + result['category'] = sample['labels'].get('category', '') + result['difficulty'] = sample['labels'].get('difficulty', '') + results.append(result) + for result in results: + result['pass_at_k'] = 1 if pass_at_k else 0 + result['all_k_correct'] = 1 if all_k_correct else 0 + return results async def main(): @@ -93,6 +96,12 @@ async def main(): choices=['en', 'zh'], help='Natural language of the dataset (en or zh)' ) + parser.add_argument( + '--num-workers', + type=int, + default=10, + help='Maximum number of concurrent evaluation requests (default: 10)' + ) args = parser.parse_args() @@ -109,7 +118,9 @@ async def main(): logger.info(f"Found {len(samples)} samples to evaluate") logger.info("Starting evaluation...") - tasks = [evaluate_sample(sample, args.language) for sample in samples] + # Limit concurrent requests to avoid "Too many open files" error + semaphore = asyncio.Semaphore(args.num_workers) + tasks = [evaluate_sample(sample, args.language, semaphore) for sample in samples] results = [] for result in await tqdm_asyncio.gather(*tasks, desc="Evaluating"): From c8c5b33628dde8ef4229b4bac04d790dc166113c Mon Sep 17 00:00:00 2001 From: Zifan Zhu <55147873+dogeggz@users.noreply.github.com> Date: Fri, 23 Jan 2026 14:08:31 -0500 Subject: [PATCH 5/5] fix: make sure num workers at least 1 Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/evaluate.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/evaluate.py b/src/evaluate.py index 9ea93ae..2a1858f 100644 --- a/src/evaluate.py +++ b/src/evaluate.py @@ -117,6 +117,9 @@ async def main(): samples = read_jsonl(args.inference_file) logger.info(f"Found {len(samples)} samples to evaluate") + if args.num_workers < 1: + logger.error(f"--num-workers must be at least 1, got {args.num_workers}") + return 1 logger.info("Starting evaluation...") # Limit concurrent requests to avoid "Too many open files" error semaphore = asyncio.Semaphore(args.num_workers)