From 1834086895892d0ca69523176d19e27cf88776a1 Mon Sep 17 00:00:00 2001 From: AUTHENSOR Date: Sat, 5 Sep 2026 02:59:54 -0500 Subject: [PATCH] Honor start_date/end_date and release_version on every loader path --not_fast silently ignored --start_date/--end_date (no date parameters at all) and never forwarded --release_version to load_dataset, so the original-benchmark path loaded the full dataset at the Hub's default revision with no signal. selfrepair dropped the dates; testoutputprediction and codeexecution accepted release_version but never used it. Thread the flags through every loader: not_fast now forwards version_tag and applies the same contest-date window as the fast path, selfrepair passes its dates through, and the test-generation and execution loaders forward version_tag. Fixes #153 --- lcb_runner/benchmarks/code_execution.py | 2 +- lcb_runner/benchmarks/code_generation.py | 12 ++++++++++-- lcb_runner/benchmarks/test_output_prediction.py | 2 +- lcb_runner/runner/scenario_router.py | 12 ++++++++++-- 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/lcb_runner/benchmarks/code_execution.py b/lcb_runner/benchmarks/code_execution.py index f37a03b3..b8c81897 100644 --- a/lcb_runner/benchmarks/code_execution.py +++ b/lcb_runner/benchmarks/code_execution.py @@ -57,7 +57,7 @@ def get_evaluation_sample(self) -> dict: def load_code_execution_dataset(release_version="release_v1") -> list[CodeExecutionProblem]: - dataset = load_dataset("livecodebench/execution-v2", split="test") + dataset = load_dataset("livecodebench/execution-v2", split="test", version_tag=release_version, trust_remote_code=True) dataset = [CodeExecutionProblem(**p) for p in dataset] # type: ignore print(f"Loaded {len(dataset)} problems") return dataset diff --git a/lcb_runner/benchmarks/code_generation.py b/lcb_runner/benchmarks/code_generation.py index e2e48a73..c1860632 100644 --- a/lcb_runner/benchmarks/code_generation.py +++ b/lcb_runner/benchmarks/code_generation.py @@ -136,9 +136,17 @@ def load_code_generation_dataset(release_version="release_v1", start_date=None, return dataset -def load_code_generation_dataset_not_fast(release_version="release_v1") -> list[CodeGenerationProblem]: - dataset = load_dataset("livecodebench/code_generation", split="test") +def load_code_generation_dataset_not_fast(release_version="release_v1", start_date=None, end_date=None) -> list[CodeGenerationProblem]: + dataset = load_dataset("livecodebench/code_generation", split="test", version_tag=release_version, trust_remote_code=True) dataset = [CodeGenerationProblem(**p) for p in dataset] # type: ignore + if start_date is not None: + p_start_date = datetime.strptime(start_date, "%Y-%m-%d") + dataset = [e for e in dataset if p_start_date <= e.contest_date] + + if end_date is not None: + p_end_date = datetime.strptime(end_date, "%Y-%m-%d") + dataset = [e for e in dataset if e.contest_date <= p_end_date] + print(f"Loaded {len(dataset)} problems") return dataset diff --git a/lcb_runner/benchmarks/test_output_prediction.py b/lcb_runner/benchmarks/test_output_prediction.py index 12154ada..d6e5ecdc 100644 --- a/lcb_runner/benchmarks/test_output_prediction.py +++ b/lcb_runner/benchmarks/test_output_prediction.py @@ -60,7 +60,7 @@ def get_evaluation_sample(self) -> dict: def load_test_prediction_dataset(release_version="release_v1") -> list[TestOutputPredictionProblem]: - dataset = load_dataset("livecodebench/test_generation", split="test") # type: ignore + dataset = load_dataset("livecodebench/test_generation", split="test", version_tag=release_version, trust_remote_code=True) # type: ignore dataset = [TestOutputPredictionProblem(**d) for d in dataset] print(f"Loaded {len(dataset)} prediction problems") return dataset diff --git a/lcb_runner/runner/scenario_router.py b/lcb_runner/runner/scenario_router.py index ffcfe0e7..bd3c2b23 100644 --- a/lcb_runner/runner/scenario_router.py +++ b/lcb_runner/runner/scenario_router.py @@ -50,7 +50,11 @@ def build_prompt_benchmark( if scenario == Scenario.codegeneration: not_fast: bool = args.not_fast if not_fast: - benchmark = load_code_generation_dataset_not_fast(args.release_version) + benchmark = load_code_generation_dataset_not_fast( + args.release_version, + start_date=args.start_date, + end_date=args.end_date, + ) else: benchmark = load_code_generation_dataset( args.release_version, @@ -64,7 +68,11 @@ def build_prompt_benchmark( benchmark = sorted(benchmark, key=lambda x: (x.question_id, x.test_id)) format_prompt = format_prompt_test_output elif scenario == Scenario.selfrepair: - benchmark = load_code_generation_dataset(args.release_version) + benchmark = load_code_generation_dataset( + args.release_version, + start_date=args.start_date, + end_date=args.end_date, + ) benchmark = sorted(benchmark, key=lambda x: x.question_id) format_prompt = format_prompt_self_repair elif scenario == Scenario.codeexecution: