diff --git a/INSTALL.md b/INSTALL.md
new file mode 100644
index 00000000..88042331
--- /dev/null
+++ b/INSTALL.md
@@ -0,0 +1,16 @@
+# Install 
+
+Follow instructions in README.md
+
+## Manual data setup
+
+- Circumvents https://github.com/LiveCodeBench/LiveCodeBench/issues/108
+- Also, code_generation.py was modified to ensure that `release_v{x}` contains only the datapoints in `test{x}.jsonl` (i.e., it is not cumulative anymore).
+
+```
+Clone the repo to benchmarks/livecodebench
+cd livecodebench
+git clone https://huggingface.co/datasets/livecodebench/code_generation_lite data
+```
+
+Currently, the latest release is v6. When some newer version is released, it should be just a matter of updating the repo inside `data` (a `git pull` should do it)
diff --git a/data/lcb_category_tag.csv b/data/lcb_category_tag.csv
new file mode 100644
index 00000000..7dd51152
--- /dev/null
+++ b/data/lcb_category_tag.csv
@@ -0,0 +1,176 @@
+question_id,platform,category,tag
+3674,leetcode,"Data Structures,Common Techniques","Queue,Two-Pointer Technique"
+3677,leetcode,"Dynamic Programming,Algorithm Basics","Other Dynamic Programming Methods,Recurrence"
+3687,leetcode,"Search Algorithms,Mathematics,Algorithm Basics,Common Techniques","Depth-First Search (DFS),Pigeonhole Principle,Recursion,Two-Pointer Technique"
+3692,leetcode,"String Algorithms,Dynamic Programming","Knuth-Morris-Pratt (KMP) Algorithm,Other Dynamic Programming Methods"
+3696,leetcode,"Mathematics,Dynamic Programming,Data Structures,Common Techniques,Algorithm Basics","Combinatorics,Digit Dynamic Programming,Hash Table,Prefix Sum & Difference,Recurrence"
+3697,leetcode,"Search Algorithms,Algorithm Basics,Mathematics,Algorithm Basics,Algorithm Basics","Backtracking,Brute Force,Combinatorics,Enumeration,Recursion"
+3701,leetcode,"Search Algorithms,Dynamic Programming,Algorithm Basics","Depth-First Search (DFS),Memoization Search,Recursion"
+3704,leetcode,Algorithm Basics,Ad-hoc
+3705,leetcode,"Algorithm Basics,Data Structures","Brute Force,Hash Table"
+3708,leetcode,Algorithm Basics,Simulation
+3709,leetcode,,
+3714,leetcode,"Mathematics,Algorithm Basics","Combinatorics,Sorting"
+3717,leetcode,"Mathematics,Dynamic Programming,Algorithm Basics","Order Theory,Other Dynamic Programming Methods,Sorting"
+3720,leetcode,"Algorithm Basics,Search Algorithms,Graph Theory,Graph Theory,Data Structures,Data Structures,Algorithm Basics","Binary Search,Breadth-First Search (BFS),Connectivity,Graph Representation,Hash Table,Queue,Simulation"
+3721,leetcode,"Algorithm Basics,Algorithm Basics","Simulation,Sorting"
+3722,leetcode,"Dynamic Programming,Common Techniques,Algorithm Basics","Other Dynamic Programming Methods,Prefix Sum & Difference,Recurrence"
+3723,leetcode,Algorithm Basics,Ad-hoc
+3725,leetcode,"Mathematics,Data Structures","Combinatorics,Monotonic Stacks"
+3731,leetcode,Common Techniques,Prefix Sum & Difference
+3733,leetcode,"Common Techniques,Search Algorithms,Dynamic Programming,Algorithm Basics","Bit Manipulation,Depth-First Search (DFS),Memoization Search,Recursion"
+3736,leetcode,"Algorithm Basics,Data Structures","Brute Force,Hash Table"
+3737,leetcode,"Search Algorithms,Common Techniques,Dynamic Programming,Algorithm Basics,Algorithm Basics","Backtracking,Meet in Middle,Memoization Search,Recurrence,Recursion"
+3739,leetcode,"Algorithm Basics,Mathematics,Algorithm Basics,Computational Geometry","Brute Force,Combinatorics,Enumeration,Manhattan Distance"
+3743,leetcode,"Algorithm Basics,Algorithm Basics,Common Techniques","Ad-hoc,Greedy,Prefix Sum & Difference"
+3744,leetcode,"Dynamic Programming,Common Techniques,Algorithm Basics","Other Dynamic Programming Methods,Prefix Sum & Difference,Recurrence"
+3747,leetcode,,
+3748,leetcode,"Data Structures,Algorithm Basics","Heap,Sorting"
+3750,leetcode,"Algorithm Basics,Data Structures","Constructive Algorithms,Hash Table"
+3751,leetcode,Common Techniques,Prefix Sum & Difference
+3753,leetcode,"Algorithm Basics,Data Structures","Greedy,Hash Table"
+3754,leetcode,"Algorithm Basics,Computational Geometry,Algorithm Basics","Greedy,Manhattan Distance,Simulation"
+3759,leetcode,"Algorithm Basics,Data Structures","Greedy,Heap"
+3760,leetcode,"Algorithm Basics,Data Structures,Mathematics","Constructive Algorithms,Hash Table,Sieve Method"
+3762,leetcode,"Algorithm Basics,Algorithm Basics","Binary Search,Greedy"
+3763,leetcode,"Common Techniques,Algorithm Basics,Common Techniques,Data Structures,Algorithm Basics,Algorithm Basics,Computational Geometry","Discretization,Divide and Conquer,Hanging Line Method,Segment Tree,Simulation,Sorting,Sweep Line"
+3764,leetcode,"Algorithm Basics,Algorithm Basics","Greedy,Sorting"
+3765,leetcode,"Search Algorithms,Dynamic Programming,Common Techniques,Algorithm Basics,Algorithm Basics","Depth-First Search (DFS),Memoization Search,Prefix Sum & Difference,Recurrence,Recursion"
+3768,leetcode,Algorithm Basics,Simulation
+3770,leetcode,"Algorithm Basics,Algorithm Basics,Algorithm Basics,String Algorithms","Constructive Algorithms,Greedy,Simulation,String Matching"
+3771,leetcode,"Algorithm Basics,Algorithm Basics,Data Structures,Algorithm Basics","Brute Force,Greedy,Hash Table,Sorting"
+3773,leetcode,"Algorithm Basics,Algorithm Basics","Greedy,Simulation"
+3776,leetcode,"Search Algorithms,Mathematics,Data Structures,Dynamic Programming,Algorithm Basics,Algorithm Basics","Backtracking,Combinatorics,Hash Table,Memoization Search,Recursion,Sorting"
+3777,leetcode,"Search Algorithms,Mathematics,Dynamic Programming,Algorithm Basics","Backtracking,Combinatorics,Memoization Search,Recursion"
+3778,leetcode,Common Techniques,Binary Thinking
+3779,leetcode,"Algorithm Basics,Algorithm Basics","Greedy,Sorting"
+3781,leetcode,"Algorithm Basics,Common Techniques,Computational Geometry,Mathematics,Algorithm Basics,Algorithm Basics","Binary Search,Discretization,Manhattan Distance,Numerical Algorithms,Simulation,Sorting"
+3783,leetcode,"Mathematics,Algorithm Basics,Dynamic Programming,Algorithm Basics,Algorithm Basics","Combinatorics,Constructive Algorithms,Counting Dynamic Programming,Recurrence,Recursion"
+3784,leetcode,"Algorithm Basics,Common Techniques,Algorithm Basics,Data Structures","Binary Search,Binary Thinking,Enumeration,Hash Table"
+3785,leetcode,"Common Techniques,Algorithm Basics","Differences,Greedy"
+3786,leetcode,"Mathematics,Dynamic Programming,Dynamic Programming,Algorithm Basics","Combinatorics,Interval Dynamic Programming,Knapsack Dynamic Programming,Recurrence"
+3788,leetcode,"Mathematics,Algorithm Basics","Binary Set Operations,Greedy"
+3789,leetcode,Algorithm Basics,Ad-hoc
+3791,leetcode,"Algorithm Basics,Data Structures,Common Techniques","Greedy,Hash Table,Two-Pointer Technique"
+3793,leetcode,"Dynamic Programming,Algorithm Basics","Memoization Search,Recursion"
+3794,leetcode,"Algorithm Basics,Common Techniques,Mathematics","Greedy,Prefix Sum & Difference,Schedules"
+3795,leetcode,"Algorithm Basics,Common Techniques","Binary Search,Prefix Sum & Difference"
+3799,leetcode,"Mathematics,Algorithm Basics","Combinatorics,Enumeration"
+3801,leetcode,"Mathematics,Dynamic Programming,Algorithm Basics","Combinatorics,Digit Dynamic Programming,Recurrence"
+3805,leetcode,Algorithm Basics,Enumeration
+3809,leetcode,"Mathematics,Algorithm Basics,Graph Theory,Data Structures","Binary Set Operations,Brute Force,Connectivity,Disjoint Set Union"
+3811,leetcode,Algorithm Basics,Ad-hoc
+3817,leetcode,Algorithm Basics,Ad-hoc
+3832,leetcode,"Algorithm Basics,Algorithm Basics","Greedy,Recurrence"
+abc387_a,atcoder,,
+abc387_b,atcoder,Algorithm Basics,Enumeration
+abc387_c,atcoder,"Mathematics,Algorithm Basics,Dynamic Programming","Combinatorics,Constructive Algorithms,Digit Dynamic Programming"
+abc387_f,atcoder,"Mathematics,Search Algorithms,Common Techniques,Algorithm Basics,Graph Theory,Graph Theory,Dynamic Programming","Combinatorics,Depth-First Search (DFS),Prefix Sum & Difference,Simulation,Strongly Connected Components,Tree Decomposition,Tree Dynamic Programming"
+abc388_a,atcoder,,
+abc388_b,atcoder,,
+abc388_c,atcoder,"Algorithm Basics,Algorithm Basics","Binary Search,Sorting"
+abc388_d,atcoder,"Algorithm Basics,Mathematics,Data Structures","Constructive Algorithms,Numerical Algorithms,Segment Tree"
+abc388_e,atcoder,Common Techniques,Two-Pointer Technique
+abc388_f,atcoder,"Search Algorithms,Data Structures,Dynamic Programming,Algorithm Basics","Backtracking,Hash Table,Memoization Search,Recursion"
+abc388_g,atcoder,"Mathematics,Data Structures,Common Techniques","Numerical Algorithms,Segment Tree,Two-Pointer Technique"
+abc389_a,atcoder,,
+abc389_b,atcoder,,
+abc389_d,atcoder,"Algorithm Basics,Mathematics","Binary Search,Numerical Algorithms"
+abc389_e,atcoder,Algorithm Basics,Binary Search
+abc389_f,atcoder,Data Structures,Segment Tree
+abc389_g,atcoder,"Mathematics,Dynamic Programming,Mathematics,Algorithm Basics,Graph Theory","Combinatorics,Counting Dynamic Programming,Polynomials,Recurrence,Shortest Path"
+abc390_a,atcoder,Algorithm Basics,Ad-hoc
+abc390_b,atcoder,Algorithm Basics,Ad-hoc
+abc390_c,atcoder,Algorithm Basics,Ad-hoc
+abc390_d,atcoder,"Search Algorithms,Mathematics,Search Algorithms","Backtracking,Combinatorics,Depth-First Search (DFS)"
+abc390_e,atcoder,"Algorithm Basics,Common Techniques,Dynamic Programming,Algorithm Basics","Binary Search,Binary Thinking,Knapsack Dynamic Programming,Recurrence"
+abc390_f,atcoder,"Algorithm Basics,Data Structures","Greedy,Hash Table"
+abc390_g,atcoder,"Mathematics,Algorithm Basics,Dynamic Programming,Mathematics,Algorithm Basics,Data Structures,Mathematics","Combinatorics,Divide and Conquer,Dynamic Programming Optimization,Fast Fourier Transform (FFT),Greedy,Heap,Polynomials"
+abc391_a,atcoder,Algorithm Basics,Ad-hoc
+abc391_b,atcoder,Algorithm Basics,Brute Force
+abc391_d,atcoder,Algorithm Basics,Sorting
+abc391_e,atcoder,"Algorithm Basics,Dynamic Programming,Algorithm Basics,Data Structures","Divide and Conquer,Interval Dynamic Programming,Recursion,Segment Tree"
+abc391_f,atcoder,"Algorithm Basics,Data Structures,Data Structures,Search Algorithms,Algorithm Basics","Greedy,Hash Table,Heap,Heuristic Search,Sorting"
+abc391_g,atcoder,"Mathematics,Dynamic Programming","Combinatorics,Counting Dynamic Programming"
+abc392_a,atcoder,Algorithm Basics,Enumeration
+abc392_b,atcoder,,
+abc392_c,atcoder,,
+abc392_d,atcoder,"Data Structures,Mathematics","Hash Table,Probability Theory"
+abc392_f,atcoder,Data Structures,Block Structure
+abc392_g,atcoder,Mathematics,Fast Fourier Transform (FFT)
+abc393_a,atcoder,Algorithm Basics,Ad-hoc
+abc393_b,atcoder,"Algorithm Basics,String Algorithms","Brute Force,String Matching"
+abc393_d,atcoder,Algorithm Basics,Greedy
+abc393_e,atcoder,Mathematics,Greatest Common Divisor (GCD) and Extended GCD (ExGCD)
+abc393_f,atcoder,"Algorithm Basics,Algorithm Basics,Dynamic Programming","Binary Search,Greedy,Other Dynamic Programming Methods"
+abc394_a,atcoder,,
+abc394_b,atcoder,Algorithm Basics,Sorting
+abc394_c,atcoder,"Algorithm Basics,String Algorithms","Simulation,String Matching"
+abc394_d,atcoder,"Algorithm Basics,Algorithm Basics,Data Structures","Greedy,Simulation,Stack"
+abc394_e,atcoder,"Algorithm Basics,Search Algorithms,Dynamic Programming,Data Structures,Graph Theory","Ad-hoc,Breadth-First Search (BFS),Other Dynamic Programming Methods,Queue,Shortest Path"
+abc394_f,atcoder,"Algorithm Basics,Graph Theory,Dynamic Programming","Divide and Conquer,Tree Decomposition,Tree Dynamic Programming"
+abc394_g,atcoder,"Graph Theory,Data Structures,Algorithm Basics","Connectivity,Disjoint Set Union,Doubling"
+abc395_a,atcoder,Algorithm Basics,Ad-hoc
+abc395_b,atcoder,Algorithm Basics,Simulation
+abc395_c,atcoder,Data Structures,Hash Table
+abc395_e,atcoder,"Common Techniques,Algorithm Basics,Data Structures,Dynamic Programming,Graph Theory","Binary Thinking,Greedy,Heap,Memoization Search,Shortest Path"
+abc395_f,atcoder,"Algorithm Basics,Algorithm Basics","Binary Search,Greedy"
+abc396_a,atcoder,Algorithm Basics,Brute Force
+abc396_b,atcoder,"Algorithm Basics,Data Structures","Simulation,Stack"
+abc396_c,atcoder,"Algorithm Basics,Common Techniques,Algorithm Basics,Common Techniques","Greedy,Prefix Sum & Difference,Sorting,Two-Pointer Technique"
+abc396_d,atcoder,Algorithm Basics,Enumeration
+abc396_e,atcoder,"Common Techniques,Common Techniques,Search Algorithms,Mathematics,Data Structures","Binary Thinking,Bit Manipulation,Depth-First Search (DFS),Linear Basis,Stack"
+abc396_f,atcoder,"Mathematics,Data Structures,Data Structures","Combinatorics,Fenwick Tree,Segment Tree"
+abc396_g,atcoder,"Algorithm Basics,Algorithm Basics,Mathematics","Divide and Conquer,Doubling,Fast Fourier Transform (FFT)"
+abc397_a,atcoder,Algorithm Basics,Ad-hoc
+abc397_b,atcoder,,
+abc397_c,atcoder,"Data Structures,Algorithm Basics,Common Techniques","Hash Table,Simulation,Two-Pointer Technique"
+abc397_d,atcoder,Mathematics,Indeterminate Equations
+abc397_e,atcoder,"Algorithm Basics,Graph Theory","Recursion,Tree Decomposition"
+abc397_f,atcoder,"Dynamic Programming,Common Techniques,Data Structures","Dynamic Programming Optimization,Prefix Sum & Difference,Segment Tree"
+abc397_g,atcoder,Mathematics,Linear Programming
+abc398_a,atcoder,Algorithm Basics,Constructive Algorithms
+abc398_b,atcoder,Algorithm Basics,Ad-hoc
+abc398_c,atcoder,Data Structures,Hash Table
+abc398_d,atcoder,Data Structures,Hash Table
+abc398_f,atcoder,"Algorithm Basics,String Algorithms","Greedy,Z-function"
+abc398_g,atcoder,"Algorithm Basics,Graph Theory,Search Algorithms,Mathematics,Data Structures","Ad-hoc,Bipartite Graph,Breadth-First Search (BFS),Game Theory,Queue"
+abc399_a,atcoder,,
+abc399_b,atcoder,Algorithm Basics,Brute Force
+abc399_c,atcoder,"Search Algorithms,Data Structures","Breadth-First Search (BFS),Queue"
+abc399_d,atcoder,Algorithm Basics,Ad-hoc
+abc399_e,atcoder,"Graph Theory,Data Structures","Connectivity,Disjoint Set Union"
+abc399_f,atcoder,"Mathematics,Mathematics,Common Techniques,Algorithm Basics","Combinatorics,Polynomials,Prefix Sum & Difference,Recurrence"
+abc400_a,atcoder,,
+abc400_b,atcoder,,
+abc400_c,atcoder,Mathematics,Combinatorics
+abc400_d,atcoder,"Search Algorithms,Data Structures,Graph Theory,Algorithm Basics","Breadth-First Search (BFS),Queue,Shortest Path,Simulation"
+abc400_e,atcoder,"Algorithm Basics,Algorithm Basics,Mathematics,Algorithm Basics","Binary Search,Enumeration,Sieve Method,Sorting"
+abc400_g,atcoder,"Algorithm Basics,Common Techniques,Dynamic Programming,Mathematics","Binary Search,Bit Manipulation,Bitmask Dynamic Programming,Numerical Algorithms"
+arc190_a,atcoder,"Algorithm Basics,Algorithm Basics","Greedy,Sorting"
+arc190_c,atcoder,"Mathematics,Dynamic Programming","Combinatorics,Counting Dynamic Programming"
+arc190_d,atcoder,"Algorithm Basics,Mathematics","Divide and Conquer,Fast Exponentiation"
+arc191_a,atcoder,"Algorithm Basics,Algorithm Basics","Constructive Algorithms,Greedy"
+arc191_c,atcoder,Algorithm Basics,Ad-hoc
+arc191_d,atcoder,"Search Algorithms,Algorithm Basics,Data Structures,Graph Theory,Algorithm Basics","Breadth-First Search (BFS),Constructive Algorithms,Queue,Shortest Path,Simulation"
+arc192_a,atcoder,,
+arc192_b,atcoder,"Algorithm Basics,Mathematics","Ad-hoc,Game Theory"
+arc192_d,atcoder,"Mathematics,Dynamic Programming,Mathematics,Algorithm Basics","Combinatorics,Counting Dynamic Programming,Prime Factorization,Recurrence"
+arc192_e,atcoder,Mathematics,Combinatorics
+arc193_a,atcoder,"Algorithm Basics,Common Techniques,Mathematics,Algorithm Basics","Binary Search,Discretization,Numerical Algorithms,Sorting"
+arc193_b,atcoder,"Mathematics,Dynamic Programming,Algorithm Basics","Combinatorics,Other Dynamic Programming Methods,Recurrence"
+arc193_d,atcoder,"Algorithm Basics,Common Techniques","Ad-hoc,Two-Pointer Technique"
+arc194_a,atcoder,"Common Techniques,Dynamic Programming,Algorithm Basics","Binary Thinking,Other Dynamic Programming Methods,Recurrence"
+arc194_b,atcoder,"Data Structures,Mathematics","Fenwick Tree,Numerical Algorithms"
+arc194_c,atcoder,"Algorithm Basics,Algorithm Basics,Common Techniques,Algorithm Basics","Binary Search,Greedy,Prefix Sum & Difference,Sorting"
+arc194_d,atcoder,"Mathematics,Dynamic Programming","Combinatorics,Counting Dynamic Programming"
+arc194_e,atcoder,Algorithm Basics,Ad-hoc
+arc195_a,atcoder,Algorithm Basics,Greedy
+arc195_b,atcoder,"Algorithm Basics,Data Structures,Mathematics","Enumeration,Hash Table,Pigeonhole Principle"
+arc195_c,atcoder,"Algorithm Basics,Algorithm Basics","Constructive Algorithms,Simulation"
+arc195_d,atcoder,"Algorithm Basics,Common Techniques","Greedy,Two-Pointer Technique"
+arc195_e,atcoder,"Mathematics,Common Techniques","Combinatorics,Prefix Sum & Difference"
+arc196_a,atcoder,"Algorithm Basics,Data Structures,Algorithm Basics","Greedy,Heap,Sorting"
+arc196_b,atcoder,"Search Algorithms,Mathematics,Graph Theory,Data Structures,Algorithm Basics","Breadth-First Search (BFS),Combinatorics,Connectivity,Disjoint Set Union,Simulation"
+arc196_c,atcoder,"Algorithm Basics,Dynamic Programming,Mathematics,Common Techniques","Divide and Conquer,Dynamic Programming Optimization,Fast Fourier Transform (FFT),Meet in Middle"
+arc196_d,atcoder,"Algorithm Basics,Data Structures,Mathematics,Data Structures","Greedy,Heap,Schedules,Segment Tree"
diff --git a/lcb_runner/benchmarks/code_generation.py b/lcb_runner/benchmarks/code_generation.py
index e2e48a73..8cdfeda7 100644
--- a/lcb_runner/benchmarks/code_generation.py
+++ b/lcb_runner/benchmarks/code_generation.py
@@ -1,3 +1,4 @@
+import os
 import json
 import zlib
 import pickle
@@ -6,7 +7,7 @@
 from datetime import datetime
 from dataclasses import dataclass
 
-from datasets import load_dataset
+from datasets import load_dataset, Features, Value
 
 
 class Platform(Enum):
@@ -89,16 +90,24 @@ def insert_output(self, output_list: list[str], code_list: list[str]) -> dict:
             "code_list": code_list,
         }
 
+    def get_num_test_cases(self) -> int:
+        """Get total number of test cases (public + private) for this problem."""
+        return len(self.public_test_cases) + len(self.private_test_cases)
+
     def insert_output_evaluation(
         self,
         output_list: list[str],
         code_list: list[str],
         graded_list: list[bool],
+        test_results_list: list[list] = None,
         **kwargs,
     ) -> dict:
         output = self.insert_output(output_list, code_list)
         output["graded_list"] = graded_list
         output["pass@1"] = graded_list.count(True) / len(graded_list)
+        output["num_test_cases"] = self.get_num_test_cases()
+        if test_results_list is not None:
+            output["test_results_list"] = test_results_list
         for k, v in kwargs.items():
             output[k] = v
         return output
@@ -122,7 +131,36 @@ def get_evaluation_sample(self):
 
 
 def load_code_generation_dataset(release_version="release_v1", start_date=None, end_date=None) -> list[CodeGenerationProblem]:
-    dataset = load_dataset("livecodebench/code_generation_lite", split="test", version_tag=release_version, trust_remote_code=True)
+
+    # Data folder path
+    DATA_FOLDER = os.path.abspath(os.path.join(__file__, "..", "..", "..", "data"))
+    
+    data_files = {
+        "release_v1": os.path.join(DATA_FOLDER, "test.jsonl"),
+        "release_v2": os.path.join(DATA_FOLDER, "test2.jsonl"),
+        "release_v3": os.path.join(DATA_FOLDER, "test3.jsonl"),
+        "release_v4": os.path.join(DATA_FOLDER, "test4.jsonl"),
+        "release_v5": os.path.join(DATA_FOLDER, "test5.jsonl"),
+        "release_v6": os.path.join(DATA_FOLDER, "test6.jsonl"),
+        "release_latest": os.path.join(DATA_FOLDER, "test6.jsonl"),
+    }
+
+    # As defined in https://huggingface.co/datasets/livecodebench/code_generation_lite/blob/main/code_generation_lite.py
+    features=Features({
+        "question_title": Value("string"),
+        "question_content": Value("string"),
+        "platform": Value("string"),
+        "question_id": Value("string"),
+        "contest_id": Value("string"),
+        "contest_date": Value("string"),
+        "starter_code": Value("string"),
+        "difficulty": Value("string"),
+        "public_test_cases": Value("string"),
+        "private_test_cases": Value("string"),
+        "metadata": Value("string"),
+    })
+    
+    dataset = load_dataset("json", data_files=data_files[release_version], split="train", features=features)
     dataset = [CodeGenerationProblem(**p) for p in dataset]  # type: ignore
     if start_date is not None:
         p_start_date = datetime.strptime(start_date, "%Y-%m-%d")
diff --git a/lcb_runner/evaluation/__init__.py b/lcb_runner/evaluation/__init__.py
index cad106ea..14798072 100644
--- a/lcb_runner/evaluation/__init__.py
+++ b/lcb_runner/evaluation/__init__.py
@@ -3,4 +3,4 @@
 from lcb_runner.evaluation.compute_test_output_prediction_metrics import (
     test_output_metrics,
 )
-from lcb_runner.evaluation.pass_k_utils import extract_instance_results
+from lcb_runner.evaluation.pass_k_utils import extract_instance_results, extract_test_results
diff --git a/lcb_runner/evaluation/compute_code_generation_metrics.py b/lcb_runner/evaluation/compute_code_generation_metrics.py
index b8de33ee..5855e80f 100644
--- a/lcb_runner/evaluation/compute_code_generation_metrics.py
+++ b/lcb_runner/evaluation/compute_code_generation_metrics.py
@@ -18,6 +18,9 @@
 
 from lcb_runner.evaluation.testing_util import run_test
 from lcb_runner.evaluation.pass_k_utils import compute_metrics_from_results
+from lcb_runner.utils.logger import setup_logger, truncate
+
+logger = setup_logger(__name__)
 
 
 def _temp_run(sample, generation, debug, result, metadata_list, timeout):
@@ -49,7 +52,7 @@ def check_correctness(sample, generation, timeout, debug=True):
         # consider that all tests failed
         result = [[-1 for i in range(len(in_outs["inputs"]))]]
         if debug:
-            print(f"global timeout")
+            logger.warning("Global timeout for task")
     return result[0], metadata_list[0]
 
 
@@ -59,6 +62,12 @@ def evaluate_generations_by_problem(args):
     debug: bool = args[2]
     timeout: int = args[3]
 
+    # NOTE: The debug flag is intentionally overridden to True here.
+    # We override the passed 'debug' argument to get detailed logging.
+    # LCB accepts a --debug flag, but it does not work for this function because its value is not passed down.
+    # This override also conveniently avoids the flag's unwanted side effects (like disabling parallelism).
+    debug = True
+
     res = []
     metadata = []
     for o_idx, o in enumerate(problem_generations):
@@ -68,7 +77,7 @@ def evaluate_generations_by_problem(args):
                 sample, o, timeout=timeout, debug=debug
             )
             if debug:
-                print(f"\nSuccessful compilation of task {o_idx}!")
+                logger.debug("Successful compilation of task %d", o_idx)
             fixed = []
             for e in curr_res:
                 if isinstance(e, np.ndarray):
@@ -79,10 +88,10 @@ def evaluate_generations_by_problem(args):
             curr_res = fixed
             if not np.all(curr_res):
                 if debug:
-                    print(f"Results were not True for all test cases {curr_res=}\n")
+                    logger.debug("Results were not True for all test cases: %s", curr_res)
         except Exception as e:
             if debug:
-                print(f"Compilation failed, test framework exception = {repr(e)}{e}\n")
+                logger.debug("Compilation failed, test framework exception = %r", e)
             # break
             curr_metadata = {
                 "error": repr(e),
@@ -96,12 +105,7 @@ def evaluate_generations_by_problem(args):
             metadata.append(curr_metadata)
     if debug:
         for i, r in enumerate(problem_generations):
-            print("Sample\n")
-            print(r)
-            print("\n")
-            print("Result\n")
-            print(res[i])
-            print("*" * 30 + "\n\n")
+            logger.debug("Sample:\n%s\nResult:\n%s", truncate(r), truncate(res[i]))
     return res, metadata
 
 
@@ -178,7 +182,7 @@ def codegen_metrics(
             generations_linear.append([generation])
             remap_index.append(idx)
 
-    print(f"Evaluating {len(samples_linear)}...")
+    logger.info("Evaluating %d samples...", len(samples_linear))
 
     results_linear, metadatas_linear = evaluate_generations(
         samples_linear,
diff --git a/lcb_runner/evaluation/pass_k_utils.py b/lcb_runner/evaluation/pass_k_utils.py
index a49382f0..55d438c0 100644
--- a/lcb_runner/evaluation/pass_k_utils.py
+++ b/lcb_runner/evaluation/pass_k_utils.py
@@ -64,3 +64,13 @@ def extract_instance_results(results):
         v for _, v in sorted(instance_wise_grades.items(), key=lambda item: item[0])
     ]
     return instance_wise_grades
+
+def extract_test_results(results):
+    test_results_dict = {}
+    for task_id, res in results.items():
+        test_results_dict[task_id] = res
+
+    test_results_list = [
+        v for _, v in sorted(test_results_dict.items(), key=lambda item: item[0])
+    ]
+    return test_results_list
\ No newline at end of file
diff --git a/lcb_runner/evaluation/testing_util.py b/lcb_runner/evaluation/testing_util.py
index ecc89474..40fa4776 100644
--- a/lcb_runner/evaluation/testing_util.py
+++ b/lcb_runner/evaluation/testing_util.py
@@ -24,6 +24,10 @@
 from decimal import Decimal
 import time
 
+from lcb_runner.utils.logger import setup_logger
+
+logger = setup_logger(__name__)
+
 import_string = "from string import *\nfrom re import *\nfrom datetime import *\nfrom collections import *\nfrom heapq import *\nfrom bisect import *\nfrom copy import *\nfrom math import *\nfrom random import *\nfrom statistics import *\nfrom itertools import *\nfrom functools import *\nfrom operator import *\nfrom io import *\nfrom sys import *\nfrom json import *\nfrom builtins import *\nfrom typing import *\nimport string\nimport re\nimport datetime\nimport collections\nimport heapq\nimport bisect\nimport copy\nimport math\nimport random\nimport statistics\nimport itertools\nimport functools\nimport operator\nimport io\nimport sys\nimport json\nsys.setrecursionlimit(50000)\n"
 
 
@@ -49,7 +53,9 @@ class TimeoutException(Exception):
 
 
 def timeout_handler(signum, frame):
-    print("timeout occured: alarm went off")
+    # NOTE: Do NOT print/log here — this is a signal handler.
+    # Calling print() in a signal handler can trigger BlockingIOError
+    # under heavy I/O, and logging is not async-signal-safe either.
     raise TimeoutException
 
 
@@ -231,16 +237,24 @@ def grade_call_based(
 ):
     # call-based clean up logic
     # need to wrap in try-catch logic after to catch the correct errors, but for now this is fine.
+    num_test_cases = len(all_inputs)
+
     code = import_string + "\n\n" + code
     compiled_sol = compile_code(code, timeout)
 
     if compiled_sol is None:
-        return
+        return [-4] * num_test_cases, {
+            "error_code": -4,
+            "error_message": "Compilation failed",
+        }
 
     method = get_function(compiled_sol, fn_name)
 
     if method is None:
-        return
+        return [-4] * num_test_cases, {
+            "error_code": -4,
+            "error_message": f"Function '{fn_name}' not found",
+        }
 
     all_inputs = [
         [json.loads(line) for line in inputs.split("\n")] for inputs in all_inputs
@@ -250,6 +264,8 @@ def grade_call_based(
 
     total_execution = 0
     all_results = []
+    failed_test_info = []
+
     for idx, (gt_inp, gt_out) in enumerate(zip(all_inputs, all_outputs)):
         signal.alarm(timeout)
         faulthandler.enable()
@@ -272,39 +288,46 @@ def grade_call_based(
             all_results.append(tmp_result)
 
             if not tmp_result:
-                return all_results, {
+                failed_test_info.append({
+                    "test_idx": idx,
                     "output": truncatefn(prediction),
                     "inputs": truncatefn(gt_inp),
                     "expected": truncatefn(gt_out),
                     "error_code": -2,
                     "error_message": "Wrong Answer",
-                }
+                })
         except Exception as e:
             signal.alarm(0)
             if "timeoutexception" in repr(e).lower():
                 all_results.append(-3)
-                return all_results, {
+                failed_test_info.append({
+                    "test_idx": idx,
                     "error": repr(e),
                     "error_code": -3,
                     "error_message": "Time Limit Exceeded",
                     "inputs": truncatefn(gt_inp),
                     "expected": truncatefn(gt_out),
-                }
+                })
             else:
                 all_results.append(-4)
-                return all_results, {
+                failed_test_info.append({
+                    "test_idx": idx,
                     "error": repr(e),
                     "error_code": -4,
                     "error_message": "Runtime Error",
                     "inputs": truncatefn(gt_inp),
                     "expected": truncatefn(gt_out),
-                }
+                })
 
         finally:
             signal.alarm(0)
             faulthandler.disable()
 
-    return all_results, {"execution time": total_execution}
+    metadata = {"execution time": total_execution}
+    if failed_test_info:
+        metadata["failed_tests"] = failed_test_info
+
+    return all_results, metadata
 
 
 def grade_stdio(
@@ -313,6 +336,8 @@ def grade_stdio(
     all_outputs: list,
     timeout: int,
 ):
+    num_test_cases = len(all_inputs)
+
     ## runtime doesn't interact well with __name__ == '__main__'
     code = clean_if_name(code)
 
@@ -321,15 +346,23 @@ def grade_stdio(
 
     compiled_sol = compile_code(code, timeout)
     if compiled_sol is None:
-        return
+        return [-4] * num_test_cases, {
+            "error_code": -4,
+            "error_message": "Compilation failed",
+        }
 
     method = get_function(compiled_sol, "wrapped_function")
 
     if method is None:
-        return
+        return [-4] * num_test_cases, {
+            "error_code": -4,
+            "error_message": "Function 'wrapped_function' not found",
+        }
 
     all_results = []
     total_execution_time = 0
+    failed_test_info = []
+
     for idx, (gt_inp, gt_out) in enumerate(zip(all_inputs, all_outputs)):
         signal.alarm(timeout)
         faulthandler.enable()
@@ -346,22 +379,26 @@ def grade_stdio(
                 signal.alarm(0)
                 if "timeoutexception" in repr(e).lower():
                     all_results.append(-3)
-                    return all_results, {
+                    failed_test_info.append({
+                        "test_idx": idx,
                         "error": repr(e),
                         "error_code": -3,
                         "error_message": "Time Limit Exceeded",
                         "inputs": truncatefn(gt_inp),
                         "expected": truncatefn(gt_out),
-                    }
+                    })
+                    continue
                 else:
                     all_results.append(-4)
-                    return all_results, {
+                    failed_test_info.append({
+                        "test_idx": idx,
                         "error": repr(e),
                         "error_code": -4,
                         "error_message": "Runtime Error",
                         "inputs": truncatefn(gt_inp),
                         "expected": truncatefn(gt_out),
-                    }
+                    })
+                    continue
 
             finally:
                 signal.alarm(0)
@@ -375,16 +412,20 @@ def grade_stdio(
         ## WA happens in multiple circumstances
         ## so cache the return to make it clean!
         WA_send_args = {
+            "test_idx": idx,
             "output": truncatefn(prediction),
             "inputs": truncatefn(gt_inp),
             "expected": truncatefn(gt_out),
             "error_code": -2,
         }
 
+        test_passed = True
+
         if len(stripped_prediction_lines) != len(stripped_gt_out_lines):
             all_results.append(-2)
             WA_send_args["error_message"] = "Wrong answer: mismatched output length"
-            return all_results, WA_send_args
+            failed_test_info.append(WA_send_args)
+            continue
 
         for output_line_idx, (
             stripped_prediction_line,
@@ -409,20 +450,33 @@ def grade_stdio(
             )
             if not success:
                 all_results.append(-2)
-                return all_results, WA_send_args
+                failed_test_info.append(WA_send_args)
+                test_passed = False
+                break
+
             success, decimal_gtout_line = convert_line_to_decimals(stripped_gt_out_line)
             if not success:
                 all_results.append(-2)
-                return all_results, WA_send_args
+                failed_test_info.append(WA_send_args)
+                test_passed = False
+                break
 
             if decimal_prediction_line == decimal_gtout_line:
                 continue
 
             all_results.append(-2)
-            return all_results, WA_send_args
-        all_results.append(True)
+            failed_test_info.append(WA_send_args)
+            test_passed = False
+            break
+
+        if test_passed:
+            all_results.append(True)
 
-    return all_results, {"execution time": total_execution_time}
+    metadata = {"execution time": total_execution_time}
+    if failed_test_info:
+        metadata["failed_tests"] = failed_test_info
+
+    return all_results, metadata
 
 
 def run_test(sample, test=None, debug=False, timeout=6):
@@ -437,7 +491,7 @@ def run_test(sample, test=None, debug=False, timeout=6):
     reliability_guard()
 
     if debug:
-        print(f"start = {datetime.now().time()}")
+        logger.debug("timing: %s", datetime.now().time())
 
     try:
         in_outs = json.loads(sample["input_output"])
@@ -455,16 +509,21 @@ def run_test(sample, test=None, debug=False, timeout=6):
             method_name = in_outs["fn_name"]
 
     if debug:
-        print(f"loaded input_output = {datetime.now().time()}")
+        logger.debug("timing: %s", datetime.now().time())
 
     if test is None:
         assert False, "should not happen: test code is none"
         return in_outs, {"error": "no test code provided"}
+    elif test == "":
+        return [-6], {
+            "error_code": -6,
+            "error_message": "code is empty; model output may be empty due to failed inference requests or incorrect format"
+        }
     elif test is not None:
         results = []
         sol = import_string
         if debug:
-            print(f"loading test code = {datetime.now().time()}")
+            logger.debug("loading test code: %s", datetime.now().time())
 
         if which_type == CODE_TYPE.call_based:
             signal.alarm(timeout)
diff --git a/lcb_runner/lm_styles.py b/lcb_runner/lm_styles.py
index d10b2117..9788179a 100644
--- a/lcb_runner/lm_styles.py
+++ b/lcb_runner/lm_styles.py
@@ -867,5 +867,15 @@ def to_dict(self) -> dict:
     lm.model_name: lm for lm in LanguageModelList
 }
 
+def create_generic_openai_model(model_name: str) -> LanguageModel:
+    """Create a generic OpenAI model representation."""
+    return LanguageModel(
+        model_name=model_name,
+        model_repr=model_name,
+        model_style=LMStyle.OpenAIChat,
+        release_date=datetime.now(),
+        link="https://openai.com"
+    )
+
 if __name__ == "__main__":
     print(list(LanguageModelStore.keys()))
diff --git a/lcb_runner/prompts/self_repair.py b/lcb_runner/prompts/self_repair.py
index 21ded044..c9360fbf 100644
--- a/lcb_runner/prompts/self_repair.py
+++ b/lcb_runner/prompts/self_repair.py
@@ -33,31 +33,30 @@ class PromptConstants:
 
 
 def get_check_prompt(question: str, result, metadata):
-    ## assumes i/o examples are already truncated!
-    ## less pressure on storing 10 MB json because on a single large input-output pair
-    # result_by_test_case = result
-    # assert len(metadata) == 1, f"metadata = {metadata}"
-    # metadata = metadata[0]
     metadata = json.loads(metadata)
-    if "error_code" not in metadata:
+
+    # Check if there are any failed tests
+    if "failed_tests" not in metadata or len(metadata["failed_tests"]) == 0:
         return ""
-    if metadata["error_code"] == -1:
-        # time limit exceeded
-        message = f"The above code is incorrect and got the following compilation error.\n{metadata['error']}"
-    elif metadata["error_code"] == -2:
+
+    # Use the first failed test for repair
+    first_failure = metadata["failed_tests"][0]
+    error_code = first_failure["error_code"]
+
+    if error_code == -2:
         # wrong answer
-        message = f"The above code is incorrect and got a wrong answer.\nInput: {metadata['inputs']}\nGenerated Output: {metadata['output']}\nExpected: {metadata['expected']}"
-    elif metadata["error_code"] == -3:
+        message = f"The above code is incorrect and got a wrong answer.\nInput: {first_failure['inputs']}\nGenerated Output: {first_failure['output']}\nExpected: {first_failure['expected']}"
+    elif error_code == -3:
         # time limit exceeded
-        message = f"The above code is incorrect and got time limit exceeded.\n{metadata['error']}\nInput: {metadata['inputs']}\nExpected: {metadata['expected']}"
-        pass
-    elif metadata["error_code"] == -4:
+        message = f"The above code is incorrect and got time limit exceeded.\n{first_failure['error']}\nInput: {first_failure['inputs']}\nExpected: {first_failure['expected']}"
+    elif error_code == -4:
         # runtime error
-        message = f"The above code is incorrect and got a runtime error.\nInput: {metadata['inputs']}\nExpected: {metadata['expected']}\n{metadata['error']}"
+        message = f"The above code is incorrect and got a runtime error.\nInput: {first_failure['inputs']}\nExpected: {first_failure['expected']}\n{first_failure['error']}"
     else:
         raise NotImplementedError(
-            f"metadata['error_code'] = {metadata['error_code']} not implemented || {metadata=}"
+            f"error_code = {error_code} not implemented || {first_failure=}"
         )
+
     return message
 
 
diff --git a/lcb_runner/runner/base_runner.py b/lcb_runner/runner/base_runner.py
index d99c8243..8e4ed3f4 100644
--- a/lcb_runner/runner/base_runner.py
+++ b/lcb_runner/runner/base_runner.py
@@ -2,12 +2,14 @@
 import json
 from abc import ABC, abstractmethod
 
-from tqdm import tqdm
-
 from lcb_runner.lm_styles import LanguageModel
-from lcb_runner.utils.path_utils import get_cache_path
+from lcb_runner.utils.path_utils import get_cache_path, get_progress_path
+from lcb_runner.utils.progress import TeeTqdm as tqdm
 from lcb_runner.utils.multiprocess import run_tasks_in_parallel
 from lcb_runner.runner.scenario_router import Scenario
+from lcb_runner.utils.logger import setup_logger, truncate
+
+logger = setup_logger(__name__)
 
 
 class BaseRunner(ABC):
@@ -53,15 +55,17 @@ def run_single(combined_args) -> list[str]:
         elif isinstance(prompt, tuple):
             prompt_cache = prompt[0] + json.dumps(prompt[1])
         else:
-            prompt_cache = prompt      
+            prompt_cache = prompt
 
         if cache is not None and prompt_cache in cache:
             if len(cache[prompt_cache]) == args.n:
+                logger.debug("(Cache Hit) Prompt: %s\nResult: %s", truncate(prompt_cache), truncate(str(cache[prompt_cache])))
                 return cache[prompt_cache]
 
         result = call_method(prompt)
         assert len(result) == args.n
 
+        logger.debug("Prompt: %s\nResult: %s", truncate(prompt_cache), truncate(str(result)))
         return result
 
     def run_batch(self, prompts: list[str | list[dict[str, str]]]) -> list[list[str]]:
@@ -75,23 +79,24 @@ def run_batch(self, prompts: list[str | list[dict[str, str]]]) -> list[list[str]
             )
             for prompt in prompts
         ]
+        progress_file_path = get_progress_path(self.args)
         if self.args.multiprocess > 1:
             parallel_outputs = run_tasks_in_parallel(
                 self.run_single,
                 arguments,
                 self.args.multiprocess,
                 use_progress_bar=True,
+                progress_file=progress_file_path
             )
             for output in parallel_outputs:
                 if output.is_success():
                     outputs.append(output.result)
                 else:
-                    print("Failed to run the model for some prompts")
-                    print(output.status)
-                    print(output.exception_tb)
-                    outputs.extend([""] * self.args.n)
+                    logger.error("Failed to run the model for some prompts: %s\n%s", output.status, truncate(output.exception_tb))
+                    outputs.append([""] * self.args.n)
         else:
-            outputs = [self.run_single(argument) for argument in tqdm(arguments)]
+            outputs = [self.run_single(argument) for argument in
+                       tqdm(arguments, progress_file=progress_file_path)]
 
         if self.args.use_cache:
             for prompt, output in zip(prompts, outputs):
@@ -139,7 +144,7 @@ def run_main_repair(self, benchmark: list, format_prompt: callable) -> list[list
         for problem_idx, problem in enumerate(benchmark):
             for check_metadata_idx, check_metadata in enumerate(check_metadata_list):
                 if problem.question_id == check_metadata['question_id']:
-                    count += 1 
+                    count += 1
                     question_content = check_metadata["question_content"]
                     code_list = check_metadata["code_list"]
                     output_list = check_metadata["output_list"]
diff --git a/lcb_runner/runner/main.py b/lcb_runner/runner/main.py
index 07902b3e..204797c8 100644
--- a/lcb_runner/runner/main.py
+++ b/lcb_runner/runner/main.py
@@ -3,26 +3,32 @@
 
 from lcb_runner.runner.parser import get_args
 from lcb_runner.utils.scenarios import Scenario
-from lcb_runner.lm_styles import LanguageModelStore
+from lcb_runner.lm_styles import create_generic_openai_model, LanguageModelStore
 from lcb_runner.runner.runner_utils import build_runner
 from lcb_runner.utils.path_utils import get_output_path
-from lcb_runner.evaluation import extract_instance_results
+from lcb_runner.evaluation import extract_instance_results, extract_test_results
 from lcb_runner.runner.scenario_router import (
     build_prompt_benchmark,
     combine_results,
     sort_and_extract_save_results,
     get_metrics,
 )
+from lcb_runner.utils.logger import setup_logger
+
+logger = setup_logger(__name__)
 
 
 def main():
     args = get_args()
 
-    model = LanguageModelStore[args.model]
+    if args.use_generic_openai_server:
+        model = create_generic_openai_model(args.model)
+    else:
+        model = LanguageModelStore[args.model]
     benchmark, format_prompt = build_prompt_benchmark(args)
     if args.debug:
-        print(f"Running with {len(benchmark)} instances in debug mode")
         benchmark = benchmark[:15]
+        logger.info("Running with %d instances in debug mode", len(benchmark))
 
     output_path = get_output_path(model.model_repr, args)
     eval_file = output_path.replace(".json", "_eval.json")
@@ -36,9 +42,7 @@ def main():
             with open(eval_all_file, "r") as f:
                 old_save_results = json.load(f)
         else:
-            print(
-                f"File {output_path} does not exist in --continue_existing, starting from scratch"
-            )
+            logger.warning("File %s does not exist in --continue_existing, starting from scratch", output_path)
             old_save_results = []
 
         old_save_results = [
@@ -54,9 +58,7 @@ def main():
             for instance in benchmark
             if instance.question_id not in old_save_results_question_ids
         ]
-        print(
-            f"Found {len(old_save_results)} existing generations, continuing with {len(remaining_benchmark)} remaining"
-        )
+        logger.info("Found %d existing generations, continuing with %d remaining", len(old_save_results), len(remaining_benchmark))
     else:
         old_save_results = []
         remaining_benchmark = benchmark
@@ -67,6 +69,9 @@ def main():
     else:
         results = []
 
+    # If model outputs are empty (probably due to failed inference requests),
+    # add args.n empty strings to ensure failed tasks are still evaluated
+    results = [r if r else [""] * args.n for r in results]
     combined_results = combine_results(
         args.scenario, results, model, args.cot_code_execution
     )
@@ -130,7 +135,7 @@ def main():
             if new_eval_size == 0:
                 return
 
-            print(f"Found {old_eval_size}, running evals for {new_eval_size} problems")
+            logger.info("Found %d, running evals for %d problems", old_eval_size, new_eval_size)
 
             metrics = get_metrics(args.scenario, args, benchmark, combined_results)
             graded = extract_instance_results(metrics[1])
@@ -153,7 +158,7 @@ def main():
                         }
                 metrics[1] = {**metrics[1], **old_eval_results[1]}
             else:
-                print("Old eval file not present, cannot update eval file")
+                logger.warning("Old eval file not present, cannot update eval file")
                 metrics = {}
 
         else:
@@ -165,14 +170,16 @@ def main():
         if args.scenario == Scenario.codegeneration:
             if metrics:
                 metadatas = metrics[2]
+                test_results = extract_test_results(metrics[1])
             else:
                 metadatas = [[] for _ in benchmark]
+                test_results = [[] for _ in benchmark]
             save_eval_results = [
                 instance.insert_output_evaluation(
-                    outputs_list, extracted_list, graded_list, metadata=meta
+                    outputs_list, extracted_list, graded_list, metadata=meta, test_results_list=test_result
                 )
-                for instance, (outputs_list, extracted_list), graded_list, meta in zip(
-                    benchmark, combined_results, graded, metadatas
+                for instance, (outputs_list, extracted_list), graded_list, meta, test_result in zip(
+                    benchmark, combined_results, graded, metadatas, test_results
                 )
             ]
             if metrics and old_eval_results:
@@ -222,6 +229,99 @@ def main():
         with open(eval_all_file, "w") as f:
             json.dump(save_eval_results, f, indent=4)
 
+        # Generate detailed results file with separated reasoning and output
+        detailed_results_file = eval_all_file.replace("_eval_all.json", "_detailed_results.json")
+        detailed_results = generate_detailed_results(save_eval_results, output_path)
+        with open(detailed_results_file, "w") as f:
+            json.dump(detailed_results, f, indent=2)
+        logger.info("Generated detailed results file: %s", detailed_results_file)
+
+
+def generate_detailed_results(eval_results, output_path):
+    """
+    Generate detailed results with separated reasoning and output for each sample.
+
+    Transforms task-level eval_all.json entries (with arrays) into sample-level entries.
+
+    Args:
+        eval_results: List of evaluation results from eval_all.json (task-level)
+        output_path: Path to the output file (used to extract exp_id)
+
+    Returns:
+        List of detailed result entries (sample-level) with fields:
+        - exp_id, task_id, kth_sample, reasoning, output
+        - question_title, difficulty, platform, contest_id, contest_date
+        - passed (bool), test_results (array), metadata (dict)
+    """
+    from pathlib import Path
+
+    output_dir = Path(output_path).parent.name
+    exp_id = output_dir.split('--', 1)[1] if '--' in output_dir else output_dir
+
+    # Load reasoning content if available
+    reasoning_map = {}
+    reasoning_file = Path(output_path).parent / "reasoning_result.jsonl"
+    if reasoning_file.exists():
+        try:
+            malformed_lines = 0
+            with open(reasoning_file, 'r') as f:
+                for line_num, line in enumerate(f, 1):
+                    try:
+                        data = json.loads(line.strip())
+                        for response in data.get('responses', []):
+                            content = response.get('content', '')
+                            reasoning_content = response.get('reasoning_content', '')
+                            # Only require content to be non-empty (reasoning_content already filtered when saving)
+                            if content:
+                                reasoning_map[content] = reasoning_content
+                    except json.JSONDecodeError as e:
+                        malformed_lines += 1
+                        if malformed_lines == 1:  # Only log first error to avoid spam
+                            logger.warning("Malformed JSON on line %d in reasoning file: %s", line_num, e)
+                        continue
+
+            if malformed_lines > 0:
+                logger.warning("Skipped %d malformed lines in reasoning file", malformed_lines)
+            logger.info("Loaded %d reasoning entries from %s", len(reasoning_map), reasoning_file)
+        except Exception as e:
+            logger.warning("Failed to load reasoning file: %s", e)
+
+    detailed_results = []
+
+    for entry in eval_results:
+        task_id = entry.get('question_id', 'unknown')
+        output_list = entry.get('output_list', [])
+        test_results_list = entry.get('test_results_list', [])
+        metadata_list = entry.get('metadata', [])
+        graded_list = entry.get('graded_list', [])
+
+        for kth_sample in range(len(output_list)):
+            output = output_list[kth_sample] if kth_sample < len(output_list) else ""
+            test_results = test_results_list[kth_sample] if kth_sample < len(test_results_list) else []
+            metadata = metadata_list[kth_sample] if kth_sample < len(metadata_list) else {}
+            passed = graded_list[kth_sample] if kth_sample < len(graded_list) else False
+
+            # Look up reasoning content
+            reasoning_content = reasoning_map.get(output, "")
+
+            detailed_results.append({
+                'exp_id': exp_id,
+                'task_id': str(task_id),
+                'kth_sample': kth_sample,
+                'output': output,
+                'reasoning_content': reasoning_content,
+                'question_title': entry.get('question_title', ''),
+                'difficulty': entry.get('difficulty', ''),
+                'platform': entry.get('platform', ''),
+                'contest_id': entry.get('contest_id', ''),
+                'contest_date': entry.get('contest_date', ''),
+                'passed': passed,
+                'test_results': test_results,
+                'metadata': metadata,
+            })
+
+    return detailed_results
+
 
 if __name__ == "__main__":
     main()
diff --git a/lcb_runner/runner/oai_runner.py b/lcb_runner/runner/oai_runner.py
index 59e20747..907fde4d 100644
--- a/lcb_runner/runner/oai_runner.py
+++ b/lcb_runner/runner/oai_runner.py
@@ -1,25 +1,35 @@
-import os
+import json
+import fcntl
 from time import sleep
+from pathlib import Path
 
-try:
-    import openai
-    from openai import OpenAI
-except ImportError as e:
-    pass
+import openai
+from openai import OpenAI
 
 from lcb_runner.lm_styles import LMStyle
 from lcb_runner.runner.base_runner import BaseRunner
+from lcb_runner.utils.logger import setup_logger, truncate
+
+logger = setup_logger(__name__)
 
 
 class OpenAIRunner(BaseRunner):
-    client = OpenAI(
-        api_key=os.getenv("OPENAI_KEY"),
-    )
+    client = OpenAI()
 
     def __init__(self, args, model):
         super().__init__(args, model)
+
+        # Parse user-provided extra_body and extra_headers
+        extra_body = json.loads(args.extra_body) if args.extra_body else {}
+        extra_headers = json.loads(args.extra_headers) if args.extra_headers else {}
+
+        # Store stream setting
+        self.stream = getattr(args, 'stream', False)
+        if self.stream:
+            logger.info("Streaming is enabled.")
+
         if model.model_style == LMStyle.OpenAIReasonPreview:
-            self.client_kwargs: dict[str | str] = {
+            self.client_kwargs = {
                 "model": args.model,
                 "max_completion_tokens": 25000,
             }
@@ -28,35 +38,60 @@ def __init__(self, args, model):
                 "__" in args.model
             ), f"Model {args.model} is not a valid OpenAI Reasoning model as we require reasoning effort in model name."
             model, reasoning_effort = args.model.split("__")
-            self.client_kwargs: dict[str | str] = {
+            self.client_kwargs = {
                 "model": model,
                 "reasoning_effort": reasoning_effort,
             }
         else:
-            self.client_kwargs: dict[str | str] = {
+            if args.top_k is not None:
+                extra_body["top_k"] = args.top_k
+            if args.repetition_penalty is not None:
+                extra_body["repetition_penalty"] = args.repetition_penalty
+
+            self.client_kwargs = {
                 "model": args.model,
                 "temperature": args.temperature,
-                "max_tokens": args.max_tokens,
+                #"max_tokens": args.max_tokens,
                 "top_p": args.top_p,
-                "frequency_penalty": 0,
-                "presence_penalty": 0,
                 "n": args.n,
                 "timeout": args.openai_timeout,
                 # "stop": args.stop, --> stop is only used for base models currently
             }
 
+            if args.presence_penalty is not None:
+                self.client_kwargs["presence_penalty"] = args.presence_penalty
+            if args.max_tokens is not None:
+                self.client_kwargs["max_tokens"] = args.max_tokens
+
+            # Only add extra_body/extra_headers if they have content
+            if extra_body:
+                self.client_kwargs["extra_body"] = extra_body
+            if extra_headers:
+                self.client_kwargs["extra_headers"] = extra_headers
+
+        logger.info("Inference parameters: %s", self.client_kwargs)
+
     def _run_single(self, prompt: list[dict[str, str]], n: int = 10) -> list[str]:
         assert isinstance(prompt, list)
 
         if n == 0:
-            print("Max retries reached. Returning empty response.")
+            logger.warning("Max retries reached. Returning empty response.")
             return []
 
         try:
-            response = OpenAIRunner.client.chat.completions.create(
-                messages=prompt,
-                **self.client_kwargs,
-            )
+            if not self.stream:
+                response = OpenAIRunner.client.chat.completions.create(
+                    messages=prompt,
+                    **self.client_kwargs,
+                )
+            else:
+                # Handle streaming
+                with OpenAIRunner.client.chat.completions.stream(
+                    messages=prompt,
+                    **self.client_kwargs,
+                ) as stream:
+                    response = stream.get_final_completion() # This will wait for the full response to be received
+
         except (
             openai.APIError,
             openai.RateLimitError,
@@ -67,13 +102,70 @@ def _run_single(self, prompt: list[dict[str, str]], n: int = 10) -> list[str]:
             openai.InternalServerError,
             openai.APIConnectionError,
         ) as e:
-            print("Exception: ", repr(e))
-            print("Sleeping for 30 seconds...")
-            print("Consider reducing the number of parallel processes.")
+            logger.warning("API error (retries left=%d): %s. Sleeping 30s...", n - 1, truncate(repr(e)))
             sleep(30)
             return self._run_single(prompt, n=n - 1)
         except Exception as e:
-            print(f"Failed to run the model for {prompt}!")
-            print("Exception: ", repr(e))
+            logger.error("Failed to run the model: %s", truncate(repr(e)))
             raise e
-        return [c.message.content for c in response.choices]
+        
+        # Hack: Log reasoning content if available
+        for choice in response.choices:
+            message = choice.message
+            if hasattr(message, 'reasoning_content') and message.reasoning_content:
+                logger.debug("Reasoning content detected (len=%d)", len(str(message.reasoning_content)))
+            else:
+                logger.debug("No reasoning content detected")
+
+        self._save_reasoning_content(prompt, response)
+
+        return [c.message.content or "" for c in response.choices]
+
+    def _save_reasoning_content(self, prompt: list[dict[str, str]], response):
+        """Save reasoning content to a separate JSON file for post-processing."""
+        # Check if any response has reasoning_content
+        has_reasoning = any(
+            hasattr(c.message, 'reasoning_content') and c.message.reasoning_content
+            for c in response.choices
+        )
+
+        # Only save if there's actual reasoning content
+        if not has_reasoning:
+            return
+
+        try:
+            output_dir = Path(self.args.output_dir)
+            output_dir.mkdir(parents=True, exist_ok=True)
+            reasoning_file = output_dir / "reasoning_result.jsonl"
+
+            # Prepare reasoning data (only save responses with reasoning content)
+            reasoning_data = {
+                "prompt": prompt,
+                "model": self.client_kwargs.get("model", "unknown"),
+                "responses": []
+            }
+
+            for c in response.choices:
+                content = c.message.content or ""
+                reasoning_content = getattr(c.message, 'reasoning_content', None) or ""
+
+                # Only append if there's reasoning content (since we already checked has_reasoning)
+                if reasoning_content:
+                    reasoning_data["responses"].append({
+                        "content": content,
+                        "reasoning_content": reasoning_content,
+                    })
+
+            line = json.dumps(reasoning_data) + '\n'
+
+            with open(reasoning_file, 'a') as f:
+                fcntl.flock(f.fileno(), fcntl.LOCK_EX)
+                try:
+                    f.write(line)
+                    f.flush()
+                finally:
+                    fcntl.flock(f.fileno(), fcntl.LOCK_UN)
+
+        except Exception as e:
+            # Don't fail the entire run just because we couldn't save reasoning
+            logger.warning("Failed to save reasoning content: %s", e)
diff --git a/lcb_runner/runner/parser.py b/lcb_runner/runner/parser.py
index a047fc0d..fc2c503d 100644
--- a/lcb_runner/runner/parser.py
+++ b/lcb_runner/runner/parser.py
@@ -4,7 +4,6 @@
 
 from lcb_runner.utils.scenarios import Scenario
 
-
 def get_args():
     parser = argparse.ArgumentParser()
     parser.add_argument(
@@ -59,8 +58,13 @@ def get_args():
         "--temperature", type=float, default=0.2, help="Temperature for sampling"
     )
     parser.add_argument("--top_p", type=float, default=0.95, help="Top p for sampling")
+
+    parser.add_argument("--top_k", type=int, help="Top k for sampling")
+    parser.add_argument("--repetition_penalty", type=float, help="Repetition penalty")
+    parser.add_argument("--presence_penalty", type=float, help="Presence penalty")
+
     parser.add_argument(
-        "--max_tokens", type=int, default=2000, help="Max tokens for sampling"
+        "--max_tokens", type=int, default=None, help="Max tokens for sampling"
     )
     parser.add_argument(
         "--multiprocess",
@@ -131,6 +135,35 @@ def get_args():
         default=None,
         help="End date for the contest to filter the evaluation file (format - YYYY-MM-DD)",
     )
+    parser.add_argument(
+        "--use_generic_openai_server",
+        action="store_true",
+        help="Use generic OpenAI server (set both OPENAI_API_KEY and OPENAI_BASE_URL environment variables)",
+    )
+    parser.add_argument(
+        "--extra_body",
+        type=str,
+        default=None,
+        help="Extra body parameters as JSON string",
+    )
+    parser.add_argument(
+        "--extra_headers",
+        type=str,
+        default=None,
+        help="Extra headers as JSON string",
+    )
+    parser.add_argument(
+        "--stream",
+        action="store_true",
+        help="Enable streaming API for inference",
+    )
+
+    parser.add_argument(
+        "--output_dir",
+        type=str,
+        required=True,
+        help="The directory storing the outputs",
+    )
 
     args = parser.parse_args()
 
diff --git a/lcb_runner/runner/scenario_router.py b/lcb_runner/runner/scenario_router.py
index ffcfe0e7..e02b2736 100644
--- a/lcb_runner/runner/scenario_router.py
+++ b/lcb_runner/runner/scenario_router.py
@@ -2,6 +2,9 @@
 
 from lcb_runner.utils.scenarios import Scenario
 from lcb_runner.lm_styles import LanguageModel
+from lcb_runner.utils.logger import setup_logger
+
+logger = setup_logger(__name__)
 from lcb_runner.evaluation import (
     codegen_metrics,
     test_output_metrics,
@@ -212,6 +215,6 @@ def get_metrics(
     else:
         raise ValueError(f"Scenario {scenario} not implemented")
 
-    print(metrics[0]["pass@1"])
+    logger.info("pass@1: %s", metrics[0]["pass@1"])
 
     return metrics
diff --git a/lcb_runner/utils/logger.py b/lcb_runner/utils/logger.py
new file mode 100644
index 00000000..8443c56c
--- /dev/null
+++ b/lcb_runner/utils/logger.py
@@ -0,0 +1,64 @@
+"""Centralized logging for lcb_runner.
+
+Replaces print() calls with proper logging to avoid BlockingIOError (errno 11)
+when stdout buffer is full under heavy I/O with many parallel processes.
+"""
+
+import logging
+import os
+import sys
+
+# Default max length for truncated log messages
+_MAX_LOG_STR_LEN = 500
+
+# Read log level from env: LCB_LOG_LEVEL=DEBUG|INFO|WARNING|ERROR (default: INFO)
+_LOG_LEVEL = getattr(logging, os.environ.get("LCB_LOG_LEVEL", "INFO").upper(), logging.INFO)
+
+
+def truncate(s, max_len: int = _MAX_LOG_STR_LEN) -> str:
+    """Truncate a string to max_len, showing head and tail with '...' in between."""
+    text = str(s)
+    if len(text) <= max_len:
+        return text
+    half = max_len // 2
+    return text[:half] + f"...<truncated {len(text) - max_len} chars>..." + text[-half:]
+
+
+def setup_logger(name: str = "lcb_runner", level: int = _LOG_LEVEL) -> logging.Logger:
+    """Get or create a logger with a non-blocking stderr handler.
+
+    Uses stderr instead of stdout because:
+    - stdout may be redirected/piped (triggering non-blocking I/O)
+    - stderr is line-buffered by default and less likely to block
+
+    The handler is configured to silently drop messages rather than
+    raise BlockingIOError when the buffer is full.
+    """
+    logger = logging.getLogger(name)
+
+    if not logger.handlers:
+        handler = logging.StreamHandler(sys.stderr)
+        handler.setFormatter(
+            logging.Formatter(
+                "%(asctime)s [%(levelname)s] %(name)s: %(message)s",
+                datefmt="%Y-%m-%d %H:%M:%S",
+            )
+        )
+        # Override emit to swallow BlockingIOError
+        _original_emit = handler.emit
+
+        def _safe_emit(record):
+            try:
+                _original_emit(record)
+            except BlockingIOError:
+                pass
+
+        handler.emit = _safe_emit
+        logger.addHandler(handler)
+        logger.setLevel(level)
+
+    return logger
+
+
+# Module-level default logger
+logger = setup_logger()
diff --git a/lcb_runner/utils/multiprocess.py b/lcb_runner/utils/multiprocess.py
index 835c9c84..5690f470 100644
--- a/lcb_runner/utils/multiprocess.py
+++ b/lcb_runner/utils/multiprocess.py
@@ -6,12 +6,13 @@
 import traceback
 from enum import Enum
 from typing import Callable, Optional, Dict, Any, List, Iterator
-from concurrent.futures import TimeoutError
+from concurrent.futures import TimeoutError, as_completed
 
 import attrs
-import tqdm
 from pebble import concurrent, ProcessPool, ProcessExpired
 
+from lcb_runner.utils.progress import TeeTqdm as tqdm
+
 
 class FuncTimeoutError(TimeoutError):
     pass
@@ -105,6 +106,7 @@ def run_tasks_in_parallel_iter(
     max_tasks_per_worker: Optional[int] = None,
     use_spawn: bool = True,
     max_mem: int = 1024 * 1024 * 1024 * 4,
+    progress_file=None
 ) -> Iterator[TaskResult]:
     """
     Args:
@@ -129,37 +131,51 @@ def run_tasks_in_parallel_iter(
         max_tasks=0 if max_tasks_per_worker is None else max_tasks_per_worker,
         context=mp.get_context(mode),
     ) as pool:
-        future = pool.map(func, tasks, timeout=timeout_per_task)
+        # Schedule all tasks
+        futures = [pool.schedule(func, args=(task,), timeout=timeout_per_task) for task in tasks]
 
-        iterator = future.result()
         if use_progress_bar:
-            pbar = tqdm.tqdm(
+            pbar = tqdm(
                 desc=progress_bar_desc,
                 total=len(tasks),
                 dynamic_ncols=True,
-                file=sys.stdout,
+                file=sys.stderr,
+                progress_file=progress_file
             )
         else:
             pbar = None
 
-        succ = timeouts = exceptions = expirations = 0
-
-        while True:
+        succ = timeouts = exceptions = expirations = 0      
+        
+        # Create a mapping from future to original index
+        future_to_index = {future: idx for idx, future in enumerate(futures)}
+        
+        # Store results with their original indices
+        results_with_indices = []
+        
+        # Process results as they complete
+        for future in as_completed(futures):
+            original_index = future_to_index[future]
+            
             try:
-                result = next(iterator)
+                result = future.result()
+
+                task_result = TaskResult(
+                    status=TaskRunStatus.SUCCESS,
+                    result=result,
+                )
 
-            except StopIteration:
-                break
+                succ += 1
 
             except TimeoutError as error:
-                yield TaskResult(
+                task_result = TaskResult(
                     status=TaskRunStatus.TIMEOUT,
                 )
 
                 timeouts += 1
 
             except ProcessExpired as error:
-                yield TaskResult(
+                task_result = TaskResult(
                     status=TaskRunStatus.PROCESS_EXPIRED,
                 )
                 expirations += 1
@@ -167,27 +183,29 @@ def run_tasks_in_parallel_iter(
             except Exception as error:
                 exception_tb = traceback.format_exc()
 
-                yield TaskResult(
+                task_result = TaskResult(
                     status=TaskRunStatus.EXCEPTION,
                     exception_tb=exception_tb,
                 )
                 exceptions += 1
 
-            else:
-                yield TaskResult(
-                    status=TaskRunStatus.SUCCESS,
-                    result=result,
-                )
-
-                succ += 1
+            results_with_indices.append((original_index, task_result))
 
             if pbar is not None:
                 pbar.update(1)
                 pbar.set_postfix(
                     succ=succ, timeouts=timeouts, exc=exceptions, p_exp=expirations
                 )
-                sys.stdout.flush()
-                sys.stderr.flush()
+                try:
+                    sys.stdout.flush()
+                    sys.stderr.flush()
+                except BlockingIOError:
+                    pass
+        
+        # Sort by original index and yield in submission order
+        results_with_indices.sort(key=lambda x: x[0])
+        for _, task_result in results_with_indices:
+            yield task_result
 
 
 def run_tasks_in_parallel(
@@ -199,6 +217,7 @@ def run_tasks_in_parallel(
     progress_bar_desc: Optional[str] = None,
     max_tasks_per_worker: Optional[int] = None,
     use_spawn: bool = True,
+    progress_file: Optional[str] = None
 ) -> List[TaskResult]:
     """
     Args:
@@ -226,6 +245,7 @@ def run_tasks_in_parallel(
             progress_bar_desc=progress_bar_desc,
             max_tasks_per_worker=max_tasks_per_worker,
             use_spawn=use_spawn,
+            progress_file=progress_file
         )
     )
 
diff --git a/lcb_runner/utils/path_utils.py b/lcb_runner/utils/path_utils.py
index 9f28a1a6..2a4eff45 100644
--- a/lcb_runner/utils/path_utils.py
+++ b/lcb_runner/utils/path_utils.py
@@ -1,3 +1,4 @@
+import os.path
 import pathlib
 
 from lcb_runner.lm_styles import LanguageModel, LMStyle
@@ -11,12 +12,24 @@ def ensure_dir(path: str, is_file=True):
         pathlib.Path(path).mkdir(parents=True, exist_ok=True)
     return
 
+def get_cache_dir(args) -> str:
+    if os.path.isabs(args.output_dir):
+        return os.path.join(args.output_dir, "cache")
+    else:
+        return f"cache/{args.output_dir}"
+
+def get_output_dir(args) -> str:
+    if os.path.isabs(args.output_dir):
+        return args.output_dir
+    else:
+        return f"output/{args.output_dir}"
+
 
 def get_cache_path(model_repr:str, args) -> str:
     scenario: Scenario = args.scenario
     n = args.n
     temperature = args.temperature
-    path = f"cache/{model_repr}/{scenario}_{n}_{temperature}.json"
+    path = f"{get_cache_dir(args)}/{scenario}_{n}_{temperature}.json"
     ensure_dir(path)
     return path
 
@@ -26,7 +39,7 @@ def get_output_path(model_repr:str, args) -> str:
     n = args.n
     temperature = args.temperature
     cot_suffix = "_cot" if args.cot_code_execution else ""
-    path = f"output/{model_repr}/{scenario}_{n}_{temperature}{cot_suffix}.json"
+    path = f"{get_output_dir(args)}/{scenario}_{n}_{temperature}{cot_suffix}.json"
     ensure_dir(path)
     return path
 
@@ -36,5 +49,10 @@ def get_eval_all_output_path(model_repr:str, args) -> str:
     n = args.n
     temperature = args.temperature
     cot_suffix = "_cot" if args.cot_code_execution else ""
-    path = f"output/{model_repr}/{scenario}_{n}_{temperature}{cot_suffix}_eval_all.json"
+    path = f"{get_output_dir(args)}/{scenario}_{n}_{temperature}{cot_suffix}_eval_all.json"
     return path
+
+def get_progress_path(args) -> str:
+    path = f"{get_output_dir(args)}/progress.json"
+    ensure_dir(path)
+    return path
\ No newline at end of file
diff --git a/lcb_runner/utils/progress.py b/lcb_runner/utils/progress.py
new file mode 100644
index 00000000..2245462d
--- /dev/null
+++ b/lcb_runner/utils/progress.py
@@ -0,0 +1,20 @@
+import json
+
+from tqdm import tqdm
+
+
+class TeeTqdm(tqdm):
+    def __init__(self, *args, progress_file=None, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.progress_file = progress_file
+        self.write_progress_to_file()
+
+    def update(self, n=1):
+        super().update(n)
+        self.write_progress_to_file()
+
+    def write_progress_to_file(self):
+        if self.progress_file:
+            progress_dict = self.format_dict
+            with open(self.progress_file, "w") as f:
+                json.dump(progress_dict, f)
\ No newline at end of file