#!/usr/bin/env python3

"""

Evaluation script for information retrieval results.

This script evaluates search results against qrels (relevance judgments) using

standard IR metrics including MRR, Recall, NDCG, and MAP.

"""

from dataclasses import dataclass

from pathlib import Path

import tyro

from python.metrics import (

compute_mrr_at_k,

compute_recall_at_k,

compute_ndcg_at_k,

compute_map,

)

from python.metrics.utils import load_qrels, load_results

@dataclass

class EvalConfig:

"""Configuration for evaluation."""

qrels: Path

"""Path to qrels file (relevance judgments)"""

results: Path

"""Path to results file (search output)"""

use_map: bool = False

"""Use MAP instead of NDCG (for binary relevance judgments)"""

mrr_k: int = 10

"""Cutoff k for MRR@k metric"""

recall_k: int = 100

"""Cutoff k for Recall@k metric"""

ndcg_k: tuple[int, ...] = (10, 100)

"""Cutoff k values for NDCG@k metric"""

verbose: bool = True

"""Print detailed evaluation information"""

def evaluate(config: EvalConfig) -> dict:

"""

Evaluate search results against qrels.

Args:

config: Evaluation configuration

Returns:

Dictionary containing all computed metrics

"""

if config.verbose:

print(f"\n{'=' * 70}")

print(f" Evaluating: {config.results.name}")

print(f" Qrels: {config.qrels.name}")

print(f"{'=' * 70}")

# Load data

qrels = load_qrels(str(config.qrels))

results = load_results(str(config.results))

if config.verbose:

print(f"Loaded {len(qrels)} queries from qrels")

print(f"Loaded results for {len(results)} queries")

# Compute metrics

metrics = {}

# MRR

mrr = compute_mrr_at_k(qrels, results, k=config.mrr_k)

metrics[f"mrr@{config.mrr_k}"] = mrr

# Recall

recall = compute_recall_at_k(qrels, results, k=config.recall_k)

metrics[f"recall@{config.recall_k}"] = recall

if config.verbose:

print("\nMetrics:")

print(f" MRR@{config.mrr_k}: {mrr:.4f}")

print(f" Recall@{config.recall_k}: {recall:.4f}")

if config.use_map:

# For binary relevance (qrels.dev.tsv)

map_score = compute_map(qrels, results)

metrics["map"] = map_score

if config.verbose:

print(f" MAP: {map_score:.4f}")

else:

# For graded relevance (qrels.eval.one/two.tsv)

for k in config.ndcg_k:

ndcg = compute_ndcg_at_k(qrels, results, k=k)

metrics[f"ndcg@{k}"] = ndcg

if config.verbose:

print(f" NDCG@{k}: {ndcg:.4f}")

if config.verbose:

print(f"{'=' * 70}\n")

return metrics

def main(config: EvalConfig) -> None:

"""Main evaluation function."""

# Check if files exist

if not config.qrels.exists():

raise FileNotFoundError(f"Qrels file not found: {config.qrels}")

if not config.results.exists():

raise FileNotFoundError(f"Results file not found: {config.results}")

# Run evaluation

evaluate(config)

if __name__ == "__main__":

tyro.cli(main)