-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathmrr.py
More file actions
51 lines (40 loc) · 1.46 KB
/
Copy pathmrr.py
File metadata and controls
51 lines (40 loc) · 1.46 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
"""
Mean Reciprocal Rank (MRR) metric
"""
import numpy as np
from typing import Dict, List, Tuple
def compute_mrr_at_k(
qrels: Dict[str, Dict[str, int]],
results: Dict[str, List[Tuple[str, int, float]]],
k: int = 10,
) -> float:
"""
Compute Mean Reciprocal Rank at k.
MRR@k measures the average of the reciprocal ranks of the first relevant
document found within the top-k results for each query.
Args:
qrels: Dictionary mapping query_id to {passage_id: relevance}
results: Dictionary mapping query_id to [(passage_id, rank, score), ...]
k: Cutoff rank (default: 10)
Returns:
float: MRR@k score (0.0 to 1.0)
Example:
If the first relevant document appears at rank 3, RR = 1/3 = 0.333
MRR is the average of RR across all queries.
"""
reciprocal_ranks = []
for query_id, query_qrels in qrels.items():
if query_id not in results:
reciprocal_ranks.append(0.0)
continue
# Get top-k results
top_k_results = results[query_id][:k]
# Find first relevant document
for passage_id, rank, _ in top_k_results:
if passage_id in query_qrels and query_qrels[passage_id] > 0:
reciprocal_ranks.append(1.0 / rank)
break
else:
# No relevant document found in top-k
reciprocal_ranks.append(0.0)
return np.mean(reciprocal_ranks) if reciprocal_ranks else 0.0