-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathmap.py
More file actions
80 lines (59 loc) · 2.33 KB
/
Copy pathmap.py
File metadata and controls
80 lines (59 loc) · 2.33 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
"""
Mean Average Precision (MAP) metric
"""
import numpy as np
from typing import Dict, List, Tuple
def compute_average_precision(
qrels_for_query: Dict[str, int], results_for_query: List[Tuple[str, int, float]]
) -> float:
"""
Compute Average Precision for a single query.
Average Precision (AP) is the average of precision values computed at
each position where a relevant document is retrieved.
Args:
qrels_for_query: Dictionary mapping passage_id to relevance for one query
results_for_query: List of (passage_id, rank, score) for one query
Returns:
float: Average Precision (0.0 to 1.0)
Formula:
AP = (sum of P@k for each relevant doc) / (total # relevant docs)
where P@k is precision at position k
"""
relevant_docs = set(pid for pid, rel in qrels_for_query.items() if rel > 0)
total_relevant = len(relevant_docs)
if total_relevant == 0:
return 0.0
num_relevant_seen = 0
sum_precisions = 0.0
for i, (passage_id, _, _) in enumerate(results_for_query, start=1):
if passage_id in relevant_docs:
num_relevant_seen += 1
precision_at_i = num_relevant_seen / i
sum_precisions += precision_at_i
return sum_precisions / total_relevant
def compute_map(
qrels: Dict[str, Dict[str, int]], results: Dict[str, List[Tuple[str, int, float]]]
) -> float:
"""
Compute Mean Average Precision.
MAP is the mean of Average Precision scores across all queries.
It's particularly useful for binary relevance judgments.
Args:
qrels: Dictionary mapping query_id to {passage_id: relevance}
results: Dictionary mapping query_id to [(passage_id, rank, score), ...]
Returns:
float: MAP score (0.0 to 1.0)
Formula:
MAP = average of AP across all queries
Note:
MAP is preferred over NDCG when relevance judgments are binary (0 or 1)
as it gives equal weight to all relevant documents.
"""
average_precisions = []
for query_id, query_qrels in qrels.items():
if query_id not in results:
average_precisions.append(0.0)
continue
ap = compute_average_precision(query_qrels, results[query_id])
average_precisions.append(ap)
return np.mean(average_precisions) if average_precisions else 0.0