Skip to content

Commit fe1ef7f

Browse files
Roman Rakusclaude
Roman Rakus
and
claude
committed
feat(eval): robust summary scoring + multi-case example dataset
- evaluator: score must_not_include via plain presence-detection then invert, instead of asking the judge to reason about avoidance under an "EXPECTED OUTPUT" label (which inverted verdicts on negative criteria) - reporting: make the FAIL note evaluator-agnostic (list whichever boolean checks are False) instead of the visualization-only "no visualization created" - examples: replace the single template with three self-describing cases — full-dashboard, selected-visualizations (scoping), and format-hint-brief (format adherence) — each with a small gating set and rubric aligned to the endpoint's actual output Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent 62bf443 commit fe1ef7f

7 files changed

Lines changed: 101 additions & 39 deletions

File tree

‎packages/gooddata-eval/examples/summary_dataset/dashboard_summary_example.json‎

Lines changed: 0 additions & 22 deletions
This file was deleted.

‎packages/gooddata-eval/examples/summary_dataset/summary_format_hint_brief.json‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
{
2+
"id": "summary-format-hint-brief",
3+
"dataset_name": "summary_pilot",
4+
"question": "Give a brief executive summary of the Top 10 Products.",
5+
"test_kind": "dashboard_summary",
6+
"summary_input": {
7+
"dashboard_id": "b2f2d436-9831-4fe0-81df-8c59fd33242b",
8+
"visualizations": ["top_10_products"],
9+
"format_hint": "A brief executive summary: at most 3 sentences, no headings or bullet points."
10+
},
11+
"expected_output": {
12+
"must_include": [
13+
"Cites specific numeric values from the Top 10 Products data"
14+
],
15+
"must_not_include": [
16+
"Uses section headings (such as 'Summary' or 'Key Insights') or bullet/numbered lists",
17+
"Reports specific data or findings about visualizations other than Top 10 Products"
18+
],
19+
"rubric": [
20+
"Respects the requested brevity (roughly three sentences or fewer)",
21+
"Conveys the main product or category concentration insight (e.g. Outdoor or Neptide dominance)",
22+
"Notes that only the Top 10 Products visualization was analyzed",
23+
"Reads as fluent prose rather than a list of raw values"
24+
]
25+
}
26+
}

‎packages/gooddata-eval/examples/summary_dataset/summary_full_dashboard.json‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
{
2+
"id": "summary-full-dashboard",
3+
"dataset_name": "summary_pilot",
4+
"test_kind": "dashboard_summary",
5+
"question": "Summarize the All visualizations 1 with custom filter dashboard.",
6+
"summary_input": {
7+
"dashboard_id": "b2f2d436-9831-4fe0-81df-8c59fd33242b"
8+
},
9+
"expected_output": {
10+
"must_include": [
11+
"Includes a Summary, Key Insights, and Areas to Watch section",
12+
"Cites specific numeric values from the dashboard data"
13+
],
14+
"must_not_include": [
15+
"Includes a markdown heading dedicated to filters (a line such as '## Filter Context' or '### Filters'), separate from the Summary, Key Insights, and Areas to Watch sections. Filters mentioned within the prose of those sections do NOT count."
16+
],
17+
"rubric": [
18+
"The Summary section captures the overall state across the dashboard",
19+
"Key Insights call out top and/or bottom performers and their relative contribution (shares or percentages)",
20+
"Key Insights describe a direction of change over time (growing, declining, or flat) for at least one metric",
21+
"Areas to Watch highlights at least one risk, decline, or concentration issue and explains why it matters for business decisions",
22+
"Findings reference the active filter context (e.g. the time window or selected segments) in the wording",
23+
"Synthesizes insights across multiple visualizations rather than describing each one in isolation",
24+
"Reads as a cohesive business narrative connecting related metrics, not a list of raw values"
25+
]
26+
}
27+
}

‎packages/gooddata-eval/examples/summary_dataset/summary_selected_visualizations.json‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
{
2+
"id": "summary-selected-visualizations",
3+
"dataset_name": "summary_pilot",
4+
"test_kind": "dashboard_summary",
5+
"question": "Summarize only the Top 10 Customers and Top 10 Products on the dashboard.",
6+
"summary_input": {
7+
"dashboard_id": "b2f2d436-9831-4fe0-81df-8c59fd33242b",
8+
"visualizations": ["top_10_customers", "top_10_products"]
9+
},
10+
"expected_output": {
11+
"must_include": [
12+
"Includes a Summary, Key Insights, and Areas to Watch section",
13+
"Cites specific numeric values for both the customers and the products data"
14+
],
15+
"must_not_include": [
16+
"Reports specific data or findings about dashboard visualizations other than Top 10 Customers and Top 10 Products"
17+
],
18+
"rubric": [
19+
"Identifies the top customer (highest-revenue account) with their revenue or contribution",
20+
"Identifies the top product with its revenue and share",
21+
"Highlights product or category concentration (e.g. a single product or category dominating)",
22+
"Notes that only the two requested visualizations were analyzed and other dashboard data was not included",
23+
"Reads as a cohesive business narrative connecting customers and products, not a list of raw values"
24+
]
25+
}
26+
}

‎packages/gooddata-eval/src/gooddata_eval/core/evaluators/summary.py‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -33,11 +33,16 @@
3333
"Score 0 if the criterion is missing, contradicted, or only partially addressed.",
3434
]
3535

36-
_NEGATIVE_STEPS = [
37-
"Read the INPUT (the user's request) and the EXPECTED OUTPUT, which describes something the summary must NOT do.",
36+
# For must_not_include we ask the judge a plain presence question and invert the
37+
# result in code. Scoring "does the summary AVOID X?" via a field labelled
38+
# EXPECTED OUTPUT is unreliable: the model reads the forbidden behaviour as
39+
# desired and flips the verdict. Detecting presence (no negation, no
40+
# contradictory label) is far more robust.
41+
_VIOLATION_STEPS = [
42+
"Read the CHARACTERISTIC described in EXPECTED OUTPUT.",
3843
"Read the ACTUAL OUTPUT (the generated summary).",
39-
"Score 1 if the actual output correctly AVOIDS the described issue.",
40-
"Score 0 if the actual output violates it (e.g. invents numbers or mentions segments not in the data).",
44+
"Score 1 if the actual output clearly exhibits the described characteristic.",
45+
"Score 0 if it does not exhibit it.",
4146
]
4247

4348

@@ -46,7 +51,7 @@ class DashboardSummaryEvaluator:
4651

4752
def __init__(self):
4853
self._positive_judge = LLMJudge(evaluation_steps=_POSITIVE_STEPS)
49-
self._negative_judge = LLMJudge(evaluation_steps=_NEGATIVE_STEPS)
54+
self._violation_judge = LLMJudge(evaluation_steps=_VIOLATION_STEPS)
5055

5156
@staticmethod
5257
def _criteria(expected_output: Any) -> tuple[list[str], list[str], list[str]]:
@@ -74,8 +79,9 @@ def evaluate(self, item: DatasetItem, chat_result: ChatResult) -> ItemEvaluation
7479
passed = passed and ok
7580

7681
for i, criterion in enumerate(must_not_include):
77-
ok, reason = self._negative_judge.score(item.question, criterion, actual)
78-
detail[f"exclude_{i}"] = ok # True == correctly avoided
82+
violated, reason = self._violation_judge.score(item.question, criterion, actual)
83+
ok = not violated # True == characteristic absent == correctly avoided
84+
detail[f"exclude_{i}"] = ok
7985
detail[f"exclude_{i}_reason"] = reason
8086
passed = passed and ok
8187

‎packages/gooddata-eval/src/gooddata_eval/core/reporting/console.py‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -32,13 +32,11 @@ def render_console(report: EvalReport, *, console: Console | None = None) -> str
3232
elif item.pass_at_k:
3333
result, notes = "PASS", ""
3434
else:
35-
d = item.best_detail
36-
failing = [
37-
k
38-
for k in ("metrics_correct", "dimensions_correct", "filters_correct", "viz_type_hard")
39-
if d.get(k) is False
40-
]
41-
notes = "failed: " + ", ".join(failing) if failing else "no visualization created"
35+
# Evaluator-agnostic: report whichever boolean checks came back False
36+
# (visualization uses metrics_correct/…; dashboard_summary uses
37+
# include_*/exclude_*/rubric_*). Falls back to a generic message.
38+
failing = [k for k, v in item.best_detail.items() if v is False]
39+
notes = "failed: " + ", ".join(failing) if failing else "did not pass strict checks"
4240
result = "FAIL"
4341
latency = "-" if item.runs == 0 else f"{item.latency_s:.2f}s"
4442
avg = "-" if item.runs == 0 else f"{item.avg_latency_s:.2f}s"

‎packages/gooddata-eval/tests/test_summary_evaluator.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ def _chat(text: str = "Revenue grew QoQ; West is the top region.") -> ChatResult
2727
def test_passes_when_all_criteria_satisfied():
2828
ev = _make_evaluator()
2929
ev._positive_judge.score = MagicMock(return_value=(True, "ok"))
30-
ev._negative_judge.score = MagicMock(return_value=(True, "avoided"))
30+
ev._violation_judge.score = MagicMock(return_value=(False, "characteristic absent"))
3131

3232
item = _item({"must_include": ["a", "b"], "must_not_include": ["x"], "rubric": ["r"]})
3333
res = ev.evaluate(item, _chat())
@@ -40,7 +40,8 @@ def test_passes_when_all_criteria_satisfied():
4040
def test_fails_when_must_not_include_violated():
4141
ev = _make_evaluator()
4242
ev._positive_judge.score = MagicMock(return_value=(True, "ok"))
43-
ev._negative_judge.score = MagicMock(return_value=(False, "hallucinated a number"))
43+
# violation judge detects the forbidden characteristic is present -> avoided=False
44+
ev._violation_judge.score = MagicMock(return_value=(True, "has a separate filter section"))
4445

4546
item = _item({"must_include": ["a"], "must_not_include": ["x"]})
4647
res = ev.evaluate(item, _chat())
@@ -53,7 +54,7 @@ def test_fails_when_must_not_include_violated():
5354
def test_fails_when_a_must_include_is_missing():
5455
ev = _make_evaluator()
5556
ev._positive_judge.score = MagicMock(side_effect=[(True, "ok"), (False, "missing")])
56-
ev._negative_judge.score = MagicMock(return_value=(True, "avoided"))
57+
ev._violation_judge.score = MagicMock(return_value=(False, "characteristic absent"))
5758

5859
item = _item({"must_include": ["a", "b"]})
5960
res = ev.evaluate(item, _chat())

0 commit comments

Comments
 (0)