ci: add helpful remarks to filter benchmark results (#4824)

a10y · web-flow · commit 0a96ace8a05d · 2025-10-02T10:30:58.000-04:00
The top-line summaries on PR benchmarks are very helpful, but we could still use more help to triage results. For things like TPC-DS, we get back a ~500 element table that you need to visually scan to find individual queries with signficant changes. Here we add a 🚀 emoji for queries that improved by >= 30%, and a 🚨 for ones that decreased performance by >= 30%. **Example** <img width="885" height="810" alt="image" src="https://github.com/user-attachments/assets/dbfd716b-0b83-42ba-afd7-d528aa2afae7" /> Signed-off-by: Andrew Duffy <andrew@a10y.dev>
diff --git a/scripts/compare-benchmark-jsons.py b/scripts/compare-benchmark-jsons.py
@@ -58,6 +58,13 @@ def extract_dataset_key(df):
 
 # Generate summary statistics
 df3["ratio"] = df3["value_pr"] / df3["value_base"]
+df3["remark"] = pd.Series([""] * len(df3))
+df3["remark"] = df3["remark"].case_when(
+    [
+        (df3["ratio"] >= 1.3, "🚨"),
+        (df3["ratio"] <= 0.7, "🚀"),
+    ]
+)
 
 # Filter for different target combinations for summary statistics
 vortex_df = df3[df3["name"].str.contains("vortex", case=False, na=False)]
@@ -180,6 +187,7 @@ def format_performance(ratio, target_name):
         f"base {base_commit_id[:8]}": df3["value_base"],
         "ratio (PR/base)": df3["ratio"],
         "unit": df3["unit_base"],
+        "remark": df3["remark"],
     }
 )
 
