Find the latest ingested coverage revision from the gcp bucket directly

marco-c · marco-c · commit 5cfafdac40de · 2026-03-04T23:44:01.000+01:00
Fixes #2976
diff --git a/bot/code_coverage_bot/hgmo.py b/bot/code_coverage_bot/hgmo.py
@@ -6,6 +6,9 @@
 import requests
 import structlog
 
+from typing import Iterable
+
+
 logger = structlog.get_logger(__name__)
 
 
@@ -56,10 +59,14 @@ def __exit__(self, type, value, traceback):
         logger.info("hgmo has been killed")
 
     def get_pushes(
-        self, startID=None, startDate=None, changeset=None, full=True, tipsonly=False
+        self,
+        startID=None,
+        endID=None,
+        startDate=None,
+        changeset=None,
+        full=True,
+        tipsonly=False,
     ):
-        assert startID is not None or startDate is not None or changeset is not None
-
         params = {"version": 2}
 
         if full:
@@ -71,6 +78,9 @@ def get_pushes(
         if startID is not None:
             params["startID"] = startID
 
+        if endID is not None:
+            params["endID"] = endID
+
         if startDate is not None:
             params["startdate"] = startDate
 
@@ -96,3 +106,33 @@ def get_automation_relevance_changesets(self, changeset):
         )
         r.raise_for_status()
         return r.json()["changesets"]
+
+
+def iter_pushes(
+    server_address: str,
+) -> Iterable[tuple[int, dict]]:
+    """Yield pushes from newest to oldest push-id."""
+    start_id = None
+    end_id = None
+
+    with HGMO(server_address=server_address) as hgmo_server:
+        while True:
+            data = hgmo_server.get_pushes(
+                startID=start_id, endID=end_id, full=False, tipsonly=True
+            )
+            pushes = data.get("pushes", {})
+
+            if not pushes:
+                return
+
+            push_ids = sorted((int(push_id) for push_id in pushes.keys()), reverse=True)
+            for push_id in push_ids:
+                yield push_id, pushes[str(push_id)]
+
+            oldest_seen = push_ids[-1]
+            if oldest_seen <= 1:
+                return
+
+            # json-pushes treats startID as exclusive and endID as inclusive.
+            end_id = oldest_seen - 1
+            start_id = max(1, end_id - len(pushes))
diff --git a/bot/code_coverage_bot/hooks/cron.py b/bot/code_coverage_bot/hooks/cron.py
@@ -24,7 +24,7 @@ class CronHook(Hook):
     def __init__(self, *args, **kwargs):
         # Retrieve latest ingested revision
         try:
-            revision = uploader.gcp_latest("mozilla-central")[0]["revision"]
+            revision = uploader.gcp_latest(config.MOZILLA_CENTRAL_REPOSITORY)
         except Exception as e:
             logger.warn("Failed to retrieve the latest reports ingested: {}".format(e))
             raise
diff --git a/bot/code_coverage_bot/hooks/crontrigger.py b/bot/code_coverage_bot/hooks/crontrigger.py
@@ -23,7 +23,7 @@ class CronTriggerHook(Hook):
     def __init__(self, *args, **kwargs):
         # Retrieve latest ingested revision
         try:
-            revision = uploader.gcp_latest("mozilla-central")[0]["revision"]
+            revision = uploader.gcp_latest(config.MOZILLA_CENTRAL_REPOSITORY)
         except Exception as e:
             logger.warn("Failed to retrieve the latest reports ingested: {}".format(e))
             raise
diff --git a/bot/code_coverage_bot/uploader.py b/bot/code_coverage_bot/uploader.py
@@ -2,13 +2,13 @@
 import itertools
 import os.path
 
-import requests
 import structlog
 import zstandard as zstd
 from google.cloud.storage.bucket import Bucket
 
 from code_coverage_bot.secrets import secrets
 from code_coverage_bot.gcp import get_bucket
+from code_coverage_bot import hgmo
 
 logger = structlog.get_logger(__name__)
 GCP_COVDIR_PATH = "{repository}/{revision}/{platform}:{suite}.json.zstd"
@@ -60,15 +60,24 @@ def gcp_covdir_exists(
     return blob.exists()
 
 
-def gcp_latest(repository):
+def gcp_latest(repo_url):
     """
     List the latest reports ingested on the backend
     """
-    params = {"repository": repository}
-    backend_host = secrets[secrets.BACKEND_HOST]
-    resp = requests.get("{}/v2/latest".format(backend_host), params=params)
-    resp.raise_for_status()
-    return resp.json()
+    assert (
+        secrets[secrets.GOOGLE_CLOUD_STORAGE] is not None
+    ), "Missing GOOGLE_CLOUD_STORAGE secret"
+    bucket = get_bucket(secrets[secrets.GOOGLE_CLOUD_STORAGE])
+
+    for push_id, push_data in hgmo.iter_pushes(server_address=repo_url):
+        changesets: list[str] = push_data.get("changesets", [])
+        if not changesets:
+            continue
+
+        if gcp_covdir_exists(bucket, "mozilla-central", changesets[-1], "all", "all"):
+            return changesets[-1]
+
+    return None
 
 
 def covdir_paths(report):
