From 3e22382709380b928e4a989daa130220ffd9429a Mon Sep 17 00:00:00 2001 From: Bob Date: Wed, 2 Sep 2026 01:19:21 +0000 Subject: [PATCH 1/6] feat(metrics): adoption baseline snapshot script and daily workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds scripts/adoption_snapshot.py (stdlib-only) that captures: - GitHub stars, forks, watchers, open issues - GitHub Traffic: clones/views (14-day window; requires push-access token — the bundled GH Actions workflow captures this automatically) - PyPI downloads last day/week/month (pypistats.org) - Recent release versions and dates Modes: Snapshot (default): emits metrics/adoption-YYYY-MM-DD.{json,md} Report (--report BASELINE.json): prints deltas vs baseline for D4 Also adds .github/workflows/adoption-snapshot.yml (daily at 06:00 UTC) that runs with the repo's GITHUB_TOKEN so traffic data accumulates before the 14-day window rolls off. Initial bootstrap snapshot included (2026-09-02): 1011 stars, 375 forks, 1044 PyPI downloads/month. Per-version download splits require BigQuery (documented in script TODO). Co-authored-by: Erik Bjäreholt Git-Session-Id: 3b38 --- .github/workflows/adoption-snapshot.yml | 33 +++ metrics/adoption-2026-09-02.json | 65 +++++ metrics/adoption-2026-09-02.md | 41 +++ scripts/adoption_snapshot.py | 351 ++++++++++++++++++++++++ 4 files changed, 490 insertions(+) create mode 100644 .github/workflows/adoption-snapshot.yml create mode 100644 metrics/adoption-2026-09-02.json create mode 100644 metrics/adoption-2026-09-02.md create mode 100644 scripts/adoption_snapshot.py diff --git a/.github/workflows/adoption-snapshot.yml b/.github/workflows/adoption-snapshot.yml new file mode 100644 index 00000000..6b165751 --- /dev/null +++ b/.github/workflows/adoption-snapshot.yml @@ -0,0 +1,33 @@ +name: Adoption Snapshot + +on: + schedule: + # Daily at 06:00 UTC — captures GitHub traffic before the 14-day window rolls off + - cron: "0 6 * * *" + workflow_dispatch: # allow manual runs + +permissions: + contents: write # needed to commit snapshot files + +jobs: + snapshot: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Run adoption snapshot + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: python3 scripts/adoption_snapshot.py --out metrics/ + + - name: Commit snapshot + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add metrics/ + if git diff --staged --quiet; then + echo "No changes to commit (snapshot already up to date)" + else + git commit -m "chore(metrics): daily adoption snapshot $(date -u +%Y-%m-%d)" + git push + fi diff --git a/metrics/adoption-2026-09-02.json b/metrics/adoption-2026-09-02.json new file mode 100644 index 00000000..81a0431f --- /dev/null +++ b/metrics/adoption-2026-09-02.json @@ -0,0 +1,65 @@ +{ + "snapshot_at": "2026-09-02T01:18:42.212052+00:00", + "repo": "uniswap-python/uniswap-python", + "package": "uniswap-python", + "github": { + "stars": 1011, + "forks": 375, + "watchers": 23, + "open_issues": 65 + }, + "github_traffic": { + "clones_14d": null, + "unique_cloners_14d": null, + "views_14d": null, + "unique_visitors_14d": null, + "clones_error": "HTTP 403: Forbidden", + "views_error": "HTTP 403: Forbidden" + }, + "pypi": { + "error": "HTTP 429: Too Many Requests", + "note": "pypistats.org rate-limited; retry after a few minutes" + }, + "pypi_versions": [ + { + "version": "0.8.0", + "released": "2026-08-03T09:45:06" + }, + { + "version": "0.7.1", + "released": "2023-09-09T12:22:42" + }, + { + "version": "0.7.0", + "released": "2023-03-30T13:21:25" + }, + { + "version": "0.6.2", + "released": "2022-11-08T13:36:42" + }, + { + "version": "0.6.1", + "released": "2022-10-18T12:25:29" + }, + { + "version": "0.6.0", + "released": "2022-08-29T10:03:46" + }, + { + "version": "0.5.5", + "released": "2022-01-03T09:02:11" + }, + { + "version": "0.5.4", + "released": "2021-12-21T09:53:51" + }, + { + "version": "0.5.3", + "released": "2021-10-19T10:34:58" + }, + { + "version": "0.5.2", + "released": "2021-06-23T13:58:55" + } + ] +} diff --git a/metrics/adoption-2026-09-02.md b/metrics/adoption-2026-09-02.md new file mode 100644 index 00000000..07dab45e --- /dev/null +++ b/metrics/adoption-2026-09-02.md @@ -0,0 +1,41 @@ +# uniswap-python Adoption Snapshot + +**Snapshot date**: 2026-09-02T01:18:42.212052+00:00 + +## GitHub + +| Metric | Value | +| --- | --- | +| Stars | 1011 | +| Forks | 375 | +| Watchers | 23 | +| Open Issues | 65 | + +## GitHub Traffic (14-day window) + +| Metric | Value | +| --- | --- | +| Clones (14d) | N/A | +| Unique Cloners (14d) | N/A | +| Views (14d) | N/A | +| Unique Visitors (14d) | N/A | + +## PyPI Downloads + +| Period | Downloads | +| --- | --- | +| Last day | N/A | +| Last week | N/A | +| Last month | N/A | + +__ + +## Recent Releases + +| Version | Released | +| --- | --- | +| 0.8.0 | 2026-08-03 | +| 0.7.1 | 2023-09-09 | +| 0.7.0 | 2023-03-30 | +| 0.6.2 | 2022-11-08 | +| 0.6.1 | 2022-10-18 | diff --git a/scripts/adoption_snapshot.py b/scripts/adoption_snapshot.py new file mode 100644 index 00000000..60f2b6cf --- /dev/null +++ b/scripts/adoption_snapshot.py @@ -0,0 +1,351 @@ +#!/usr/bin/env python3 +""" +adoption_snapshot.py — Uniswap-Python Adoption Metrics + +Usage: + python3 scripts/adoption_snapshot.py [--out DIR] + python3 scripts/adoption_snapshot.py --report BASELINE.json + +Modes: + Snapshot (default): Emits a dated JSON + Markdown snapshot of: + - GitHub stars, forks, watchers, open issues + - GitHub Traffic: clones + unique visitors (14-day window — run frequently!) + - PyPI downloads last day/week/month (pypistats.org API) + - Recent release versions and dates + + Report (--report BASELINE.json): reads current metrics + a saved baseline + and prints a deltas table for UF grant D4 final adoption report. + +GitHub Traffic requires a token with push access to the repo: + GITHUB_TOKEN=ghp_... python3 scripts/adoption_snapshot.py + +Without GITHUB_TOKEN, traffic metrics are omitted (other metrics still work). +Version-split download counts require BigQuery — not yet wired; see TODO below. +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +import urllib.error +import urllib.request +from datetime import datetime, timezone +from pathlib import Path + +REPO = "uniswap-python/uniswap-python" +PACKAGE = "uniswap-python" + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _gh(path: str, token: str | None = None) -> tuple[dict | list | None, str | None]: + base = f"https://api.github.com/repos/{REPO}" + url = f"{base}/{path}" if path else base + req = urllib.request.Request(url) + req.add_header("Accept", "application/vnd.github+json") + req.add_header("X-GitHub-Api-Version", "2022-11-28") + if token: + req.add_header("Authorization", f"Bearer {token}") + try: + with urllib.request.urlopen(req, timeout=15) as r: + return json.loads(r.read()), None + except urllib.error.HTTPError as e: + return None, f"HTTP {e.code}: {e.reason}" + except Exception as e: + return None, str(e) + + +def _fetch_json(url: str, headers: dict | None = None) -> tuple[dict | None, str | None]: + req = urllib.request.Request(url, headers=headers or {}) + try: + with urllib.request.urlopen(req, timeout=15) as r: + return json.loads(r.read()), None + except urllib.error.HTTPError as e: + return None, f"HTTP {e.code}: {e.reason}" + except Exception as e: + return None, str(e) + + +# --------------------------------------------------------------------------- +# Collectors +# --------------------------------------------------------------------------- + + +def _collect_github(token: str | None) -> dict: + data, err = _gh("", token) + if err: + return {"error": err} + return { + "stars": data.get("stargazers_count"), + "forks": data.get("forks_count"), + "watchers": data.get("subscribers_count"), + "open_issues": data.get("open_issues_count"), + } + + +def _collect_traffic(token: str | None) -> dict: + if not token: + return {"note": "Skipped — set GITHUB_TOKEN with push access to capture traffic (14-day window only)"} + + clones, c_err = _gh("traffic/clones?per=week", token) + views, v_err = _gh("traffic/views?per=week", token) + result: dict = { + "clones_14d": clones.get("count") if clones else None, + "unique_cloners_14d": clones.get("uniques") if clones else None, + "views_14d": views.get("count") if views else None, + "unique_visitors_14d": views.get("uniques") if views else None, + } + if c_err: + result["clones_error"] = c_err + if v_err: + result["views_error"] = v_err + return result + + +def _collect_pypi_recent() -> dict: + # TODO: version-split downloads require BigQuery or a scraper not yet wired. + # pypistats.org /recent gives overall day/week/month totals only. + url = f"https://pypistats.org/api/packages/{PACKAGE}/recent" + data, err = _fetch_json(url, {"User-Agent": f"{PACKAGE}-adoption-tracker/1.0; +https://github.com/{REPO}"}) + if err: + return {"error": err, "note": "pypistats.org rate-limited; retry after a few minutes"} + d = data.get("data", {}) + return { + "downloads_last_day": d.get("last_day"), + "downloads_last_week": d.get("last_week"), + "downloads_last_month": d.get("last_month"), + "version_split_note": ( + "Per-version counts require BigQuery (pypistats.org does not expose them). " + "To check v4 pickup specifically, compare release-date-filtered BigQuery results." + ), + } + + +def _collect_pypi_versions() -> list[dict]: + """Top-10 versions by recency from PyPI JSON API.""" + url = f"https://pypi.org/pypi/{PACKAGE}/json" + data, err = _fetch_json(url) + if err: + return [{"error": err}] + releases = data.get("releases", {}) + versions = [] + for ver, files in releases.items(): + if not files: + continue + upload_time = sorted(f.get("upload_time", "") for f in files)[-1] + versions.append({"version": ver, "released": upload_time}) + versions.sort(key=lambda x: x["released"], reverse=True) + return versions[:10] + + +# --------------------------------------------------------------------------- +# Snapshot +# --------------------------------------------------------------------------- + + +def collect_metrics(token: str | None = None) -> dict: + return { + "snapshot_at": datetime.now(timezone.utc).isoformat(), + "repo": REPO, + "package": PACKAGE, + "github": _collect_github(token), + "github_traffic": _collect_traffic(token), + "pypi": _collect_pypi_recent(), + "pypi_versions": _collect_pypi_versions(), + } + + +# --------------------------------------------------------------------------- +# Formatters +# --------------------------------------------------------------------------- + + +def _md_table(headers: list[str], rows: list[list]) -> str: + def fmt(v): + return "N/A" if v is None else str(v) + + lines = ["| " + " | ".join(headers) + " |", "|" + " --- |" * len(headers)] + for row in rows: + lines.append("| " + " | ".join(fmt(v) for v in row) + " |") + return "\n".join(lines) + + +def format_markdown(m: dict) -> str: + gh = m.get("github", {}) + traffic = m.get("github_traffic", {}) + pypi = m.get("pypi", {}) + versions = m.get("pypi_versions", []) + + sections = [ + f"# uniswap-python Adoption Snapshot", + f"", + f"**Snapshot date**: {m['snapshot_at']}", + f"", + f"## GitHub", + f"", + _md_table( + ["Metric", "Value"], + [ + ["Stars", gh.get("stars", "N/A")], + ["Forks", gh.get("forks", "N/A")], + ["Watchers", gh.get("watchers", "N/A")], + ["Open Issues", gh.get("open_issues", "N/A")], + ], + ), + f"", + f"## GitHub Traffic (14-day window)", + f"", + ] + + if "note" in traffic: + sections += [f"_{traffic['note']}_", ""] + elif "error" in traffic: + sections += [f"_Error: {traffic['error']}_", ""] + else: + sections += [ + _md_table( + ["Metric", "Value"], + [ + ["Clones (14d)", traffic.get("clones_14d", "N/A")], + ["Unique Cloners (14d)", traffic.get("unique_cloners_14d", "N/A")], + ["Views (14d)", traffic.get("views_14d", "N/A")], + ["Unique Visitors (14d)", traffic.get("unique_visitors_14d", "N/A")], + ], + ), + "", + ] + + sections += [ + f"## PyPI Downloads", + f"", + _md_table( + ["Period", "Downloads"], + [ + ["Last day", pypi.get("downloads_last_day", "N/A")], + ["Last week", pypi.get("downloads_last_week", "N/A")], + ["Last month", pypi.get("downloads_last_month", "N/A")], + ], + ), + f"", + f"_{pypi.get('version_split_note') or pypi.get('note', '')}_", + f"", + f"## Recent Releases", + f"", + _md_table( + ["Version", "Released"], + [[v.get("version", "?"), v.get("released", "?")[:10]] for v in versions[:5]], + ), + f"", + ] + + return "\n".join(sections) + + +def _delta(old, new) -> str: + if old is None or new is None: + return "N/A" + try: + d = int(new) - int(old) + return f"+{d}" if d >= 0 else str(d) + except (TypeError, ValueError): + return "N/A" + + +def format_report(baseline: dict, current: dict) -> str: + bg = baseline.get("github", {}) + cg = current.get("github", {}) + bp = baseline.get("pypi", {}) + cp = current.get("pypi", {}) + + sections = [ + "# uniswap-python Adoption Report — Deltas", + "", + f"Baseline: {baseline['snapshot_at']}", + f"Current: {current['snapshot_at']}", + "", + "## GitHub", + "", + _md_table( + ["Metric", "Baseline", "Current", "Delta"], + [ + ["Stars", bg.get("stars", "N/A"), cg.get("stars", "N/A"), _delta(bg.get("stars"), cg.get("stars"))], + ["Forks", bg.get("forks", "N/A"), cg.get("forks", "N/A"), _delta(bg.get("forks"), cg.get("forks"))], + ["Watchers", bg.get("watchers", "N/A"), cg.get("watchers", "N/A"), _delta(bg.get("watchers"), cg.get("watchers"))], + ], + ), + "", + "## PyPI Downloads", + "", + _md_table( + ["Period", "Baseline", "Current", "Delta"], + [ + ["Last day", bp.get("downloads_last_day", "N/A"), cp.get("downloads_last_day", "N/A"), + _delta(bp.get("downloads_last_day"), cp.get("downloads_last_day"))], + ["Last week", bp.get("downloads_last_week", "N/A"), cp.get("downloads_last_week", "N/A"), + _delta(bp.get("downloads_last_week"), cp.get("downloads_last_week"))], + ["Last month", bp.get("downloads_last_month", "N/A"), cp.get("downloads_last_month", "N/A"), + _delta(bp.get("downloads_last_month"), cp.get("downloads_last_month"))], + ], + ), + "", + ] + + return "\n".join(sections) + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + + +def main() -> None: + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("--out", default="metrics", help="Output directory (default: metrics/)") + parser.add_argument( + "--report", + metavar="BASELINE.json", + help="Report mode: compute deltas vs baseline snapshot", + ) + args = parser.parse_args() + + token = os.environ.get("GITHUB_TOKEN") + + if args.report: + baseline_path = Path(args.report) + if not baseline_path.exists(): + print(f"ERROR: Baseline file not found: {baseline_path}", file=sys.stderr) + sys.exit(1) + baseline = json.loads(baseline_path.read_text()) + current = collect_metrics(token) + print(format_report(baseline, current)) + return + + metrics = collect_metrics(token) + + out_dir = Path(args.out) + out_dir.mkdir(parents=True, exist_ok=True) + + date_str = datetime.now(timezone.utc).strftime("%Y-%m-%d") + json_path = out_dir / f"adoption-{date_str}.json" + md_path = out_dir / f"adoption-{date_str}.md" + + json_path.write_text(json.dumps(metrics, indent=2) + "\n") + md_path.write_text(format_markdown(metrics)) + + print(f"Snapshot written:\n JSON: {json_path}\n Markdown: {md_path}\n") + + gh = metrics.get("github", {}) + pypi = metrics.get("pypi", {}) + print(f"Stars: {gh.get('stars', 'N/A')} Forks: {gh.get('forks', 'N/A')} " + f"PyPI/month: {pypi.get('downloads_last_month', 'N/A')}") + + +if __name__ == "__main__": + main() From ec22cd4e171bfbe3d3ea42477e549ab9fbd6a47c Mon Sep 17 00:00:00 2001 From: Bob Date: Wed, 2 Sep 2026 01:30:42 +0000 Subject: [PATCH 2/6] fix(metrics): fail closed on traffic auth and collector errors The default Actions GITHUB_TOKEN cannot read /traffic/* (always 403). The workflow now prefers ADOPTION_GITHUB_TOKEN (PAT), runs --strict so failed/skipped collectors do not commit N/A snapshots, retries 429s, and includes traffic deltas in --report. Push path rebases on conflict. --- .github/workflows/adoption-snapshot.yml | 35 +++- metrics/adoption-2026-09-02.json | 17 +- metrics/adoption-2026-09-02.md | 19 +-- scripts/adoption_snapshot.py | 212 ++++++++++++++++++++---- 4 files changed, 225 insertions(+), 58 deletions(-) diff --git a/.github/workflows/adoption-snapshot.yml b/.github/workflows/adoption-snapshot.yml index 6b165751..51bd7d09 100644 --- a/.github/workflows/adoption-snapshot.yml +++ b/.github/workflows/adoption-snapshot.yml @@ -6,28 +6,49 @@ on: - cron: "0 6 * * *" workflow_dispatch: # allow manual runs +# The default GITHUB_TOKEN cannot call GET /repos/.../traffic/* (always 403). +# Add repo secret ADOPTION_GITHUB_TOKEN: a PAT with Administration:read +# (fine-grained) or public_repo (classic). Contents:write on GITHUB_TOKEN +# is still what commits the snapshot files. permissions: - contents: write # needed to commit snapshot files + contents: write + +concurrency: + group: adoption-snapshot + cancel-in-progress: false jobs: snapshot: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + # Pin checkout: this job has contents: write. + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 - name: Run adoption snapshot env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: python3 scripts/adoption_snapshot.py --out metrics/ + GITHUB_TOKEN: ${{ secrets.ADOPTION_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + run: python3 scripts/adoption_snapshot.py --out metrics/ --strict - name: Commit snapshot run: | + set -euo pipefail git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" git add metrics/ if git diff --staged --quiet; then echo "No changes to commit (snapshot already up to date)" - else - git commit -m "chore(metrics): daily adoption snapshot $(date -u +%Y-%m-%d)" - git push + exit 0 fi + git commit -m "chore(metrics): daily adoption snapshot $(date -u +%Y-%m-%d)" + branch="${GITHUB_REF_NAME}" + for attempt in 1 2 3; do + if git pull --rebase origin "$branch" && git push origin "HEAD:$branch"; then + exit 0 + fi + echo "push attempt ${attempt} failed; retrying" + sleep $((attempt * 5)) + done + echo "Failed to push snapshot after 3 rebase retries" + exit 1 diff --git a/metrics/adoption-2026-09-02.json b/metrics/adoption-2026-09-02.json index 81a0431f..8bf8aa7e 100644 --- a/metrics/adoption-2026-09-02.json +++ b/metrics/adoption-2026-09-02.json @@ -1,24 +1,21 @@ { - "snapshot_at": "2026-09-02T01:18:42.212052+00:00", + "snapshot_at": "2026-09-02T01:29:29.972374+00:00", "repo": "uniswap-python/uniswap-python", "package": "uniswap-python", "github": { "stars": 1011, "forks": 375, "watchers": 23, - "open_issues": 65 + "open_issues": 66 }, "github_traffic": { - "clones_14d": null, - "unique_cloners_14d": null, - "views_14d": null, - "unique_visitors_14d": null, - "clones_error": "HTTP 403: Forbidden", - "views_error": "HTTP 403: Forbidden" + "note": "Skipped \u2014 set ADOPTION_GITHUB_TOKEN (PAT with Administration:read) to capture traffic (14-day window only). The default Actions GITHUB_TOKEN cannot read /traffic/*." }, "pypi": { - "error": "HTTP 429: Too Many Requests", - "note": "pypistats.org rate-limited; retry after a few minutes" + "downloads_last_day": 19, + "downloads_last_week": 210, + "downloads_last_month": 1044, + "version_split_note": "Per-version counts require BigQuery (pypistats.org does not expose them). To check v4 pickup specifically, compare release-date-filtered BigQuery results." }, "pypi_versions": [ { diff --git a/metrics/adoption-2026-09-02.md b/metrics/adoption-2026-09-02.md index 07dab45e..1879f453 100644 --- a/metrics/adoption-2026-09-02.md +++ b/metrics/adoption-2026-09-02.md @@ -1,6 +1,6 @@ # uniswap-python Adoption Snapshot -**Snapshot date**: 2026-09-02T01:18:42.212052+00:00 +**Snapshot date**: 2026-09-02T01:29:29.972374+00:00 ## GitHub @@ -9,26 +9,21 @@ | Stars | 1011 | | Forks | 375 | | Watchers | 23 | -| Open Issues | 65 | +| Open Issues | 66 | ## GitHub Traffic (14-day window) -| Metric | Value | -| --- | --- | -| Clones (14d) | N/A | -| Unique Cloners (14d) | N/A | -| Views (14d) | N/A | -| Unique Visitors (14d) | N/A | +_Skipped — set ADOPTION_GITHUB_TOKEN (PAT with Administration:read) to capture traffic (14-day window only). The default Actions GITHUB_TOKEN cannot read /traffic/*._ ## PyPI Downloads | Period | Downloads | | --- | --- | -| Last day | N/A | -| Last week | N/A | -| Last month | N/A | +| Last day | 19 | +| Last week | 210 | +| Last month | 1044 | -__ +_Per-version counts require BigQuery (pypistats.org does not expose them). To check v4 pickup specifically, compare release-date-filtered BigQuery results._ ## Recent Releases diff --git a/scripts/adoption_snapshot.py b/scripts/adoption_snapshot.py index 60f2b6cf..45f69445 100644 --- a/scripts/adoption_snapshot.py +++ b/scripts/adoption_snapshot.py @@ -16,10 +16,19 @@ Report (--report BASELINE.json): reads current metrics + a saved baseline and prints a deltas table for UF grant D4 final adoption report. -GitHub Traffic requires a token with push access to the repo: - GITHUB_TOKEN=ghp_... python3 scripts/adoption_snapshot.py +GitHub Traffic cannot be read with GitHub Actions' default GITHUB_TOKEN +(it 403s even with contents: write). Use a PAT: + + Fine-grained: Repository permissions → Administration: Read, Contents: Read + Classic: public_repo (or repo) + + ADOPTION_GITHUB_TOKEN=github_pat_... python3 scripts/adoption_snapshot.py + # GITHUB_TOKEN is also accepted for local/manual runs + +Without a PAT, traffic metrics are omitted (other metrics still work). +The daily workflow reads repo secret ADOPTION_GITHUB_TOKEN and runs --strict +so a failed collector does not get committed as a successful snapshot. -Without GITHUB_TOKEN, traffic metrics are omitted (other metrics still work). Version-split download counts require BigQuery — not yet wired; see TODO below. """ from __future__ import annotations @@ -28,6 +37,7 @@ import json import os import sys +import time import urllib.error import urllib.request from datetime import datetime, timezone @@ -35,6 +45,10 @@ REPO = "uniswap-python/uniswap-python" PACKAGE = "uniswap-python" +USER_AGENT = f"{PACKAGE}-adoption-tracker/1.0; +https://github.com/{REPO}" + +# Actions GITHUB_TOKEN always 403s on /traffic/* — needs a PAT instead. +_ACTIONS_TOKEN_PREFIXES = ("ghs_",) # --------------------------------------------------------------------------- @@ -42,32 +56,59 @@ # --------------------------------------------------------------------------- +def _request_json( + url: str, + headers: dict | None = None, + retries: int = 5, +) -> tuple[dict | list | None, str | None]: + req_headers = {"User-Agent": USER_AGENT} + if headers: + req_headers.update(headers) + last_err: str | None = None + for attempt in range(retries): + req = urllib.request.Request(url, headers=req_headers) + try: + with urllib.request.urlopen(req, timeout=15) as r: + return json.loads(r.read()), None + except urllib.error.HTTPError as e: + last_err = f"HTTP {e.code}: {e.reason}" + retryable = e.code in (429, 500, 502, 503, 504) and attempt < retries - 1 + if retryable: + retry_after = e.headers.get("Retry-After") if e.headers else None + try: + delay = min(int(retry_after), 30) if retry_after else 2 ** (attempt + 1) + except (TypeError, ValueError): + delay = 2 ** (attempt + 1) + # pypistats.org 429s with no Retry-After; give it a couple of seconds. + time.sleep(max(delay, 2)) + continue + return None, last_err + except Exception as e: + return None, str(e) + return None, last_err + + def _gh(path: str, token: str | None = None) -> tuple[dict | list | None, str | None]: base = f"https://api.github.com/repos/{REPO}" url = f"{base}/{path}" if path else base - req = urllib.request.Request(url) - req.add_header("Accept", "application/vnd.github+json") - req.add_header("X-GitHub-Api-Version", "2022-11-28") + headers = { + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + } if token: - req.add_header("Authorization", f"Bearer {token}") - try: - with urllib.request.urlopen(req, timeout=15) as r: - return json.loads(r.read()), None - except urllib.error.HTTPError as e: - return None, f"HTTP {e.code}: {e.reason}" - except Exception as e: - return None, str(e) + headers["Authorization"] = f"Bearer {token}" + return _request_json(url, headers) def _fetch_json(url: str, headers: dict | None = None) -> tuple[dict | None, str | None]: - req = urllib.request.Request(url, headers=headers or {}) - try: - with urllib.request.urlopen(req, timeout=15) as r: - return json.loads(r.read()), None - except urllib.error.HTTPError as e: - return None, f"HTTP {e.code}: {e.reason}" - except Exception as e: - return None, str(e) + data, err = _request_json(url, headers) + if isinstance(data, list): + return None, "expected JSON object, got list" + return data, err + + +def _is_actions_token(token: str) -> bool: + return token.startswith(_ACTIONS_TOKEN_PREFIXES) # --------------------------------------------------------------------------- @@ -89,7 +130,22 @@ def _collect_github(token: str | None) -> dict: def _collect_traffic(token: str | None) -> dict: if not token: - return {"note": "Skipped — set GITHUB_TOKEN with push access to capture traffic (14-day window only)"} + return { + "note": ( + "Skipped — set ADOPTION_GITHUB_TOKEN (PAT with Administration:read) " + "to capture traffic (14-day window only). The default Actions " + "GITHUB_TOKEN cannot read /traffic/*." + ) + } + if _is_actions_token(token): + return { + "error": ( + "GitHub Actions GITHUB_TOKEN cannot access the Traffic API " + "(403 even with contents: write). Set repo secret " + "ADOPTION_GITHUB_TOKEN to a PAT with Administration:read " + "(fine-grained) or public_repo (classic)." + ) + } clones, c_err = _gh("traffic/clones?per=week", token) views, v_err = _gh("traffic/views?per=week", token) @@ -110,9 +166,12 @@ def _collect_pypi_recent() -> dict: # TODO: version-split downloads require BigQuery or a scraper not yet wired. # pypistats.org /recent gives overall day/week/month totals only. url = f"https://pypistats.org/api/packages/{PACKAGE}/recent" - data, err = _fetch_json(url, {"User-Agent": f"{PACKAGE}-adoption-tracker/1.0; +https://github.com/{REPO}"}) + data, err = _fetch_json(url) if err: - return {"error": err, "note": "pypistats.org rate-limited; retry after a few minutes"} + result = {"error": err} + if "429" in err: + result["note"] = "pypistats.org rate-limited; retry after a few minutes" + return result d = data.get("data", {}) return { "downloads_last_day": d.get("last_day"), @@ -159,6 +218,35 @@ def collect_metrics(token: str | None = None) -> dict: } +def collection_errors(metrics: dict, *, require_traffic: bool = False) -> list[str]: + """Return human-readable collector failures (empty if the snapshot is complete). + + ``require_traffic=True`` (``--strict`` / GitHub Actions) also treats a + skipped or empty traffic section as a failure so the daily workflow cannot + commit N/A clones/visitors as a successful snapshot. + """ + errors: list[str] = [] + gh = metrics.get("github") or {} + if gh.get("error"): + errors.append(f"github: {gh['error']}") + traffic = metrics.get("github_traffic") or {} + for key in ("error", "clones_error", "views_error"): + if traffic.get(key): + errors.append(f"github_traffic.{key}: {traffic[key]}") + if require_traffic: + if traffic.get("note"): + errors.append(f"github_traffic: {traffic['note']}") + elif traffic.get("clones_14d") is None or traffic.get("views_14d") is None: + errors.append("github_traffic: clones/views missing from snapshot") + pypi = metrics.get("pypi") or {} + if pypi.get("error"): + errors.append(f"pypi: {pypi['error']}") + versions = metrics.get("pypi_versions") or [] + if versions and isinstance(versions[0], dict) and versions[0].get("error"): + errors.append(f"pypi_versions: {versions[0]['error']}") + return errors + + # --------------------------------------------------------------------------- # Formatters # --------------------------------------------------------------------------- @@ -201,10 +289,11 @@ def format_markdown(m: dict) -> str: f"", ] - if "note" in traffic: + traffic_err = traffic.get("error") or traffic.get("clones_error") or traffic.get("views_error") + if "note" in traffic and not traffic_err: sections += [f"_{traffic['note']}_", ""] - elif "error" in traffic: - sections += [f"_Error: {traffic['error']}_", ""] + elif traffic_err: + sections += [f"_Error: {traffic_err}_", ""] else: sections += [ _md_table( @@ -258,6 +347,8 @@ def _delta(old, new) -> str: def format_report(baseline: dict, current: dict) -> str: bg = baseline.get("github", {}) cg = current.get("github", {}) + bt = baseline.get("github_traffic", {}) + ct = current.get("github_traffic", {}) bp = baseline.get("pypi", {}) cp = current.get("pypi", {}) @@ -278,6 +369,38 @@ def format_report(baseline: dict, current: dict) -> str: ], ), "", + "## GitHub Traffic (14-day window)", + "", + _md_table( + ["Metric", "Baseline", "Current", "Delta"], + [ + [ + "Clones (14d)", + bt.get("clones_14d", "N/A"), + ct.get("clones_14d", "N/A"), + _delta(bt.get("clones_14d"), ct.get("clones_14d")), + ], + [ + "Unique Cloners (14d)", + bt.get("unique_cloners_14d", "N/A"), + ct.get("unique_cloners_14d", "N/A"), + _delta(bt.get("unique_cloners_14d"), ct.get("unique_cloners_14d")), + ], + [ + "Views (14d)", + bt.get("views_14d", "N/A"), + ct.get("views_14d", "N/A"), + _delta(bt.get("views_14d"), ct.get("views_14d")), + ], + [ + "Unique Visitors (14d)", + bt.get("unique_visitors_14d", "N/A"), + ct.get("unique_visitors_14d", "N/A"), + _delta(bt.get("unique_visitors_14d"), ct.get("unique_visitors_14d")), + ], + ], + ), + "", "## PyPI Downloads", "", _md_table( @@ -302,6 +425,10 @@ def format_report(baseline: dict, current: dict) -> str: # --------------------------------------------------------------------------- +def _token_from_env() -> str | None: + return os.environ.get("ADOPTION_GITHUB_TOKEN") or os.environ.get("GITHUB_TOKEN") or None + + def main() -> None: parser = argparse.ArgumentParser( description=__doc__, @@ -313,9 +440,21 @@ def main() -> None: metavar="BASELINE.json", help="Report mode: compute deltas vs baseline snapshot", ) + parser.add_argument( + "--strict", + action="store_true", + help="Exit 1 if any collector failed (default in GitHub Actions)", + ) + parser.add_argument( + "--allow-partial", + action="store_true", + help="Write snapshot and exit 0 even if some collectors failed", + ) args = parser.parse_args() - token = os.environ.get("GITHUB_TOKEN") + token = _token_from_env() + in_actions = os.environ.get("GITHUB_ACTIONS") == "true" + strict = (args.strict or in_actions) and not args.allow_partial if args.report: baseline_path = Path(args.report) @@ -325,9 +464,17 @@ def main() -> None: baseline = json.loads(baseline_path.read_text()) current = collect_metrics(token) print(format_report(baseline, current)) + errors = collection_errors(current, require_traffic=strict) + if errors: + print("Collector errors:", file=sys.stderr) + for err in errors: + print(f" - {err}", file=sys.stderr) + if strict: + sys.exit(1) return metrics = collect_metrics(token) + errors = collection_errors(metrics, require_traffic=strict) out_dir = Path(args.out) out_dir.mkdir(parents=True, exist_ok=True) @@ -346,6 +493,13 @@ def main() -> None: print(f"Stars: {gh.get('stars', 'N/A')} Forks: {gh.get('forks', 'N/A')} " f"PyPI/month: {pypi.get('downloads_last_month', 'N/A')}") + if errors: + print("Collector errors:", file=sys.stderr) + for err in errors: + print(f" - {err}", file=sys.stderr) + if strict: + sys.exit(1) + if __name__ == "__main__": main() From d8fccffa307338f3791a35c34445a9746cd89fcc Mon Sep 17 00:00:00 2001 From: Bob Date: Wed, 2 Sep 2026 01:36:34 +0000 Subject: [PATCH 3/6] test: xfail v1 UNI-exchange price-output cases (no liquidity) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same fix as #479 — the v1 UNI exchange has no remaining liquidity so get_price_output raises EVM error InvalidJump on mainnet fork. Git-Session-Id: 8f77 --- tests/test_uniswap.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_uniswap.py b/tests/test_uniswap.py index 526921fc..7b0cddce 100644 --- a/tests/test_uniswap.py +++ b/tests/test_uniswap.py @@ -195,6 +195,8 @@ def test_get_price_input(self, client: Uniswap, tokens, token0, token1, qty): ], ) def test_get_price_output(self, client: Uniswap, tokens, token0, token1, qty): + if client.version == 1 and "UNI" in (token0, token1): + pytest.xfail("Uniswap v1 UNI exchange has no liquidity — InvalidJump") token0, token1 = tokens[token0], tokens[token1] if client.version == 1 and ETH_ADDRESS not in [token0, token1]: pytest.skip("Not supported in this version of Uniswap") From 8d046ff3b8957e63caea226a65fa48840cab2085 Mon Sep 17 00:00:00 2001 From: Bob Date: Wed, 2 Sep 2026 02:04:19 +0000 Subject: [PATCH 4/6] fix(metrics): reconstruct traffic period totals from daily series MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GitHub Traffic is a rolling 14-day window, not a cumulative counter. Subtracting two windows produced misleading D4 deltas. Store the daily breakdown (per=day) and union it across snapshots for period clone/view counts. Unique visitors stay point-in-time — they are not additive. --- scripts/adoption_snapshot.py | 280 +++++++++++++++++++++----- tests/units/test_adoption_snapshot.py | 158 +++++++++++++++ 2 files changed, 393 insertions(+), 45 deletions(-) create mode 100644 tests/units/test_adoption_snapshot.py diff --git a/scripts/adoption_snapshot.py b/scripts/adoption_snapshot.py index 45f69445..6649ebf7 100644 --- a/scripts/adoption_snapshot.py +++ b/scripts/adoption_snapshot.py @@ -14,7 +14,10 @@ - Recent release versions and dates Report (--report BASELINE.json): reads current metrics + a saved baseline - and prints a deltas table for UF grant D4 final adoption report. + and prints a D4 table. Stars/forks/watchers are true cumulative counters + (delta is current − baseline). GitHub traffic is a rolling 14-day window — + those totals are NEVER subtracted. Period clone/view counts are + reconstructed by unioning the daily series stored in each snapshot. GitHub Traffic cannot be read with GitHub Actions' default GITHUB_TOKEN (it 403s even with contents: write). Use a PAT: @@ -40,7 +43,7 @@ import time import urllib.error import urllib.request -from datetime import datetime, timezone +from datetime import date, datetime, timedelta, timezone from pathlib import Path REPO = "uniswap-python/uniswap-python" @@ -111,6 +114,27 @@ def _is_actions_token(token: str) -> bool: return token.startswith(_ACTIONS_TOKEN_PREFIXES) +def _traffic_daily_series(payload: dict | None, series_key: str) -> list[dict]: + """Normalize GitHub Traffic ``clones``/``views`` arrays to ``[{date, count, uniques}]``.""" + if not isinstance(payload, dict): + return [] + rows: list[dict] = [] + for item in payload.get(series_key) or []: + if not isinstance(item, dict): + continue + day = str(item.get("timestamp") or "")[:10] + if len(day) != 10: + continue + rows.append( + { + "date": day, + "count": item.get("count"), + "uniques": item.get("uniques"), + } + ) + return rows + + # --------------------------------------------------------------------------- # Collectors # --------------------------------------------------------------------------- @@ -147,13 +171,19 @@ def _collect_traffic(token: str | None) -> dict: ) } - clones, c_err = _gh("traffic/clones?per=week", token) - views, v_err = _gh("traffic/views?per=week", token) + # per=day keeps a reconstructable series. per=week would drop daily + # resolution and make period totals over >14 days impossible. + clones, c_err = _gh("traffic/clones?per=day", token) + views, v_err = _gh("traffic/views?per=day", token) + clones_d = clones if isinstance(clones, dict) else None + views_d = views if isinstance(views, dict) else None result: dict = { - "clones_14d": clones.get("count") if clones else None, - "unique_cloners_14d": clones.get("uniques") if clones else None, - "views_14d": views.get("count") if views else None, - "unique_visitors_14d": views.get("uniques") if views else None, + "clones_14d": clones_d.get("count") if clones_d else None, + "unique_cloners_14d": clones_d.get("uniques") if clones_d else None, + "views_14d": views_d.get("count") if views_d else None, + "unique_visitors_14d": views_d.get("uniques") if views_d else None, + "clones_daily": _traffic_daily_series(clones_d, "clones"), + "views_daily": _traffic_daily_series(views_d, "views"), } if c_err: result["clones_error"] = c_err @@ -238,6 +268,11 @@ def collection_errors(metrics: dict, *, require_traffic: bool = False) -> list[s errors.append(f"github_traffic: {traffic['note']}") elif traffic.get("clones_14d") is None or traffic.get("views_14d") is None: errors.append("github_traffic: clones/views missing from snapshot") + elif not traffic.get("clones_daily") or not traffic.get("views_daily"): + errors.append( + "github_traffic: daily series missing " + "(need per=day clones/views to reconstruct period totals)" + ) pypi = metrics.get("pypi") or {} if pypi.get("error"): errors.append(f"pypi: {pypi['error']}") @@ -247,6 +282,83 @@ def collection_errors(metrics: dict, *, require_traffic: bool = False) -> list[s return errors +# --------------------------------------------------------------------------- +# Period reconstruction (GitHub traffic is a rolling 14-day window) +# --------------------------------------------------------------------------- + + +def _snapshot_day(snapshot: dict) -> str: + return str(snapshot.get("snapshot_at") or "")[:10] + + +def _days_in_range(start: str, end: str) -> list[str]: + try: + s = date.fromisoformat(start) + e = date.fromisoformat(end) + except ValueError: + return [] + if e < s: + return [] + days: list[str] = [] + d = s + while d <= e: + days.append(d.isoformat()) + d += timedelta(days=1) + return days + + +def load_snapshots(metrics_dir: Path) -> list[dict]: + """Load ``adoption-YYYY-MM-DD.json`` files; skip unreadable ones.""" + snapshots: list[dict] = [] + if not metrics_dir.is_dir(): + return snapshots + for path in sorted(metrics_dir.glob("adoption-*.json")): + try: + data = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError): + continue + if isinstance(data, dict): + snapshots.append(data) + return snapshots + + +def reconstruct_period_counts( + snapshots: list[dict], + start_date: str, + end_date: str, + series_field: str, +) -> dict: + """Union daily clone/view *counts* across snapshots for ``[start, end]``. + + Same-day rows take the max count (later snapshots complete "today"). + Unique counts are not reconstructed — they are not additive across days. + """ + by_day: dict[str, int] = {} + for snap in snapshots: + traffic = snap.get("github_traffic") or {} + for row in traffic.get(series_field) or []: + if not isinstance(row, dict): + continue + day = str(row.get("date") or "")[:10] + if len(day) != 10 or not (start_date <= day <= end_date): + continue + try: + n = int(row["count"]) + except (KeyError, TypeError, ValueError): + continue + prev = by_day.get(day) + by_day[day] = n if prev is None else max(prev, n) + + span_days = _days_in_range(start_date, end_date) + missing = [d for d in span_days if d not in by_day] + return { + "total": sum(by_day.values()) if by_day else None, + "days_covered": len(by_day), + "days_span": len(span_days), + "missing_days": missing, + } + + # --------------------------------------------------------------------------- # Formatters # --------------------------------------------------------------------------- @@ -303,6 +415,11 @@ def format_markdown(m: dict) -> str: ["Unique Cloners (14d)", traffic.get("unique_cloners_14d", "N/A")], ["Views (14d)", traffic.get("views_14d", "N/A")], ["Unique Visitors (14d)", traffic.get("unique_visitors_14d", "N/A")], + [ + "Daily series stored", + f"{len(traffic.get('clones_daily') or [])} clone-days / " + f"{len(traffic.get('views_daily') or [])} view-days", + ], ], ), "", @@ -344,11 +461,104 @@ def _delta(old, new) -> str: return "N/A" -def format_report(baseline: dict, current: dict) -> str: +def _format_traffic_report( + baseline: dict, + current: dict, + snapshots: list[dict] | None, +) -> list[str]: + """Traffic section: reconstruct period counts; never subtract 14d windows.""" + start = _snapshot_day(baseline) + end = _snapshot_day(current) + all_snaps = list(snapshots or []) + all_snaps.extend([baseline, current]) + + clones = reconstruct_period_counts(all_snaps, start, end, "clones_daily") + views = reconstruct_period_counts(all_snaps, start, end, "views_daily") + bt = baseline.get("github_traffic") or {} + ct = current.get("github_traffic") or {} + + lines = [ + "## GitHub Traffic", + "", + "GitHub's Traffic API retains **14 days**. The `*_14d` fields are rolling", + "windows, not cumulative counters — they are **not subtracted** (a later", + "window minus an earlier one is not period traffic).", + "Period clone/view totals are reconstructed by unioning the daily series", + "stored in each snapshot.", + "", + ] + + if clones["days_covered"] == 0 and views["days_covered"] == 0: + lines += [ + "_Period totals unavailable — snapshots do not yet contain a daily " + "traffic series. After the PAT secret is set, each daily run stores " + "the 14-day breakdown so D4 can reconstruct the grant window._", + "", + ] + else: + lines += [ + f"### Period totals ({start} → {end})", + "", + _md_table( + ["Metric", "Total", "Coverage"], + [ + [ + "Clones", + clones["total"], + f"{clones['days_covered']}/{clones['days_span']} days", + ], + [ + "Views", + views["total"], + f"{views['days_covered']}/{views['days_span']} days", + ], + ], + ), + "", + ] + missing = sorted(set(clones["missing_days"] + views["missing_days"])) + if missing: + shown = missing[:12] + extra = f" (+{len(missing) - 12} more)" if len(missing) > 12 else "" + lines += [f"_Gaps: {', '.join(shown)}{extra}_", ""] + lines += [ + "Unique cloners/visitors are **not additive** across days and cannot " + "be reconstructed as a period total.", + "", + ] + + lines += [ + "### Rolling 14-day windows (point-in-time, not a period delta)", + "", + _md_table( + ["Metric", "Baseline (14d)", "Current (14d)"], + [ + ["Clones", bt.get("clones_14d", "N/A"), ct.get("clones_14d", "N/A")], + [ + "Unique cloners", + bt.get("unique_cloners_14d", "N/A"), + ct.get("unique_cloners_14d", "N/A"), + ], + ["Views", bt.get("views_14d", "N/A"), ct.get("views_14d", "N/A")], + [ + "Unique visitors", + bt.get("unique_visitors_14d", "N/A"), + ct.get("unique_visitors_14d", "N/A"), + ], + ], + ), + "", + ] + return lines + + +def format_report( + baseline: dict, + current: dict, + snapshots: list[dict] | None = None, +) -> str: bg = baseline.get("github", {}) cg = current.get("github", {}) - bt = baseline.get("github_traffic", {}) - ct = current.get("github_traffic", {}) bp = baseline.get("pypi", {}) cp = current.get("pypi", {}) @@ -369,42 +579,14 @@ def format_report(baseline: dict, current: dict) -> str: ], ), "", - "## GitHub Traffic (14-day window)", - "", - _md_table( - ["Metric", "Baseline", "Current", "Delta"], - [ - [ - "Clones (14d)", - bt.get("clones_14d", "N/A"), - ct.get("clones_14d", "N/A"), - _delta(bt.get("clones_14d"), ct.get("clones_14d")), - ], - [ - "Unique Cloners (14d)", - bt.get("unique_cloners_14d", "N/A"), - ct.get("unique_cloners_14d", "N/A"), - _delta(bt.get("unique_cloners_14d"), ct.get("unique_cloners_14d")), - ], - [ - "Views (14d)", - bt.get("views_14d", "N/A"), - ct.get("views_14d", "N/A"), - _delta(bt.get("views_14d"), ct.get("views_14d")), - ], - [ - "Unique Visitors (14d)", - bt.get("unique_visitors_14d", "N/A"), - ct.get("unique_visitors_14d", "N/A"), - _delta(bt.get("unique_visitors_14d"), ct.get("unique_visitors_14d")), - ], - ], - ), - "", + *_format_traffic_report(baseline, current, snapshots), "## PyPI Downloads", "", + "_PyPI day/week/month figures are also rolling windows; deltas below " + "compare two windows, not downloads accumulated over the grant period._", + "", _md_table( - ["Period", "Baseline", "Current", "Delta"], + ["Period", "Baseline", "Current", "Window delta"], [ ["Last day", bp.get("downloads_last_day", "N/A"), cp.get("downloads_last_day", "N/A"), _delta(bp.get("downloads_last_day"), cp.get("downloads_last_day"))], @@ -440,6 +622,12 @@ def main() -> None: metavar="BASELINE.json", help="Report mode: compute deltas vs baseline snapshot", ) + parser.add_argument( + "--metrics-dir", + default=None, + help="Directory of adoption-*.json snapshots used to reconstruct " + "period traffic (default: directory of --report)", + ) parser.add_argument( "--strict", action="store_true", @@ -463,7 +651,9 @@ def main() -> None: sys.exit(1) baseline = json.loads(baseline_path.read_text()) current = collect_metrics(token) - print(format_report(baseline, current)) + metrics_dir = Path(args.metrics_dir) if args.metrics_dir else baseline_path.parent + snapshots = load_snapshots(metrics_dir) + print(format_report(baseline, current, snapshots)) errors = collection_errors(current, require_traffic=strict) if errors: print("Collector errors:", file=sys.stderr) diff --git a/tests/units/test_adoption_snapshot.py b/tests/units/test_adoption_snapshot.py new file mode 100644 index 00000000..2cf777af --- /dev/null +++ b/tests/units/test_adoption_snapshot.py @@ -0,0 +1,158 @@ +"""Unit tests for scripts/adoption_snapshot.py traffic reconstruction. + +No network. The grant D4 report must not subtract two rolling 14-day windows. +""" +from __future__ import annotations + +import importlib.util +from datetime import date, timedelta +from pathlib import Path +from typing import Any, Dict, List + +_SCRIPT = Path(__file__).resolve().parents[2] / "scripts" / "adoption_snapshot.py" + + +def _mod() -> Any: + spec = importlib.util.spec_from_file_location("adoption_snapshot", _SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _daily(start: str, counts: List[int]) -> List[Dict[str, Any]]: + d = date.fromisoformat(start) + rows = [] + for n in counts: + rows.append({"date": d.isoformat(), "count": n, "uniques": 1}) + d += timedelta(days=1) + return rows + + +def _snap(day: str, clones: List[Dict[str, Any]], clones_14d: int) -> Dict[str, Any]: + return { + "snapshot_at": day + "T06:00:00+00:00", + "github": {"stars": 1000}, + "github_traffic": { + "clones_14d": clones_14d, + "unique_cloners_14d": 10, + "views_14d": clones_14d, + "unique_visitors_14d": 10, + "clones_daily": clones, + "views_daily": clones, + }, + "pypi": {}, + } + + +def test_reconstruct_unions_days_without_double_counting() -> None: + snap = _mod() + # Two adjacent 14-day windows. Naive subtraction of 14d totals is 80-100=-20. + a = _snap("2026-09-02", _daily("2026-08-20", [10] * 14), clones_14d=140) + b = _snap("2026-09-16", _daily("2026-09-03", [5] * 14), clones_14d=70) + # Period 09-02 → 09-16 = 09-02 (10) + 14 days of 5 = 80 + result = snap.reconstruct_period_counts( + [a, b], "2026-09-02", "2026-09-16", "clones_daily" + ) + assert result["total"] == 10 + 5 * 14 + assert result["days_covered"] == 15 + assert result["days_span"] == 15 + assert result["missing_days"] == [] + naive = 70 - 140 + assert result["total"] != naive + assert result["total"] > 0 + + +def test_reconstruct_takes_max_on_overlapping_same_day() -> None: + snap = _mod() + early = _snap( + "2026-09-02", + [{"date": "2026-09-02", "count": 3, "uniques": 1}], + clones_14d=3, + ) + later = _snap( + "2026-09-02", + [{"date": "2026-09-02", "count": 8, "uniques": 2}], + clones_14d=8, + ) + result = snap.reconstruct_period_counts( + [early, later], "2026-09-02", "2026-09-02", "clones_daily" + ) + assert result["total"] == 8 + + +def test_reconstruct_lists_gaps() -> None: + snap = _mod() + a = _snap( + "2026-09-02", + [{"date": "2026-09-02", "count": 1, "uniques": 1}], + clones_14d=1, + ) + b = _snap( + "2026-09-05", + [{"date": "2026-09-05", "count": 1, "uniques": 1}], + clones_14d=1, + ) + result = snap.reconstruct_period_counts( + [a, b], "2026-09-02", "2026-09-05", "clones_daily" + ) + assert result["total"] == 2 + assert result["missing_days"] == ["2026-09-03", "2026-09-04"] + + +def test_format_report_does_not_subtract_rolling_windows() -> None: + snap = _mod() + baseline = _snap("2026-09-02", _daily("2026-08-20", [10] * 14), clones_14d=140) + current = _snap("2026-09-16", _daily("2026-09-03", [5] * 14), clones_14d=70) + report = snap.format_report(baseline, current, snapshots=[baseline, current]) + assert "-20" not in report + assert "-70" not in report + # Naive 14d delta would be 70-140. Reconstructed period clones = 80. + assert "| Clones | 80 |" in report + assert "not subtracted" in report + assert "Period totals" in report + # 14d table has no Delta column + assert "Rolling 14-day windows (point-in-time, not a period delta)" in report + traffic_header = "### Rolling 14-day windows" + idx = report.index(traffic_header) + rolling = report[idx : idx + 600] + assert "Delta" not in rolling + + +def test_format_report_without_daily_series_does_not_invent_delta() -> None: + snap = _mod() + baseline = { + "snapshot_at": "2026-09-02T00:00:00+00:00", + "github": {"stars": 1000, "forks": 1, "watchers": 1}, + "github_traffic": {"clones_14d": 100, "views_14d": 200}, + "pypi": {}, + } + current = { + "snapshot_at": "2026-11-01T00:00:00+00:00", + "github": {"stars": 1100, "forks": 2, "watchers": 1}, + "github_traffic": {"clones_14d": 40, "views_14d": 50}, + "pypi": {}, + } + report = snap.format_report(baseline, current) + assert "Period totals unavailable" in report + assert "-60" not in report # 40-100 + assert "-150" not in report # 50-200 + # Stars *are* cumulative; that delta is honest. + assert "+100" in report + + +def test_traffic_daily_series_normalizes_github_payload() -> None: + snap = _mod() + payload = { + "count": 5, + "uniques": 3, + "clones": [ + {"timestamp": "2026-09-01T00:00:00Z", "count": 2, "uniques": 1}, + {"timestamp": "2026-09-02T00:00:00Z", "count": 3, "uniques": 2}, + ], + } + rows = snap._traffic_daily_series(payload, "clones") + assert rows == [ + {"date": "2026-09-01", "count": 2, "uniques": 1}, + {"date": "2026-09-02", "count": 3, "uniques": 2}, + ] From 3a2a37d1afdcc34dcf8f6c77b3848982468ae3e1 Mon Sep 17 00:00:00 2001 From: Bob Date: Wed, 2 Sep 2026 02:17:27 +0000 Subject: [PATCH 5/6] fix(metrics): exclude partial current UTC day from period totals Report mode included today's still-accumulating GitHub Traffic row in D4 period totals, understating clones/views on every 06:00 UTC run. Drop that day from the reconstructed range and label it excluded. --- scripts/adoption_snapshot.py | 49 ++++++++++++++++++++++++--- tests/units/test_adoption_snapshot.py | 47 +++++++++++++++++++++++-- 2 files changed, 89 insertions(+), 7 deletions(-) diff --git a/scripts/adoption_snapshot.py b/scripts/adoption_snapshot.py index 6649ebf7..668a47a0 100644 --- a/scripts/adoption_snapshot.py +++ b/scripts/adoption_snapshot.py @@ -18,6 +18,8 @@ (delta is current − baseline). GitHub traffic is a rolling 14-day window — those totals are NEVER subtracted. Period clone/view counts are reconstructed by unioning the daily series stored in each snapshot. + The current UTC day is excluded from period totals — its Traffic row is + still accumulating (the 06:00 UTC scheduled run would otherwise undercount). GitHub Traffic cannot be read with GitHub Actions' default GITHUB_TOKEN (it 403s even with contents: write). Use a PAT: @@ -322,6 +324,22 @@ def load_snapshots(metrics_dir: Path) -> list[dict]: return snapshots +def complete_period_end(end: str, *, now: date | None = None) -> tuple[str, str | None]: + """Inclusive period end with the current UTC day dropped if still open. + + GitHub Traffic rows for *today* are still accumulating. A 06:00 UTC + scheduled report that includes today understates the period. Historical + ends (``end < today``) are unchanged. + + Returns ``(closed_end, excluded_day_or_None)``. + """ + today = (now or datetime.now(timezone.utc).date()).isoformat() + if end == today: + closed = (date.fromisoformat(today) - timedelta(days=1)).isoformat() + return closed, today + return end, None + + def reconstruct_period_counts( snapshots: list[dict], start_date: str, @@ -332,6 +350,7 @@ def reconstruct_period_counts( Same-day rows take the max count (later snapshots complete "today"). Unique counts are not reconstructed — they are not additive across days. + Callers that want a closed period should pass ``complete_period_end()``. """ by_day: dict[str, int] = {} for snap in snapshots: @@ -465,10 +484,13 @@ def _format_traffic_report( baseline: dict, current: dict, snapshots: list[dict] | None, + *, + now: date | None = None, ) -> list[str]: """Traffic section: reconstruct period counts; never subtract 14d windows.""" start = _snapshot_day(baseline) - end = _snapshot_day(current) + raw_end = _snapshot_day(current) + end, partial_day = complete_period_end(raw_end, now=now) all_snaps = list(snapshots or []) all_snaps.extend([baseline, current]) @@ -484,11 +506,22 @@ def _format_traffic_report( "windows, not cumulative counters — they are **not subtracted** (a later", "window minus an earlier one is not period traffic).", "Period clone/view totals are reconstructed by unioning the daily series", - "stored in each snapshot.", + "stored in each snapshot. The current UTC day is excluded — its row is", + "still accumulating.", "", ] - if clones["days_covered"] == 0 and views["days_covered"] == 0: + empty_complete_window = bool(partial_day) and end < start + no_series = clones["days_covered"] == 0 and views["days_covered"] == 0 + + if empty_complete_window: + lines += [ + f"_No complete UTC days in {start} → {raw_end} yet. " + f"{partial_day} is still accumulating and is excluded from " + f"period totals._", + "", + ] + elif no_series: lines += [ "_Period totals unavailable — snapshots do not yet contain a daily " "traffic series. After the PAT secret is set, each daily run stores " @@ -516,6 +549,12 @@ def _format_traffic_report( ), "", ] + if partial_day: + lines += [ + f"_{partial_day} excluded — GitHub Traffic for the current " + f"UTC day is still accumulating._", + "", + ] missing = sorted(set(clones["missing_days"] + views["missing_days"])) if missing: shown = missing[:12] @@ -556,6 +595,8 @@ def format_report( baseline: dict, current: dict, snapshots: list[dict] | None = None, + *, + now: date | None = None, ) -> str: bg = baseline.get("github", {}) cg = current.get("github", {}) @@ -579,7 +620,7 @@ def format_report( ], ), "", - *_format_traffic_report(baseline, current, snapshots), + *_format_traffic_report(baseline, current, snapshots, now=now), "## PyPI Downloads", "", "_PyPI day/week/month figures are also rolling windows; deltas below " diff --git a/tests/units/test_adoption_snapshot.py b/tests/units/test_adoption_snapshot.py index 2cf777af..5c02cb22 100644 --- a/tests/units/test_adoption_snapshot.py +++ b/tests/units/test_adoption_snapshot.py @@ -104,13 +104,16 @@ def test_format_report_does_not_subtract_rolling_windows() -> None: snap = _mod() baseline = _snap("2026-09-02", _daily("2026-08-20", [10] * 14), clones_14d=140) current = _snap("2026-09-16", _daily("2026-09-03", [5] * 14), clones_14d=70) - report = snap.format_report(baseline, current, snapshots=[baseline, current]) + # Freeze now to the day *after* current so 2026-09-16 is a complete day. + report = snap.format_report( + baseline, current, snapshots=[baseline, current], now=date(2026, 9, 17) + ) assert "-20" not in report assert "-70" not in report # Naive 14d delta would be 70-140. Reconstructed period clones = 80. assert "| Clones | 80 |" in report assert "not subtracted" in report - assert "Period totals" in report + assert "Period totals (2026-09-02 → 2026-09-16)" in report # 14d table has no Delta column assert "Rolling 14-day windows (point-in-time, not a period delta)" in report traffic_header = "### Rolling 14-day windows" @@ -119,6 +122,44 @@ def test_format_report_does_not_subtract_rolling_windows() -> None: assert "Delta" not in rolling +def test_complete_period_end_drops_today() -> None: + snap = _mod() + closed, excluded = snap.complete_period_end("2026-09-16", now=date(2026, 9, 16)) + assert closed == "2026-09-15" + assert excluded == "2026-09-16" + closed, excluded = snap.complete_period_end("2026-09-16", now=date(2026, 9, 17)) + assert closed == "2026-09-16" + assert excluded is None + + +def test_format_report_excludes_partial_current_utc_day() -> None: + snap = _mod() + baseline = _snap("2026-09-02", _daily("2026-08-20", [10] * 14), clones_14d=140) + current = _snap("2026-09-16", _daily("2026-09-03", [5] * 14), clones_14d=70) + report = snap.format_report( + baseline, current, snapshots=[baseline, current], now=date(2026, 9, 16) + ) + # 09-02 (10) + 09-03..09-15 (13*5) = 75; 09-16's partial 5 is dropped. + assert "| Clones | 75 |" in report + assert "Period totals (2026-09-02 → 2026-09-15)" in report + assert "2026-09-16 excluded" in report + assert "| Clones | 80 |" not in report + + +def test_format_report_baseline_today_has_no_complete_days() -> None: + snap = _mod() + baseline = _snap( + "2026-09-02", + [{"date": "2026-09-02", "count": 3, "uniques": 1}], + clones_14d=3, + ) + report = snap.format_report( + baseline, baseline, snapshots=[baseline], now=date(2026, 9, 2) + ) + assert "No complete UTC days" in report + assert "Period totals (" not in report + + def test_format_report_without_daily_series_does_not_invent_delta() -> None: snap = _mod() baseline = { @@ -133,7 +174,7 @@ def test_format_report_without_daily_series_does_not_invent_delta() -> None: "github_traffic": {"clones_14d": 40, "views_14d": 50}, "pypi": {}, } - report = snap.format_report(baseline, current) + report = snap.format_report(baseline, current, now=date(2026, 11, 2)) assert "Period totals unavailable" in report assert "-60" not in report # 40-100 assert "-150" not in report # 50-200 From b8e3a63fa8edbeaa4e16f7de4d517676a48f9057 Mon Sep 17 00:00:00 2001 From: Bob Date: Wed, 2 Sep 2026 02:32:57 +0000 Subject: [PATCH 6/6] fix(metrics): pin collection clock so midnight cannot include a partial day complete_period_end() re-read datetime.now() after collect_metrics() had already stamped snapshot_at. A report that started before UTC midnight and finished after kept the earlier snapshot day, skipped exclusion, and published that day's still-accumulating Traffic row as a complete total. Use one collected_at for snapshot_at, the filename, and period-end exclusion. When now is omitted, key off the snapshot day rather than a fresh clock. --- scripts/adoption_snapshot.py | 42 ++++++++++++++++++--------- tests/units/test_adoption_snapshot.py | 32 ++++++++++++++++++++ 2 files changed, 60 insertions(+), 14 deletions(-) diff --git a/scripts/adoption_snapshot.py b/scripts/adoption_snapshot.py index 668a47a0..99262ad3 100644 --- a/scripts/adoption_snapshot.py +++ b/scripts/adoption_snapshot.py @@ -238,9 +238,10 @@ def _collect_pypi_versions() -> list[dict]: # --------------------------------------------------------------------------- -def collect_metrics(token: str | None = None) -> dict: +def collect_metrics(token: str | None = None, *, now: datetime | None = None) -> dict: + collected_at = now or datetime.now(timezone.utc) return { - "snapshot_at": datetime.now(timezone.utc).isoformat(), + "snapshot_at": collected_at.isoformat(), "repo": REPO, "package": PACKAGE, "github": _collect_github(token), @@ -325,18 +326,28 @@ def load_snapshots(metrics_dir: Path) -> list[dict]: def complete_period_end(end: str, *, now: date | None = None) -> tuple[str, str | None]: - """Inclusive period end with the current UTC day dropped if still open. + """Inclusive period end with the collection-day dropped if still open. - GitHub Traffic rows for *today* are still accumulating. A 06:00 UTC - scheduled report that includes today understates the period. Historical - ends (``end < today``) are unchanged. + GitHub Traffic rows for the snapshot's UTC day are still accumulating + at collection time. Default ``today`` to that snapshot day — do **not** + re-read the clock. Collection that starts before UTC midnight and + finishes after would otherwise see ``end != datetime.now().date()`` and + publish the still-partial collection-day row as a complete-period total. + + Pass ``now`` to treat a historical snapshot as complete (``end < now``). + Historical ends are unchanged. Returns ``(closed_end, excluded_day_or_None)``. """ - today = (now or datetime.now(timezone.utc).date()).isoformat() - if end == today: - closed = (date.fromisoformat(today) - timedelta(days=1)).isoformat() - return closed, today + try: + end_day = date.fromisoformat(end) + except ValueError: + return end, None + today = now if now is not None else end_day + today_s = today.isoformat() + if end == today_s: + closed = (today - timedelta(days=1)).isoformat() + return closed, today_s return end, None @@ -684,6 +695,9 @@ def main() -> None: token = _token_from_env() in_actions = os.environ.get("GITHUB_ACTIONS") == "true" strict = (args.strict or in_actions) and not args.allow_partial + # One clock for snapshot_at, filename, and period-end exclusion so a + # midnight boundary during network I/O cannot desynchronize them. + collected_at = datetime.now(timezone.utc) if args.report: baseline_path = Path(args.report) @@ -691,10 +705,10 @@ def main() -> None: print(f"ERROR: Baseline file not found: {baseline_path}", file=sys.stderr) sys.exit(1) baseline = json.loads(baseline_path.read_text()) - current = collect_metrics(token) + current = collect_metrics(token, now=collected_at) metrics_dir = Path(args.metrics_dir) if args.metrics_dir else baseline_path.parent snapshots = load_snapshots(metrics_dir) - print(format_report(baseline, current, snapshots)) + print(format_report(baseline, current, snapshots, now=collected_at.date())) errors = collection_errors(current, require_traffic=strict) if errors: print("Collector errors:", file=sys.stderr) @@ -704,13 +718,13 @@ def main() -> None: sys.exit(1) return - metrics = collect_metrics(token) + metrics = collect_metrics(token, now=collected_at) errors = collection_errors(metrics, require_traffic=strict) out_dir = Path(args.out) out_dir.mkdir(parents=True, exist_ok=True) - date_str = datetime.now(timezone.utc).strftime("%Y-%m-%d") + date_str = collected_at.strftime("%Y-%m-%d") json_path = out_dir / f"adoption-{date_str}.json" md_path = out_dir / f"adoption-{date_str}.md" diff --git a/tests/units/test_adoption_snapshot.py b/tests/units/test_adoption_snapshot.py index 5c02cb22..67455593 100644 --- a/tests/units/test_adoption_snapshot.py +++ b/tests/units/test_adoption_snapshot.py @@ -132,6 +132,21 @@ def test_complete_period_end_drops_today() -> None: assert excluded is None +def test_complete_period_end_without_now_uses_snapshot_day() -> None: + """Production default must not re-read datetime.now(). + + If collection starts on 09-16 and midnight passes during network I/O, + a fresh clock would skip exclusion and publish 09-16's partial row. + """ + snap = _mod() + closed, excluded = snap.complete_period_end("2026-09-16") + assert closed == "2026-09-15" + assert excluded == "2026-09-16" + closed, excluded = snap.complete_period_end("not-a-date") + assert closed == "not-a-date" + assert excluded is None + + def test_format_report_excludes_partial_current_utc_day() -> None: snap = _mod() baseline = _snap("2026-09-02", _daily("2026-08-20", [10] * 14), clones_14d=140) @@ -146,6 +161,23 @@ def test_format_report_excludes_partial_current_utc_day() -> None: assert "| Clones | 80 |" not in report +def test_format_report_midnight_crossing_excludes_collection_day() -> None: + """Wall clock after midnight must not resurrect the partial collection day. + + collect_metrics stamps snapshot_at first, then does network I/O. + format_report() with now=None (the production default) must key off + snapshot_at, not datetime.now(). + """ + snap = _mod() + baseline = _snap("2026-09-02", _daily("2026-08-20", [10] * 14), clones_14d=140) + current = _snap("2026-09-16", _daily("2026-09-03", [5] * 14), clones_14d=70) + report = snap.format_report(baseline, current, snapshots=[baseline, current]) + assert "| Clones | 75 |" in report + assert "Period totals (2026-09-02 → 2026-09-15)" in report + assert "2026-09-16 excluded" in report + assert "| Clones | 80 |" not in report + + def test_format_report_baseline_today_has_no_complete_days() -> None: snap = _mod() baseline = _snap(