From cf25635a75982d6b443c0cef2542bada9097a986 Mon Sep 17 00:00:00 2001 From: Max Isbey <224885523+maxisbey@users.noreply.github.com> Date: Sun, 16 Aug 2026 12:40:33 +0000 Subject: [PATCH 1/5] ci: add a weekly newest-dependencies canary Users installing `mcp` get the newest release of every dependency the day it ships, but PR CI only tests uv.lock and the floors, so an upstream release that breaks the SDK is currently noticed by users first (median 11 days across past incidents). Running "highest" on every PR was tried and removed (#1869) because a half-uploaded release turns unrelated PRs red; the weekly lock-bump PR that replaced it (#1874) never got merged. This adds a scheduled workflow instead. Every Monday it re-resolves the runtime closure of mcp[cli,rich] to the newest versions the specifiers allow (test tooling stays at uv.lock, releases younger than a day are ignored), runs the suite on ubuntu 3.10/3.14 and windows 3.14, and keeps a single tracking issue in sync: opened and assigned when newest-allowed breaks, refreshed while it stays broken, closed once it passes again. Each cell re-runs failures serially and once more with deprecation warnings demoted, so the issue says whether users are actually broken or a dependency merely deprecated something. The report lists what changed since the last green run, what is held below latest by someone else's cap, and the exact command to reproduce the resolution. It never runs on pull requests and never opens a PR adding a ceiling; the issue carries the runbook and a maintainer decides. No-Verification-Needed: CI-only change (workflow + scripts/ci); scripts exercised locally, workflow verified by a branch run --- .github/workflows/dependency-canary.yml | 316 ++++++++++++++++++++++++ .gitignore | 5 + scripts/ci/canary_cell.sh | 113 +++++++++ scripts/ci/canary_lock_diff.py | 112 +++++++++ scripts/ci/canary_report.sh | 247 ++++++++++++++++++ 5 files changed, 793 insertions(+) create mode 100644 .github/workflows/dependency-canary.yml create mode 100755 scripts/ci/canary_cell.sh create mode 100644 scripts/ci/canary_lock_diff.py create mode 100755 scripts/ci/canary_report.sh diff --git a/.github/workflows/dependency-canary.yml b/.github/workflows/dependency-canary.yml new file mode 100644 index 0000000000..0eeefab3c4 --- /dev/null +++ b/.github/workflows/dependency-canary.yml @@ -0,0 +1,316 @@ +name: Dependency canary + +# Weekly: re-resolve the runtime dependencies of `mcp[cli,rich]` to the newest +# versions our (floors-only) specifiers allow, ignoring uv.lock, run the test +# suite against them, and keep ONE tracking issue in sync with the result — +# opened (and assigned) when newest-allowed breaks, refreshed weekly while it +# stays broken, closed automatically once it passes again. +# +# Why this exists: users who `pip install mcp` get the newest release of every +# dependency the day it ships, while PR CI only ever sees uv.lock (`locked`) and +# the floors (`lowest-direct`). This is deliberately NOT part of PR CI, so an +# upstream release can never turn an unrelated PR red; the price is up to a +# week of latency, which the incident history says is fine. +# +# What it does not do, on purpose: +# - open a PR adding a ceiling. A cap only helps once released, resolvers +# route around retroactive caps by picking an older uncapped mcp, and the +# bot cannot tell which package (or interaction) is at fault. The issue's +# "What to do" section is the runbook; a human decides. +# - float test tooling (pytest, ruff, pyright, coverage, ...). Those stay at +# uv.lock so a pytest major cannot masquerade as an SDK break; Dependabot +# owns moving them. +# - test pre-releases on the schedule. `workflow_dispatch` with +# `prerelease: true` does that on demand and never files an issue. +# - bisect. The issue lists what changed since the last green run (usually +# one to three packages) and the one-line command to pin a suspect back. +# +# Known blind spots: Python 3.11-3.13 and macOS are not run; runtime deps that a +# *dev* dependency caps (e.g. logfire pins opentelemetry-sdk, which pins +# opentelemetry-api) cannot reach their newest release here — the issue's "Not +# tested at their newest release" section lists them each run. Dependency +# groups outside `default-groups` (translate, codegen) are stripped before +# resolving so their caps (anthropic: pydantic<3) do not apply. +# +# Notifications: assignees get the issue traffic. GitHub additionally e-mails +# scheduled-run failures only to whoever last edited the `cron:` line below. + +on: + schedule: + - cron: "23 5 * * 1" # Mondays 05:23 UTC + workflow_dispatch: + inputs: + prerelease: + description: "Also consider pre-releases (investigative run; never files an issue)" + type: boolean + default: false + file-issue: + description: "Create/update/close the tracking issue exactly as a scheduled run would" + type: boolean + default: false + # TEMPORARY while this workflow is under review: exercise it on the PR branch. + # Report-only (push runs never touch issues). Remove before merging. + push: + branches: ["ci/dependency-canary"] + +permissions: {} + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + COLUMNS: 150 + UV_VERSION: "0.11.33" + # Releases younger than this are invisible to the run: skips half-uploaded + # releases (the ruff 0.14.12 incident that got the per-PR "highest" leg + # removed in #1869) and same-day yanks. A weekly job loses nothing by it. + CANARY_LAG: "24 hours" + CANARY_LABEL: dependency-canary + CANARY_ASSIGNEES: "maxisbey,Kludex" + +jobs: + resolve: + # Don't run the schedule on forks. + if: github.repository == 'modelcontextprotocol/python-sdk' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + actions: read # `gh run list`: find the last green scheduled run to diff against + outputs: + cutoff: ${{ steps.cutoffs.outputs.cutoff }} + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + # setup-uv's manifest fetch is a single request with a hard 5s timeout + # (astral-sh/setup-uv#869); retry once. Drop when upstream adds a retry. + - name: Install uv + id: setup-uv + continue-on-error: true + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + with: + enable-cache: false + version: ${{ env.UV_VERSION }} + + - name: Install uv (retry) + if: steps.setup-uv.outcome == 'failure' + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + with: + enable-cache: false + version: ${{ env.UV_VERSION }} + + - name: Compute cutoffs + id: cutoffs + env: + GH_TOKEN: ${{ github.token }} + run: | + mkdir -p canary-resolve + cutoff=$(date -u -d "-$CANARY_LAG" +%Y-%m-%dT%H:%M:%SZ) + echo "cutoff=$cutoff" >>"$GITHUB_OUTPUT" + echo "$cutoff" >canary-resolve/cutoff.txt + uv self version >canary-resolve/uv-version.txt + # Baseline = what the last green *scheduled* run saw (its start time minus the same lag). + last_green=$(gh run list --repo "$GITHUB_REPOSITORY" --workflow dependency-canary.yml \ + --branch main --event schedule --status success --limit 1 --json startedAt --jq '.[0].startedAt // empty') + if [ -n "$last_green" ]; then + date -u -d "$last_green -$CANARY_LAG" +%Y-%m-%dT%H:%M:%SZ >canary-resolve/baseline.txt + else + : >canary-resolve/baseline.txt + fi + echo "cutoff=$cutoff baseline=$(cat canary-resolve/baseline.txt)" + + - name: Work out what to float + run: | + # The runtime closure of mcp[cli,rich] as currently locked: exactly the set + # `pip install "mcp[cli,rich]"` pulls in. New transitive deps that a newer + # release introduces have no lock entry and so resolve to newest anyway. + uv export --frozen --no-default-groups --all-extras --no-emit-workspace \ + --no-hashes --no-header --no-annotate | sed -E 's/[=; @].*//' | sort -u >canary-resolve/closure.txt + echo "Floating $(wc -l canary-resolve/strip.sh + import re, tomllib + project = tomllib.load(open("pyproject.toml", "rb")) + keep = set(project.get("tool", {}).get("uv", {}).get("default-groups", [])) + for group, deps in project.get("dependency-groups", {}).items(): + names = [re.match(r"[A-Za-z0-9._-]+", d).group(0) for d in deps if isinstance(d, str)] + if group not in keep and names: + print("uv remove --frozen --group", group, *names) + EOF + cat canary-resolve/strip.sh + bash -e canary-resolve/strip.sh + + - name: Resolve newest allowed versions + env: + PRERELEASE: ${{ inputs.prerelease && 'allow' || '' }} + run: | + set -o pipefail + args=(--exclude-newer "$(cat canary-resolve/cutoff.txt)") + if [ -n "$PRERELEASE" ]; then args+=(--prerelease "$PRERELEASE"); fi + while read -r pkg; do args+=(-P "$pkg"); done &1 | tee canary-resolve/baseline.log; then + cp uv.lock canary-resolve/baseline.lock + else + echo "::warning::could not re-resolve the last-green baseline; the report will only diff against uv.lock" + : >canary-resolve/baseline.txt + fi + cp canary-resolve/committed.lock uv.lock + echo "::endgroup::" + fi + + uv lock "${args[@]}" 2>&1 | tee canary-resolve/lock.log + cp uv.lock canary-resolve/uv.lock + + - name: Summarise what moved + run: | + python3 scripts/ci/canary_lock_diff.py canary-resolve/committed.lock uv.lock \ + --old-label "uv.lock" --new-label "this run" --suspects canary-resolve/suspects-vs-lock.txt >canary-resolve/vs-lock.md + if [ -f canary-resolve/baseline.lock ]; then + python3 scripts/ci/canary_lock_diff.py canary-resolve/baseline.lock uv.lock \ + --old-label "last green" --new-label "this run" --suspects canary-resolve/suspects-since-green.txt >canary-resolve/since-green.md + fi + # Direct runtime deps that could not reach their newest release (capped by something else in the resolution). + uv tree --frozen --outdated --depth 1 --package mcp >canary-resolve/tree.txt 2>/dev/null || true + { + echo "| Package | Resolved | Latest |" + echo "| --- | --- | --- |" + sed -nE 's/^[^A-Za-z0-9]*([A-Za-z0-9._-]+)(\[[^]]*\])? v([^ ]+)( \(extra: [^)]*\))? \(latest: v([^)]+)\)$/| \1 | \3 | \5 |/p' canary-resolve/tree.txt + } >canary-resolve/held-back.md + if [ "$(wc -l canary-resolve/held-back.md; fi + { + echo "## Resolution (cutoff $(cat canary-resolve/cutoff.txt))" + echo + if [ -s canary-resolve/since-green.md ]; then echo "### Since last green ($(cat canary-resolve/baseline.txt))"; cat canary-resolve/since-green.md; echo; fi + echo "### vs uv.lock"; cat canary-resolve/vs-lock.md; echo + if [ -s canary-resolve/held-back.md ]; then echo "### Held back below latest"; cat canary-resolve/held-back.md; fi + } >>"$GITHUB_STEP_SUMMARY" + + - name: Upload resolution + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: canary-resolve + path: canary-resolve/ + retention-days: 90 + if-no-files-found: error + + test: + name: test (${{ matrix.cell }}) + needs: resolve + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + include: + # Oldest and newest supported Python bracket the marker forks in the + # lock (deps drop 3.10 first; 3.14 gets wheels last). Windows/newest is + # where a fresh release most often lacks a wheel, and pywin32 lives there. + - { cell: ubuntu-3.10, os: ubuntu-latest, python: "3.10" } + - { + cell: ubuntu-3.14, + os: ubuntu-latest, + python: "3.14", + smoke: "1", + pyright: "1", + } + - { cell: windows-3.14, os: windows-latest, python: "3.14" } + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Install uv + id: setup-uv + continue-on-error: true + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + with: + enable-cache: false + version: ${{ env.UV_VERSION }} + + - name: Install uv (retry) + if: steps.setup-uv.outcome == 'failure' + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + with: + enable-cache: false + version: ${{ env.UV_VERSION }} + + - name: Fetch the resolved lock + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: canary-resolve + path: canary-resolve + + - name: Install and test + shell: bash + env: + CANARY_CELL: ${{ matrix.cell }} + CANARY_PYTHON: ${{ matrix.python }} + CANARY_PYRIGHT: ${{ matrix.pyright }} + # Same switches as PR CI: real stdio/uvicorn subprocess smoke tests on one + # cell, and PEP 597 EncodingWarnings surfaced (as an env var so xdist workers inherit it). + MCP_EXAMPLES_SMOKE: ${{ matrix.smoke }} + PYTHONWARNDEFAULTENCODING: "1" + run: | + cp canary-resolve/uv.lock uv.lock + # The lock was resolved with these groups stripped; keep pyproject consistent so --frozen holds. + bash -e canary-resolve/strip.sh + bash scripts/ci/canary_cell.sh + + - name: Upload cell result + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: canary-cell-${{ matrix.cell }} + path: | + canary-out/status + canary-out/cell.md + retention-days: 30 + if-no-files-found: error + + report: + needs: [resolve, test] + # always(): a red resolve/test job is exactly when this must run. + if: always() && github.repository == 'modelcontextprotocol/python-sdk' && needs.resolve.result != 'cancelled' && needs.resolve.result != 'skipped' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read # checkout, for scripts/ci/canary_report.sh + issues: write # open / refresh / close the tracking issue + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + sparse-checkout: scripts/ci + + - name: Collect results + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: canary-* + path: artifacts + + - name: Report + env: + GH_TOKEN: ${{ github.token }} + CANARY_ARTIFACTS: artifacts + CANARY_RESOLVE_RESULT: ${{ needs.resolve.result }} + CANARY_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + # Only the schedule (or an explicit dispatch asking for it) touches issues, and never a pre-release run. + CANARY_FILE_ISSUES: ${{ !inputs.prerelease && (github.event_name == 'schedule' || inputs.file-issue) && 'true' || 'false' }} + run: bash scripts/ci/canary_report.sh diff --git a/.gitignore b/.gitignore index a63ec932ca..4c4d092fc2 100644 --- a/.gitignore +++ b/.gitignore @@ -183,3 +183,8 @@ results/ # conformance CI local runs conformance-results/ + +# dependency canary local runs (scripts/ci/canary_*.sh) +canary-out/ +canary-resolve/ +canary-body.md diff --git a/scripts/ci/canary_cell.sh b/scripts/ci/canary_cell.sh new file mode 100755 index 0000000000..7f1fff5204 --- /dev/null +++ b/scripts/ci/canary_cell.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +# One dependency-canary test cell: install the resolved lock, run the suite, +# and classify the outcome so the report can tell a hard break from a new +# deprecation warning or a flake. Driven by .github/workflows/dependency-canary.yml. +# +# Inputs (env): CANARY_CELL (label, e.g. "ubuntu-3.14"), CANARY_PYTHON (e.g. "3.14"), +# CANARY_PYRIGHT=1 to append an informational `pyright src/mcp` result. +# Outputs: canary-out/status — one of: pass | flaky | warnings-only | error | install-failed +# canary-out/cell.md — Markdown section for the issue body / job summary +# Exit status: 0 for pass/flaky, 1 otherwise (so the job shows red). + +set -uo pipefail + +cell="${CANARY_CELL:?}" +python="${CANARY_PYTHON:?}" +out=canary-out +mkdir -p "$out" +md="$out/cell.md" +status=pass +export COLUMNS=200 # keeps pytest from truncating the -r summary lines quoted in the report + +pytest_cmd=(uv run --frozen --no-sync pytest -p no:pretty -q --no-header -rfE --color=no -o log_cli=false) +# Demote only the deprecation family: if failures vanish under these, the newest +# versions still work and merely announce a future removal. +demote=(-W default::DeprecationWarning -W default::PendingDeprecationWarning -W default::FutureWarning) + +summary_of() { # print the short-summary FAILED/ERROR lines of a pytest log, capped + grep -E '^(FAILED|ERROR) ' "$1" | head -n 40 + local n + n=$(grep -cE '^(FAILED|ERROR) ' "$1") + if [ "$n" -gt 40 ]; then echo "... and $((n - 40)) more"; fi +} + +details_of() { # collapsed tail of a log + printf '
%s\n\n```text\n' "$2" + tail -n "${3:-60}" "$1" | sed 's/```/` ` `/g' + printf '```\n\n
\n' +} + +{ + echo "#### $cell" + echo +} >"$md" + +if ! uv sync --frozen --all-extras --python "$python" >"$out/sync.log" 2>&1; then + status=install-failed + { + echo "**Install failed** — \`uv sync --frozen --all-extras --python $python\` could not install the resolved set on this platform." + echo + details_of "$out/sync.log" "uv sync output" 40 + } >>"$md" +else + installed=$(uv run --frozen --no-sync python -V 2>/dev/null) + if "${pytest_cmd[@]}" -n auto >"$out/run1.log" 2>&1; then + echo "All tests pass ($installed)." >>"$md" + else + rc=$? + if [ "$rc" -eq 1 ]; then + # Ordinary test failures: re-run just those, serially, to drop xdist/ordering flakes. + rerun=("${pytest_cmd[@]}" --lf --last-failed-no-failures none -p no:xdist) + else + # Collection/usage/internal error (e.g. a warning raised at import time): no + # last-failed set to narrow to, so re-run everything. + rerun=("${pytest_cmd[@]}" -n auto) + fi + if [ "$rc" -eq 1 ] && "${rerun[@]}" >"$out/run2.log" 2>&1; then + status=flaky + { + echo "Passed on a serial re-run ($installed); first attempt had failures (treated as flaky, not reported):" + echo + echo '```text' + summary_of "$out/run1.log" + echo '```' + } >>"$md" + else + [ -f "$out/run2.log" ] || cp "$out/run1.log" "$out/run2.log" + if "${rerun[@]}" "${demote[@]}" >"$out/run3.log" 2>&1; then + status=warnings-only + { + echo "**Deprecation warnings only** ($installed) — the failures below disappear when" + echo "\`DeprecationWarning\`/\`PendingDeprecationWarning\`/\`FutureWarning\` are not errors, so newest versions still work but announce a removal we need to get ahead of." + } >>"$md" + else + status=error + echo "**Hard failures** ($installed) — these persist on a serial re-run and with deprecation warnings demoted:" >>"$md" + fi + { + echo + echo '```text' + summary_of "$out/run2.log" + echo '```' + echo + details_of "$out/run2.log" "pytest output (tail)" 80 + } >>"$md" + fi + fi + + if [ "${CANARY_PYRIGHT:-}" = "1" ]; then + if uv run --frozen --no-sync pyright src/mcp >"$out/pyright.log" 2>&1; then + printf '\npyright on `src/mcp` against these versions: clean (informational).\n' >>"$md" + else + { + echo + details_of "$out/pyright.log" "pyright on src/mcp against these versions: $(tail -n1 "$out/pyright.log") (informational, never filed on its own)" 30 + } >>"$md" + fi + fi +fi + +echo "$status" >"$out/status" +echo "canary cell $cell: $status" +if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then cat "$md" >>"$GITHUB_STEP_SUMMARY"; fi +case "$status" in pass | flaky) exit 0 ;; *) exit 1 ;; esac diff --git a/scripts/ci/canary_lock_diff.py b/scripts/ci/canary_lock_diff.py new file mode 100644 index 0000000000..3ca06b9d19 --- /dev/null +++ b/scripts/ci/canary_lock_diff.py @@ -0,0 +1,112 @@ +"""Diff two uv.lock files for the dependency canary report. + +Usage: python scripts/ci/canary_lock_diff.py OLD NEW [--old-label L] [--new-label L] [--suspects FILE] + +Prints a Markdown table of every package whose locked version(s) differ between +OLD and NEW, tagged by its role relative to the root `mcp` package: a direct +runtime dependency, a transitive runtime dependency (reachable from `mcp` with +all extras, ignoring markers), or tooling (only reachable through a dependency +group). Prints nothing when the two locks agree. With --suspects, also writes a +one-line summary of the changed runtime dependencies (direct first) to FILE, +for use in an issue title. +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path +from typing import Any + +import tomllib + +Package = dict[str, Any] + +ROOT = "mcp" + + +def load(path: Path) -> list[Package]: + return tomllib.loads(path.read_text(encoding="utf-8")).get("package", []) + + +def versions(packages: list[Package]) -> dict[str, list[str]]: + """name -> sorted distinct versions (a name can be locked more than once across marker forks).""" + out: dict[str, set[str]] = {} + for pkg in packages: + if "version" in pkg and "editable" not in pkg.get("source", {}) and "virtual" not in pkg.get("source", {}): + out.setdefault(pkg["name"], set()).add(pkg["version"]) + return {name: sorted(vs) for name, vs in out.items()} + + +def runtime_roles(packages: list[Package]) -> tuple[set[str], set[str]]: + """(direct, closure): names mcp depends on directly (any extra), and everything reachable from them.""" + by_name: dict[str, list[Package]] = {} + for pkg in packages: + by_name.setdefault(pkg["name"], []).append(pkg) + + def edges(name: str, extras: frozenset[str]) -> list[tuple[str, frozenset[str]]]: + found: list[tuple[str, frozenset[str]]] = [] + for pkg in by_name.get(name, []): + deps = list(pkg.get("dependencies", [])) + for extra in extras: + deps += pkg.get("optional-dependencies", {}).get(extra, []) + found += [(d["name"], frozenset(d.get("extra", []))) for d in deps] + return found + + root_extras = frozenset().union(*(pkg.get("optional-dependencies", {}).keys() for pkg in by_name.get(ROOT, []))) + direct = {name for name, _ in edges(ROOT, root_extras)} + closure: set[str] = set() + seen: set[tuple[str, frozenset[str]]] = set() + todo = [(ROOT, root_extras)] + while todo: + node = todo.pop() + if node in seen: + continue + seen.add(node) + closure.add(node[0]) + todo += edges(*node) + closure.discard(ROOT) + return direct, closure + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("old", type=Path) + parser.add_argument("new", type=Path) + parser.add_argument("--old-label", default="before") + parser.add_argument("--new-label", default="after") + parser.add_argument("--suspects", type=Path, help="write a one-line title summary of changed runtime deps here") + args = parser.parse_args() + + old_packages, new_packages = load(args.old), load(args.new) + old, new = versions(old_packages), versions(new_packages) + # Union of both locks' graphs, so a dependency dropped by the new resolution keeps its old role. + direct, closure = (a | b for a, b in zip(runtime_roles(old_packages), runtime_roles(new_packages))) + + def role(name: str) -> tuple[int, str]: + if name in direct: + return 0, "runtime (direct)" + if name in closure: + return 1, "runtime (transitive)" + return 2, "tooling" + + changed = sorted((role(n), n) for n in old.keys() | new.keys() if old.get(n) != new.get(n)) + # Tooling that merely appears or disappears (e.g. a dependency group stripped before resolving) is noise here. + changed = [c for c in changed if c[0][0] < 2 or (c[1] in old and c[1] in new)] + if changed: + print(f"| Package | {args.old_label} | {args.new_label} | Role |") + print("| --- | --- | --- | --- |") + for (_, label), name in changed: + before = ", ".join(old.get(name, [])) or "(absent)" + after = ", ".join(new.get(name, [])) or "(removed)" + print(f"| {name} | {before} | {after} | {label} |") + + if args.suspects: + runtime = [f"{n} {new[n][-1]}" for (rank, _), n in changed if rank < 2 and n in new] + summary = ", ".join(runtime[:3]) + (f" (+{len(runtime) - 3} more)" if len(runtime) > 3 else "") + args.suspects.write_text(summary + "\n", encoding="utf-8") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci/canary_report.sh b/scripts/ci/canary_report.sh new file mode 100755 index 0000000000..53104bf21b --- /dev/null +++ b/scripts/ci/canary_report.sh @@ -0,0 +1,247 @@ +#!/usr/bin/env bash +# Dependency-canary reporter: folds the resolve + per-cell artifacts into one +# Markdown report, then keeps a single tracking issue in sync with it (open on +# red, refresh while red, close on green). Driven by .github/workflows/dependency-canary.yml. +# +# Inputs (env): +# CANARY_ARTIFACTS directory holding canary-resolve/ and canary-cell-*/ artifacts +# CANARY_RESOLVE_RESULT result of the resolve job (success|failure|cancelled|skipped) +# CANARY_FILE_ISSUES "true" to create/update/close the tracking issue; anything else = report only +# CANARY_LABEL label that identifies the tracking issue (created if missing) +# CANARY_ASSIGNEES comma-separated logins assigned when an issue is opened +# CANARY_RUN_URL link to this workflow run +# GH_TOKEN, GITHUB_REPOSITORY, GITHUB_STEP_SUMMARY (standard) + +set -euo pipefail + +artifacts="${CANARY_ARTIFACTS:?}" +resolve_dir="$artifacts/canary-resolve" +label="${CANARY_LABEL:?}" +run_url="${CANARY_RUN_URL:?}" +today=$(date -u +%Y-%m-%d) +body=canary-body.md + +read_or() { if [ -s "$1" ]; then cat "$1"; else printf '%s' "$2"; fi; } + +# ---- classify ------------------------------------------------------------- + +cells=() statuses=() +for f in "$artifacts"/canary-cell-*/status; do + [ -f "$f" ] || continue + cells+=("$(basename "$(dirname "$f")" | sed 's/^canary-cell-//')") + statuses+=("$(tr -d '[:space:]' <"$f")") +done + +count() { + local n=0 s + for s in "${statuses[@]}"; do if [ "$s" = "$1" ]; then n=$((n + 1)); fi; done + echo "$n" +} +n_cells=${#statuses[@]} +n_error=$(count error) +n_install=$(count install-failed) +n_warn=$(count warnings-only) + +if [ "${CANARY_RESOLVE_RESULT:-}" != "success" ]; then + overall=unresolvable +elif [ "$n_cells" -eq 0 ]; then + overall=no-results +elif [ "$n_error" -gt 0 ]; then + overall=error +elif [ "$n_install" -gt 0 ]; then + overall=install-failed +elif [ "$n_warn" -gt 0 ]; then + overall=warnings-only +else + overall=green +fi +# P0 only when every cell hard-fails: a fresh `pip install mcp` is broken everywhere today. +p0=false +if [ "$n_cells" -gt 0 ] && [ $((n_error + n_install)) -eq "$n_cells" ]; then p0=true; fi + +suspects=$(read_or "$resolve_dir/suspects-since-green.txt" "") +[ -n "$suspects" ] || suspects=$(read_or "$resolve_dir/suspects-vs-lock.txt" "") +[ -n "$suspects" ] || suspects="see run" +cutoff=$(read_or "$resolve_dir/cutoff.txt" "unknown") +baseline=$(read_or "$resolve_dir/baseline.txt" "") +uv_version=$(read_or "$resolve_dir/uv-version.txt" "unknown") + +case "$overall" in + error) title="Newest dependency versions fail the test suite: $suspects" ;; + install-failed) title="Newest dependency versions fail to install: $suspects" ;; + warnings-only) title="Newest dependency versions raise deprecation warnings: $suspects" ;; + unresolvable) title="Newest dependency versions cannot be resolved together" ;; + no-results) title="Dependency canary produced no test results" ;; + green) title="Newest dependency versions pass" ;; +esac + +# ---- report body --------------------------------------------------------- + +{ + echo "" + case "$overall" in + green) echo "**Status: passing** as of $today." ;; + error) echo "**Status: failing** — hard test failures on $n_error/$n_cells cells as of $today." ;; + install-failed) echo "**Status: failing** — the resolved set does not install on $n_install/$n_cells cells as of $today." ;; + warnings-only) echo "**Status: deprecations** — tests fail only because new deprecation warnings are errors under our pytest config ($n_warn/$n_cells cells) as of $today. Nothing is broken for users yet." ;; + unresolvable) echo "**Status: unresolvable** — uv could not resolve mcp's runtime dependencies to their newest allowed versions as of $today." ;; + no-results) echo "**Status: unknown** — the resolve step succeeded but no test cell reported (infrastructure problem; see the run)." ;; + esac + echo + echo "[Workflow run]($run_url) · newest versions published before \`$cutoff\` (releases younger than a day are skipped) · uv \`$uv_version\`" + echo + echo "This is the weekly dependency canary: it re-resolves the runtime dependencies of \`mcp[cli,rich]\` (direct and transitive) to the newest versions our specifiers allow, ignoring \`uv.lock\`, keeps test tooling at the locked versions, and runs the test suite. PR CI never does this, so this issue is the only signal that a new upstream release breaks the SDK for users installing it today." + echo + + if [ "$overall" = "unresolvable" ]; then + echo "### Resolution failure" + echo + if [ -s "$resolve_dir/lock.log" ]; then + echo '```text' + tail -n 60 "$resolve_dir/lock.log" + echo '```' + else + echo "The resolve job produced no log; see the workflow run." + fi + echo + fi + + echo "### What changed since the last green run" + echo + if [ -z "$baseline" ]; then + echo "No earlier successful scheduled run to compare against yet — use the full diff against \`uv.lock\` below." + elif [ -s "$resolve_dir/since-green.md" ]; then + echo "Compared with the versions that were newest at the last green run (cutoff \`$baseline\`). **Start here** — the culprit is almost always in this table." + echo + cat "$resolve_dir/since-green.md" + else + echo "Nothing in the runtime closure changed since the last green run (cutoff \`$baseline\`). If tests fail anyway, a change on \`main\` since then is the likely cause — compare with the \`locked\` leg of PR CI." + fi + echo + + if [ "$n_cells" -gt 0 ]; then + echo "### Results" + echo + for i in "${!cells[@]}"; do echo "- \`${cells[$i]}\`: **${statuses[$i]}**"; done + echo + for f in "$artifacts"/canary-cell-*/cell.md; do + if [ -f "$f" ]; then + cat "$f" + echo + fi + done + fi + + if [ -s "$resolve_dir/held-back.md" ]; then + echo "### Not tested at their newest release" + echo + echo "Something else in the resolution caps these runtime dependencies below their latest version, so this run says nothing about the versions listed as latest:" + echo + cat "$resolve_dir/held-back.md" + echo + fi + + if [ -s "$resolve_dir/vs-lock.md" ]; then + n_diff=$(($(wc -l <"$resolve_dir/vs-lock.md") - 2)) + echo "
All differences from the committed uv.lock ($n_diff packages)" + echo + cat "$resolve_dir/vs-lock.md" + echo + echo "
" + echo + fi + + echo "### Reproduce locally" + echo + echo "Either download the \`canary-resolve\` artifact from the run and drop its \`uv.lock\` over yours, or re-resolve the same way (same uv version, same cutoff):" + echo + echo '```bash' + echo "uv self version # the canary used $uv_version" + if [ -s "$resolve_dir/closure.txt" ]; then + printf 'uv lock --exclude-newer %s' "$cutoff" + while read -r pkg; do if [ -n "$pkg" ]; then printf ' -P %s' "$pkg"; fi; done <"$resolve_dir/closure.txt" + echo + fi + echo "uv sync --frozen --all-extras --python 3.14 # or the failing cell's Python" + echo "uv run --frozen --no-sync pytest # plus the failing test ids above" + echo '```' + echo + echo "To confirm a suspect, put just that package back and re-run: \`uv lock -P '=='\`, then sync and test again." + echo + echo "### What to do" + echo + echo "1. **Deprecation warnings only** → not urgent for users; migrate off the deprecated API (supporting both old and new versions) before the removal lands. No ceiling." + echo "2. **Hard failures / install failures** → users running \`pip install mcp\` today get this combination. Prefer a fix that supports both the old and new version of the dependency, and release it." + echo "3. Only if users are broken *and* a real fix will take more than a day or two: add a temporary ceiling (\`Generated by .github/workflows/dependency-canary.yml — adjust the workflow, not this text." +} >"$body" + +# GitHub caps issue bodies at 65536 characters; keep headroom. +if [ "$(wc -c <"$body")" -gt 60000 ]; then + head -c 59000 "$body" >"$body.tmp" + printf '\n\n…(report truncated; full version in the workflow run summary)\n' >>"$body.tmp" + mv "$body.tmp" "$body" +fi + +if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + { + echo "## $title" + echo + cat "$body" + } >>"$GITHUB_STEP_SUMMARY" +fi +echo "canary: overall=$overall p0=$p0 cells=${cells[*]:-none} statuses=${statuses[*]:-none}" +echo "canary: title: $title" + +# ---- tracking issue ------------------------------------------------------ + +if [ "${CANARY_FILE_ISSUES:-}" != "true" ]; then + echo "canary: report-only run; not touching issues." + exit 0 +fi + +repo="${GITHUB_REPOSITORY:?}" +# The label plus bot authorship is the identity of the tracking issue; the title is free to change. +bot_issues="repos/$repo/issues?labels=$label&creator=github-actions%5Bbot%5D&per_page=10" +open_issue=$(gh api "$bot_issues&state=open" --jq '[.[] | select(has("pull_request") | not)][0].number // empty') + +if [ "$overall" = "green" ]; then + if [ -n "$open_issue" ]; then + gh issue comment "$open_issue" --repo "$repo" --body "Newest allowed dependency versions pass again as of $today ([run]($run_url)). Closing." + gh issue close "$open_issue" --repo "$repo" --reason completed + echo "canary: closed #$open_issue" + else + echo "canary: green and no open issue; nothing to do." + fi + exit 0 +fi + +if [ -n "$open_issue" ]; then + gh issue edit "$open_issue" --repo "$repo" --title "$title" --body-file "$body" + if [ "$p0" = "true" ]; then gh issue edit "$open_issue" --repo "$repo" --add-label P0; fi + gh issue comment "$open_issue" --repo "$repo" --body "Still red on $today: **$overall** — $suspects ([run]($run_url)). The issue body above now shows this run." + echo "canary: updated #$open_issue" + exit 0 +fi + +gh label create "$label" --repo "$repo" --force --color FBCA04 \ + --description "Filed by the weekly newest-dependencies canary (.github/workflows/dependency-canary.yml)" +previous=$(gh api "$bot_issues&state=closed&sort=updated" --jq '[.[] | select(has("pull_request") | not)][0] | if . then "Previous incident: #\(.number) (closed \(.closed_at[:10]))." else empty end') +if [ -n "$previous" ]; then + { + echo "$previous" + echo + cat "$body" + } >"$body.tmp" + mv "$body.tmp" "$body" +fi +labels=(--label "$label" --label dependencies) +if [ "$p0" = "true" ]; then labels+=(--label P0); fi +assignees=() +if [ -n "${CANARY_ASSIGNEES:-}" ]; then assignees=(--assignee "$CANARY_ASSIGNEES"); fi +url=$(gh issue create --repo "$repo" --title "$title" --body-file "$body" "${labels[@]}" "${assignees[@]}") +echo "canary: opened $url" From 1b0d968d13101df8b47ff89dbe9169c13e6f27e6 Mon Sep 17 00:00:00 2001 From: Max Isbey <224885523+maxisbey@users.noreply.github.com> Date: Sun, 16 Aug 2026 12:40:59 +0000 Subject: [PATCH 2/5] ci: run CI on uv 0.11.33 The canary relies on `--exclude-newer` leaving locked versions in place for packages it does not explicitly upgrade, which uv only guarantees from 0.10 (astral-sh/uv#17721), and on relative cutoffs. Move every workflow to the same pin so the canary and PR CI cannot disagree about resolution semantics. 0.10/0.11 carry no breaking changes that touch how this repo uses uv (frozen syncs, lowest-direct resolution, lock --check). No-Verification-Needed: CI configuration only; exercised by PR CI itself --- .github/workflows/conformance.yml | 4 ++-- .github/workflows/deploy-docs.yml | 2 +- .github/workflows/docs-preview.yml | 2 +- .github/workflows/publish-pypi.yml | 2 +- .github/workflows/shared.yml | 16 ++++++++-------- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 65b014b631..b365d41423 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -44,7 +44,7 @@ jobs: - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: enable-cache: true - version: 0.9.5 + version: 0.11.33 - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: 24 @@ -90,7 +90,7 @@ jobs: - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: enable-cache: true - version: 0.9.5 + version: 0.11.33 - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: 24 diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index 28ae351feb..37ca3e0fab 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -50,7 +50,7 @@ jobs: uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: enable-cache: true - version: 0.9.5 + version: 0.11.33 - name: Build combined docs (main at / and /v2/, v1.x at /v1/) run: bash scripts/build-docs.sh site diff --git a/.github/workflows/docs-preview.yml b/.github/workflows/docs-preview.yml index 278aa4423d..18729c69f3 100644 --- a/.github/workflows/docs-preview.yml +++ b/.github/workflows/docs-preview.yml @@ -129,7 +129,7 @@ jobs: # a cache populated while untrusted PR code ran would let it poison # later trusted workflows. Mirrors publish-pypi.yml. enable-cache: false - version: 0.9.5 + version: 0.11.33 # pull_request_target runs this workflow file from the base branch, so # the whole recipe — dependency sync included — must come from the diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index 41b127f923..7a43981dd9 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -21,7 +21,7 @@ jobs: uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: enable-cache: false - version: 0.9.5 + version: 0.11.33 - name: Set up Python 3.12 run: uv python install 3.12 diff --git a/.github/workflows/shared.yml b/.github/workflows/shared.yml index 85023dbfac..b602fa9d93 100644 --- a/.github/workflows/shared.yml +++ b/.github/workflows/shared.yml @@ -25,14 +25,14 @@ jobs: uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: enable-cache: true - version: 0.9.5 + version: 0.11.33 - name: Install uv (retry) if: steps.setup-uv.outcome == 'failure' uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: enable-cache: true - version: 0.9.5 + version: 0.11.33 - name: Install dependencies run: uv sync --frozen --all-extras --python 3.10 @@ -81,14 +81,14 @@ jobs: uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: enable-cache: true - version: 0.9.5 + version: 0.11.33 - name: Install uv (retry) if: steps.setup-uv.outcome == 'failure' uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: enable-cache: true - version: 0.9.5 + version: 0.11.33 - name: Install the project run: uv sync ${{ matrix.dep-resolution.install-flags }} --all-extras --python ${{ matrix.python-version }} @@ -127,14 +127,14 @@ jobs: uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: enable-cache: true - version: 0.9.5 + version: 0.11.33 - name: Install uv (retry) if: steps.setup-uv.outcome == 'failure' uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: enable-cache: true - version: 0.9.5 + version: 0.11.33 - name: Install dependencies run: uv sync --frozen --all-extras --python 3.10 @@ -167,14 +167,14 @@ jobs: uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: enable-cache: true - version: 0.9.5 + version: 0.11.33 - name: Install uv (retry) if: steps.setup-uv.outcome == 'failure' uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: enable-cache: true - version: 0.9.5 + version: 0.11.33 - name: Build the docs in strict mode run: bash scripts/docs/build.sh From 02dfbf5fb8e0db1f91ea5ed56158a73e66d0e5b7 Mon Sep 17 00:00:00 2001 From: Max Isbey <224885523+maxisbey@users.noreply.github.com> Date: Sun, 16 Aug 2026 12:44:52 +0000 Subject: [PATCH 3/5] ci: drop the canary's temporary branch trigger The workflow ran green end to end on the branch (run 31947729778), so remove the push trigger that exercised it. Also read pyright's error count rather than its last output line for the informational summary, and silence its new-version nag. No-Verification-Needed: CI-only change (workflow trigger + report cosmetics) --- .github/workflows/dependency-canary.yml | 4 ---- scripts/ci/canary_cell.sh | 5 +++-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/dependency-canary.yml b/.github/workflows/dependency-canary.yml index 0eeefab3c4..26a3ae3406 100644 --- a/.github/workflows/dependency-canary.yml +++ b/.github/workflows/dependency-canary.yml @@ -48,10 +48,6 @@ on: description: "Create/update/close the tracking issue exactly as a scheduled run would" type: boolean default: false - # TEMPORARY while this workflow is under review: exercise it on the PR branch. - # Report-only (push runs never touch issues). Remove before merging. - push: - branches: ["ci/dependency-canary"] permissions: {} diff --git a/scripts/ci/canary_cell.sh b/scripts/ci/canary_cell.sh index 7f1fff5204..fd6128c995 100755 --- a/scripts/ci/canary_cell.sh +++ b/scripts/ci/canary_cell.sh @@ -96,12 +96,13 @@ else fi if [ "${CANARY_PYRIGHT:-}" = "1" ]; then - if uv run --frozen --no-sync pyright src/mcp >"$out/pyright.log" 2>&1; then + # Typing-only drift (e.g. a dependency tightening a signature) never fails the cell; it is context for the reader. + if PYRIGHT_PYTHON_IGNORE_WARNINGS=1 uv run --frozen --no-sync pyright src/mcp >"$out/pyright.log" 2>&1; then printf '\npyright on `src/mcp` against these versions: clean (informational).\n' >>"$md" else { echo - details_of "$out/pyright.log" "pyright on src/mcp against these versions: $(tail -n1 "$out/pyright.log") (informational, never filed on its own)" 30 + details_of "$out/pyright.log" "pyright on src/mcp against these versions: $(grep -Eo '^[0-9]+ errors?' "$out/pyright.log" | tail -n1) (informational, never filed on its own)" 30 } >>"$md" fi fi From 00f5e50a813b6fc4528f981f1c5db90aba74b2d4 Mon Sep 17 00:00:00 2001 From: Max Isbey <224885523+maxisbey@users.noreply.github.com> Date: Sun, 16 Aug 2026 13:09:01 +0000 Subject: [PATCH 4/5] ci: make the canary report reason over the planned matrix Review feedback, taken as one structural change rather than patches: the report classified whatever artifacts happened to arrive, so a cell that timed out or lost its runner simply vanished (two passing cells read as green and would have closed a live incident), a cancelled run could still write to the issue, and any resolve-job failure was reported as "cannot be resolved". Now the matrix is defined once and published by the resolve job, the report checks every planned cell against `needs.test.result`, a cell writes a provisional status before doing anything, `uv lock` records whether it was the thing that failed, and incomplete runs are their own class that comments on an open incident instead of rewriting or closing it. Artifacts carry their own top-level directory and are merged on download, so the layout no longer depends on how many of them exist. Smaller corrections from the same review: per-package cutoffs instead of a global --exclude-newer (a freshly bumped exact pin elsewhere could otherwise fail the resolution); the since-last-green diff uses the lock the last green run actually uploaded rather than re-resolving today's tree at an old cutoff; the issue's reproduce block is the literal commands the job ran (group strip included) under `uvx uv@`; per-cell output is size-bounded at the source and the full report goes to the step summary before any truncation; the flake/deprecation re-runs key on whether pytest recorded failures rather than on its exit code; titles say "nothing changed since last green" when that is the case; bash runs with pipefail everywhere; the group-strip generator follows include-group and default-groups = "all"; wording says tooling is preferred at uv.lock, not frozen. The temporary branch trigger is back for one more end-to-end run. No-Verification-Needed: CI-only change (workflow + scripts/ci); scripts exercised locally, workflow verified by a branch run --- .github/workflows/dependency-canary.yml | 226 ++++++++++++++---------- scripts/ci/canary_cell.sh | 149 +++++++++------- scripts/ci/canary_lock_diff.py | 3 +- scripts/ci/canary_report.sh | 207 +++++++++++++--------- 4 files changed, 340 insertions(+), 245 deletions(-) diff --git a/.github/workflows/dependency-canary.yml b/.github/workflows/dependency-canary.yml index 26a3ae3406..ddf5717d71 100644 --- a/.github/workflows/dependency-canary.yml +++ b/.github/workflows/dependency-canary.yml @@ -17,9 +17,10 @@ name: Dependency canary # route around retroactive caps by picking an older uncapped mcp, and the # bot cannot tell which package (or interaction) is at fault. The issue's # "What to do" section is the runbook; a human decides. -# - float test tooling (pytest, ruff, pyright, coverage, ...). Those stay at -# uv.lock so a pytest major cannot masquerade as an SDK break; Dependabot -# owns moving them. +# - float test tooling (pytest, ruff, pyright, coverage, ...). Only the +# runtime closure is upgraded; everything else keeps its uv.lock version as +# a preference and moves only when a floated runtime dependency forces it +# (the report tags such rows "tooling"). Dependabot owns moving tooling. # - test pre-releases on the schedule. `workflow_dispatch` with # `prerelease: true` does that on demand and never files an issue. # - bisect. The issue lists what changed since the last green run (usually @@ -29,11 +30,13 @@ name: Dependency canary # *dev* dependency caps (e.g. logfire pins opentelemetry-sdk, which pins # opentelemetry-api) cannot reach their newest release here — the issue's "Not # tested at their newest release" section lists them each run. Dependency -# groups outside `default-groups` (translate, codegen) are stripped before -# resolving so their caps (anthropic: pydantic<3) do not apply. +# groups that `uv sync` does not install (translate, codegen) are stripped +# before resolving so their caps (anthropic: pydantic<3) do not apply. # # Notifications: assignees get the issue traffic. GitHub additionally e-mails # scheduled-run failures only to whoever last edited the `cron:` line below. +# The P0 label is added automatically when every cell hard-fails and is never +# removed automatically; de-escalation is a human call. on: schedule: @@ -48,12 +51,22 @@ on: description: "Create/update/close the tracking issue exactly as a scheduled run would" type: boolean default: false + # TEMPORARY while this workflow is under review: exercise it on the PR branch. + # Report-only (push runs never touch issues). Remove before merging. + push: + branches: ["ci/dependency-canary"] permissions: {} concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + # One canary at a time, and never cancel one in flight: a half-finished run + # must not be what decides the tracking issue's fate. Runs are short; queue. + group: ${{ github.workflow }} + cancel-in-progress: false + +defaults: + run: + shell: bash # -eo pipefail everywhere, and Git-Bash on the Windows cell env: COLUMNS: 150 @@ -62,6 +75,16 @@ env: # releases (the ruff 0.14.12 incident that got the per-PR "highest" leg # removed in #1869) and same-day yanks. A weekly job loses nothing by it. CANARY_LAG: "24 hours" + # Single source of truth for the test matrix; the report checks every one of + # these produced a result. Oldest and newest supported Python bracket the + # marker forks in the lock (deps drop 3.10 first; 3.14 gets wheels last), and + # Windows/newest is where a fresh release most often lacks a wheel. + CANARY_CELLS: >- + [ + {"cell": "ubuntu-3.10", "os": "ubuntu-latest", "python": "3.10", "smoke": "", "pyright": ""}, + {"cell": "ubuntu-3.14", "os": "ubuntu-latest", "python": "3.14", "smoke": "1", "pyright": "1"}, + {"cell": "windows-3.14", "os": "windows-latest", "python": "3.14", "smoke": "", "pyright": ""} + ] CANARY_LABEL: dependency-canary CANARY_ASSIGNEES: "maxisbey,Kludex" @@ -73,10 +96,19 @@ jobs: timeout-minutes: 15 permissions: contents: read - actions: read # `gh run list`: find the last green scheduled run to diff against + actions: read # find and download the last green scheduled run's resolution to diff against outputs: - cutoff: ${{ steps.cutoffs.outputs.cutoff }} + cells: ${{ steps.plan.outputs.cells }} steps: + - name: Publish the test matrix + id: plan + run: | + { + echo 'cells<>"$GITHUB_OUTPUT" + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false @@ -98,109 +130,119 @@ jobs: enable-cache: false version: ${{ env.UV_VERSION }} - - name: Compute cutoffs - id: cutoffs + - name: Fetch the last green run's resolution env: GH_TOKEN: ${{ github.token }} run: | - mkdir -p canary-resolve - cutoff=$(date -u -d "-$CANARY_LAG" +%Y-%m-%dT%H:%M:%SZ) - echo "cutoff=$cutoff" >>"$GITHUB_OUTPUT" - echo "$cutoff" >canary-resolve/cutoff.txt - uv self version >canary-resolve/uv-version.txt - # Baseline = what the last green *scheduled* run saw (its start time minus the same lag). - last_green=$(gh run list --repo "$GITHUB_REPOSITORY" --workflow dependency-canary.yml \ - --branch main --event schedule --status success --limit 1 --json startedAt --jq '.[0].startedAt // empty') - if [ -n "$last_green" ]; then - date -u -d "$last_green -$CANARY_LAG" +%Y-%m-%dT%H:%M:%SZ >canary-resolve/baseline.txt - else - : >canary-resolve/baseline.txt + mkdir -p out/canary-resolve + cd out/canary-resolve + date -u -d "-$CANARY_LAG" +%Y-%m-%dT%H:%M:%SZ >cutoff.txt + (uv self version --short 2>/dev/null || uv --version | awk '{print $2}') >uv-version.txt + : >baseline.txt + # Baseline = the uv.lock the last successful *scheduled* run uploaded, so the + # report can list only what moved since then. Best effort (artifacts expire). + gh run list --repo "$GITHUB_REPOSITORY" --workflow dependency-canary.yml --branch main \ + --event schedule --status success --limit 1 --json databaseId,startedAt,url >last-green.json || echo '[]' >last-green.json + run_id=$(jq -r '.[0].databaseId // empty' last-green.json) + if [ -n "$run_id" ] && gh run download "$run_id" --repo "$GITHUB_REPOSITORY" -n canary-resolve -D baseline-dl; then + if [ -f baseline-dl/canary-resolve/uv.lock ]; then + cp baseline-dl/canary-resolve/uv.lock baseline.lock + jq -r '.[0] | "[\(.startedAt[:10]) run](\(.url))"' last-green.json >baseline.txt + fi fi - echo "cutoff=$cutoff baseline=$(cat canary-resolve/baseline.txt)" + rm -rf baseline-dl + echo "cutoff=$(cat cutoff.txt) uv=$(cat uv-version.txt) baseline=$(cat baseline.txt)" - name: Work out what to float run: | + r=out/canary-resolve # The runtime closure of mcp[cli,rich] as currently locked: exactly the set # `pip install "mcp[cli,rich]"` pulls in. New transitive deps that a newer # release introduces have no lock entry and so resolve to newest anyway. uv export --frozen --no-default-groups --all-extras --no-emit-workspace \ - --no-hashes --no-header --no-annotate | sed -E 's/[=; @].*//' | sort -u >canary-resolve/closure.txt - echo "Floating $(wc -l $r/closure.txt + if [ ! -s $r/closure.txt ]; then echo "::error::uv export produced an empty runtime closure"; exit 1; fi + echo "Floating $(wc -l <$r/closure.txt) packages:"; tr '\n' ' ' <$r/closure.txt; echo # Dependency groups that `uv sync` does not install still constrain the - # resolution (uv.lock is universal). Strip the non-default ones so e.g. the - # translate group's `anthropic` cannot hold pydantic below a new major. - python3 - <<'EOF' >canary-resolve/strip.sh + # resolution (uv.lock is universal). Strip them so e.g. the translate + # group's `anthropic` cannot hold pydantic below a new major. + python3 - <<'EOF' >$r/strip.sh import re, tomllib project = tomllib.load(open("pyproject.toml", "rb")) - keep = set(project.get("tool", {}).get("uv", {}).get("default-groups", [])) - for group, deps in project.get("dependency-groups", {}).items(): - names = [re.match(r"[A-Za-z0-9._-]+", d).group(0) for d in deps if isinstance(d, str)] + groups = project.get("dependency-groups", {}) + default = project.get("tool", {}).get("uv", {}).get("default-groups", ["dev"]) + keep = set(groups) if default == "all" else set(default) + todo = list(keep) + while todo: # groups pulled in via {include-group = "..."} are installed too + for entry in groups.get(todo.pop(), []): + if isinstance(entry, dict) and entry.get("include-group") not in keep | {None}: + keep.add(entry["include-group"]) + todo.append(entry["include-group"]) + for group, entries in groups.items(): + names = [re.match(r"[A-Za-z0-9._-]+", e).group(0) for e in entries if isinstance(e, str)] if group not in keep and names: print("uv remove --frozen --group", group, *names) EOF - cat canary-resolve/strip.sh - bash -e canary-resolve/strip.sh + cat $r/strip.sh + bash -e $r/strip.sh - name: Resolve newest allowed versions env: PRERELEASE: ${{ inputs.prerelease && 'allow' || '' }} run: | - set -o pipefail - args=(--exclude-newer "$(cat canary-resolve/cutoff.txt)") + r=out/canary-resolve + cutoff=$(cat $r/cutoff.txt) + cp uv.lock $r/committed.lock + # Per-package cutoffs rather than a global --exclude-newer, so an exact pin + # elsewhere (docs group, build constraints) bumped the day before the run + # cannot make the resolution fail. + args=() if [ -n "$PRERELEASE" ]; then args+=(--prerelease "$PRERELEASE"); fi - while read -r pkg; do args+=(-P "$pkg"); done &1 | tee canary-resolve/baseline.log; then - cp uv.lock canary-resolve/baseline.lock - else - echo "::warning::could not re-resolve the last-green baseline; the report will only diff against uv.lock" - : >canary-resolve/baseline.txt - fi - cp canary-resolve/committed.lock uv.lock - echo "::endgroup::" + while read -r pkg; do args+=(-P "$pkg" --exclude-newer-package "$pkg=$cutoff"); done <$r/closure.txt + { printf 'uv lock'; printf ' %q' "${args[@]}"; echo; } >$r/lock-cmd.sh + if uv lock "${args[@]}" 2>&1 | tee $r/lock.log; then + echo ok >$r/lock-status + cp uv.lock $r/uv.lock + else + # Distinguishes "uv ran and could not resolve" from every other way this job can fail. + echo failed >$r/lock-status + exit 1 fi - uv lock "${args[@]}" 2>&1 | tee canary-resolve/lock.log - cp uv.lock canary-resolve/uv.lock - - name: Summarise what moved run: | - python3 scripts/ci/canary_lock_diff.py canary-resolve/committed.lock uv.lock \ - --old-label "uv.lock" --new-label "this run" --suspects canary-resolve/suspects-vs-lock.txt >canary-resolve/vs-lock.md - if [ -f canary-resolve/baseline.lock ]; then - python3 scripts/ci/canary_lock_diff.py canary-resolve/baseline.lock uv.lock \ - --old-label "last green" --new-label "this run" --suspects canary-resolve/suspects-since-green.txt >canary-resolve/since-green.md + r=out/canary-resolve + python3 scripts/ci/canary_lock_diff.py $r/committed.lock uv.lock \ + --old-label "uv.lock" --new-label "this run" --suspects $r/suspects-vs-lock.txt >$r/vs-lock.md + if [ -f $r/baseline.lock ]; then + python3 scripts/ci/canary_lock_diff.py $r/baseline.lock uv.lock \ + --old-label "last green" --new-label "this run" --suspects $r/suspects-since-green.txt >$r/since-green.md fi # Direct runtime deps that could not reach their newest release (capped by something else in the resolution). - uv tree --frozen --outdated --depth 1 --package mcp >canary-resolve/tree.txt 2>/dev/null || true + uv tree --frozen --outdated --universal --depth 1 --package mcp >$r/tree.txt { echo "| Package | Resolved | Latest |" echo "| --- | --- | --- |" - sed -nE 's/^[^A-Za-z0-9]*([A-Za-z0-9._-]+)(\[[^]]*\])? v([^ ]+)( \(extra: [^)]*\))? \(latest: v([^)]+)\)$/| \1 | \3 | \5 |/p' canary-resolve/tree.txt - } >canary-resolve/held-back.md - if [ "$(wc -l canary-resolve/held-back.md; fi + sed -nE 's/^[^A-Za-z0-9]*([A-Za-z0-9._-]+)(\[[^]]*\])? v([^ ]+) .*\(latest: v([^)]+)\)$/\1 \3 \4/p' $r/tree.txt | + sort -u | while read -r name have latest; do + if grep -qxF "$name" $r/closure.txt; then echo "| $name | $have | $latest |"; fi + done + } >$r/held-back.md + if [ "$(wc -l <$r/held-back.md)" -le 2 ]; then : >$r/held-back.md; fi { - echo "## Resolution (cutoff $(cat canary-resolve/cutoff.txt))" + echo "## Resolution (cutoff $(cat $r/cutoff.txt))" echo - if [ -s canary-resolve/since-green.md ]; then echo "### Since last green ($(cat canary-resolve/baseline.txt))"; cat canary-resolve/since-green.md; echo; fi - echo "### vs uv.lock"; cat canary-resolve/vs-lock.md; echo - if [ -s canary-resolve/held-back.md ]; then echo "### Held back below latest"; cat canary-resolve/held-back.md; fi + if [ -s $r/since-green.md ]; then echo "### Since last green ($(cat $r/baseline.txt))"; cat $r/since-green.md; echo; fi + echo "### vs uv.lock"; cat $r/vs-lock.md; echo + if [ -s $r/held-back.md ]; then echo "### Held back below latest"; cat $r/held-back.md; fi } >>"$GITHUB_STEP_SUMMARY" - name: Upload resolution - if: always() + if: ${{ !cancelled() }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: canary-resolve - path: canary-resolve/ + path: out/ retention-days: 90 if-no-files-found: error @@ -208,25 +250,13 @@ jobs: name: test (${{ matrix.cell }}) needs: resolve runs-on: ${{ matrix.os }} - timeout-minutes: 20 + timeout-minutes: 25 permissions: contents: read strategy: fail-fast: false matrix: - include: - # Oldest and newest supported Python bracket the marker forks in the - # lock (deps drop 3.10 first; 3.14 gets wheels last). Windows/newest is - # where a fresh release most often lacks a wheel, and pywin32 lives there. - - { cell: ubuntu-3.10, os: ubuntu-latest, python: "3.10" } - - { - cell: ubuntu-3.14, - os: ubuntu-latest, - python: "3.14", - smoke: "1", - pyright: "1", - } - - { cell: windows-3.14, os: windows-latest, python: "3.14" } + include: ${{ fromJSON(needs.resolve.outputs.cells) }} steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: @@ -251,10 +281,12 @@ jobs: uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: canary-resolve - path: canary-resolve + path: . - name: Install and test - shell: bash + # The suite takes ~2 min; a dependency that makes it hang should surface as + # this cell's "incomplete" status in the report, not as a 25-minute job kill. + timeout-minutes: 15 env: CANARY_CELL: ${{ matrix.cell }} CANARY_PYTHON: ${{ matrix.python }} @@ -265,25 +297,22 @@ jobs: PYTHONWARNDEFAULTENCODING: "1" run: | cp canary-resolve/uv.lock uv.lock - # The lock was resolved with these groups stripped; keep pyproject consistent so --frozen holds. - bash -e canary-resolve/strip.sh bash scripts/ci/canary_cell.sh - name: Upload cell result - if: always() + if: ${{ !cancelled() }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: canary-cell-${{ matrix.cell }} - path: | - canary-out/status - canary-out/cell.md + path: out/ retention-days: 30 if-no-files-found: error report: needs: [resolve, test] - # always(): a red resolve/test job is exactly when this must run. - if: always() && github.repository == 'modelcontextprotocol/python-sdk' && needs.resolve.result != 'cancelled' && needs.resolve.result != 'skipped' + # !cancelled(): a red resolve/test job is exactly when this must run, but a + # cancelled run must never decide the issue's fate from partial results. + if: ${{ !cancelled() && github.repository == 'modelcontextprotocol/python-sdk' }} runs-on: ubuntu-latest timeout-minutes: 10 permissions: @@ -299,6 +328,10 @@ jobs: uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: pattern: canary-* + # Every artifact already carries its own top-level directory, so merging + # yields artifacts/canary-resolve/ and artifacts/canary-cell-*/ no matter + # how many artifacts exist (a lone match would otherwise be flattened). + merge-multiple: true path: artifacts - name: Report @@ -306,6 +339,7 @@ jobs: GH_TOKEN: ${{ github.token }} CANARY_ARTIFACTS: artifacts CANARY_RESOLVE_RESULT: ${{ needs.resolve.result }} + CANARY_TEST_RESULT: ${{ needs.test.result }} CANARY_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} # Only the schedule (or an explicit dispatch asking for it) touches issues, and never a pre-release run. CANARY_FILE_ISSUES: ${{ !inputs.prerelease && (github.event_name == 'schedule' || inputs.file-issue) && 'true' || 'false' }} diff --git a/scripts/ci/canary_cell.sh b/scripts/ci/canary_cell.sh index fd6128c995..fd64d6e9b2 100755 --- a/scripts/ci/canary_cell.sh +++ b/scripts/ci/canary_cell.sh @@ -5,18 +5,22 @@ # # Inputs (env): CANARY_CELL (label, e.g. "ubuntu-3.14"), CANARY_PYTHON (e.g. "3.14"), # CANARY_PYRIGHT=1 to append an informational `pyright src/mcp` result. -# Outputs: canary-out/status — one of: pass | flaky | warnings-only | error | install-failed -# canary-out/cell.md — Markdown section for the issue body / job summary +# Outputs under out/canary-cell-$CANARY_CELL/: +# status — pass | flaky | warnings-only | error | install-failed +# ("incomplete" is written first, so a killed cell still says something) +# cell.md — Markdown section for the issue body / job summary (bounded to ~20 kB) +# *.log — raw logs, kept in the artifact for debugging # Exit status: 0 for pass/flaky, 1 otherwise (so the job shows red). set -uo pipefail cell="${CANARY_CELL:?}" python="${CANARY_PYTHON:?}" -out=canary-out +out="out/canary-cell-$cell" mkdir -p "$out" md="$out/cell.md" -status=pass +echo incomplete >"$out/status" +printf '#### %s\n\nDid not finish (see the workflow run).\n' "$cell" >"$md" export COLUMNS=200 # keeps pytest from truncating the -r summary lines quoted in the report pytest_cmd=(uv run --frozen --no-sync pytest -p no:pretty -q --no-header -rfE --color=no -o log_cli=false) @@ -24,91 +28,102 @@ pytest_cmd=(uv run --frozen --no-sync pytest -p no:pretty -q --no-header -rfE -- # versions still work and merely announce a future removal. demote=(-W default::DeprecationWarning -W default::PendingDeprecationWarning -W default::FutureWarning) -summary_of() { # print the short-summary FAILED/ERROR lines of a pytest log, capped - grep -E '^(FAILED|ERROR) ' "$1" | head -n 40 +summary_of() { # the short-summary FAILED/ERROR lines of a pytest log, capped; falls back to the log tail + local lines + lines=$(grep -E '^(FAILED|ERROR) ' "$1" | awk '!seen[$0]++' | cut -c1-240) # xdist repeats collection errors per worker + if [ -z "$lines" ]; then + tail -n 15 "$1" | cut -c1-240 + return + fi + head -n 40 <<<"$lines" local n - n=$(grep -cE '^(FAILED|ERROR) ' "$1") + n=$(wc -l <<<"$lines") if [ "$n" -gt 40 ]; then echo "... and $((n - 40)) more"; fi } -details_of() { # collapsed tail of a log +details_of() { # collapsed, size-bounded tail of a log: file, summary text, max lines printf '
%s\n\n```text\n' "$2" - tail -n "${3:-60}" "$1" | sed 's/```/` ` `/g' - printf '```\n\n
\n' + tail -n "${3:-60}" "$1" | cut -c1-240 | head -c 12000 | sed 's/```/` ` `/g' + printf '\n```\n\n\n' +} + +finish() { + echo "$1" >"$out/status" + echo "canary cell $cell: $1" + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then cat "$md" >>"$GITHUB_STEP_SUMMARY"; fi + case "$1" in pass | flaky) exit 0 ;; *) exit 1 ;; esac } -{ - echo "#### $cell" - echo -} >"$md" +printf '#### %s\n\n' "$cell" >"$md" if ! uv sync --frozen --all-extras --python "$python" >"$out/sync.log" 2>&1; then - status=install-failed { echo "**Install failed** — \`uv sync --frozen --all-extras --python $python\` could not install the resolved set on this platform." echo details_of "$out/sync.log" "uv sync output" 40 } >>"$md" + finish install-failed +fi +installed=$(uv run --frozen --no-sync python -V 2>/dev/null || echo "Python $python") + +# Phase 1: the suite as CI runs it (from a clean last-failed record, which phase 2 keys on). +rm -f .pytest_cache/v/cache/lastfailed +if "${pytest_cmd[@]}" -n auto >"$out/run1.log" 2>&1; then + echo "All tests pass ($installed)." >>"$md" + status=pass else - installed=$(uv run --frozen --no-sync python -V 2>/dev/null) - if "${pytest_cmd[@]}" -n auto >"$out/run1.log" 2>&1; then - echo "All tests pass ($installed)." >>"$md" + # Phase 2: again, to drop flakes. Narrow to the recorded failures (serially) when + # pytest got far enough to record any; a collection/usage error leaves none. + if grep -qs '::' .pytest_cache/v/cache/lastfailed; then + rerun=("${pytest_cmd[@]}" --lf --last-failed-no-failures none -p no:xdist) + rerun_desc="a serial re-run of just the failing tests" else - rc=$? - if [ "$rc" -eq 1 ]; then - # Ordinary test failures: re-run just those, serially, to drop xdist/ordering flakes. - rerun=("${pytest_cmd[@]}" --lf --last-failed-no-failures none -p no:xdist) - else - # Collection/usage/internal error (e.g. a warning raised at import time): no - # last-failed set to narrow to, so re-run everything. - rerun=("${pytest_cmd[@]}" -n auto) - fi - if [ "$rc" -eq 1 ] && "${rerun[@]}" >"$out/run2.log" 2>&1; then - status=flaky + rerun=("${pytest_cmd[@]}" -n auto) + rerun_desc="a full re-run" + fi + if "${rerun[@]}" >"$out/run2.log" 2>&1; then + { + echo "Passed on $rerun_desc ($installed); the first attempt failed as below (treated as flaky, not reported):" + echo + echo '```text' + summary_of "$out/run1.log" + echo '```' + } >>"$md" + status=flaky + else + # Phase 3: same again with the deprecation family demoted, to tell "broken" from "deprecated". + if "${rerun[@]}" "${demote[@]}" >"$out/run3.log" 2>&1; then { - echo "Passed on a serial re-run ($installed); first attempt had failures (treated as flaky, not reported):" - echo - echo '```text' - summary_of "$out/run1.log" - echo '```' + echo "**Deprecation warnings only** ($installed) — the failures below persist on $rerun_desc but disappear once" + echo "\`DeprecationWarning\`/\`PendingDeprecationWarning\`/\`FutureWarning\` are not errors, so the newest versions still work and are announcing a removal we need to get ahead of." } >>"$md" + status=warnings-only else - [ -f "$out/run2.log" ] || cp "$out/run1.log" "$out/run2.log" - if "${rerun[@]}" "${demote[@]}" >"$out/run3.log" 2>&1; then - status=warnings-only - { - echo "**Deprecation warnings only** ($installed) — the failures below disappear when" - echo "\`DeprecationWarning\`/\`PendingDeprecationWarning\`/\`FutureWarning\` are not errors, so newest versions still work but announce a removal we need to get ahead of." - } >>"$md" - else - status=error - echo "**Hard failures** ($installed) — these persist on a serial re-run and with deprecation warnings demoted:" >>"$md" - fi - { - echo - echo '```text' - summary_of "$out/run2.log" - echo '```' - echo - details_of "$out/run2.log" "pytest output (tail)" 80 - } >>"$md" + echo "**Hard failures** ($installed) — the failures below persist on $rerun_desc and with deprecation warnings demoted:" >>"$md" + status=error fi + { + echo + echo '```text' + summary_of "$out/run2.log" + echo '```' + echo + details_of "$out/run2.log" "pytest output (tail)" 80 + } >>"$md" fi +fi - if [ "${CANARY_PYRIGHT:-}" = "1" ]; then - # Typing-only drift (e.g. a dependency tightening a signature) never fails the cell; it is context for the reader. - if PYRIGHT_PYTHON_IGNORE_WARNINGS=1 uv run --frozen --no-sync pyright src/mcp >"$out/pyright.log" 2>&1; then - printf '\npyright on `src/mcp` against these versions: clean (informational).\n' >>"$md" - else - { - echo - details_of "$out/pyright.log" "pyright on src/mcp against these versions: $(grep -Eo '^[0-9]+ errors?' "$out/pyright.log" | tail -n1) (informational, never filed on its own)" 30 - } >>"$md" - fi +if [ "${CANARY_PYRIGHT:-}" = "1" ]; then + # Typing-only drift (e.g. a dependency tightening a signature) never fails the cell; it is context for the reader. + if PYRIGHT_PYTHON_IGNORE_WARNINGS=1 uv run --frozen --no-sync pyright src/mcp >"$out/pyright.log" 2>&1; then + printf '\npyright on `src/mcp` against these versions: clean (informational).\n' >>"$md" + else + count=$(grep -Eo '^[0-9]+ errors?' "$out/pyright.log" | tail -n1) + { + echo + details_of "$out/pyright.log" "pyright on src/mcp against these versions: ${count:-did not complete} (informational, never filed on its own)" 30 + } >>"$md" fi fi -echo "$status" >"$out/status" -echo "canary cell $cell: $status" -if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then cat "$md" >>"$GITHUB_STEP_SUMMARY"; fi -case "$status" in pass | flaky) exit 0 ;; *) exit 1 ;; esac +finish "$status" diff --git a/scripts/ci/canary_lock_diff.py b/scripts/ci/canary_lock_diff.py index 3ca06b9d19..5964ecd84b 100644 --- a/scripts/ci/canary_lock_diff.py +++ b/scripts/ci/canary_lock_diff.py @@ -104,7 +104,8 @@ def role(name: str) -> tuple[int, str]: if args.suspects: runtime = [f"{n} {new[n][-1]}" for (rank, _), n in changed if rank < 2 and n in new] summary = ", ".join(runtime[:3]) + (f" (+{len(runtime) - 3} more)" if len(runtime) > 3 else "") - args.suspects.write_text(summary + "\n", encoding="utf-8") + # Empty file (not a blank line) when no runtime dependency changed, so `[ -s FILE ]` means what it says. + args.suspects.write_text(summary + "\n" if summary else "", encoding="utf-8") return 0 diff --git a/scripts/ci/canary_report.sh b/scripts/ci/canary_report.sh index 53104bf21b..2bdf7aeb1c 100755 --- a/scripts/ci/canary_report.sh +++ b/scripts/ci/canary_report.sh @@ -4,13 +4,26 @@ # red, refresh while red, close on green). Driven by .github/workflows/dependency-canary.yml. # # Inputs (env): -# CANARY_ARTIFACTS directory holding canary-resolve/ and canary-cell-*/ artifacts +# CANARY_ARTIFACTS directory holding canary-resolve/ and canary-cell-/ (merged artifacts) +# CANARY_CELLS JSON list of the matrix cells that were supposed to run ({"cell": ...} objects) # CANARY_RESOLVE_RESULT result of the resolve job (success|failure|cancelled|skipped) +# CANARY_TEST_RESULT result of the test job as a whole # CANARY_FILE_ISSUES "true" to create/update/close the tracking issue; anything else = report only # CANARY_LABEL label that identifies the tracking issue (created if missing) # CANARY_ASSIGNEES comma-separated logins assigned when an issue is opened # CANARY_RUN_URL link to this workflow run # GH_TOKEN, GITHUB_REPOSITORY, GITHUB_STEP_SUMMARY (standard) +# +# Classification, in order: +# resolve job not successful -> resolve-failed if `uv lock` itself failed, else infra +# otherwise per expected cell -> its status file, or "missing" if it never reported +# any error -> error (users are broken today) +# any install-failed -> install-failed +# any warnings-only -> warnings-only (a deprecation, nothing broken yet) +# any missing -> infra (the canary, not the deps, had a problem) +# else -> green +# Only green closes the issue; infra never rewrites an open incident (comment only); +# P0 is added when every expected cell is error/install-failed and never removed here. set -euo pipefail @@ -25,11 +38,17 @@ read_or() { if [ -s "$1" ]; then cat "$1"; else printf '%s' "$2"; fi; } # ---- classify ------------------------------------------------------------- -cells=() statuses=() -for f in "$artifacts"/canary-cell-*/status; do - [ -f "$f" ] || continue - cells+=("$(basename "$(dirname "$f")" | sed 's/^canary-cell-//')") - statuses+=("$(tr -d '[:space:]' <"$f")") +mapfile -t cells < <(jq -r '.[].cell' <<<"${CANARY_CELLS:?}") +statuses=() +for cell in "${cells[@]}"; do + f="$artifacts/canary-cell-$cell/status" + s="" + if [ -f "$f" ]; then s=$(tr -d '[:space:]' <"$f"); fi + case "$s" in + pass | flaky | warnings-only | error | install-failed) ;; + *) s=missing ;; # never reported, or died part-way ("incomplete") + esac + statuses+=("$s") done count() { @@ -37,43 +56,50 @@ count() { for s in "${statuses[@]}"; do if [ "$s" = "$1" ]; then n=$((n + 1)); fi; done echo "$n" } -n_cells=${#statuses[@]} +n_cells=${#cells[@]} n_error=$(count error) n_install=$(count install-failed) n_warn=$(count warnings-only) +n_missing=$(count missing) -if [ "${CANARY_RESOLVE_RESULT:-}" != "success" ]; then - overall=unresolvable -elif [ "$n_cells" -eq 0 ]; then - overall=no-results +resolve_result="${CANARY_RESOLVE_RESULT:-unknown}" +test_result="${CANARY_TEST_RESULT:-unknown}" +if [ "$resolve_result" != "success" ]; then + if [ "$(read_or "$resolve_dir/lock-status" "")" = "failed" ]; then overall=resolve-failed; else overall=infra; fi elif [ "$n_error" -gt 0 ]; then overall=error elif [ "$n_install" -gt 0 ]; then overall=install-failed elif [ "$n_warn" -gt 0 ]; then overall=warnings-only +elif [ "$n_missing" -gt 0 ] || [ "$test_result" != "success" ]; then + overall=infra else overall=green fi -# P0 only when every cell hard-fails: a fresh `pip install mcp` is broken everywhere today. p0=false if [ "$n_cells" -gt 0 ] && [ $((n_error + n_install)) -eq "$n_cells" ]; then p0=true; fi -suspects=$(read_or "$resolve_dir/suspects-since-green.txt" "") -[ -n "$suspects" ] || suspects=$(read_or "$resolve_dir/suspects-vs-lock.txt" "") -[ -n "$suspects" ] || suspects="see run" cutoff=$(read_or "$resolve_dir/cutoff.txt" "unknown") baseline=$(read_or "$resolve_dir/baseline.txt" "") -uv_version=$(read_or "$resolve_dir/uv-version.txt" "unknown") +uv_version=$(read_or "$resolve_dir/uv-version.txt" "") +uv_version=${uv_version%% *} +if [ -e "$resolve_dir/suspects-since-green.txt" ]; then + suspects=$(tr -d '\n' <"$resolve_dir/suspects-since-green.txt") + [ -n "$suspects" ] || suspects="no dependency changed since the last green run" +else + suspects=$(read_or "$resolve_dir/suspects-vs-lock.txt" "see run" | tr -d '\n') +fi case "$overall" in error) title="Newest dependency versions fail the test suite: $suspects" ;; install-failed) title="Newest dependency versions fail to install: $suspects" ;; warnings-only) title="Newest dependency versions raise deprecation warnings: $suspects" ;; - unresolvable) title="Newest dependency versions cannot be resolved together" ;; - no-results) title="Dependency canary produced no test results" ;; + resolve-failed) title="Newest dependency versions could not be resolved" ;; + infra) title="Dependency canary could not run to completion" ;; green) title="Newest dependency versions pass" ;; esac +title=${title:0:200} # ---- report body --------------------------------------------------------- @@ -84,47 +110,51 @@ esac error) echo "**Status: failing** — hard test failures on $n_error/$n_cells cells as of $today." ;; install-failed) echo "**Status: failing** — the resolved set does not install on $n_install/$n_cells cells as of $today." ;; warnings-only) echo "**Status: deprecations** — tests fail only because new deprecation warnings are errors under our pytest config ($n_warn/$n_cells cells) as of $today. Nothing is broken for users yet." ;; - unresolvable) echo "**Status: unresolvable** — uv could not resolve mcp's runtime dependencies to their newest allowed versions as of $today." ;; - no-results) echo "**Status: unknown** — the resolve step succeeded but no test cell reported (infrastructure problem; see the run)." ;; + resolve-failed) echo "**Status: unresolvable** — \`uv lock\` could not produce a resolution with mcp's runtime dependencies at their newest allowed versions as of $today (log below; if it shows a network or index error rather than a conflict, this was infrastructure)." ;; + infra) echo "**Status: incomplete** — the canary itself did not run cleanly as of $today (resolve job: $resolve_result, test job: $test_result, cells without a result: $n_missing/$n_cells). This says nothing about the dependencies; see the run." ;; esac echo - echo "[Workflow run]($run_url) · newest versions published before \`$cutoff\` (releases younger than a day are skipped) · uv \`$uv_version\`" + echo "[Workflow run]($run_url) · newest versions published before \`$cutoff\` (releases younger than a day are skipped) · uv \`${uv_version:-unknown}\`" echo - echo "This is the weekly dependency canary: it re-resolves the runtime dependencies of \`mcp[cli,rich]\` (direct and transitive) to the newest versions our specifiers allow, ignoring \`uv.lock\`, keeps test tooling at the locked versions, and runs the test suite. PR CI never does this, so this issue is the only signal that a new upstream release breaks the SDK for users installing it today." + echo "This is the weekly dependency canary: it re-resolves the runtime dependencies of \`mcp[cli,rich]\` (direct and transitive) to the newest versions our specifiers allow, ignoring \`uv.lock\`, and runs the test suite against them. Everything else keeps its locked version unless a floated dependency forces it to move (tagged *tooling* below). PR CI never does this, so this issue is the only signal that a new upstream release breaks the SDK for people installing it today." echo - if [ "$overall" = "unresolvable" ]; then + if [ "$overall" = "resolve-failed" ]; then echo "### Resolution failure" echo - if [ -s "$resolve_dir/lock.log" ]; then - echo '```text' - tail -n 60 "$resolve_dir/lock.log" - echo '```' - else - echo "The resolve job produced no log; see the workflow run." - fi + echo '```text' + read_or "$resolve_dir/lock.log" "(no lock.log in the artifact; see the run)" | tail -n 60 | cut -c1-240 + echo '```' echo fi - echo "### What changed since the last green run" - echo - if [ -z "$baseline" ]; then - echo "No earlier successful scheduled run to compare against yet — use the full diff against \`uv.lock\` below." - elif [ -s "$resolve_dir/since-green.md" ]; then - echo "Compared with the versions that were newest at the last green run (cutoff \`$baseline\`). **Start here** — the culprit is almost always in this table." + if [ "$resolve_result" = "success" ]; then + echo "### What changed since the last green run" + echo + if [ -z "$baseline" ]; then + echo "No earlier successful scheduled run to compare against (none yet, or its artifact expired) — use the full diff against \`uv.lock\` below." + elif [ -s "$resolve_dir/since-green.md" ]; then + echo "Compared with what the last green run resolved ($baseline). **Start here** — the culprit is almost always in this table." + echo + cat "$resolve_dir/since-green.md" + else + echo "Nothing in the runtime closure changed since the last green run ($baseline). If tests fail anyway, a change on \`main\` since then is the likely cause — compare with the \`locked\` leg of PR CI." + fi echo - cat "$resolve_dir/since-green.md" - else - echo "Nothing in the runtime closure changed since the last green run (cutoff \`$baseline\`). If tests fail anyway, a change on \`main\` since then is the likely cause — compare with the \`locked\` leg of PR CI." fi - echo - if [ "$n_cells" -gt 0 ]; then + if [ "$n_cells" -gt 0 ] && [ "$resolve_result" = "success" ]; then echo "### Results" echo - for i in "${!cells[@]}"; do echo "- \`${cells[$i]}\`: **${statuses[$i]}**"; done + for i in "${!cells[@]}"; do + case "${statuses[$i]}" in + missing) echo "- \`${cells[$i]}\`: **no result** (timed out, cancelled, or lost before reporting; see the run)" ;; + *) echo "- \`${cells[$i]}\`: **${statuses[$i]}**" ;; + esac + done echo - for f in "$artifacts"/canary-cell-*/cell.md; do + for cell in "${cells[@]}"; do + f="$artifacts/canary-cell-$cell/cell.md" if [ -f "$f" ]; then cat "$f" echo @@ -145,7 +175,7 @@ esac n_diff=$(($(wc -l <"$resolve_dir/vs-lock.md") - 2)) echo "
All differences from the committed uv.lock ($n_diff packages)" echo - cat "$resolve_dir/vs-lock.md" + head -n 150 "$resolve_dir/vs-lock.md" echo echo "
" echo @@ -153,17 +183,18 @@ esac echo "### Reproduce locally" echo - echo "Either download the \`canary-resolve\` artifact from the run and drop its \`uv.lock\` over yours, or re-resolve the same way (same uv version, same cutoff):" + echo "Either download the \`canary-resolve\` artifact from the run and copy its \`canary-resolve/uv.lock\` over yours, or redo the resolution exactly as the canary did (dependency groups it does not install are dropped first; the pinned uv gives identical resolver behaviour):" echo echo '```bash' - echo "uv self version # the canary used $uv_version" - if [ -s "$resolve_dir/closure.txt" ]; then - printf 'uv lock --exclude-newer %s' "$cutoff" - while read -r pkg; do if [ -n "$pkg" ]; then printf ' -P %s' "$pkg"; fi; done <"$resolve_dir/closure.txt" - echo + if [ -s "$resolve_dir/strip.sh" ]; then cat "$resolve_dir/strip.sh"; fi + if [ -s "$resolve_dir/lock-cmd.sh" ]; then + sed "s/^uv lock/uvx uv@${uv_version:-latest} lock/" "$resolve_dir/lock-cmd.sh" + else + echo "# (the resolve job did not get as far as recording its uv lock command; see the run)" fi echo "uv sync --frozen --all-extras --python 3.14 # or the failing cell's Python" echo "uv run --frozen --no-sync pytest # plus the failing test ids above" + echo "git checkout -- pyproject.toml uv.lock # undo the group strip and the relock when done" echo '```' echo echo "To confirm a suspect, put just that package back and re-run: \`uv lock -P '=='\`, then sync and test again." @@ -175,18 +206,11 @@ esac echo "3. Only if users are broken *and* a real fix will take more than a day or two: add a temporary ceiling (\`Generated by .github/workflows/dependency-canary.yml — adjust the workflow, not this text." } >"$body" -# GitHub caps issue bodies at 65536 characters; keep headroom. -if [ "$(wc -c <"$body")" -gt 60000 ]; then - head -c 59000 "$body" >"$body.tmp" - printf '\n\n…(report truncated; full version in the workflow run summary)\n' >>"$body.tmp" - mv "$body.tmp" "$body" -fi - if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then { echo "## $title" @@ -194,7 +218,14 @@ if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then cat "$body" } >>"$GITHUB_STEP_SUMMARY" fi -echo "canary: overall=$overall p0=$p0 cells=${cells[*]:-none} statuses=${statuses[*]:-none}" +# Issue bodies cap at 65536 characters. Cells bound their own sections, so this is a +# safety net; if it ever fires, the step summary above still has everything. +if [ "$(wc -c <"$body")" -gt 60000 ]; then + head -c 59000 "$body" >"$body.tmp" + printf '\n```\n\n…(truncated — the complete report is in the workflow run summary: %s)\n' "$run_url" >>"$body.tmp" + mv "$body.tmp" "$body" +fi +echo "canary: overall=$overall p0=$p0 resolve=$resolve_result test=$test_result cells=${cells[*]:-none} statuses=${statuses[*]:-none}" echo "canary: title: $title" # ---- tracking issue ------------------------------------------------------ @@ -206,18 +237,39 @@ fi repo="${GITHUB_REPOSITORY:?}" # The label plus bot authorship is the identity of the tracking issue; the title is free to change. -bot_issues="repos/$repo/issues?labels=$label&creator=github-actions%5Bbot%5D&per_page=10" +bot_issues="repos/$repo/issues?labels=$label&creator=github-actions%5Bbot%5D&per_page=20" open_issue=$(gh api "$bot_issues&state=open" --jq '[.[] | select(has("pull_request") | not)][0].number // empty') -if [ "$overall" = "green" ]; then - if [ -n "$open_issue" ]; then - gh issue comment "$open_issue" --repo "$repo" --body "Newest allowed dependency versions pass again as of $today ([run]($run_url)). Closing." - gh issue close "$open_issue" --repo "$repo" --reason completed - echo "canary: closed #$open_issue" - else - echo "canary: green and no open issue; nothing to do." - fi - exit 0 +case "$overall" in + green) + if [ -n "$open_issue" ]; then + gh issue comment "$open_issue" --repo "$repo" --body "Newest allowed dependency versions pass again as of $today ([run]($run_url)). Closing." + gh issue close "$open_issue" --repo "$repo" --reason completed + echo "canary: closed #$open_issue" + else + echo "canary: green and no open issue; nothing to do." + fi + exit 0 + ;; + infra) + # Never let a broken canary rewrite (or close) a live incident; just say so. + if [ -n "$open_issue" ]; then + gh issue comment "$open_issue" --repo "$repo" --body "The canary could not run to completion on $today (resolve: $resolve_result, test: $test_result, cells without a result: $n_missing/$n_cells) — [run]($run_url). Leaving this issue as it was." + echo "canary: infra week; commented on #$open_issue" + exit 0 + fi + ;; +esac + +# Most recent earlier incident, for context (issues list newest-created first). +previous=$(gh api "$bot_issues&state=closed" --jq '[.[] | select(has("pull_request") | not)][0] | if . then "Previous incident: #\(.number) (closed \(.closed_at[:10]))." else empty end') +if [ -n "$previous" ]; then + { + echo "$previous" + echo + cat "$body" + } >"$body.tmp" + mv "$body.tmp" "$body" fi if [ -n "$open_issue" ]; then @@ -228,18 +280,11 @@ if [ -n "$open_issue" ]; then exit 0 fi -gh label create "$label" --repo "$repo" --force --color FBCA04 \ - --description "Filed by the weekly newest-dependencies canary (.github/workflows/dependency-canary.yml)" -previous=$(gh api "$bot_issues&state=closed&sort=updated" --jq '[.[] | select(has("pull_request") | not)][0] | if . then "Previous incident: #\(.number) (closed \(.closed_at[:10]))." else empty end') -if [ -n "$previous" ]; then - { - echo "$previous" - echo - cat "$body" - } >"$body.tmp" - mv "$body.tmp" "$body" -fi -labels=(--label "$label" --label dependencies) +# Create the label on first use; an existing (possibly customised) one is left alone. +gh label create "$label" --repo "$repo" --color FBCA04 \ + --description "Filed by the weekly newest-dependencies canary (.github/workflows/dependency-canary.yml)" 2>/dev/null || true +labels=(--label "$label") +if [ "$overall" != "infra" ]; then labels+=(--label dependencies); fi if [ "$p0" = "true" ]; then labels+=(--label P0); fi assignees=() if [ -n "${CANARY_ASSIGNEES:-}" ]; then assignees=(--assignee "$CANARY_ASSIGNEES"); fi From ea01985871516331960222cc9c4171d885e83111 Mon Sep 17 00:00:00 2001 From: Max Isbey <224885523+maxisbey@users.noreply.github.com> Date: Sun, 16 Aug 2026 13:13:00 +0000 Subject: [PATCH 5/5] ci: drop the canary's temporary branch trigger again Second end-to-end branch run (31949021363) is green with the revised layout. No-Verification-Needed: CI-only change (workflow trigger) --- .github/workflows/dependency-canary.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/dependency-canary.yml b/.github/workflows/dependency-canary.yml index ddf5717d71..c2418f4aef 100644 --- a/.github/workflows/dependency-canary.yml +++ b/.github/workflows/dependency-canary.yml @@ -51,10 +51,6 @@ on: description: "Create/update/close the tracking issue exactly as a scheduled run would" type: boolean default: false - # TEMPORARY while this workflow is under review: exercise it on the PR branch. - # Report-only (push runs never touch issues). Remove before merging. - push: - branches: ["ci/dependency-canary"] permissions: {}