diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 65b014b631..b365d41423 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -44,7 +44,7 @@ jobs: - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: enable-cache: true - version: 0.9.5 + version: 0.11.33 - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: 24 @@ -90,7 +90,7 @@ jobs: - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: enable-cache: true - version: 0.9.5 + version: 0.11.33 - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: 24 diff --git a/.github/workflows/dependency-canary.yml b/.github/workflows/dependency-canary.yml new file mode 100644 index 0000000000..c2418f4aef --- /dev/null +++ b/.github/workflows/dependency-canary.yml @@ -0,0 +1,342 @@ +name: Dependency canary + +# Weekly: re-resolve the runtime dependencies of `mcp[cli,rich]` to the newest +# versions our (floors-only) specifiers allow, ignoring uv.lock, run the test +# suite against them, and keep ONE tracking issue in sync with the result — +# opened (and assigned) when newest-allowed breaks, refreshed weekly while it +# stays broken, closed automatically once it passes again. +# +# Why this exists: users who `pip install mcp` get the newest release of every +# dependency the day it ships, while PR CI only ever sees uv.lock (`locked`) and +# the floors (`lowest-direct`). This is deliberately NOT part of PR CI, so an +# upstream release can never turn an unrelated PR red; the price is up to a +# week of latency, which the incident history says is fine. +# +# What it does not do, on purpose: +# - open a PR adding a ceiling. A cap only helps once released, resolvers +# route around retroactive caps by picking an older uncapped mcp, and the +# bot cannot tell which package (or interaction) is at fault. The issue's +# "What to do" section is the runbook; a human decides. +# - float test tooling (pytest, ruff, pyright, coverage, ...). Only the +# runtime closure is upgraded; everything else keeps its uv.lock version as +# a preference and moves only when a floated runtime dependency forces it +# (the report tags such rows "tooling"). Dependabot owns moving tooling. +# - test pre-releases on the schedule. `workflow_dispatch` with +# `prerelease: true` does that on demand and never files an issue. +# - bisect. The issue lists what changed since the last green run (usually +# one to three packages) and the one-line command to pin a suspect back. +# +# Known blind spots: Python 3.11-3.13 and macOS are not run; runtime deps that a +# *dev* dependency caps (e.g. logfire pins opentelemetry-sdk, which pins +# opentelemetry-api) cannot reach their newest release here — the issue's "Not +# tested at their newest release" section lists them each run. Dependency +# groups that `uv sync` does not install (translate, codegen) are stripped +# before resolving so their caps (anthropic: pydantic<3) do not apply. +# +# Notifications: assignees get the issue traffic. GitHub additionally e-mails +# scheduled-run failures only to whoever last edited the `cron:` line below. +# The P0 label is added automatically when every cell hard-fails and is never +# removed automatically; de-escalation is a human call. + +on: + schedule: + - cron: "23 5 * * 1" # Mondays 05:23 UTC + workflow_dispatch: + inputs: + prerelease: + description: "Also consider pre-releases (investigative run; never files an issue)" + type: boolean + default: false + file-issue: + description: "Create/update/close the tracking issue exactly as a scheduled run would" + type: boolean + default: false + +permissions: {} + +concurrency: + # One canary at a time, and never cancel one in flight: a half-finished run + # must not be what decides the tracking issue's fate. Runs are short; queue. + group: ${{ github.workflow }} + cancel-in-progress: false + +defaults: + run: + shell: bash # -eo pipefail everywhere, and Git-Bash on the Windows cell + +env: + COLUMNS: 150 + UV_VERSION: "0.11.33" + # Releases younger than this are invisible to the run: skips half-uploaded + # releases (the ruff 0.14.12 incident that got the per-PR "highest" leg + # removed in #1869) and same-day yanks. A weekly job loses nothing by it. + CANARY_LAG: "24 hours" + # Single source of truth for the test matrix; the report checks every one of + # these produced a result. Oldest and newest supported Python bracket the + # marker forks in the lock (deps drop 3.10 first; 3.14 gets wheels last), and + # Windows/newest is where a fresh release most often lacks a wheel. + CANARY_CELLS: >- + [ + {"cell": "ubuntu-3.10", "os": "ubuntu-latest", "python": "3.10", "smoke": "", "pyright": ""}, + {"cell": "ubuntu-3.14", "os": "ubuntu-latest", "python": "3.14", "smoke": "1", "pyright": "1"}, + {"cell": "windows-3.14", "os": "windows-latest", "python": "3.14", "smoke": "", "pyright": ""} + ] + CANARY_LABEL: dependency-canary + CANARY_ASSIGNEES: "maxisbey,Kludex" + +jobs: + resolve: + # Don't run the schedule on forks. + if: github.repository == 'modelcontextprotocol/python-sdk' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + actions: read # find and download the last green scheduled run's resolution to diff against + outputs: + cells: ${{ steps.plan.outputs.cells }} + steps: + - name: Publish the test matrix + id: plan + run: | + { + echo 'cells<>"$GITHUB_OUTPUT" + + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + # setup-uv's manifest fetch is a single request with a hard 5s timeout + # (astral-sh/setup-uv#869); retry once. Drop when upstream adds a retry. + - name: Install uv + id: setup-uv + continue-on-error: true + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + with: + enable-cache: false + version: ${{ env.UV_VERSION }} + + - name: Install uv (retry) + if: steps.setup-uv.outcome == 'failure' + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + with: + enable-cache: false + version: ${{ env.UV_VERSION }} + + - name: Fetch the last green run's resolution + env: + GH_TOKEN: ${{ github.token }} + run: | + mkdir -p out/canary-resolve + cd out/canary-resolve + date -u -d "-$CANARY_LAG" +%Y-%m-%dT%H:%M:%SZ >cutoff.txt + (uv self version --short 2>/dev/null || uv --version | awk '{print $2}') >uv-version.txt + : >baseline.txt + # Baseline = the uv.lock the last successful *scheduled* run uploaded, so the + # report can list only what moved since then. Best effort (artifacts expire). + gh run list --repo "$GITHUB_REPOSITORY" --workflow dependency-canary.yml --branch main \ + --event schedule --status success --limit 1 --json databaseId,startedAt,url >last-green.json || echo '[]' >last-green.json + run_id=$(jq -r '.[0].databaseId // empty' last-green.json) + if [ -n "$run_id" ] && gh run download "$run_id" --repo "$GITHUB_REPOSITORY" -n canary-resolve -D baseline-dl; then + if [ -f baseline-dl/canary-resolve/uv.lock ]; then + cp baseline-dl/canary-resolve/uv.lock baseline.lock + jq -r '.[0] | "[\(.startedAt[:10]) run](\(.url))"' last-green.json >baseline.txt + fi + fi + rm -rf baseline-dl + echo "cutoff=$(cat cutoff.txt) uv=$(cat uv-version.txt) baseline=$(cat baseline.txt)" + + - name: Work out what to float + run: | + r=out/canary-resolve + # The runtime closure of mcp[cli,rich] as currently locked: exactly the set + # `pip install "mcp[cli,rich]"` pulls in. New transitive deps that a newer + # release introduces have no lock entry and so resolve to newest anyway. + uv export --frozen --no-default-groups --all-extras --no-emit-workspace \ + --no-hashes --no-header --no-annotate | grep -E '^[A-Za-z0-9]' | sed -E 's/[=; @[].*//' | sort -u >$r/closure.txt + if [ ! -s $r/closure.txt ]; then echo "::error::uv export produced an empty runtime closure"; exit 1; fi + echo "Floating $(wc -l <$r/closure.txt) packages:"; tr '\n' ' ' <$r/closure.txt; echo + # Dependency groups that `uv sync` does not install still constrain the + # resolution (uv.lock is universal). Strip them so e.g. the translate + # group's `anthropic` cannot hold pydantic below a new major. + python3 - <<'EOF' >$r/strip.sh + import re, tomllib + project = tomllib.load(open("pyproject.toml", "rb")) + groups = project.get("dependency-groups", {}) + default = project.get("tool", {}).get("uv", {}).get("default-groups", ["dev"]) + keep = set(groups) if default == "all" else set(default) + todo = list(keep) + while todo: # groups pulled in via {include-group = "..."} are installed too + for entry in groups.get(todo.pop(), []): + if isinstance(entry, dict) and entry.get("include-group") not in keep | {None}: + keep.add(entry["include-group"]) + todo.append(entry["include-group"]) + for group, entries in groups.items(): + names = [re.match(r"[A-Za-z0-9._-]+", e).group(0) for e in entries if isinstance(e, str)] + if group not in keep and names: + print("uv remove --frozen --group", group, *names) + EOF + cat $r/strip.sh + bash -e $r/strip.sh + + - name: Resolve newest allowed versions + env: + PRERELEASE: ${{ inputs.prerelease && 'allow' || '' }} + run: | + r=out/canary-resolve + cutoff=$(cat $r/cutoff.txt) + cp uv.lock $r/committed.lock + # Per-package cutoffs rather than a global --exclude-newer, so an exact pin + # elsewhere (docs group, build constraints) bumped the day before the run + # cannot make the resolution fail. + args=() + if [ -n "$PRERELEASE" ]; then args+=(--prerelease "$PRERELEASE"); fi + while read -r pkg; do args+=(-P "$pkg" --exclude-newer-package "$pkg=$cutoff"); done <$r/closure.txt + { printf 'uv lock'; printf ' %q' "${args[@]}"; echo; } >$r/lock-cmd.sh + if uv lock "${args[@]}" 2>&1 | tee $r/lock.log; then + echo ok >$r/lock-status + cp uv.lock $r/uv.lock + else + # Distinguishes "uv ran and could not resolve" from every other way this job can fail. + echo failed >$r/lock-status + exit 1 + fi + + - name: Summarise what moved + run: | + r=out/canary-resolve + python3 scripts/ci/canary_lock_diff.py $r/committed.lock uv.lock \ + --old-label "uv.lock" --new-label "this run" --suspects $r/suspects-vs-lock.txt >$r/vs-lock.md + if [ -f $r/baseline.lock ]; then + python3 scripts/ci/canary_lock_diff.py $r/baseline.lock uv.lock \ + --old-label "last green" --new-label "this run" --suspects $r/suspects-since-green.txt >$r/since-green.md + fi + # Direct runtime deps that could not reach their newest release (capped by something else in the resolution). + uv tree --frozen --outdated --universal --depth 1 --package mcp >$r/tree.txt + { + echo "| Package | Resolved | Latest |" + echo "| --- | --- | --- |" + sed -nE 's/^[^A-Za-z0-9]*([A-Za-z0-9._-]+)(\[[^]]*\])? v([^ ]+) .*\(latest: v([^)]+)\)$/\1 \3 \4/p' $r/tree.txt | + sort -u | while read -r name have latest; do + if grep -qxF "$name" $r/closure.txt; then echo "| $name | $have | $latest |"; fi + done + } >$r/held-back.md + if [ "$(wc -l <$r/held-back.md)" -le 2 ]; then : >$r/held-back.md; fi + { + echo "## Resolution (cutoff $(cat $r/cutoff.txt))" + echo + if [ -s $r/since-green.md ]; then echo "### Since last green ($(cat $r/baseline.txt))"; cat $r/since-green.md; echo; fi + echo "### vs uv.lock"; cat $r/vs-lock.md; echo + if [ -s $r/held-back.md ]; then echo "### Held back below latest"; cat $r/held-back.md; fi + } >>"$GITHUB_STEP_SUMMARY" + + - name: Upload resolution + if: ${{ !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: canary-resolve + path: out/ + retention-days: 90 + if-no-files-found: error + + test: + name: test (${{ matrix.cell }}) + needs: resolve + runs-on: ${{ matrix.os }} + timeout-minutes: 25 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.resolve.outputs.cells) }} + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Install uv + id: setup-uv + continue-on-error: true + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + with: + enable-cache: false + version: ${{ env.UV_VERSION }} + + - name: Install uv (retry) + if: steps.setup-uv.outcome == 'failure' + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + with: + enable-cache: false + version: ${{ env.UV_VERSION }} + + - name: Fetch the resolved lock + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: canary-resolve + path: . + + - name: Install and test + # The suite takes ~2 min; a dependency that makes it hang should surface as + # this cell's "incomplete" status in the report, not as a 25-minute job kill. + timeout-minutes: 15 + env: + CANARY_CELL: ${{ matrix.cell }} + CANARY_PYTHON: ${{ matrix.python }} + CANARY_PYRIGHT: ${{ matrix.pyright }} + # Same switches as PR CI: real stdio/uvicorn subprocess smoke tests on one + # cell, and PEP 597 EncodingWarnings surfaced (as an env var so xdist workers inherit it). + MCP_EXAMPLES_SMOKE: ${{ matrix.smoke }} + PYTHONWARNDEFAULTENCODING: "1" + run: | + cp canary-resolve/uv.lock uv.lock + bash scripts/ci/canary_cell.sh + + - name: Upload cell result + if: ${{ !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: canary-cell-${{ matrix.cell }} + path: out/ + retention-days: 30 + if-no-files-found: error + + report: + needs: [resolve, test] + # !cancelled(): a red resolve/test job is exactly when this must run, but a + # cancelled run must never decide the issue's fate from partial results. + if: ${{ !cancelled() && github.repository == 'modelcontextprotocol/python-sdk' }} + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read # checkout, for scripts/ci/canary_report.sh + issues: write # open / refresh / close the tracking issue + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + sparse-checkout: scripts/ci + + - name: Collect results + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: canary-* + # Every artifact already carries its own top-level directory, so merging + # yields artifacts/canary-resolve/ and artifacts/canary-cell-*/ no matter + # how many artifacts exist (a lone match would otherwise be flattened). + merge-multiple: true + path: artifacts + + - name: Report + env: + GH_TOKEN: ${{ github.token }} + CANARY_ARTIFACTS: artifacts + CANARY_RESOLVE_RESULT: ${{ needs.resolve.result }} + CANARY_TEST_RESULT: ${{ needs.test.result }} + CANARY_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + # Only the schedule (or an explicit dispatch asking for it) touches issues, and never a pre-release run. + CANARY_FILE_ISSUES: ${{ !inputs.prerelease && (github.event_name == 'schedule' || inputs.file-issue) && 'true' || 'false' }} + run: bash scripts/ci/canary_report.sh diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index 28ae351feb..37ca3e0fab 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -50,7 +50,7 @@ jobs: uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: enable-cache: true - version: 0.9.5 + version: 0.11.33 - name: Build combined docs (main at / and /v2/, v1.x at /v1/) run: bash scripts/build-docs.sh site diff --git a/.github/workflows/docs-preview.yml b/.github/workflows/docs-preview.yml index 278aa4423d..18729c69f3 100644 --- a/.github/workflows/docs-preview.yml +++ b/.github/workflows/docs-preview.yml @@ -129,7 +129,7 @@ jobs: # a cache populated while untrusted PR code ran would let it poison # later trusted workflows. Mirrors publish-pypi.yml. enable-cache: false - version: 0.9.5 + version: 0.11.33 # pull_request_target runs this workflow file from the base branch, so # the whole recipe — dependency sync included — must come from the diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index 41b127f923..7a43981dd9 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -21,7 +21,7 @@ jobs: uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: enable-cache: false - version: 0.9.5 + version: 0.11.33 - name: Set up Python 3.12 run: uv python install 3.12 diff --git a/.github/workflows/shared.yml b/.github/workflows/shared.yml index 85023dbfac..b602fa9d93 100644 --- a/.github/workflows/shared.yml +++ b/.github/workflows/shared.yml @@ -25,14 +25,14 @@ jobs: uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: enable-cache: true - version: 0.9.5 + version: 0.11.33 - name: Install uv (retry) if: steps.setup-uv.outcome == 'failure' uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: enable-cache: true - version: 0.9.5 + version: 0.11.33 - name: Install dependencies run: uv sync --frozen --all-extras --python 3.10 @@ -81,14 +81,14 @@ jobs: uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: enable-cache: true - version: 0.9.5 + version: 0.11.33 - name: Install uv (retry) if: steps.setup-uv.outcome == 'failure' uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: enable-cache: true - version: 0.9.5 + version: 0.11.33 - name: Install the project run: uv sync ${{ matrix.dep-resolution.install-flags }} --all-extras --python ${{ matrix.python-version }} @@ -127,14 +127,14 @@ jobs: uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: enable-cache: true - version: 0.9.5 + version: 0.11.33 - name: Install uv (retry) if: steps.setup-uv.outcome == 'failure' uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: enable-cache: true - version: 0.9.5 + version: 0.11.33 - name: Install dependencies run: uv sync --frozen --all-extras --python 3.10 @@ -167,14 +167,14 @@ jobs: uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: enable-cache: true - version: 0.9.5 + version: 0.11.33 - name: Install uv (retry) if: steps.setup-uv.outcome == 'failure' uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: enable-cache: true - version: 0.9.5 + version: 0.11.33 - name: Build the docs in strict mode run: bash scripts/docs/build.sh diff --git a/.gitignore b/.gitignore index a63ec932ca..4c4d092fc2 100644 --- a/.gitignore +++ b/.gitignore @@ -183,3 +183,8 @@ results/ # conformance CI local runs conformance-results/ + +# dependency canary local runs (scripts/ci/canary_*.sh) +canary-out/ +canary-resolve/ +canary-body.md diff --git a/scripts/ci/canary_cell.sh b/scripts/ci/canary_cell.sh new file mode 100755 index 0000000000..fd64d6e9b2 --- /dev/null +++ b/scripts/ci/canary_cell.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# One dependency-canary test cell: install the resolved lock, run the suite, +# and classify the outcome so the report can tell a hard break from a new +# deprecation warning or a flake. Driven by .github/workflows/dependency-canary.yml. +# +# Inputs (env): CANARY_CELL (label, e.g. "ubuntu-3.14"), CANARY_PYTHON (e.g. "3.14"), +# CANARY_PYRIGHT=1 to append an informational `pyright src/mcp` result. +# Outputs under out/canary-cell-$CANARY_CELL/: +# status — pass | flaky | warnings-only | error | install-failed +# ("incomplete" is written first, so a killed cell still says something) +# cell.md — Markdown section for the issue body / job summary (bounded to ~20 kB) +# *.log — raw logs, kept in the artifact for debugging +# Exit status: 0 for pass/flaky, 1 otherwise (so the job shows red). + +set -uo pipefail + +cell="${CANARY_CELL:?}" +python="${CANARY_PYTHON:?}" +out="out/canary-cell-$cell" +mkdir -p "$out" +md="$out/cell.md" +echo incomplete >"$out/status" +printf '#### %s\n\nDid not finish (see the workflow run).\n' "$cell" >"$md" +export COLUMNS=200 # keeps pytest from truncating the -r summary lines quoted in the report + +pytest_cmd=(uv run --frozen --no-sync pytest -p no:pretty -q --no-header -rfE --color=no -o log_cli=false) +# Demote only the deprecation family: if failures vanish under these, the newest +# versions still work and merely announce a future removal. +demote=(-W default::DeprecationWarning -W default::PendingDeprecationWarning -W default::FutureWarning) + +summary_of() { # the short-summary FAILED/ERROR lines of a pytest log, capped; falls back to the log tail + local lines + lines=$(grep -E '^(FAILED|ERROR) ' "$1" | awk '!seen[$0]++' | cut -c1-240) # xdist repeats collection errors per worker + if [ -z "$lines" ]; then + tail -n 15 "$1" | cut -c1-240 + return + fi + head -n 40 <<<"$lines" + local n + n=$(wc -l <<<"$lines") + if [ "$n" -gt 40 ]; then echo "... and $((n - 40)) more"; fi +} + +details_of() { # collapsed, size-bounded tail of a log: file, summary text, max lines + printf '
%s\n\n```text\n' "$2" + tail -n "${3:-60}" "$1" | cut -c1-240 | head -c 12000 | sed 's/```/` ` `/g' + printf '\n```\n\n
\n' +} + +finish() { + echo "$1" >"$out/status" + echo "canary cell $cell: $1" + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then cat "$md" >>"$GITHUB_STEP_SUMMARY"; fi + case "$1" in pass | flaky) exit 0 ;; *) exit 1 ;; esac +} + +printf '#### %s\n\n' "$cell" >"$md" + +if ! uv sync --frozen --all-extras --python "$python" >"$out/sync.log" 2>&1; then + { + echo "**Install failed** — \`uv sync --frozen --all-extras --python $python\` could not install the resolved set on this platform." + echo + details_of "$out/sync.log" "uv sync output" 40 + } >>"$md" + finish install-failed +fi +installed=$(uv run --frozen --no-sync python -V 2>/dev/null || echo "Python $python") + +# Phase 1: the suite as CI runs it (from a clean last-failed record, which phase 2 keys on). +rm -f .pytest_cache/v/cache/lastfailed +if "${pytest_cmd[@]}" -n auto >"$out/run1.log" 2>&1; then + echo "All tests pass ($installed)." >>"$md" + status=pass +else + # Phase 2: again, to drop flakes. Narrow to the recorded failures (serially) when + # pytest got far enough to record any; a collection/usage error leaves none. + if grep -qs '::' .pytest_cache/v/cache/lastfailed; then + rerun=("${pytest_cmd[@]}" --lf --last-failed-no-failures none -p no:xdist) + rerun_desc="a serial re-run of just the failing tests" + else + rerun=("${pytest_cmd[@]}" -n auto) + rerun_desc="a full re-run" + fi + if "${rerun[@]}" >"$out/run2.log" 2>&1; then + { + echo "Passed on $rerun_desc ($installed); the first attempt failed as below (treated as flaky, not reported):" + echo + echo '```text' + summary_of "$out/run1.log" + echo '```' + } >>"$md" + status=flaky + else + # Phase 3: same again with the deprecation family demoted, to tell "broken" from "deprecated". + if "${rerun[@]}" "${demote[@]}" >"$out/run3.log" 2>&1; then + { + echo "**Deprecation warnings only** ($installed) — the failures below persist on $rerun_desc but disappear once" + echo "\`DeprecationWarning\`/\`PendingDeprecationWarning\`/\`FutureWarning\` are not errors, so the newest versions still work and are announcing a removal we need to get ahead of." + } >>"$md" + status=warnings-only + else + echo "**Hard failures** ($installed) — the failures below persist on $rerun_desc and with deprecation warnings demoted:" >>"$md" + status=error + fi + { + echo + echo '```text' + summary_of "$out/run2.log" + echo '```' + echo + details_of "$out/run2.log" "pytest output (tail)" 80 + } >>"$md" + fi +fi + +if [ "${CANARY_PYRIGHT:-}" = "1" ]; then + # Typing-only drift (e.g. a dependency tightening a signature) never fails the cell; it is context for the reader. + if PYRIGHT_PYTHON_IGNORE_WARNINGS=1 uv run --frozen --no-sync pyright src/mcp >"$out/pyright.log" 2>&1; then + printf '\npyright on `src/mcp` against these versions: clean (informational).\n' >>"$md" + else + count=$(grep -Eo '^[0-9]+ errors?' "$out/pyright.log" | tail -n1) + { + echo + details_of "$out/pyright.log" "pyright on src/mcp against these versions: ${count:-did not complete} (informational, never filed on its own)" 30 + } >>"$md" + fi +fi + +finish "$status" diff --git a/scripts/ci/canary_lock_diff.py b/scripts/ci/canary_lock_diff.py new file mode 100644 index 0000000000..5964ecd84b --- /dev/null +++ b/scripts/ci/canary_lock_diff.py @@ -0,0 +1,113 @@ +"""Diff two uv.lock files for the dependency canary report. + +Usage: python scripts/ci/canary_lock_diff.py OLD NEW [--old-label L] [--new-label L] [--suspects FILE] + +Prints a Markdown table of every package whose locked version(s) differ between +OLD and NEW, tagged by its role relative to the root `mcp` package: a direct +runtime dependency, a transitive runtime dependency (reachable from `mcp` with +all extras, ignoring markers), or tooling (only reachable through a dependency +group). Prints nothing when the two locks agree. With --suspects, also writes a +one-line summary of the changed runtime dependencies (direct first) to FILE, +for use in an issue title. +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path +from typing import Any + +import tomllib + +Package = dict[str, Any] + +ROOT = "mcp" + + +def load(path: Path) -> list[Package]: + return tomllib.loads(path.read_text(encoding="utf-8")).get("package", []) + + +def versions(packages: list[Package]) -> dict[str, list[str]]: + """name -> sorted distinct versions (a name can be locked more than once across marker forks).""" + out: dict[str, set[str]] = {} + for pkg in packages: + if "version" in pkg and "editable" not in pkg.get("source", {}) and "virtual" not in pkg.get("source", {}): + out.setdefault(pkg["name"], set()).add(pkg["version"]) + return {name: sorted(vs) for name, vs in out.items()} + + +def runtime_roles(packages: list[Package]) -> tuple[set[str], set[str]]: + """(direct, closure): names mcp depends on directly (any extra), and everything reachable from them.""" + by_name: dict[str, list[Package]] = {} + for pkg in packages: + by_name.setdefault(pkg["name"], []).append(pkg) + + def edges(name: str, extras: frozenset[str]) -> list[tuple[str, frozenset[str]]]: + found: list[tuple[str, frozenset[str]]] = [] + for pkg in by_name.get(name, []): + deps = list(pkg.get("dependencies", [])) + for extra in extras: + deps += pkg.get("optional-dependencies", {}).get(extra, []) + found += [(d["name"], frozenset(d.get("extra", []))) for d in deps] + return found + + root_extras = frozenset().union(*(pkg.get("optional-dependencies", {}).keys() for pkg in by_name.get(ROOT, []))) + direct = {name for name, _ in edges(ROOT, root_extras)} + closure: set[str] = set() + seen: set[tuple[str, frozenset[str]]] = set() + todo = [(ROOT, root_extras)] + while todo: + node = todo.pop() + if node in seen: + continue + seen.add(node) + closure.add(node[0]) + todo += edges(*node) + closure.discard(ROOT) + return direct, closure + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("old", type=Path) + parser.add_argument("new", type=Path) + parser.add_argument("--old-label", default="before") + parser.add_argument("--new-label", default="after") + parser.add_argument("--suspects", type=Path, help="write a one-line title summary of changed runtime deps here") + args = parser.parse_args() + + old_packages, new_packages = load(args.old), load(args.new) + old, new = versions(old_packages), versions(new_packages) + # Union of both locks' graphs, so a dependency dropped by the new resolution keeps its old role. + direct, closure = (a | b for a, b in zip(runtime_roles(old_packages), runtime_roles(new_packages))) + + def role(name: str) -> tuple[int, str]: + if name in direct: + return 0, "runtime (direct)" + if name in closure: + return 1, "runtime (transitive)" + return 2, "tooling" + + changed = sorted((role(n), n) for n in old.keys() | new.keys() if old.get(n) != new.get(n)) + # Tooling that merely appears or disappears (e.g. a dependency group stripped before resolving) is noise here. + changed = [c for c in changed if c[0][0] < 2 or (c[1] in old and c[1] in new)] + if changed: + print(f"| Package | {args.old_label} | {args.new_label} | Role |") + print("| --- | --- | --- | --- |") + for (_, label), name in changed: + before = ", ".join(old.get(name, [])) or "(absent)" + after = ", ".join(new.get(name, [])) or "(removed)" + print(f"| {name} | {before} | {after} | {label} |") + + if args.suspects: + runtime = [f"{n} {new[n][-1]}" for (rank, _), n in changed if rank < 2 and n in new] + summary = ", ".join(runtime[:3]) + (f" (+{len(runtime) - 3} more)" if len(runtime) > 3 else "") + # Empty file (not a blank line) when no runtime dependency changed, so `[ -s FILE ]` means what it says. + args.suspects.write_text(summary + "\n" if summary else "", encoding="utf-8") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci/canary_report.sh b/scripts/ci/canary_report.sh new file mode 100755 index 0000000000..2bdf7aeb1c --- /dev/null +++ b/scripts/ci/canary_report.sh @@ -0,0 +1,292 @@ +#!/usr/bin/env bash +# Dependency-canary reporter: folds the resolve + per-cell artifacts into one +# Markdown report, then keeps a single tracking issue in sync with it (open on +# red, refresh while red, close on green). Driven by .github/workflows/dependency-canary.yml. +# +# Inputs (env): +# CANARY_ARTIFACTS directory holding canary-resolve/ and canary-cell-/ (merged artifacts) +# CANARY_CELLS JSON list of the matrix cells that were supposed to run ({"cell": ...} objects) +# CANARY_RESOLVE_RESULT result of the resolve job (success|failure|cancelled|skipped) +# CANARY_TEST_RESULT result of the test job as a whole +# CANARY_FILE_ISSUES "true" to create/update/close the tracking issue; anything else = report only +# CANARY_LABEL label that identifies the tracking issue (created if missing) +# CANARY_ASSIGNEES comma-separated logins assigned when an issue is opened +# CANARY_RUN_URL link to this workflow run +# GH_TOKEN, GITHUB_REPOSITORY, GITHUB_STEP_SUMMARY (standard) +# +# Classification, in order: +# resolve job not successful -> resolve-failed if `uv lock` itself failed, else infra +# otherwise per expected cell -> its status file, or "missing" if it never reported +# any error -> error (users are broken today) +# any install-failed -> install-failed +# any warnings-only -> warnings-only (a deprecation, nothing broken yet) +# any missing -> infra (the canary, not the deps, had a problem) +# else -> green +# Only green closes the issue; infra never rewrites an open incident (comment only); +# P0 is added when every expected cell is error/install-failed and never removed here. + +set -euo pipefail + +artifacts="${CANARY_ARTIFACTS:?}" +resolve_dir="$artifacts/canary-resolve" +label="${CANARY_LABEL:?}" +run_url="${CANARY_RUN_URL:?}" +today=$(date -u +%Y-%m-%d) +body=canary-body.md + +read_or() { if [ -s "$1" ]; then cat "$1"; else printf '%s' "$2"; fi; } + +# ---- classify ------------------------------------------------------------- + +mapfile -t cells < <(jq -r '.[].cell' <<<"${CANARY_CELLS:?}") +statuses=() +for cell in "${cells[@]}"; do + f="$artifacts/canary-cell-$cell/status" + s="" + if [ -f "$f" ]; then s=$(tr -d '[:space:]' <"$f"); fi + case "$s" in + pass | flaky | warnings-only | error | install-failed) ;; + *) s=missing ;; # never reported, or died part-way ("incomplete") + esac + statuses+=("$s") +done + +count() { + local n=0 s + for s in "${statuses[@]}"; do if [ "$s" = "$1" ]; then n=$((n + 1)); fi; done + echo "$n" +} +n_cells=${#cells[@]} +n_error=$(count error) +n_install=$(count install-failed) +n_warn=$(count warnings-only) +n_missing=$(count missing) + +resolve_result="${CANARY_RESOLVE_RESULT:-unknown}" +test_result="${CANARY_TEST_RESULT:-unknown}" +if [ "$resolve_result" != "success" ]; then + if [ "$(read_or "$resolve_dir/lock-status" "")" = "failed" ]; then overall=resolve-failed; else overall=infra; fi +elif [ "$n_error" -gt 0 ]; then + overall=error +elif [ "$n_install" -gt 0 ]; then + overall=install-failed +elif [ "$n_warn" -gt 0 ]; then + overall=warnings-only +elif [ "$n_missing" -gt 0 ] || [ "$test_result" != "success" ]; then + overall=infra +else + overall=green +fi +p0=false +if [ "$n_cells" -gt 0 ] && [ $((n_error + n_install)) -eq "$n_cells" ]; then p0=true; fi + +cutoff=$(read_or "$resolve_dir/cutoff.txt" "unknown") +baseline=$(read_or "$resolve_dir/baseline.txt" "") +uv_version=$(read_or "$resolve_dir/uv-version.txt" "") +uv_version=${uv_version%% *} +if [ -e "$resolve_dir/suspects-since-green.txt" ]; then + suspects=$(tr -d '\n' <"$resolve_dir/suspects-since-green.txt") + [ -n "$suspects" ] || suspects="no dependency changed since the last green run" +else + suspects=$(read_or "$resolve_dir/suspects-vs-lock.txt" "see run" | tr -d '\n') +fi + +case "$overall" in + error) title="Newest dependency versions fail the test suite: $suspects" ;; + install-failed) title="Newest dependency versions fail to install: $suspects" ;; + warnings-only) title="Newest dependency versions raise deprecation warnings: $suspects" ;; + resolve-failed) title="Newest dependency versions could not be resolved" ;; + infra) title="Dependency canary could not run to completion" ;; + green) title="Newest dependency versions pass" ;; +esac +title=${title:0:200} + +# ---- report body --------------------------------------------------------- + +{ + echo "" + case "$overall" in + green) echo "**Status: passing** as of $today." ;; + error) echo "**Status: failing** — hard test failures on $n_error/$n_cells cells as of $today." ;; + install-failed) echo "**Status: failing** — the resolved set does not install on $n_install/$n_cells cells as of $today." ;; + warnings-only) echo "**Status: deprecations** — tests fail only because new deprecation warnings are errors under our pytest config ($n_warn/$n_cells cells) as of $today. Nothing is broken for users yet." ;; + resolve-failed) echo "**Status: unresolvable** — \`uv lock\` could not produce a resolution with mcp's runtime dependencies at their newest allowed versions as of $today (log below; if it shows a network or index error rather than a conflict, this was infrastructure)." ;; + infra) echo "**Status: incomplete** — the canary itself did not run cleanly as of $today (resolve job: $resolve_result, test job: $test_result, cells without a result: $n_missing/$n_cells). This says nothing about the dependencies; see the run." ;; + esac + echo + echo "[Workflow run]($run_url) · newest versions published before \`$cutoff\` (releases younger than a day are skipped) · uv \`${uv_version:-unknown}\`" + echo + echo "This is the weekly dependency canary: it re-resolves the runtime dependencies of \`mcp[cli,rich]\` (direct and transitive) to the newest versions our specifiers allow, ignoring \`uv.lock\`, and runs the test suite against them. Everything else keeps its locked version unless a floated dependency forces it to move (tagged *tooling* below). PR CI never does this, so this issue is the only signal that a new upstream release breaks the SDK for people installing it today." + echo + + if [ "$overall" = "resolve-failed" ]; then + echo "### Resolution failure" + echo + echo '```text' + read_or "$resolve_dir/lock.log" "(no lock.log in the artifact; see the run)" | tail -n 60 | cut -c1-240 + echo '```' + echo + fi + + if [ "$resolve_result" = "success" ]; then + echo "### What changed since the last green run" + echo + if [ -z "$baseline" ]; then + echo "No earlier successful scheduled run to compare against (none yet, or its artifact expired) — use the full diff against \`uv.lock\` below." + elif [ -s "$resolve_dir/since-green.md" ]; then + echo "Compared with what the last green run resolved ($baseline). **Start here** — the culprit is almost always in this table." + echo + cat "$resolve_dir/since-green.md" + else + echo "Nothing in the runtime closure changed since the last green run ($baseline). If tests fail anyway, a change on \`main\` since then is the likely cause — compare with the \`locked\` leg of PR CI." + fi + echo + fi + + if [ "$n_cells" -gt 0 ] && [ "$resolve_result" = "success" ]; then + echo "### Results" + echo + for i in "${!cells[@]}"; do + case "${statuses[$i]}" in + missing) echo "- \`${cells[$i]}\`: **no result** (timed out, cancelled, or lost before reporting; see the run)" ;; + *) echo "- \`${cells[$i]}\`: **${statuses[$i]}**" ;; + esac + done + echo + for cell in "${cells[@]}"; do + f="$artifacts/canary-cell-$cell/cell.md" + if [ -f "$f" ]; then + cat "$f" + echo + fi + done + fi + + if [ -s "$resolve_dir/held-back.md" ]; then + echo "### Not tested at their newest release" + echo + echo "Something else in the resolution caps these runtime dependencies below their latest version, so this run says nothing about the versions listed as latest:" + echo + cat "$resolve_dir/held-back.md" + echo + fi + + if [ -s "$resolve_dir/vs-lock.md" ]; then + n_diff=$(($(wc -l <"$resolve_dir/vs-lock.md") - 2)) + echo "
All differences from the committed uv.lock ($n_diff packages)" + echo + head -n 150 "$resolve_dir/vs-lock.md" + echo + echo "
" + echo + fi + + echo "### Reproduce locally" + echo + echo "Either download the \`canary-resolve\` artifact from the run and copy its \`canary-resolve/uv.lock\` over yours, or redo the resolution exactly as the canary did (dependency groups it does not install are dropped first; the pinned uv gives identical resolver behaviour):" + echo + echo '```bash' + if [ -s "$resolve_dir/strip.sh" ]; then cat "$resolve_dir/strip.sh"; fi + if [ -s "$resolve_dir/lock-cmd.sh" ]; then + sed "s/^uv lock/uvx uv@${uv_version:-latest} lock/" "$resolve_dir/lock-cmd.sh" + else + echo "# (the resolve job did not get as far as recording its uv lock command; see the run)" + fi + echo "uv sync --frozen --all-extras --python 3.14 # or the failing cell's Python" + echo "uv run --frozen --no-sync pytest # plus the failing test ids above" + echo "git checkout -- pyproject.toml uv.lock # undo the group strip and the relock when done" + echo '```' + echo + echo "To confirm a suspect, put just that package back and re-run: \`uv lock -P '=='\`, then sync and test again." + echo + echo "### What to do" + echo + echo "1. **Deprecation warnings only** → not urgent for users; migrate off the deprecated API (supporting both old and new versions) before the removal lands. No ceiling." + echo "2. **Hard failures / install failures** → users running \`pip install mcp\` today get this combination. Prefer a fix that supports both the old and new version of the dependency, and release it." + echo "3. Only if users are broken *and* a real fix will take more than a day or two: add a temporary ceiling (\`Generated by .github/workflows/dependency-canary.yml — adjust the workflow, not this text." +} >"$body" + +if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + { + echo "## $title" + echo + cat "$body" + } >>"$GITHUB_STEP_SUMMARY" +fi +# Issue bodies cap at 65536 characters. Cells bound their own sections, so this is a +# safety net; if it ever fires, the step summary above still has everything. +if [ "$(wc -c <"$body")" -gt 60000 ]; then + head -c 59000 "$body" >"$body.tmp" + printf '\n```\n\n…(truncated — the complete report is in the workflow run summary: %s)\n' "$run_url" >>"$body.tmp" + mv "$body.tmp" "$body" +fi +echo "canary: overall=$overall p0=$p0 resolve=$resolve_result test=$test_result cells=${cells[*]:-none} statuses=${statuses[*]:-none}" +echo "canary: title: $title" + +# ---- tracking issue ------------------------------------------------------ + +if [ "${CANARY_FILE_ISSUES:-}" != "true" ]; then + echo "canary: report-only run; not touching issues." + exit 0 +fi + +repo="${GITHUB_REPOSITORY:?}" +# The label plus bot authorship is the identity of the tracking issue; the title is free to change. +bot_issues="repos/$repo/issues?labels=$label&creator=github-actions%5Bbot%5D&per_page=20" +open_issue=$(gh api "$bot_issues&state=open" --jq '[.[] | select(has("pull_request") | not)][0].number // empty') + +case "$overall" in + green) + if [ -n "$open_issue" ]; then + gh issue comment "$open_issue" --repo "$repo" --body "Newest allowed dependency versions pass again as of $today ([run]($run_url)). Closing." + gh issue close "$open_issue" --repo "$repo" --reason completed + echo "canary: closed #$open_issue" + else + echo "canary: green and no open issue; nothing to do." + fi + exit 0 + ;; + infra) + # Never let a broken canary rewrite (or close) a live incident; just say so. + if [ -n "$open_issue" ]; then + gh issue comment "$open_issue" --repo "$repo" --body "The canary could not run to completion on $today (resolve: $resolve_result, test: $test_result, cells without a result: $n_missing/$n_cells) — [run]($run_url). Leaving this issue as it was." + echo "canary: infra week; commented on #$open_issue" + exit 0 + fi + ;; +esac + +# Most recent earlier incident, for context (issues list newest-created first). +previous=$(gh api "$bot_issues&state=closed" --jq '[.[] | select(has("pull_request") | not)][0] | if . then "Previous incident: #\(.number) (closed \(.closed_at[:10]))." else empty end') +if [ -n "$previous" ]; then + { + echo "$previous" + echo + cat "$body" + } >"$body.tmp" + mv "$body.tmp" "$body" +fi + +if [ -n "$open_issue" ]; then + gh issue edit "$open_issue" --repo "$repo" --title "$title" --body-file "$body" + if [ "$p0" = "true" ]; then gh issue edit "$open_issue" --repo "$repo" --add-label P0; fi + gh issue comment "$open_issue" --repo "$repo" --body "Still red on $today: **$overall** — $suspects ([run]($run_url)). The issue body above now shows this run." + echo "canary: updated #$open_issue" + exit 0 +fi + +# Create the label on first use; an existing (possibly customised) one is left alone. +gh label create "$label" --repo "$repo" --color FBCA04 \ + --description "Filed by the weekly newest-dependencies canary (.github/workflows/dependency-canary.yml)" 2>/dev/null || true +labels=(--label "$label") +if [ "$overall" != "infra" ]; then labels+=(--label dependencies); fi +if [ "$p0" = "true" ]; then labels+=(--label P0); fi +assignees=() +if [ -n "${CANARY_ASSIGNEES:-}" ]; then assignees=(--assignee "$CANARY_ASSIGNEES"); fi +url=$(gh issue create --repo "$repo" --title "$title" --body-file "$body" "${labels[@]}" "${assignees[@]}") +echo "canary: opened $url"