diff --git a/litellm/README.md b/litellm/README.md index 6c3afdc1..a915d2f3 100644 --- a/litellm/README.md +++ b/litellm/README.md @@ -9,6 +9,11 @@ Messages** format (`POST /v1/messages`), but the target models don't: |---|---|---| | Bedrock open-weight (GLM, DeepSeek, Kimi) | Converse | yes | | OpenRouter (any model) | OpenAI Chat Completions | yes | +| OpenAI / Azure-OpenAI direct (GPT / Codex) | OpenAI Chat Completions | yes | + +The last row is how we test **skill-activation reliability on a GPT/Codex model**: +route it through this same Claude Code SDK harness instead of the Codex SDK (which +does not run Claude-Code skills), so the native `Skill` tool fires and can be graded. **LiteLLM is the translation shim.** The SDK is pointed at LiteLLM via `ANTHROPIC_BASE_URL`; LiteLLM translates Anthropic ↔ Converse / OpenAI per model @@ -46,6 +51,7 @@ environment before launching: | `AWS_BEARER_TOKEN_BEDROCK` | Bedrock models | required if you use any `bedrock/*` model | | `AWS_REGION` | Bedrock models | defaults to `eu-north-1` | | `OPENROUTER_API_KEY` | OpenRouter models | required if you use any `openrouter/*` model | +| `CODEX_API_KEY` / `CODEX_BASE_URL` | OpenAI / Azure-OpenAI (GPT / Codex) models | required if you use the `gpt-*` OpenAI-compatible model; base URL is reused verbatim (e.g. Azure's `/openai/v1`) | | `LITELLM_AUTH_TOKEN` | the virtual key clients present | becomes the proxy's `LITELLM_MASTER_KEY`; falls back to `sk-spike-local`. It can be customly designed.| ### Start it diff --git a/litellm/litellm-config.yaml b/litellm/litellm-config.yaml index 5ec10b8e..4f52ffa1 100644 --- a/litellm/litellm-config.yaml +++ b/litellm/litellm-config.yaml @@ -18,8 +18,10 @@ # AWS_REGION=eu-north-1 — EU residency (Stockholm). # LITELLM_MASTER_KEY — the virtual key clients present as LITELLM_AUTH_TOKEN. # -# Run manually: -# uvx --from 'litellm[proxy]' litellm --config litellm/litellm-config.yaml --port 4000 +# Run manually (pin fastapi==0.140.0 — a later 0.140.x patch removed a symbol +# litellm's proxy imports; start-litellm.sh does this for you): +# uvx --from 'litellm[proxy]==1.95.0' --with 'fastapi==0.140.0' \ +# litellm --config litellm/litellm-config.yaml --port 4000 model_list: # DeepSeek V3.2 — cost lead ($0.74 / $2.22 per Mtok). @@ -92,6 +94,37 @@ model_list: allow_fallbacks: true only: ["streamlake", "gmicloud", "novita", "alibaba"] + # OpenAI / Azure-OpenAI direct — drive a GPT/Codex-family model through the SAME + # Claude Code SDK harness (LiteLLM translates Anthropic Messages <-> OpenAI Chat + # Completions) so the native `Skill` tool + progressive disclosure fire and we can + # measure skill-activation reliability on a non-Claude model — the thing the Codex + # SDK path cannot test (it does not run Claude-Code skills). + # + # The endpoint is OpenAI-compatible: Azure's `/openai/v1` surface accepts a Bearer + # key and OpenAI-format requests, so the generic `openai/` provider + a custom + # `api_base` mirrors exactly how the Codex agent already reaches it (bearer auth, + # no api-version). `model:` after the `openai/` prefix is the Azure DEPLOYMENT name. + # No usage.include/provider block — those are OpenRouter-only; OpenAI/Azure returns + # no usage.cost, so cost falls back to the static pricing table (gpt-5.6-terra is + # registered), which is exact for a flat per-token rate. + - model_name: gpt-5.6-terra + litellm_params: + model: openai/gpt-5.6-terra + api_base: os.environ/CODEX_BASE_URL + api_key: os.environ/CODEX_API_KEY + + # Gemini (Google) direct — drive a Gemini model through the SAME Claude Code SDK + # harness so the native `Skill` tool fires, to A/B skill activation against Gemini's + # own Antigravity SDK. LiteLLM's `gemini/` provider calls the Gemini Developer API + # with GEMINI_API_KEY and does the Anthropic<->Gemini translation (incl. tool_use + # <-> function-calling, which is what the Skill tool rides). No usage.include block — + # that's OpenRouter-only; Gemini returns no usage.cost, so cost falls back to the + # static pricing table (gemini-3.1-pro-preview is registered). + - model_name: gemini-3.1-pro-preview + litellm_params: + model: gemini/gemini-3.1-pro-preview + api_key: os.environ/GEMINI_API_KEY + litellm_settings: # Silently drop provider-unsupported OpenAI/Anthropic params rather than 400. drop_params: true diff --git a/litellm/start-litellm.sh b/litellm/start-litellm.sh index 169ece47..b495b30c 100755 --- a/litellm/start-litellm.sh +++ b/litellm/start-litellm.sh @@ -48,6 +48,15 @@ export AWS_REGION="${AWS_REGION:-$(read_env AWS_REGION)}" export AWS_REGION="${AWS_REGION:-eu-north-1}" # OpenRouter key for the openrouter/* models in the config (cost-optimization path). export OPENROUTER_API_KEY="${OPENROUTER_API_KEY:-$(read_env OPENROUTER_API_KEY)}" +# OpenAI / Azure-OpenAI key + base URL for the gpt-* models in the config — the +# Claude-SDK-through-LiteLLM skill-activation path. CODEX_BASE_URL is an +# OpenAI-compatible endpoint (e.g. Azure's /openai/v1), reused verbatim so the +# proxy reaches the model the same way the Codex agent does. +export CODEX_API_KEY="${CODEX_API_KEY:-$(read_env CODEX_API_KEY)}" +export CODEX_BASE_URL="${CODEX_BASE_URL:-$(read_env CODEX_BASE_URL)}" +# Gemini key for the gemini/* model in the config — the Gemini-via-Claude-SDK +# skill-activation A/B against the native Antigravity harness. +export GEMINI_API_KEY="${GEMINI_API_KEY:-$(read_env GEMINI_API_KEY)}" # Master key = the key clients present. Default to .env's LITELLM_AUTH_TOKEN so the # client and proxy match; fall back to the local dev key. export LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY:-$(read_env LITELLM_AUTH_TOKEN)}" @@ -92,8 +101,17 @@ Set these in coder_eval's .env (or shell) to use it: LITELLM_BASE_URL=http://localhost:$PORT LITELLM_AUTH_TOKEN=$LITELLM_MASTER_KEY LITELLM_COST_LOG=$LITELLM_COST_LOG # SAME path as above, so the run joins ACTUAL per-call cost (else static pricing) - # then run: coder-eval run --model zai.glm-5 (or deepseek.v3.2 / moonshotai.kimi-k2.5) + # then run: coder-eval run --model zai.glm-5 (or deepseek.v3.2 / moonshotai.kimi-k2.5 / gpt-5.6-terra) EOF -exec uvx --from 'litellm[proxy]' litellm --config "$CONFIG" --host 127.0.0.1 --port "$PORT" +# Pin the proxy deps. `uvx --from 'litellm[proxy]'` unpinned drifts: litellm 1.95.0 +# declares `fastapi>=0.136.3,<1.0`, so uvx grabs the newest fastapi — but fastapi +# dropped `get_flat_dependant` (which litellm's proxy still imports) in a 0.140.x +# PATCH (0.140.0 has it, 0.140.13 doesn't), so a range cap isn't enough and startup +# dies with a (masked) `ModuleNotFoundError: proxy_server`. Pin fastapi to an exact +# verified-good version and pin litellm so the sidecar can't silently re-break. +# Override for an upgrade: LITELLM_SPEC='litellm[proxy]==' LITELLM_FASTAPI='fastapi=='. +LITELLM_SPEC="${LITELLM_SPEC:-litellm[proxy]==1.95.0}" +LITELLM_FASTAPI="${LITELLM_FASTAPI:-fastapi==0.140.0}" +exec uvx --from "$LITELLM_SPEC" --with "$LITELLM_FASTAPI" litellm --config "$CONFIG" --host 127.0.0.1 --port "$PORT"