diff --git a/Makefile b/Makefile index d39020b5f09..3200ae9565c 100644 --- a/Makefile +++ b/Makefile @@ -169,7 +169,7 @@ _gen/bin/apikeyscopesgen: $(wildcard scripts/apikeyscopesgen/*.go) $(RBAC_GO_FIL @mkdir -p _gen/bin go build -o $@ ./scripts/apikeyscopesgen -_gen/bin/aibridgepricesgen: $(wildcard scripts/aibridgepricesgen/*.go) | _gen +_gen/bin/aibridgepricesgen: $(wildcard scripts/aibridgepricesgen/*.go) scripts/aibridgepricesgen/curation.json | _gen @mkdir -p _gen/bin go build -o $@ ./scripts/aibridgepricesgen @@ -1022,14 +1022,33 @@ gen: gen/db $(if $(GEN_SKIP_GOLDEN),,gen/golden-files) $(GEN_FILES) gen/db: $(DB_GEN_FILES) .PHONY: gen/db -# Refresh the AI Bridge pricing seed file from models.dev. Kept out of -# `make gen`. Phony so each invocation regenerates. -coderd/aibridge/prices/data/prices.json: _gen/bin/aibridgepricesgen | _gen +# Patched snapshot of the models.dev catalog. Fetched once per +# gen/aibridge-prices run, with upstream corrections applied by +# overrides.jq; both prices.json and the frontend known-models catalog are +# generated from this single snapshot. Phony so each invocation refreshes it. +_gen/models-dev.json: | _gen + set -o pipefail; $(call atomic_write,curl -fsSL https://models.dev/api.json | jq -f scripts/aibridgepricesgen/overrides.jq) +.PHONY: _gen/models-dev.json + +# Refresh the AI Bridge pricing seed file from the patched models.dev +# snapshot. Kept out of `make gen` because the output depends on live +# upstream data. Phony so each invocation regenerates. +coderd/aibridge/prices/data/prices.json: _gen/bin/aibridgepricesgen _gen/models-dev.json | _gen @mkdir -p $(dir $@) - $(call atomic_write,_gen/bin/aibridgepricesgen) + $(call atomic_write,_gen/bin/aibridgepricesgen -upstream _gen/models-dev.json) .PHONY: coderd/aibridge/prices/data/prices.json -gen/aibridge-prices: coderd/aibridge/prices/data/prices.json +# Frontend known-models catalog, generated from the same patched models.dev +# snapshot joined with the editorial curation in +# scripts/aibridgepricesgen/curation.json. Kept out of `make gen` for the +# same live-upstream-data reason as prices.json. +site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.json: _gen/bin/aibridgepricesgen _gen/models-dev.json | _gen + $(call atomic_write,_gen/bin/aibridgepricesgen -format=catalog -upstream _gen/models-dev.json,./scripts/biome_format.sh) +.PHONY: site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.json + +gen/aibridge-prices: \ + coderd/aibridge/prices/data/prices.json \ + site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.json .PHONY: gen/aibridge-prices gen/golden-files: \ diff --git a/coderd/aibridge/prices/data/README.md b/coderd/aibridge/prices/data/README.md index e5d90b34720..f92025c8f79 100644 --- a/coderd/aibridge/prices/data/README.md +++ b/coderd/aibridge/prices/data/README.md @@ -3,3 +3,11 @@ `prices.json` in this directory is generated by `make gen/aibridge-prices` and embedded into the Coder binary at build time. Do not edit it manually; the next regeneration will overwrite any changes. + +The Make target fetches models.dev once into `_gen/models-dev.json`, applying +the upstream corrections in `scripts/aibridgepricesgen/overrides.jq`. Both +`prices.json` and the frontend known-models catalog at +`site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.json` +are generated from that single patched snapshot; the catalog is additionally +joined with the editorial curation in +`scripts/aibridgepricesgen/curation.json`. diff --git a/coderd/aibridge/prices/data/prices.json b/coderd/aibridge/prices/data/prices.json index 4c8b4527e10..4748236dd50 100644 --- a/coderd/aibridge/prices/data/prices.json +++ b/coderd/aibridge/prices/data/prices.json @@ -1,67 +1,11 @@ [ { "provider": "anthropic", - "model": "claude-3-5-haiku-20241022", - "input_price": 800000, - "output_price": 4000000, - "cache_read_price": 80000, - "cache_write_price": 1000000 - }, - { - "provider": "anthropic", - "model": "claude-3-5-haiku-latest", - "input_price": 800000, - "output_price": 4000000, - "cache_read_price": 80000, - "cache_write_price": 1000000 - }, - { - "provider": "anthropic", - "model": "claude-3-5-sonnet-20240620", - "input_price": 3000000, - "output_price": 15000000, - "cache_read_price": 300000, - "cache_write_price": 3750000 - }, - { - "provider": "anthropic", - "model": "claude-3-5-sonnet-20241022", - "input_price": 3000000, - "output_price": 15000000, - "cache_read_price": 300000, - "cache_write_price": 3750000 - }, - { - "provider": "anthropic", - "model": "claude-3-7-sonnet-20250219", - "input_price": 3000000, - "output_price": 15000000, - "cache_read_price": 300000, - "cache_write_price": 3750000 - }, - { - "provider": "anthropic", - "model": "claude-3-haiku-20240307", - "input_price": 250000, - "output_price": 1250000, - "cache_read_price": 30000, - "cache_write_price": 300000 - }, - { - "provider": "anthropic", - "model": "claude-3-opus-20240229", - "input_price": 15000000, - "output_price": 75000000, - "cache_read_price": 1500000, - "cache_write_price": 18750000 - }, - { - "provider": "anthropic", - "model": "claude-3-sonnet-20240229", - "input_price": 3000000, - "output_price": 15000000, - "cache_read_price": 300000, - "cache_write_price": 300000 + "model": "claude-fable-5", + "input_price": 10000000, + "output_price": 50000000, + "cache_read_price": 1000000, + "cache_write_price": 12500000 }, { "provider": "anthropic", @@ -81,11 +25,11 @@ }, { "provider": "anthropic", - "model": "claude-opus-4-0", - "input_price": 15000000, - "output_price": 75000000, - "cache_read_price": 1500000, - "cache_write_price": 18750000 + "model": "claude-mythos-5", + "input_price": 10000000, + "output_price": 50000000, + "cache_read_price": 1000000, + "cache_write_price": 12500000 }, { "provider": "anthropic", @@ -103,14 +47,6 @@ "cache_read_price": 1500000, "cache_write_price": 18750000 }, - { - "provider": "anthropic", - "model": "claude-opus-4-20250514", - "input_price": 15000000, - "output_price": 75000000, - "cache_read_price": 1500000, - "cache_write_price": 18750000 - }, { "provider": "anthropic", "model": "claude-opus-4-5", @@ -145,19 +81,11 @@ }, { "provider": "anthropic", - "model": "claude-sonnet-4-0", - "input_price": 3000000, - "output_price": 15000000, - "cache_read_price": 300000, - "cache_write_price": 3750000 - }, - { - "provider": "anthropic", - "model": "claude-sonnet-4-20250514", - "input_price": 3000000, - "output_price": 15000000, - "cache_read_price": 300000, - "cache_write_price": 3750000 + "model": "claude-opus-4-8", + "input_price": 5000000, + "output_price": 25000000, + "cache_read_price": 500000, + "cache_write_price": 6250000 }, { "provider": "anthropic", @@ -183,12 +111,20 @@ "cache_read_price": 300000, "cache_write_price": 3750000 }, + { + "provider": "anthropic", + "model": "claude-sonnet-5", + "input_price": 2000000, + "output_price": 10000000, + "cache_read_price": 200000, + "cache_write_price": 2500000 + }, { "provider": "openai", "model": "gpt-3.5-turbo", "input_price": 500000, "output_price": 1500000, - "cache_read_price": 1250000, + "cache_read_price": 0, "cache_write_price": null }, { @@ -228,7 +164,7 @@ "model": "gpt-4.1-nano", "input_price": 100000, "output_price": 400000, - "cache_read_price": 30000, + "cache_read_price": 25000, "cache_write_price": null }, { @@ -268,7 +204,7 @@ "model": "gpt-4o-mini", "input_price": 150000, "output_price": 600000, - "cache_read_price": 80000, + "cache_read_price": 75000, "cache_write_price": null }, { @@ -284,7 +220,7 @@ "model": "gpt-5-chat-latest", "input_price": 1250000, "output_price": 10000000, - "cache_read_price": null, + "cache_read_price": 125000, "cache_write_price": null }, { @@ -324,7 +260,7 @@ "model": "gpt-5.1", "input_price": 1250000, "output_price": 10000000, - "cache_read_price": 130000, + "cache_read_price": 125000, "cache_write_price": null }, { @@ -465,23 +401,47 @@ }, { "provider": "openai", - "model": "o1", - "input_price": 15000000, - "output_price": 60000000, - "cache_read_price": 7500000, - "cache_write_price": null + "model": "gpt-5.6", + "input_price": 5000000, + "output_price": 30000000, + "cache_read_price": 500000, + "cache_write_price": 6250000 }, { "provider": "openai", - "model": "o1-mini", - "input_price": 1100000, - "output_price": 4400000, - "cache_read_price": 550000, + "model": "gpt-5.6-luna", + "input_price": 1000000, + "output_price": 6000000, + "cache_read_price": 100000, + "cache_write_price": 1250000 + }, + { + "provider": "openai", + "model": "gpt-5.6-sol", + "input_price": 5000000, + "output_price": 30000000, + "cache_read_price": 500000, + "cache_write_price": 6250000 + }, + { + "provider": "openai", + "model": "gpt-5.6-terra", + "input_price": 2500000, + "output_price": 15000000, + "cache_read_price": 250000, + "cache_write_price": 3125000 + }, + { + "provider": "openai", + "model": "gpt-image-2", + "input_price": 5000000, + "output_price": 30000000, + "cache_read_price": 1250000, "cache_write_price": null }, { "provider": "openai", - "model": "o1-preview", + "model": "o1", "input_price": 15000000, "output_price": 60000000, "cache_read_price": 7500000, @@ -532,7 +492,7 @@ "model": "o4-mini", "input_price": 1100000, "output_price": 4400000, - "cache_read_price": 280000, + "cache_read_price": 275000, "cache_write_price": null }, { diff --git a/scripts/aibridgepricesgen/catalog.go b/scripts/aibridgepricesgen/catalog.go new file mode 100644 index 00000000000..0a2ff05ea5a --- /dev/null +++ b/scripts/aibridgepricesgen/catalog.go @@ -0,0 +1,150 @@ +package main + +import ( + "cmp" + _ "embed" + "encoding/json" + "io" + + "golang.org/x/xerrors" +) + +// curationJSON is the checked-in editorial curation input for the frontend +// known-models catalog. Entry order within each provider controls suggestion +// order in the UI. Everything factual (display name, limits, pricing) is +// joined from models.dev at generation time; the curation file only carries +// editorial choices: which models to suggest, aliases, reasoning defaults, +// and overrides. +// +//go:embed curation.json +var curationJSON []byte + +// curatedModel is one entry in curation.json. +type curatedModel struct { + ModelIdentifier string `json:"modelIdentifier"` + Aliases []string `json:"aliases"` + // DisplayName overrides the upstream `name` when set. Needed where + // upstream naming does not match what we want to show (for example + // "Claude Haiku 4.5 (latest)"). + DisplayName string `json:"displayName"` + // ReasoningEffort is editorial, not from models.dev. Mutually + // exclusive with ThinkingBudgetTokens. + ReasoningEffort string `json:"reasoningEffort"` + // ThinkingBudgetTokens is Anthropic-only, for models that do not + // support adaptive thinking and use the legacy + // `thinking.budget_tokens` API instead. + ThinkingBudgetTokens int `json:"thinkingBudgetTokens"` +} + +// catalogEntry matches the frontend KnownModel shape (knownModels/types.ts). +// Costs are flat USD per million tokens, straight from models.dev; tiered +// pricing such as context_over_200k is intentionally omitted. +type catalogEntry struct { + Provider string `json:"provider"` + ModelIdentifier string `json:"modelIdentifier"` + DisplayName string `json:"displayName"` + Aliases []string `json:"aliases"` + ContextLimit *int64 `json:"contextLimit,omitempty"` + MaxOutputTokens *int64 `json:"maxOutputTokens,omitempty"` + ReasoningEffort string `json:"reasoningEffort,omitempty"` + ThinkingBudgetTokens int `json:"thinkingBudgetTokens,omitempty"` + InputCost *float64 `json:"inputCost,omitempty"` + OutputCost *float64 `json:"outputCost,omitempty"` + CacheReadCost *float64 `json:"cacheReadCost,omitempty"` + CacheWriteCost *float64 `json:"cacheWriteCost,omitempty"` +} + +// validReasoningEfforts are the values accepted for curatedModel.ReasoningEffort. +var validReasoningEfforts = map[string]bool{"low": true, "medium": true, "high": true} + +// buildCatalog joins the curation file with the upstream models.dev payload +// and returns provider-keyed ordered entry lists. +func buildCatalog(upstream map[string]upstreamProvider, curation map[string][]curatedModel) (map[string][]catalogEntry, error) { + out := make(map[string][]catalogEntry, len(curation)) + for providerID, curated := range curation { + provider, ok := upstream[providerID] + if !ok { + return nil, xerrors.Errorf("provider %q missing from upstream", providerID) + } + seenIdentifiers := make(map[string]bool, len(curated)) + seenAliases := make(map[string]bool) + entries := make([]catalogEntry, 0, len(curated)) + for _, c := range curated { + if c.ModelIdentifier == "" { + return nil, xerrors.Errorf("provider %q: entry with empty modelIdentifier", providerID) + } + if seenIdentifiers[c.ModelIdentifier] { + return nil, xerrors.Errorf("provider %q: duplicate modelIdentifier %q", providerID, c.ModelIdentifier) + } + seenIdentifiers[c.ModelIdentifier] = true + if c.ReasoningEffort != "" && !validReasoningEfforts[c.ReasoningEffort] { + return nil, xerrors.Errorf(`%s/%s: reasoningEffort %q is not one of "low", "medium", "high"`, providerID, c.ModelIdentifier, c.ReasoningEffort) + } + if c.ThinkingBudgetTokens < 0 { + return nil, xerrors.Errorf("%s/%s: thinkingBudgetTokens %d is negative", providerID, c.ModelIdentifier, c.ThinkingBudgetTokens) + } + if c.ReasoningEffort != "" && c.ThinkingBudgetTokens != 0 { + return nil, xerrors.Errorf("%s/%s: reasoningEffort and thinkingBudgetTokens are mutually exclusive", providerID, c.ModelIdentifier) + } + for _, alias := range c.Aliases { + if alias == "" { + return nil, xerrors.Errorf("%s/%s: empty-string alias", providerID, c.ModelIdentifier) + } + if seenAliases[alias] { + return nil, xerrors.Errorf("%s/%s: alias %q declared more than once in provider", providerID, c.ModelIdentifier, alias) + } + seenAliases[alias] = true + } + m, ok := provider.Models[c.ModelIdentifier] + if !ok { + return nil, xerrors.Errorf("%s/%s: model missing from upstream (patch it in via overrides.jq if intentional)", providerID, c.ModelIdentifier) + } + if !m.Cost.hasPricing() { + return nil, xerrors.Errorf("%s/%s: upstream model has no pricing data", providerID, c.ModelIdentifier) + } + if m.Limit.Context == nil || m.Limit.Output == nil { + return nil, xerrors.Errorf("%s/%s: upstream model missing limit.context or limit.output", providerID, c.ModelIdentifier) + } + displayName := cmp.Or(c.DisplayName, m.Name) + if displayName == "" { + return nil, xerrors.Errorf("%s/%s: no displayName override and upstream name is empty", providerID, c.ModelIdentifier) + } + aliases := c.Aliases + if aliases == nil { + aliases = []string{} + } + entries = append(entries, catalogEntry{ + Provider: providerID, + ModelIdentifier: c.ModelIdentifier, + DisplayName: displayName, + Aliases: aliases, + ContextLimit: m.Limit.Context, + MaxOutputTokens: m.Limit.Output, + ReasoningEffort: c.ReasoningEffort, + ThinkingBudgetTokens: c.ThinkingBudgetTokens, + InputCost: m.Cost.Input, + OutputCost: m.Cost.Output, + CacheReadCost: m.Cost.CacheRead, + CacheWriteCost: m.Cost.CacheWrite, + }) + } + // An alias resolving to a canonical identifier would make exact-alias + // lookup and canonical-id lookup disagree. + for alias := range seenAliases { + if seenIdentifiers[alias] { + return nil, xerrors.Errorf("alias %q duplicates a modelIdentifier in provider %q", alias, providerID) + } + } + out[providerID] = entries + } + return out, nil +} + +func writeCatalog(w io.Writer, catalog map[string][]catalogEntry) error { + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + if err := enc.Encode(catalog); err != nil { + return xerrors.Errorf("encode: %w", err) + } + return nil +} diff --git a/scripts/aibridgepricesgen/catalog_test.go b/scripts/aibridgepricesgen/catalog_test.go new file mode 100644 index 00000000000..cc8c5affa4e --- /dev/null +++ b/scripts/aibridgepricesgen/catalog_test.go @@ -0,0 +1,342 @@ +package main + +import ( + "bytes" + "encoding/json" + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +// fixtureUpstream returns a small upstream payload covering the join cases: +// fully priced models with limits and a costless model. +func fixtureUpstream(t *testing.T) map[string]upstreamProvider { + t.Helper() + const upstreamJSON = `{ + "anthropic": { + "models": { + "claude-fable-5": { + "name": "Claude Fable 5", + "limit": {"context": 1000000, "output": 128000}, + "cost": {"input": 10, "output": 50, "cache_read": 1, "cache_write": 12.5} + }, + "claude-mythos-5": { + "name": "Claude Mythos 5", + "limit": {"context": 1000000, "output": 128000}, + "cost": {"input": 10, "output": 50, "cache_read": 1, "cache_write": 12.5} + }, + "claude-costless": { + "name": "Claude Costless", + "limit": {"context": 200000, "output": 64000} + }, + "claude-nameless": { + "name": "", + "limit": {"context": 200000, "output": 64000}, + "cost": {"input": 1, "output": 5} + } + } + }, + "openai": { + "models": { + "gpt-5.6-sol": { + "name": "GPT-5.6 Sol", + "limit": {"context": 1050000, "output": 128000}, + "cost": {"input": 5, "output": 30, "cache_read": 0.5, "cache_write": 6.25} + }, + "gpt-partial": { + "name": "GPT Partial", + "limit": {"context": 400000, "output": 128000}, + "cost": {"input": 0.2, "output": 1.25} + }, + "gpt-limitless": { + "name": "GPT Limitless", + "limit": {"context": 400000}, + "cost": {"input": 0.2, "output": 1.25} + } + } + } + }` + var upstream map[string]upstreamProvider + require.NoError(t, json.Unmarshal([]byte(upstreamJSON), &upstream)) + return upstream +} + +func TestBuildCatalog(t *testing.T) { + t.Parallel() + + curation := map[string][]curatedModel{ + "openai": { + {ModelIdentifier: "gpt-5.6-sol", Aliases: []string{"gpt-5.6"}, ReasoningEffort: "medium"}, + {ModelIdentifier: "gpt-partial"}, + }, + "anthropic": { + {ModelIdentifier: "claude-fable-5", ReasoningEffort: "high"}, + {ModelIdentifier: "claude-mythos-5", DisplayName: "Mythos 5 Override", ThinkingBudgetTokens: 8192}, + }, + } + + catalog, err := buildCatalog(fixtureUpstream(t), curation) + require.NoError(t, err) + + var buf bytes.Buffer + require.NoError(t, writeCatalog(&buf, catalog)) + + const want = `{ + "anthropic": [ + { + "provider": "anthropic", + "modelIdentifier": "claude-fable-5", + "displayName": "Claude Fable 5", + "aliases": [], + "contextLimit": 1000000, + "maxOutputTokens": 128000, + "reasoningEffort": "high", + "inputCost": 10, + "outputCost": 50, + "cacheReadCost": 1, + "cacheWriteCost": 12.5 + }, + { + "provider": "anthropic", + "modelIdentifier": "claude-mythos-5", + "displayName": "Mythos 5 Override", + "aliases": [], + "contextLimit": 1000000, + "maxOutputTokens": 128000, + "thinkingBudgetTokens": 8192, + "inputCost": 10, + "outputCost": 50, + "cacheReadCost": 1, + "cacheWriteCost": 12.5 + } + ], + "openai": [ + { + "provider": "openai", + "modelIdentifier": "gpt-5.6-sol", + "displayName": "GPT-5.6 Sol", + "aliases": [ + "gpt-5.6" + ], + "contextLimit": 1050000, + "maxOutputTokens": 128000, + "reasoningEffort": "medium", + "inputCost": 5, + "outputCost": 30, + "cacheReadCost": 0.5, + "cacheWriteCost": 6.25 + }, + { + "provider": "openai", + "modelIdentifier": "gpt-partial", + "displayName": "GPT Partial", + "aliases": [], + "contextLimit": 400000, + "maxOutputTokens": 128000, + "inputCost": 0.2, + "outputCost": 1.25 + } + ] +} +` + require.Equal(t, want, buf.String()) +} + +func TestBuildCatalogErrors(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + curation map[string][]curatedModel + wantErr string + }{ + { + name: "MissingUpstreamModel", + curation: map[string][]curatedModel{ + "openai": {{ModelIdentifier: "gpt-nonexistent"}}, + }, + wantErr: "model missing from upstream", + }, + { + name: "NoCostBlock", + curation: map[string][]curatedModel{ + "anthropic": {{ModelIdentifier: "claude-costless"}}, + }, + wantErr: "no pricing data", + }, + { + name: "MissingUpstreamLimit", + curation: map[string][]curatedModel{ + "openai": {{ModelIdentifier: "gpt-limitless"}}, + }, + wantErr: "missing limit.context or limit.output", + }, + { + name: "EmptyUpstreamName", + curation: map[string][]curatedModel{ + "anthropic": {{ModelIdentifier: "claude-nameless"}}, + }, + wantErr: "upstream name is empty", + }, + { + name: "EffortAndBudgetBothSet", + curation: map[string][]curatedModel{ + "anthropic": {{ModelIdentifier: "claude-fable-5", ReasoningEffort: "high", ThinkingBudgetTokens: 8192}}, + }, + wantErr: "mutually exclusive", + }, + { + name: "InvalidReasoningEffort", + curation: map[string][]curatedModel{ + "anthropic": {{ModelIdentifier: "claude-fable-5", ReasoningEffort: "maximum"}}, + }, + wantErr: "is not one of", + }, + { + name: "NegativeThinkingBudget", + curation: map[string][]curatedModel{ + "anthropic": {{ModelIdentifier: "claude-fable-5", ThinkingBudgetTokens: -1}}, + }, + wantErr: "is negative", + }, + { + name: "DuplicateModelIdentifier", + curation: map[string][]curatedModel{ + "anthropic": { + {ModelIdentifier: "claude-fable-5"}, + {ModelIdentifier: "claude-fable-5"}, + }, + }, + wantErr: "duplicate modelIdentifier", + }, + { + name: "DuplicateAlias", + curation: map[string][]curatedModel{ + "anthropic": { + {ModelIdentifier: "claude-fable-5", Aliases: []string{"claude-latest"}}, + {ModelIdentifier: "claude-mythos-5", Aliases: []string{"claude-latest"}}, + }, + }, + wantErr: "declared more than once", + }, + { + name: "EmptyAlias", + curation: map[string][]curatedModel{ + "anthropic": {{ModelIdentifier: "claude-fable-5", Aliases: []string{""}}}, + }, + wantErr: "empty-string alias", + }, + { + name: "AliasShadowsModelIdentifier", + curation: map[string][]curatedModel{ + "anthropic": { + {ModelIdentifier: "claude-fable-5", Aliases: []string{"claude-mythos-5"}}, + {ModelIdentifier: "claude-mythos-5"}, + }, + }, + wantErr: "duplicates a modelIdentifier", + }, + { + name: "MissingProvider", + curation: map[string][]curatedModel{ + "google": {{ModelIdentifier: "gemini"}}, + }, + wantErr: `provider "google" missing`, + }, + { + name: "EmptyModelIdentifier", + curation: map[string][]curatedModel{ + "openai": {{}}, + }, + wantErr: "empty modelIdentifier", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + _, err := buildCatalog(fixtureUpstream(t), tc.curation) + require.Error(t, err) + require.Contains(t, err.Error(), tc.wantErr) + }) + } +} + +// TestCurationMatchesGeneratedCatalog is a drift test: the editorial fields +// (per provider, in order) in the embedded curation.json must exactly match +// their projection in the checked-in generated frontend catalog. Fails when +// curation.json changes without running `make gen/aibridge-prices`. +func TestCurationMatchesGeneratedCatalog(t *testing.T) { + t.Parallel() + + curation := embeddedCuration(t) + + data, err := os.ReadFile("../../site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.json") + require.NoError(t, err) + var generated map[string][]catalogEntry + require.NoError(t, json.Unmarshal(data, &generated)) + + // editorial is the curation-owned projection of an entry. displayName is + // only compared when the curation sets an override; otherwise it comes + // from upstream and is not the curation's to pin. + type editorial struct { + ModelIdentifier string + Aliases []string + DisplayName string + ReasoningEffort string + ThinkingBudgetTokens int + } + + curatedProjection := make(map[string][]editorial, len(curation)) + for providerID, entries := range curation { + projected := make([]editorial, 0, len(entries)) + for _, c := range entries { + aliases := c.Aliases + if aliases == nil { + aliases = []string{} + } + projected = append(projected, editorial{ + ModelIdentifier: c.ModelIdentifier, + Aliases: aliases, + DisplayName: c.DisplayName, + ReasoningEffort: c.ReasoningEffort, + ThinkingBudgetTokens: c.ThinkingBudgetTokens, + }) + } + curatedProjection[providerID] = projected + } + + generatedProjection := make(map[string][]editorial, len(generated)) + for providerID, entries := range generated { + curated := map[string]curatedModel{} + for _, c := range curation[providerID] { + curated[c.ModelIdentifier] = c + } + projected := make([]editorial, 0, len(entries)) + for _, e := range entries { + displayName := "" + if curated[e.ModelIdentifier].DisplayName != "" { + displayName = e.DisplayName + } + projected = append(projected, editorial{ + ModelIdentifier: e.ModelIdentifier, + Aliases: e.Aliases, + DisplayName: displayName, + ReasoningEffort: e.ReasoningEffort, + ThinkingBudgetTokens: e.ThinkingBudgetTokens, + }) + } + generatedProjection[providerID] = projected + } + + require.Equal(t, curatedProjection, generatedProjection, + "curation.json and knownModelsGenerated.json disagree; run `make gen/aibridge-prices`") +} + +func embeddedCuration(t *testing.T) map[string][]curatedModel { + t.Helper() + var curation map[string][]curatedModel + require.NoError(t, json.Unmarshal(curationJSON, &curation)) + return curation +} diff --git a/scripts/aibridgepricesgen/curation.json b/scripts/aibridgepricesgen/curation.json new file mode 100644 index 00000000000..b528c8d0d33 --- /dev/null +++ b/scripts/aibridgepricesgen/curation.json @@ -0,0 +1,81 @@ +{ + "openai": [ + { + "modelIdentifier": "gpt-5.6-sol", + "aliases": ["gpt-5.6"], + "reasoningEffort": "medium" + }, + { + "modelIdentifier": "gpt-5.6-terra", + "reasoningEffort": "medium" + }, + { + "modelIdentifier": "gpt-5.6-luna", + "reasoningEffort": "medium" + }, + { + "modelIdentifier": "gpt-5.5", + "reasoningEffort": "medium" + }, + { + "modelIdentifier": "gpt-5.5-pro", + "reasoningEffort": "high" + }, + { + "modelIdentifier": "gpt-5.4" + }, + { + "modelIdentifier": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + { + "modelIdentifier": "gpt-5.4-nano" + }, + { + "modelIdentifier": "gpt-5.3-codex", + "reasoningEffort": "medium" + } + ], + "anthropic": [ + { + "modelIdentifier": "claude-fable-5", + "reasoningEffort": "high" + }, + { + "modelIdentifier": "claude-mythos-5", + "reasoningEffort": "high" + }, + { + "modelIdentifier": "claude-opus-4-8", + "reasoningEffort": "high" + }, + { + "modelIdentifier": "claude-opus-4-7", + "reasoningEffort": "high" + }, + { + "modelIdentifier": "claude-opus-4-6", + "reasoningEffort": "high" + }, + { + "modelIdentifier": "claude-sonnet-5", + "reasoningEffort": "high" + }, + { + "modelIdentifier": "claude-sonnet-4-6", + "reasoningEffort": "medium" + }, + { + "modelIdentifier": "claude-haiku-4-5", + "aliases": ["claude-haiku-4-5-20251001"], + "displayName": "Claude Haiku 4.5", + "thinkingBudgetTokens": 8192 + }, + { + "modelIdentifier": "claude-sonnet-4-5", + "aliases": ["claude-sonnet-4-5-20250929"], + "displayName": "Claude Sonnet 4.5", + "thinkingBudgetTokens": 8192 + } + ] +} diff --git a/scripts/aibridgepricesgen/main.go b/scripts/aibridgepricesgen/main.go index 20a26c0f1b2..19778c357f3 100644 --- a/scripts/aibridgepricesgen/main.go +++ b/scripts/aibridgepricesgen/main.go @@ -1,37 +1,30 @@ -// aibridgepricesgen fetches model pricing from models.dev and writes a JSON -// seed file consumable by the AI Bridge cost-control loader. Output is sorted -// by (provider, model) so regenerations produce minimal diffs. +// aibridgepricesgen converts a models.dev api.json snapshot into generated +// artifacts, selected by -format: // -// Run via the gen/aibridge-prices Make target. Kept out of `make gen` because -// the output depends on live upstream data; refreshing prices should land in -// dedicated, reviewable commits rather than appearing as drift on unrelated -// gen runs. +// - "prices": a JSON seed file consumable by the AI Gateway cost-control +// loader, sorted by (provider, model) so regenerations produce minimal +// diffs. +// - "catalog": the frontend known-models JSON, joining the snapshot with +// the editorial curation in curation.json and preserving its entry order. +// +// Run via the gen/aibridge-prices Make target, which fetches and patches the +// snapshot (_gen/models-dev.json). Kept out of `make gen` because the output +// depends on live upstream data; refreshing prices should land in dedicated, +// reviewable commits rather than appearing as drift on unrelated gen runs. package main import ( - "context" "encoding/json" + "flag" "fmt" "io" "math" - "net/http" "os" "sort" - "time" "golang.org/x/xerrors" ) -const ( - sourceURL = "https://models.dev/api.json" - fetchTimeout = 30 * time.Second - // Cap the upstream body read. The current api.json is ~2 MiB, so 100 - // MiB is pure defense-in-depth against a misbehaving upstream eating - // arbitrary memory on developer or CI machines. An overflow surfaces - // as a JSON parse error (LimitReader truncates silently at the cap). - maxBodyBytes = 100 << 20 -) - // supportedProviders lists the providers we ship prices for. Adding a // provider here is enough to include it on the next regeneration. var supportedProviders = []string{"anthropic", "openai"} @@ -42,10 +35,18 @@ type upstreamProvider struct { } type upstreamModel struct { - Cost *upstreamCost `json:"cost"` + Name string `json:"name"` + Limit upstreamLimit `json:"limit"` + Cost *upstreamCost `json:"cost"` +} + +// Pointer fields in upstreamLimit and upstreamCost distinguish "key absent" +// (nil) from "key present and zero" (0). +type upstreamLimit struct { + Context *int64 `json:"context"` + Output *int64 `json:"output"` } -// Pointers distinguish "key absent" (nil) from "key present and zero" (0). type upstreamCost struct { Input *float64 `json:"input"` Output *float64 `json:"output"` @@ -80,17 +81,51 @@ type priceRow struct { } func main() { - if err := run(); err != nil { + format := flag.String("format", "prices", `output format: "prices" (cost-control seed) or "catalog" (frontend known-models JSON)`) + upstreamPath := flag.String("upstream", "", "path to a models.dev api.json snapshot (required)") + flag.Parse() + if err := run(*format, *upstreamPath); err != nil { _, _ = fmt.Fprintf(os.Stderr, "aibridgepricesgen: %v\n", err) os.Exit(1) } } -func run() error { - upstream, err := fetch() +func run(format, upstreamPath string) error { + // Validate flags before touching the filesystem so a typo fails fast. + switch format { + case "prices", "catalog": + default: + return xerrors.Errorf(`unknown -format %q (want "prices" or "catalog")`, format) + } + if upstreamPath == "" { + return xerrors.New("-upstream is required; run via `make gen/aibridge-prices`, which fetches and patches the snapshot") + } + + upstream, err := readUpstream(upstreamPath) + if err != nil { + return xerrors.Errorf("read %s: %w", upstreamPath, err) + } + if format == "catalog" { + return runCatalog(upstream) + } + return runPrices(upstream) +} + +// readUpstream loads a models.dev api.json snapshot from disk, typically the +// Makefile's _gen/models-dev.json (fetched once and patched by overrides.jq). +func readUpstream(path string) (map[string]upstreamProvider, error) { + data, err := os.ReadFile(path) if err != nil { - return xerrors.Errorf("fetch %s: %w", sourceURL, err) + return nil, err } + var upstream map[string]upstreamProvider + if err := json.Unmarshal(data, &upstream); err != nil { + return nil, xerrors.Errorf("parse: %w", err) + } + return upstream, nil +} + +func runPrices(upstream map[string]upstreamProvider) error { rows, err := convert(upstream, supportedProviders) if err != nil { return err @@ -105,28 +140,20 @@ func run() error { return nil } -func fetch() (map[string]upstreamProvider, error) { - ctx, cancel := context.WithTimeout(context.Background(), fetchTimeout) - defer cancel() - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, sourceURL, nil) - if err != nil { - return nil, err +func runCatalog(upstream map[string]upstreamProvider) error { + var curation map[string][]curatedModel + if err := json.Unmarshal(curationJSON, &curation); err != nil { + return xerrors.Errorf("parse embedded curation.json: %w", err) } - resp, err := http.DefaultClient.Do(req) + catalog, err := buildCatalog(upstream, curation) if err != nil { - return nil, err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return nil, xerrors.Errorf("status %d", resp.StatusCode) + return err } - - var data map[string]upstreamProvider - if err := json.NewDecoder(io.LimitReader(resp.Body, maxBodyBytes)).Decode(&data); err != nil { - return nil, xerrors.Errorf("parse: %w", err) + if err := writeCatalog(os.Stdout, catalog); err != nil { + return err } - return data, nil + _, _ = fmt.Fprintf(os.Stderr, "aibridgepricesgen: wrote catalog for %d provider(s)\n", len(catalog)) + return nil } // convert flattens the upstream map into table-shaped rows for the configured diff --git a/scripts/aibridgepricesgen/overrides.jq b/scripts/aibridgepricesgen/overrides.jq new file mode 100644 index 00000000000..cc1b424a1f8 --- /dev/null +++ b/scripts/aibridgepricesgen/overrides.jq @@ -0,0 +1,32 @@ +# Patches applied to the raw models.dev api.json before aibridgepricesgen +# consumes it. The Makefile pipes the fetched payload through this filter +# (jq -f scripts/aibridgepricesgen/overrides.jq) and both generated outputs +# (prices.json and knownModelsGenerated.json) read the patched snapshot. +# +# Every patch guards its assumption about upstream, so a stale override +# fails the pipeline loudly instead of silently patching nothing. + +# claude-sonnet-4-5: models.dev advertises a 1M-token context window, which +# is incorrect. Anthropic retired the 1M context window beta on May 1st, +# 2026. Ref: https://platform.claude.com/docs/en/about-claude/models/overview +if .anthropic.models | has("claude-sonnet-4-5") then + .anthropic.models."claude-sonnet-4-5".limit.context = 200000 +else + error("overrides.jq: claude-sonnet-4-5 gone from upstream; drop or update its context pin") +end + +# claude-mythos-5: not listed on models.dev. Anthropic documents it as sharing +# claude-fable-5's specs and pricing, so inject it as a copy with its own +# id and display name. +# Ref: https://platform.claude.com/docs/en/about-claude/pricing#model-pricing +| if (.anthropic.models | has("claude-fable-5") | not) then + error("overrides.jq: claude-fable-5 gone from upstream; the claude-mythos-5 copy has no source") + elif (.anthropic.models | has("claude-mythos-5")) then + error("overrides.jq: claude-mythos-5 now present upstream; drop the injection") + else + .anthropic.models."claude-mythos-5" = ( + .anthropic.models."claude-fable-5" + | .id = "claude-mythos-5" + | .name = "Claude Mythos 5" + ) + end diff --git a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/anthropic.test.ts b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/anthropic.test.ts deleted file mode 100644 index e8bc04b4208..00000000000 --- a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/anthropic.test.ts +++ /dev/null @@ -1,91 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { anthropicKnownModels } from "./anthropic"; -import { getKnownModelsForProvider } from "./index"; -import type { KnownModel } from "./types"; - -const anthropicKnownModelList: readonly KnownModel[] = anthropicKnownModels; - -const requireAnthropicKnownModel = (modelIdentifier: string): KnownModel => { - const knownModel = anthropicKnownModelList.find( - (knownModel) => knownModel.modelIdentifier === modelIdentifier, - ); - if (knownModel === undefined) { - throw new Error(`missing Anthropic Known Model: ${modelIdentifier}`); - } - return knownModel; -}; - -describe("anthropicKnownModels", () => { - it("returns Anthropic canonical IDs in declared order", () => { - expect( - getKnownModelsForProvider("anthropic").map( - (knownModel) => knownModel.modelIdentifier, - ), - ).toEqual([ - "claude-fable-5", - "claude-mythos-5", - "claude-opus-4-8", - "claude-opus-4-7", - "claude-opus-4-6", - "claude-sonnet-4-6", - "claude-haiku-4-5", - "claude-sonnet-4-5", - ]); - }); - - it("declares Anthropic reasoning defaults by API support", () => { - for (const modelIdentifier of [ - "claude-fable-5", - "claude-mythos-5", - "claude-opus-4-8", - "claude-opus-4-7", - "claude-opus-4-6", - ]) { - const knownModel = requireAnthropicKnownModel(modelIdentifier); - - expect(knownModel.reasoningEffort).toBe("high"); - expect(knownModel.thinkingBudgetTokens).toBeUndefined(); - } - - const sonnet46 = requireAnthropicKnownModel("claude-sonnet-4-6"); - expect(sonnet46.reasoningEffort).toBe("medium"); - expect(sonnet46.thinkingBudgetTokens).toBeUndefined(); - - for (const modelIdentifier of ["claude-haiku-4-5", "claude-sonnet-4-5"]) { - const knownModel = requireAnthropicKnownModel(modelIdentifier); - - expect(knownModel.reasoningEffort).toBeUndefined(); - expect(knownModel.thinkingBudgetTokens).toBe(8192); - } - }); - - it("has source metadata, provider equality, and declared order", () => { - expect( - anthropicKnownModels.map((knownModel) => knownModel.modelIdentifier), - ).toEqual([ - "claude-fable-5", - "claude-mythos-5", - "claude-opus-4-8", - "claude-opus-4-7", - "claude-opus-4-6", - "claude-sonnet-4-6", - "claude-haiku-4-5", - "claude-sonnet-4-5", - ]); - - for (const knownModel of anthropicKnownModels) { - expect(knownModel.provider).toBe("anthropic"); - expect(knownModel.sourceMetadata.sourceName).toBe("models.dev"); - expect(knownModel.sourceMetadata.sourceRetrievedAt).not.toBe(""); - expect(knownModel.sourceMetadata.lastUpdated).not.toBe(""); - } - - expect( - requireAnthropicKnownModel("claude-opus-4-8").sourceMetadata, - ).toEqual({ - sourceName: "models.dev", - sourceRetrievedAt: "2026-05-29", - lastUpdated: "2026-05-28", - }); - }); -}); diff --git a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/anthropic.ts b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/anthropic.ts deleted file mode 100644 index 760ee578d6a..00000000000 --- a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/anthropic.ts +++ /dev/null @@ -1,169 +0,0 @@ -import type { KnownModel } from "./types"; - -// Array order controls suggestion order. Keep sourceMetadata.lastUpdated in -// sync with the corresponding models.dev last_updated value for each model. -// Coder currently persists flat pricing only. Tiered models.dev pricing, -// such as context_over_200k, is intentionally omitted. -// -// The `reasoningEffort` value is editorial, not from models.dev. It reflects -// the provider's documented default for reasoning-capable models in this -// catalog and should be reviewed when the catalog is refreshed. -// -// Reasoning configuration is split per model based on Anthropic API support: -// models that support adaptive thinking (Fable 5, Mythos 5, Opus 4.8, -// Opus 4.7, Opus 4.6, Sonnet 4.6) carry `reasoningEffort`, which Coder maps to -// `thinking.type: "adaptive"` with the `effort` parameter. Models that do not -// (Haiku 4.5, Sonnet 4.5) -// carry `thinkingBudgetTokens` instead, which Coder maps to the legacy -// `thinking.type: "enabled"` path with `budget_tokens`. Setting `effort` on -// the legacy path produces an "adaptive thinking is not supported on this -// model" HTTP 400 from Anthropic. -export const anthropicKnownModels = [ - { - provider: "anthropic", - modelIdentifier: "claude-fable-5", - displayName: "Claude Fable 5", - aliases: [], - contextLimit: 1_000_000, - maxOutputTokens: 128_000, - reasoningEffort: "high", - inputCost: 10, - outputCost: 50, - cacheReadCost: 1, - cacheWriteCost: 12.5, - sourceMetadata: { - sourceName: "models.dev", - sourceRetrievedAt: "2026-07-09", - lastUpdated: "2026-06-09", - }, - }, - // models.dev does not list claude-mythos-5. Anthropic documents that - // Mythos 5 shares Fable 5's specs and pricing, so this entry mirrors - // the claude-fable-5 models.dev entry. - { - provider: "anthropic", - modelIdentifier: "claude-mythos-5", - displayName: "Claude Mythos 5", - aliases: [], - contextLimit: 1_000_000, - maxOutputTokens: 128_000, - reasoningEffort: "high", - inputCost: 10, - outputCost: 50, - cacheReadCost: 1, - cacheWriteCost: 12.5, - sourceMetadata: { - sourceName: "models.dev", - sourceRetrievedAt: "2026-07-09", - lastUpdated: "2026-06-09", - }, - }, - { - provider: "anthropic", - modelIdentifier: "claude-opus-4-8", - displayName: "Claude Opus 4.8", - aliases: [], - contextLimit: 1_000_000, - maxOutputTokens: 128_000, - reasoningEffort: "high", - inputCost: 5, - outputCost: 25, - cacheReadCost: 0.5, - cacheWriteCost: 6.25, - sourceMetadata: { - sourceName: "models.dev", - sourceRetrievedAt: "2026-05-29", - lastUpdated: "2026-05-28", - }, - }, - { - provider: "anthropic", - modelIdentifier: "claude-opus-4-7", - displayName: "Claude Opus 4.7", - aliases: [], - contextLimit: 1_000_000, - maxOutputTokens: 128_000, - reasoningEffort: "high", - inputCost: 5, - outputCost: 25, - cacheReadCost: 0.5, - cacheWriteCost: 6.25, - sourceMetadata: { - sourceName: "models.dev", - sourceRetrievedAt: "2026-04-30", - lastUpdated: "2026-04-16", - }, - }, - { - provider: "anthropic", - modelIdentifier: "claude-opus-4-6", - displayName: "Claude Opus 4.6", - aliases: [], - contextLimit: 1_000_000, - maxOutputTokens: 128_000, - reasoningEffort: "high", - inputCost: 5, - outputCost: 25, - cacheReadCost: 0.5, - cacheWriteCost: 6.25, - sourceMetadata: { - sourceName: "models.dev", - sourceRetrievedAt: "2026-04-30", - lastUpdated: "2026-03-13", - }, - }, - { - provider: "anthropic", - modelIdentifier: "claude-sonnet-4-6", - displayName: "Claude Sonnet 4.6", - aliases: [], - contextLimit: 1_000_000, - maxOutputTokens: 64_000, - reasoningEffort: "medium", - inputCost: 3, - outputCost: 15, - cacheReadCost: 0.3, - cacheWriteCost: 3.75, - sourceMetadata: { - sourceName: "models.dev", - sourceRetrievedAt: "2026-04-30", - lastUpdated: "2026-03-13", - }, - }, - { - provider: "anthropic", - modelIdentifier: "claude-haiku-4-5", - displayName: "Claude Haiku 4.5", - aliases: ["claude-haiku-4-5-20251001"], - contextLimit: 200_000, - maxOutputTokens: 64_000, - thinkingBudgetTokens: 8192, - inputCost: 1, - outputCost: 5, - cacheReadCost: 0.1, - cacheWriteCost: 1.25, - sourceMetadata: { - sourceName: "models.dev", - sourceRetrievedAt: "2026-04-30", - lastUpdated: "2025-10-15", - }, - }, - { - provider: "anthropic", - modelIdentifier: "claude-sonnet-4-5", - displayName: "Claude Sonnet 4.5", - aliases: ["claude-sonnet-4-5-20250929"], - contextLimit: 200_000, - maxOutputTokens: 64_000, - thinkingBudgetTokens: 8192, - inputCost: 3, - outputCost: 15, - cacheReadCost: 0.3, - cacheWriteCost: 3.75, - sourceMetadata: { - sourceName: "models.dev", - sourceRetrievedAt: "2026-04-30", - lastUpdated: "2025-09-29", - }, - }, -] as const satisfies readonly KnownModel[]; diff --git a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/applyKnownModelDefaults.test.ts b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/applyKnownModelDefaults.test.ts index e3ad0bd36c7..67178b03e2a 100644 --- a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/applyKnownModelDefaults.test.ts +++ b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/applyKnownModelDefaults.test.ts @@ -6,11 +6,7 @@ import { type ApplyKnownModelDefaultsResult, applyKnownModelDefaults, } from "./applyKnownModelDefaults"; -import { - findKnownModelByCanonicalId, - type KnownModel, - type KnownModelSourceMetadata, -} from "./index"; +import { findKnownModelByCanonicalId, type KnownModel } from "./index"; const requireKnownModel = ( provider: string, @@ -62,18 +58,11 @@ const applyDefaults = ( parameters: ApplyKnownModelDefaultsParameters, ): ApplyKnownModelDefaultsResult => applyKnownModelDefaults(parameters); -const testSourceMetadata = (): KnownModelSourceMetadata => ({ - sourceName: "models.dev", - sourceRetrievedAt: "2026-04-30", - lastUpdated: "2026-04-30", -}); - const customKnownModel = (overrides: Partial): KnownModel => ({ provider: "openai", modelIdentifier: "test-model", displayName: "Test Model", aliases: [], - sourceMetadata: testSourceMetadata(), ...overrides, }); diff --git a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/index.ts b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/index.ts index 76161aaa346..164a8810df3 100644 --- a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/index.ts +++ b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/index.ts @@ -1,14 +1,20 @@ import { normalizeProvider } from "#/modules/aiModels/helpers"; -import { anthropicKnownModels } from "./anthropic"; -import { openAIKnownModels } from "./openai"; -import type { KnownModel, KnownModelSourceMetadata } from "./types"; +import knownModelsGenerated from "./knownModelsGenerated.json"; +import type { KnownModel } from "./types"; -export type { KnownModel, KnownModelSourceMetadata }; +export type { KnownModel }; -const knownModelsByProvider = { - anthropic: anthropicKnownModels, - openai: openAIKnownModels, -} as const satisfies Record; +// knownModelsGenerated.json is produced by `make gen/aibridge-prices` from +// models.dev joined with the editorial curation in +// scripts/aibridgepricesgen/curation.json. Do not edit it manually. JSON +// imports widen literal types (e.g. reasoningEffort becomes string), so this +// cast is the single typed boundary; knownModelsGenerated.test.ts validates +// shape and enum values for every entry. The keyof cast preserves the +// literal provider-key union so isKnownProvider narrows usefully. +const knownModelsByProvider = knownModelsGenerated as Record< + keyof typeof knownModelsGenerated, + readonly KnownModel[] +>; type KnownProvider = keyof typeof knownModelsByProvider; diff --git a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.json b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.json new file mode 100644 index 00000000000..bfcb8c0a23a --- /dev/null +++ b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.json @@ -0,0 +1,231 @@ +{ + "anthropic": [ + { + "provider": "anthropic", + "modelIdentifier": "claude-fable-5", + "displayName": "Claude Fable 5", + "aliases": [], + "contextLimit": 1000000, + "maxOutputTokens": 128000, + "reasoningEffort": "high", + "inputCost": 10, + "outputCost": 50, + "cacheReadCost": 1, + "cacheWriteCost": 12.5 + }, + { + "provider": "anthropic", + "modelIdentifier": "claude-mythos-5", + "displayName": "Claude Mythos 5", + "aliases": [], + "contextLimit": 1000000, + "maxOutputTokens": 128000, + "reasoningEffort": "high", + "inputCost": 10, + "outputCost": 50, + "cacheReadCost": 1, + "cacheWriteCost": 12.5 + }, + { + "provider": "anthropic", + "modelIdentifier": "claude-opus-4-8", + "displayName": "Claude Opus 4.8", + "aliases": [], + "contextLimit": 1000000, + "maxOutputTokens": 128000, + "reasoningEffort": "high", + "inputCost": 5, + "outputCost": 25, + "cacheReadCost": 0.5, + "cacheWriteCost": 6.25 + }, + { + "provider": "anthropic", + "modelIdentifier": "claude-opus-4-7", + "displayName": "Claude Opus 4.7", + "aliases": [], + "contextLimit": 1000000, + "maxOutputTokens": 128000, + "reasoningEffort": "high", + "inputCost": 5, + "outputCost": 25, + "cacheReadCost": 0.5, + "cacheWriteCost": 6.25 + }, + { + "provider": "anthropic", + "modelIdentifier": "claude-opus-4-6", + "displayName": "Claude Opus 4.6", + "aliases": [], + "contextLimit": 1000000, + "maxOutputTokens": 128000, + "reasoningEffort": "high", + "inputCost": 5, + "outputCost": 25, + "cacheReadCost": 0.5, + "cacheWriteCost": 6.25 + }, + { + "provider": "anthropic", + "modelIdentifier": "claude-sonnet-5", + "displayName": "Claude Sonnet 5", + "aliases": [], + "contextLimit": 1000000, + "maxOutputTokens": 128000, + "reasoningEffort": "high", + "inputCost": 2, + "outputCost": 10, + "cacheReadCost": 0.2, + "cacheWriteCost": 2.5 + }, + { + "provider": "anthropic", + "modelIdentifier": "claude-sonnet-4-6", + "displayName": "Claude Sonnet 4.6", + "aliases": [], + "contextLimit": 1000000, + "maxOutputTokens": 128000, + "reasoningEffort": "medium", + "inputCost": 3, + "outputCost": 15, + "cacheReadCost": 0.3, + "cacheWriteCost": 3.75 + }, + { + "provider": "anthropic", + "modelIdentifier": "claude-haiku-4-5", + "displayName": "Claude Haiku 4.5", + "aliases": ["claude-haiku-4-5-20251001"], + "contextLimit": 200000, + "maxOutputTokens": 64000, + "thinkingBudgetTokens": 8192, + "inputCost": 1, + "outputCost": 5, + "cacheReadCost": 0.1, + "cacheWriteCost": 1.25 + }, + { + "provider": "anthropic", + "modelIdentifier": "claude-sonnet-4-5", + "displayName": "Claude Sonnet 4.5", + "aliases": ["claude-sonnet-4-5-20250929"], + "contextLimit": 200000, + "maxOutputTokens": 64000, + "thinkingBudgetTokens": 8192, + "inputCost": 3, + "outputCost": 15, + "cacheReadCost": 0.3, + "cacheWriteCost": 3.75 + } + ], + "openai": [ + { + "provider": "openai", + "modelIdentifier": "gpt-5.6-sol", + "displayName": "GPT-5.6 Sol", + "aliases": ["gpt-5.6"], + "contextLimit": 1050000, + "maxOutputTokens": 128000, + "reasoningEffort": "medium", + "inputCost": 5, + "outputCost": 30, + "cacheReadCost": 0.5, + "cacheWriteCost": 6.25 + }, + { + "provider": "openai", + "modelIdentifier": "gpt-5.6-terra", + "displayName": "GPT-5.6 Terra", + "aliases": [], + "contextLimit": 1050000, + "maxOutputTokens": 128000, + "reasoningEffort": "medium", + "inputCost": 2.5, + "outputCost": 15, + "cacheReadCost": 0.25, + "cacheWriteCost": 3.125 + }, + { + "provider": "openai", + "modelIdentifier": "gpt-5.6-luna", + "displayName": "GPT-5.6 Luna", + "aliases": [], + "contextLimit": 1050000, + "maxOutputTokens": 128000, + "reasoningEffort": "medium", + "inputCost": 1, + "outputCost": 6, + "cacheReadCost": 0.1, + "cacheWriteCost": 1.25 + }, + { + "provider": "openai", + "modelIdentifier": "gpt-5.5", + "displayName": "GPT-5.5", + "aliases": [], + "contextLimit": 1050000, + "maxOutputTokens": 128000, + "reasoningEffort": "medium", + "inputCost": 5, + "outputCost": 30, + "cacheReadCost": 0.5 + }, + { + "provider": "openai", + "modelIdentifier": "gpt-5.5-pro", + "displayName": "GPT-5.5 Pro", + "aliases": [], + "contextLimit": 1050000, + "maxOutputTokens": 128000, + "reasoningEffort": "high", + "inputCost": 30, + "outputCost": 180 + }, + { + "provider": "openai", + "modelIdentifier": "gpt-5.4", + "displayName": "GPT-5.4", + "aliases": [], + "contextLimit": 1050000, + "maxOutputTokens": 128000, + "inputCost": 2.5, + "outputCost": 15, + "cacheReadCost": 0.25 + }, + { + "provider": "openai", + "modelIdentifier": "gpt-5.4-mini", + "displayName": "GPT-5.4 mini", + "aliases": [], + "contextLimit": 400000, + "maxOutputTokens": 128000, + "reasoningEffort": "medium", + "inputCost": 0.75, + "outputCost": 4.5, + "cacheReadCost": 0.075 + }, + { + "provider": "openai", + "modelIdentifier": "gpt-5.4-nano", + "displayName": "GPT-5.4 nano", + "aliases": [], + "contextLimit": 400000, + "maxOutputTokens": 128000, + "inputCost": 0.2, + "outputCost": 1.25, + "cacheReadCost": 0.02 + }, + { + "provider": "openai", + "modelIdentifier": "gpt-5.3-codex", + "displayName": "GPT-5.3 Codex", + "aliases": [], + "contextLimit": 400000, + "maxOutputTokens": 128000, + "reasoningEffort": "medium", + "inputCost": 1.75, + "outputCost": 14, + "cacheReadCost": 0.175 + } + ] +} diff --git a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.test.ts b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.test.ts new file mode 100644 index 00000000000..c42ccecc851 --- /dev/null +++ b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, it } from "vitest"; +import knownModelsGenerated from "./knownModelsGenerated.json"; + +// knownModelsGenerated.json crosses a typed boundary via a cast in index.ts, +// so this suite validates the shape and enum values of every generated entry. +const providers = Object.entries(knownModelsGenerated); + +describe("knownModelsGenerated", () => { + it("pins the Anthropic thinking-mode split", () => { + // Anthropic has two mutually exclusive thinking APIs. Models on the + // legacy `thinking.budget_tokens` API must not be sent + // reasoningEffort: setting effort on a legacy-thinking model returns + // HTTP 400 from Anthropic. This map pins which side of the split each + // curated model is on; update it deliberately when curating models. + const thinkingMode = Object.fromEntries( + knownModelsGenerated.anthropic.map((model) => { + let mode = "none"; + if ("thinkingBudgetTokens" in model) { + mode = "thinkingBudgetTokens"; + } else if ("reasoningEffort" in model) { + mode = "reasoningEffort"; + } + return [model.modelIdentifier, mode]; + }), + ); + expect(thinkingMode).toEqual({ + "claude-fable-5": "reasoningEffort", + "claude-mythos-5": "reasoningEffort", + "claude-opus-4-8": "reasoningEffort", + "claude-opus-4-7": "reasoningEffort", + "claude-opus-4-6": "reasoningEffort", + "claude-sonnet-5": "reasoningEffort", + "claude-sonnet-4-6": "reasoningEffort", + "claude-haiku-4-5": "thinkingBudgetTokens", + "claude-sonnet-4-5": "thinkingBudgetTokens", + }); + }); + + it("pins the claude-sonnet-4-5 context limit override", () => { + // Pinned to the flat-priced 200k tier by + // scripts/aibridgepricesgen/overrides.jq; guards the override at the + // generated-artifact layer. + const sonnet45 = knownModelsGenerated.anthropic.find( + (model) => model.modelIdentifier === "claude-sonnet-4-5", + ); + expect(sonnet45?.contextLimit).toBe(200000); + }); + + it.each(providers)("validates every %s entry", (provider, models) => { + expect(models.length).toBeGreaterThan(0); + for (const model of models) { + expect(model.provider).toBe(provider); + expect(model.modelIdentifier).not.toBe(""); + expect(model.displayName).not.toBe(""); + expect(Array.isArray(model.aliases)).toBe(true); + + const record = model as Record; + if (record.reasoningEffort !== undefined) { + expect(["low", "medium", "high"]).toContain(record.reasoningEffort); + } + expect( + record.reasoningEffort !== undefined && + record.thinkingBudgetTokens !== undefined, + ).toBe(false); + + // Token limits and budgets must be strictly positive; costs may + // legitimately be zero upstream (e.g. gpt-3.5-turbo cache_read). + for (const field of [ + "contextLimit", + "maxOutputTokens", + "thinkingBudgetTokens", + ]) { + const value = record[field]; + if (value !== undefined) { + expect(typeof value, field).toBe("number"); + expect(value, field).toBeGreaterThan(0); + } + } + for (const field of [ + "inputCost", + "outputCost", + "cacheReadCost", + "cacheWriteCost", + ]) { + const value = record[field]; + if (value !== undefined) { + expect(typeof value, field).toBe("number"); + expect(value, field).toBeGreaterThanOrEqual(0); + } + } + } + }); +}); diff --git a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/openai.test.ts b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/openai.test.ts deleted file mode 100644 index e1bd79b2d31..00000000000 --- a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/openai.test.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { getKnownModelsForProvider, type KnownModel } from "./index"; -import { openAIKnownModels } from "./openai"; - -describe("openAIKnownModels", () => { - it("returns OpenAI canonical IDs in declared order", () => { - expect( - getKnownModelsForProvider("openai").map( - (knownModel) => knownModel.modelIdentifier, - ), - ).toEqual([ - "gpt-5.5", - "gpt-5.5-pro", - "gpt-5.4", - "gpt-5.4-mini", - "gpt-5.4-nano", - "gpt-5.3-codex", - ]); - }); - - it("declares reasoning effort only for reasoning-capable models", () => { - const knownModels: readonly KnownModel[] = openAIKnownModels; - const reasoningEffortByModel = Object.fromEntries( - knownModels.map((knownModel) => [ - knownModel.modelIdentifier, - knownModel.reasoningEffort, - ]), - ); - - expect(reasoningEffortByModel).toEqual({ - "gpt-5.5": "medium", - "gpt-5.5-pro": "high", - "gpt-5.4": undefined, - "gpt-5.4-mini": "medium", - "gpt-5.4-nano": undefined, - "gpt-5.3-codex": "medium", - }); - }); - - it("has source metadata, provider equality, and declared order", () => { - expect( - openAIKnownModels.map((knownModel) => knownModel.modelIdentifier), - ).toEqual([ - "gpt-5.5", - "gpt-5.5-pro", - "gpt-5.4", - "gpt-5.4-mini", - "gpt-5.4-nano", - "gpt-5.3-codex", - ]); - - for (const knownModel of openAIKnownModels) { - expect(knownModel.provider).toBe("openai"); - expect(knownModel.sourceMetadata.sourceName).toBe("models.dev"); - expect(knownModel.sourceMetadata.sourceRetrievedAt).toBe("2026-04-30"); - expect(knownModel.sourceMetadata.lastUpdated).not.toBe(""); - } - }); -}); diff --git a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/openai.ts b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/openai.ts deleted file mode 100644 index dc03aa31974..00000000000 --- a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/openai.ts +++ /dev/null @@ -1,111 +0,0 @@ -import type { KnownModel } from "./types"; - -// Array order controls suggestion order. Keep sourceMetadata.lastUpdated in -// sync with the corresponding models.dev last_updated value for each model. -// Coder currently persists flat pricing only. Tiered models.dev pricing, -// such as context_over_200k, is intentionally omitted. -// -// The `reasoningEffort` value is editorial, not from models.dev. It reflects -// the provider's documented default for reasoning-capable models in this -// catalog and should be reviewed when the catalog is refreshed. -export const openAIKnownModels = [ - { - provider: "openai", - modelIdentifier: "gpt-5.5", - displayName: "GPT-5.5", - aliases: [], - contextLimit: 1_050_000, - maxOutputTokens: 128_000, - reasoningEffort: "medium", - inputCost: 5, - outputCost: 30, - cacheReadCost: 0.5, - sourceMetadata: { - sourceName: "models.dev", - sourceRetrievedAt: "2026-04-30", - lastUpdated: "2026-04-23", - }, - }, - { - provider: "openai", - modelIdentifier: "gpt-5.5-pro", - displayName: "GPT-5.5 Pro", - aliases: [], - contextLimit: 1_050_000, - maxOutputTokens: 128_000, - reasoningEffort: "high", - inputCost: 30, - outputCost: 180, - sourceMetadata: { - sourceName: "models.dev", - sourceRetrievedAt: "2026-04-30", - lastUpdated: "2026-04-23", - }, - }, - { - provider: "openai", - modelIdentifier: "gpt-5.4", - displayName: "GPT-5.4", - aliases: [], - contextLimit: 1_050_000, - maxOutputTokens: 128_000, - inputCost: 2.5, - outputCost: 15, - cacheReadCost: 0.25, - sourceMetadata: { - sourceName: "models.dev", - sourceRetrievedAt: "2026-04-30", - lastUpdated: "2026-03-05", - }, - }, - { - provider: "openai", - modelIdentifier: "gpt-5.4-mini", - displayName: "GPT-5.4 mini", - aliases: [], - contextLimit: 400_000, - maxOutputTokens: 128_000, - reasoningEffort: "medium", - inputCost: 0.75, - outputCost: 4.5, - cacheReadCost: 0.075, - sourceMetadata: { - sourceName: "models.dev", - sourceRetrievedAt: "2026-04-30", - lastUpdated: "2026-03-17", - }, - }, - { - provider: "openai", - modelIdentifier: "gpt-5.4-nano", - displayName: "GPT-5.4 nano", - aliases: [], - contextLimit: 400_000, - maxOutputTokens: 128_000, - inputCost: 0.2, - outputCost: 1.25, - cacheReadCost: 0.02, - sourceMetadata: { - sourceName: "models.dev", - sourceRetrievedAt: "2026-04-30", - lastUpdated: "2026-03-17", - }, - }, - { - provider: "openai", - modelIdentifier: "gpt-5.3-codex", - displayName: "GPT-5.3 Codex", - aliases: [], - contextLimit: 400_000, - maxOutputTokens: 128_000, - reasoningEffort: "medium", - inputCost: 1.75, - outputCost: 14, - cacheReadCost: 0.175, - sourceMetadata: { - sourceName: "models.dev", - sourceRetrievedAt: "2026-04-30", - lastUpdated: "2026-02-05", - }, - }, -] as const satisfies readonly KnownModel[]; diff --git a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/types.ts b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/types.ts index 08037f54cd5..2e7c753ce9a 100644 --- a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/types.ts +++ b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/types.ts @@ -1,9 +1,3 @@ -export type KnownModelSourceMetadata = { - sourceName: "models.dev"; - sourceRetrievedAt: string; - lastUpdated: string; -}; - export type KnownModel = { provider: string; modelIdentifier: string; @@ -24,5 +18,4 @@ export type KnownModel = { outputCost?: number; cacheReadCost?: number; cacheWriteCost?: number; - sourceMetadata: KnownModelSourceMetadata; };