From 6385157af075b55192bc20d17b457d6c46e92124 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 10 Jul 2026 07:46:30 +0000 Subject: [PATCH 01/14] chore(coderd/aibridge/prices): regenerate price seed for gpt-5.6 models --- coderd/aibridge/prices/data/prices.json | 164 +++++++++--------------- 1 file changed, 58 insertions(+), 106 deletions(-) diff --git a/coderd/aibridge/prices/data/prices.json b/coderd/aibridge/prices/data/prices.json index 4c8b4527e10..6851a433bd2 100644 --- a/coderd/aibridge/prices/data/prices.json +++ b/coderd/aibridge/prices/data/prices.json @@ -1,67 +1,11 @@ [ { "provider": "anthropic", - "model": "claude-3-5-haiku-20241022", - "input_price": 800000, - "output_price": 4000000, - "cache_read_price": 80000, - "cache_write_price": 1000000 - }, - { - "provider": "anthropic", - "model": "claude-3-5-haiku-latest", - "input_price": 800000, - "output_price": 4000000, - "cache_read_price": 80000, - "cache_write_price": 1000000 - }, - { - "provider": "anthropic", - "model": "claude-3-5-sonnet-20240620", - "input_price": 3000000, - "output_price": 15000000, - "cache_read_price": 300000, - "cache_write_price": 3750000 - }, - { - "provider": "anthropic", - "model": "claude-3-5-sonnet-20241022", - "input_price": 3000000, - "output_price": 15000000, - "cache_read_price": 300000, - "cache_write_price": 3750000 - }, - { - "provider": "anthropic", - "model": "claude-3-7-sonnet-20250219", - "input_price": 3000000, - "output_price": 15000000, - "cache_read_price": 300000, - "cache_write_price": 3750000 - }, - { - "provider": "anthropic", - "model": "claude-3-haiku-20240307", - "input_price": 250000, - "output_price": 1250000, - "cache_read_price": 30000, - "cache_write_price": 300000 - }, - { - "provider": "anthropic", - "model": "claude-3-opus-20240229", - "input_price": 15000000, - "output_price": 75000000, - "cache_read_price": 1500000, - "cache_write_price": 18750000 - }, - { - "provider": "anthropic", - "model": "claude-3-sonnet-20240229", - "input_price": 3000000, - "output_price": 15000000, - "cache_read_price": 300000, - "cache_write_price": 300000 + "model": "claude-fable-5", + "input_price": 10000000, + "output_price": 50000000, + "cache_read_price": 1000000, + "cache_write_price": 12500000 }, { "provider": "anthropic", @@ -79,14 +23,6 @@ "cache_read_price": 100000, "cache_write_price": 1250000 }, - { - "provider": "anthropic", - "model": "claude-opus-4-0", - "input_price": 15000000, - "output_price": 75000000, - "cache_read_price": 1500000, - "cache_write_price": 18750000 - }, { "provider": "anthropic", "model": "claude-opus-4-1", @@ -103,14 +39,6 @@ "cache_read_price": 1500000, "cache_write_price": 18750000 }, - { - "provider": "anthropic", - "model": "claude-opus-4-20250514", - "input_price": 15000000, - "output_price": 75000000, - "cache_read_price": 1500000, - "cache_write_price": 18750000 - }, { "provider": "anthropic", "model": "claude-opus-4-5", @@ -145,19 +73,11 @@ }, { "provider": "anthropic", - "model": "claude-sonnet-4-0", - "input_price": 3000000, - "output_price": 15000000, - "cache_read_price": 300000, - "cache_write_price": 3750000 - }, - { - "provider": "anthropic", - "model": "claude-sonnet-4-20250514", - "input_price": 3000000, - "output_price": 15000000, - "cache_read_price": 300000, - "cache_write_price": 3750000 + "model": "claude-opus-4-8", + "input_price": 5000000, + "output_price": 25000000, + "cache_read_price": 500000, + "cache_write_price": 6250000 }, { "provider": "anthropic", @@ -183,12 +103,20 @@ "cache_read_price": 300000, "cache_write_price": 3750000 }, + { + "provider": "anthropic", + "model": "claude-sonnet-5", + "input_price": 2000000, + "output_price": 10000000, + "cache_read_price": 200000, + "cache_write_price": 2500000 + }, { "provider": "openai", "model": "gpt-3.5-turbo", "input_price": 500000, "output_price": 1500000, - "cache_read_price": 1250000, + "cache_read_price": 0, "cache_write_price": null }, { @@ -228,7 +156,7 @@ "model": "gpt-4.1-nano", "input_price": 100000, "output_price": 400000, - "cache_read_price": 30000, + "cache_read_price": 25000, "cache_write_price": null }, { @@ -268,7 +196,7 @@ "model": "gpt-4o-mini", "input_price": 150000, "output_price": 600000, - "cache_read_price": 80000, + "cache_read_price": 75000, "cache_write_price": null }, { @@ -284,7 +212,7 @@ "model": "gpt-5-chat-latest", "input_price": 1250000, "output_price": 10000000, - "cache_read_price": null, + "cache_read_price": 125000, "cache_write_price": null }, { @@ -324,7 +252,7 @@ "model": "gpt-5.1", "input_price": 1250000, "output_price": 10000000, - "cache_read_price": 130000, + "cache_read_price": 125000, "cache_write_price": null }, { @@ -465,23 +393,47 @@ }, { "provider": "openai", - "model": "o1", - "input_price": 15000000, - "output_price": 60000000, - "cache_read_price": 7500000, - "cache_write_price": null + "model": "gpt-5.6", + "input_price": 5000000, + "output_price": 30000000, + "cache_read_price": 500000, + "cache_write_price": 6250000 }, { "provider": "openai", - "model": "o1-mini", - "input_price": 1100000, - "output_price": 4400000, - "cache_read_price": 550000, + "model": "gpt-5.6-luna", + "input_price": 1000000, + "output_price": 6000000, + "cache_read_price": 100000, + "cache_write_price": 1250000 + }, + { + "provider": "openai", + "model": "gpt-5.6-sol", + "input_price": 5000000, + "output_price": 30000000, + "cache_read_price": 500000, + "cache_write_price": 6250000 + }, + { + "provider": "openai", + "model": "gpt-5.6-terra", + "input_price": 2500000, + "output_price": 15000000, + "cache_read_price": 250000, + "cache_write_price": 3125000 + }, + { + "provider": "openai", + "model": "gpt-image-2", + "input_price": 5000000, + "output_price": 30000000, + "cache_read_price": 1250000, "cache_write_price": null }, { "provider": "openai", - "model": "o1-preview", + "model": "o1", "input_price": 15000000, "output_price": 60000000, "cache_read_price": 7500000, @@ -532,7 +484,7 @@ "model": "o4-mini", "input_price": 1100000, "output_price": 4400000, - "cache_read_price": 280000, + "cache_read_price": 275000, "cache_write_price": null }, { From abbac7ebfdd01174b74c77235b88a9769937642c Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 10 Jul 2026 10:13:48 +0000 Subject: [PATCH 02/14] feat(site/src/pages/AgentsPage): add gpt-5.6 family to known OpenAI models --- .../knownModels/openai.test.ts | 11 +++- .../ChatModelAdminPanel/knownModels/openai.ts | 56 +++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/openai.test.ts b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/openai.test.ts index e1bd79b2d31..a90d090808b 100644 --- a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/openai.test.ts +++ b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/openai.test.ts @@ -9,6 +9,9 @@ describe("openAIKnownModels", () => { (knownModel) => knownModel.modelIdentifier, ), ).toEqual([ + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-5.6-luna", "gpt-5.5", "gpt-5.5-pro", "gpt-5.4", @@ -28,6 +31,9 @@ describe("openAIKnownModels", () => { ); expect(reasoningEffortByModel).toEqual({ + "gpt-5.6-sol": "medium", + "gpt-5.6-terra": "medium", + "gpt-5.6-luna": "medium", "gpt-5.5": "medium", "gpt-5.5-pro": "high", "gpt-5.4": undefined, @@ -41,6 +47,9 @@ describe("openAIKnownModels", () => { expect( openAIKnownModels.map((knownModel) => knownModel.modelIdentifier), ).toEqual([ + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-5.6-luna", "gpt-5.5", "gpt-5.5-pro", "gpt-5.4", @@ -52,7 +61,7 @@ describe("openAIKnownModels", () => { for (const knownModel of openAIKnownModels) { expect(knownModel.provider).toBe("openai"); expect(knownModel.sourceMetadata.sourceName).toBe("models.dev"); - expect(knownModel.sourceMetadata.sourceRetrievedAt).toBe("2026-04-30"); + expect(knownModel.sourceMetadata.sourceRetrievedAt).not.toBe(""); expect(knownModel.sourceMetadata.lastUpdated).not.toBe(""); } }); diff --git a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/openai.ts b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/openai.ts index dc03aa31974..3fcfc26883e 100644 --- a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/openai.ts +++ b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/openai.ts @@ -9,6 +9,62 @@ import type { KnownModel } from "./types"; // the provider's documented default for reasoning-capable models in this // catalog and should be reviewed when the catalog is refreshed. export const openAIKnownModels = [ + { + provider: "openai", + modelIdentifier: "gpt-5.6-sol", + displayName: "GPT-5.6 Sol", + // models.dev lists gpt-5.6 with specs and pricing identical to + // gpt-5.6-sol, so it resolves to this entry. + aliases: ["gpt-5.6"], + contextLimit: 1_050_000, + maxOutputTokens: 128_000, + reasoningEffort: "medium", + inputCost: 5, + outputCost: 30, + cacheReadCost: 0.5, + cacheWriteCost: 6.25, + sourceMetadata: { + sourceName: "models.dev", + sourceRetrievedAt: "2026-07-10", + lastUpdated: "2026-07-09", + }, + }, + { + provider: "openai", + modelIdentifier: "gpt-5.6-terra", + displayName: "GPT-5.6 Terra", + aliases: [], + contextLimit: 1_050_000, + maxOutputTokens: 128_000, + reasoningEffort: "medium", + inputCost: 2.5, + outputCost: 15, + cacheReadCost: 0.25, + cacheWriteCost: 3.125, + sourceMetadata: { + sourceName: "models.dev", + sourceRetrievedAt: "2026-07-10", + lastUpdated: "2026-07-09", + }, + }, + { + provider: "openai", + modelIdentifier: "gpt-5.6-luna", + displayName: "GPT-5.6 Luna", + aliases: [], + contextLimit: 1_050_000, + maxOutputTokens: 128_000, + reasoningEffort: "medium", + inputCost: 1, + outputCost: 6, + cacheReadCost: 0.1, + cacheWriteCost: 1.25, + sourceMetadata: { + sourceName: "models.dev", + sourceRetrievedAt: "2026-07-10", + lastUpdated: "2026-07-09", + }, + }, { provider: "openai", modelIdentifier: "gpt-5.5", From be4e32898eaf91e0358c4ae48f3d838850fae077 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 10 Jul 2026 14:24:17 +0000 Subject: [PATCH 03/14] feat(scripts/aibridgepricesgen): add -format=catalog for known models --- scripts/aibridgepricesgen/catalog.go | 145 ++++++++++ scripts/aibridgepricesgen/catalog.json | 79 ++++++ scripts/aibridgepricesgen/catalog_test.go | 254 ++++++++++++++++++ scripts/aibridgepricesgen/main.go | 48 +++- .../knownModels/anthropic.test.ts | 91 ------- .../knownModels/anthropic.ts | 169 ------------ .../knownModels/openai.test.ts | 68 ----- .../ChatModelAdminPanel/knownModels/openai.ts | 167 ------------ 8 files changed, 523 insertions(+), 498 deletions(-) create mode 100644 scripts/aibridgepricesgen/catalog.go create mode 100644 scripts/aibridgepricesgen/catalog.json create mode 100644 scripts/aibridgepricesgen/catalog_test.go delete mode 100644 site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/anthropic.test.ts delete mode 100644 site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/anthropic.ts delete mode 100644 site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/openai.test.ts delete mode 100644 site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/openai.ts diff --git a/scripts/aibridgepricesgen/catalog.go b/scripts/aibridgepricesgen/catalog.go new file mode 100644 index 00000000000..146159c7ca3 --- /dev/null +++ b/scripts/aibridgepricesgen/catalog.go @@ -0,0 +1,145 @@ +package main + +import ( + _ "embed" + "encoding/json" + "io" + "time" + + "golang.org/x/xerrors" +) + +// catalogJSON is the checked-in editorial curation input for the frontend +// known-models catalog. Entry order within each provider controls suggestion +// order in the UI. Everything factual (display name, limits, pricing, +// last_updated) is joined from models.dev at generation time; the curation +// file only carries editorial choices: which models to suggest, aliases, +// reasoning defaults, and overrides. +// +//go:embed catalog.json +var catalogJSON []byte + +// curatedModel is one entry in catalog.json. +type curatedModel struct { + ModelIdentifier string `json:"modelIdentifier"` + Aliases []string `json:"aliases"` + // DisplayName overrides the upstream `name` when set. Needed where + // upstream naming does not match what we want to show (for example + // "Claude Haiku 4.5 (latest)"), and for mirrored models. + DisplayName string `json:"displayName"` + // ReasoningEffort is editorial, not from models.dev. Mutually + // exclusive with ThinkingBudgetTokens. + ReasoningEffort string `json:"reasoningEffort"` + // ThinkingBudgetTokens is Anthropic-only, for models that do not + // support adaptive thinking and use the legacy + // `thinking.budget_tokens` API instead. + ThinkingBudgetTokens int `json:"thinkingBudgetTokens"` + // MirrorOf names another upstream model whose specs and pricing this + // entry copies. Used for models absent from models.dev (for example + // claude-mythos-5, which Anthropic documents as sharing + // claude-fable-5's specs and pricing). + MirrorOf string `json:"mirrorOf"` +} + +// catalogEntry matches the frontend KnownModel shape (knownModels/types.ts). +// Costs are flat USD per million tokens, straight from models.dev; tiered +// pricing such as context_over_200k is intentionally omitted. +type catalogEntry struct { + Provider string `json:"provider"` + ModelIdentifier string `json:"modelIdentifier"` + DisplayName string `json:"displayName"` + Aliases []string `json:"aliases"` + ContextLimit *int64 `json:"contextLimit,omitempty"` + MaxOutputTokens *int64 `json:"maxOutputTokens,omitempty"` + ReasoningEffort string `json:"reasoningEffort,omitempty"` + ThinkingBudgetTokens int `json:"thinkingBudgetTokens,omitempty"` + InputCost *float64 `json:"inputCost,omitempty"` + OutputCost *float64 `json:"outputCost,omitempty"` + CacheReadCost *float64 `json:"cacheReadCost,omitempty"` + CacheWriteCost *float64 `json:"cacheWriteCost,omitempty"` + SourceMetadata catalogMetadata `json:"sourceMetadata"` +} + +type catalogMetadata struct { + SourceName string `json:"sourceName"` + SourceRetrievedAt string `json:"sourceRetrievedAt"` + LastUpdated string `json:"lastUpdated"` +} + +// buildCatalog joins the curation file with the upstream models.dev payload +// and returns provider-keyed ordered entry lists. now supplies the +// sourceRetrievedAt date so output is deterministic under test. +func buildCatalog(upstream map[string]upstreamProvider, curation map[string][]curatedModel, now time.Time) (map[string][]catalogEntry, error) { + retrievedAt := now.UTC().Format("2006-01-02") + out := make(map[string][]catalogEntry, len(curation)) + for providerID, curated := range curation { + provider, ok := upstream[providerID] + if !ok { + return nil, xerrors.Errorf("provider %q missing from upstream", providerID) + } + entries := make([]catalogEntry, 0, len(curated)) + for _, c := range curated { + if c.ModelIdentifier == "" { + return nil, xerrors.Errorf("provider %q: entry with empty modelIdentifier", providerID) + } + if c.ReasoningEffort != "" && c.ThinkingBudgetTokens != 0 { + return nil, xerrors.Errorf("%s/%s: reasoningEffort and thinkingBudgetTokens are mutually exclusive", providerID, c.ModelIdentifier) + } + sourceID := c.ModelIdentifier + if c.MirrorOf != "" { + sourceID = c.MirrorOf + } + m, ok := provider.Models[sourceID] + if !ok { + if c.MirrorOf != "" { + return nil, xerrors.Errorf("%s/%s: mirrorOf target %q missing from upstream", providerID, c.ModelIdentifier, c.MirrorOf) + } + return nil, xerrors.Errorf("%s/%s: model missing from upstream (use mirrorOf if intentional)", providerID, c.ModelIdentifier) + } + if !m.Cost.hasPricing() { + return nil, xerrors.Errorf("%s/%s: upstream model %q has no cost block", providerID, c.ModelIdentifier, sourceID) + } + displayName := c.DisplayName + if displayName == "" { + displayName = m.Name + } + if displayName == "" { + return nil, xerrors.Errorf("%s/%s: no displayName override and upstream name is empty", providerID, c.ModelIdentifier) + } + aliases := c.Aliases + if aliases == nil { + aliases = []string{} + } + entries = append(entries, catalogEntry{ + Provider: providerID, + ModelIdentifier: c.ModelIdentifier, + DisplayName: displayName, + Aliases: aliases, + ContextLimit: m.Limit.Context, + MaxOutputTokens: m.Limit.Output, + ReasoningEffort: c.ReasoningEffort, + ThinkingBudgetTokens: c.ThinkingBudgetTokens, + InputCost: m.Cost.Input, + OutputCost: m.Cost.Output, + CacheReadCost: m.Cost.CacheRead, + CacheWriteCost: m.Cost.CacheWrite, + SourceMetadata: catalogMetadata{ + SourceName: "models.dev", + SourceRetrievedAt: retrievedAt, + LastUpdated: m.LastUpdated, + }, + }) + } + out[providerID] = entries + } + return out, nil +} + +func writeCatalog(w io.Writer, catalog map[string][]catalogEntry) error { + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + if err := enc.Encode(catalog); err != nil { + return xerrors.Errorf("encode: %w", err) + } + return nil +} diff --git a/scripts/aibridgepricesgen/catalog.json b/scripts/aibridgepricesgen/catalog.json new file mode 100644 index 00000000000..cfcf7c8b818 --- /dev/null +++ b/scripts/aibridgepricesgen/catalog.json @@ -0,0 +1,79 @@ +{ + "openai": [ + { + "modelIdentifier": "gpt-5.6-sol", + "aliases": ["gpt-5.6"], + "reasoningEffort": "medium" + }, + { + "modelIdentifier": "gpt-5.6-terra", + "reasoningEffort": "medium" + }, + { + "modelIdentifier": "gpt-5.6-luna", + "reasoningEffort": "medium" + }, + { + "modelIdentifier": "gpt-5.5", + "reasoningEffort": "medium" + }, + { + "modelIdentifier": "gpt-5.5-pro", + "reasoningEffort": "high" + }, + { + "modelIdentifier": "gpt-5.4" + }, + { + "modelIdentifier": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + { + "modelIdentifier": "gpt-5.4-nano" + }, + { + "modelIdentifier": "gpt-5.3-codex", + "reasoningEffort": "medium" + } + ], + "anthropic": [ + { + "modelIdentifier": "claude-fable-5", + "reasoningEffort": "high" + }, + { + "modelIdentifier": "claude-mythos-5", + "mirrorOf": "claude-fable-5", + "displayName": "Claude Mythos 5", + "reasoningEffort": "high" + }, + { + "modelIdentifier": "claude-opus-4-8", + "reasoningEffort": "high" + }, + { + "modelIdentifier": "claude-opus-4-7", + "reasoningEffort": "high" + }, + { + "modelIdentifier": "claude-opus-4-6", + "reasoningEffort": "high" + }, + { + "modelIdentifier": "claude-sonnet-4-6", + "reasoningEffort": "medium" + }, + { + "modelIdentifier": "claude-haiku-4-5", + "aliases": ["claude-haiku-4-5-20251001"], + "displayName": "Claude Haiku 4.5", + "thinkingBudgetTokens": 8192 + }, + { + "modelIdentifier": "claude-sonnet-4-5", + "aliases": ["claude-sonnet-4-5-20250929"], + "displayName": "Claude Sonnet 4.5", + "thinkingBudgetTokens": 8192 + } + ] +} diff --git a/scripts/aibridgepricesgen/catalog_test.go b/scripts/aibridgepricesgen/catalog_test.go new file mode 100644 index 00000000000..9c34960d5ee --- /dev/null +++ b/scripts/aibridgepricesgen/catalog_test.go @@ -0,0 +1,254 @@ +package main + +import ( + "bytes" + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// fixtureUpstream returns a small upstream payload covering the join cases: +// a fully priced model with limits, a mirror target, and a costless model. +func fixtureUpstream(t *testing.T) map[string]upstreamProvider { + t.Helper() + const upstreamJSON = `{ + "anthropic": { + "models": { + "claude-fable-5": { + "name": "Claude Fable 5", + "limit": {"context": 1000000, "output": 128000}, + "cost": {"input": 10, "output": 50, "cache_read": 1, "cache_write": 12.5}, + "last_updated": "2026-06-09" + }, + "claude-costless": { + "name": "Claude Costless", + "limit": {"context": 200000, "output": 64000}, + "last_updated": "2026-01-01" + } + } + }, + "openai": { + "models": { + "gpt-5.6-sol": { + "name": "GPT-5.6 Sol", + "limit": {"context": 1050000, "output": 128000}, + "cost": {"input": 5, "output": 30, "cache_read": 0.5, "cache_write": 6.25}, + "last_updated": "2026-07-09" + }, + "gpt-partial": { + "name": "GPT Partial", + "limit": {"context": 400000, "output": 128000}, + "cost": {"input": 0.2, "output": 1.25}, + "last_updated": "2026-03-17" + } + } + } + }` + var upstream map[string]upstreamProvider + require.NoError(t, json.Unmarshal([]byte(upstreamJSON), &upstream)) + return upstream +} + +var fixedNow = time.Date(2026, 7, 10, 12, 34, 56, 0, time.UTC) + +func TestBuildCatalog(t *testing.T) { + t.Parallel() + + curation := map[string][]curatedModel{ + "openai": { + {ModelIdentifier: "gpt-5.6-sol", Aliases: []string{"gpt-5.6"}, ReasoningEffort: "medium"}, + {ModelIdentifier: "gpt-partial"}, + }, + "anthropic": { + {ModelIdentifier: "claude-fable-5", ReasoningEffort: "high"}, + {ModelIdentifier: "claude-mythos-5", MirrorOf: "claude-fable-5", DisplayName: "Claude Mythos 5", ReasoningEffort: "high"}, + }, + } + + catalog, err := buildCatalog(fixtureUpstream(t), curation, fixedNow) + require.NoError(t, err) + + var buf bytes.Buffer + require.NoError(t, writeCatalog(&buf, catalog)) + + const want = `{ + "anthropic": [ + { + "provider": "anthropic", + "modelIdentifier": "claude-fable-5", + "displayName": "Claude Fable 5", + "aliases": [], + "contextLimit": 1000000, + "maxOutputTokens": 128000, + "reasoningEffort": "high", + "inputCost": 10, + "outputCost": 50, + "cacheReadCost": 1, + "cacheWriteCost": 12.5, + "sourceMetadata": { + "sourceName": "models.dev", + "sourceRetrievedAt": "2026-07-10", + "lastUpdated": "2026-06-09" + } + }, + { + "provider": "anthropic", + "modelIdentifier": "claude-mythos-5", + "displayName": "Claude Mythos 5", + "aliases": [], + "contextLimit": 1000000, + "maxOutputTokens": 128000, + "reasoningEffort": "high", + "inputCost": 10, + "outputCost": 50, + "cacheReadCost": 1, + "cacheWriteCost": 12.5, + "sourceMetadata": { + "sourceName": "models.dev", + "sourceRetrievedAt": "2026-07-10", + "lastUpdated": "2026-06-09" + } + } + ], + "openai": [ + { + "provider": "openai", + "modelIdentifier": "gpt-5.6-sol", + "displayName": "GPT-5.6 Sol", + "aliases": [ + "gpt-5.6" + ], + "contextLimit": 1050000, + "maxOutputTokens": 128000, + "reasoningEffort": "medium", + "inputCost": 5, + "outputCost": 30, + "cacheReadCost": 0.5, + "cacheWriteCost": 6.25, + "sourceMetadata": { + "sourceName": "models.dev", + "sourceRetrievedAt": "2026-07-10", + "lastUpdated": "2026-07-09" + } + }, + { + "provider": "openai", + "modelIdentifier": "gpt-partial", + "displayName": "GPT Partial", + "aliases": [], + "contextLimit": 400000, + "maxOutputTokens": 128000, + "inputCost": 0.2, + "outputCost": 1.25, + "sourceMetadata": { + "sourceName": "models.dev", + "sourceRetrievedAt": "2026-07-10", + "lastUpdated": "2026-03-17" + } + } + ] +} +` + require.Equal(t, want, buf.String()) +} + +func TestBuildCatalogDeterministic(t *testing.T) { + t.Parallel() + + curation := map[string][]curatedModel{ + "openai": {{ModelIdentifier: "gpt-5.6-sol"}}, + } + a, err := buildCatalog(fixtureUpstream(t), curation, fixedNow) + require.NoError(t, err) + b, err := buildCatalog(fixtureUpstream(t), curation, fixedNow) + require.NoError(t, err) + require.Equal(t, a, b) + + // A different injected clock changes only sourceRetrievedAt. + later, err := buildCatalog(fixtureUpstream(t), curation, fixedNow.AddDate(0, 0, 1)) + require.NoError(t, err) + require.Equal(t, "2026-07-11", later["openai"][0].SourceMetadata.SourceRetrievedAt) +} + +func TestBuildCatalogErrors(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + curation map[string][]curatedModel + wantErr string + }{ + { + name: "MissingUpstreamModel", + curation: map[string][]curatedModel{ + "openai": {{ModelIdentifier: "gpt-nonexistent"}}, + }, + wantErr: "missing from upstream", + }, + { + name: "DanglingMirrorOf", + curation: map[string][]curatedModel{ + "anthropic": {{ModelIdentifier: "claude-mythos-5", MirrorOf: "claude-nonexistent"}}, + }, + wantErr: "mirrorOf target", + }, + { + name: "NoCostBlock", + curation: map[string][]curatedModel{ + "anthropic": {{ModelIdentifier: "claude-costless"}}, + }, + wantErr: "no cost block", + }, + { + name: "EffortAndBudgetBothSet", + curation: map[string][]curatedModel{ + "anthropic": {{ModelIdentifier: "claude-fable-5", ReasoningEffort: "high", ThinkingBudgetTokens: 8192}}, + }, + wantErr: "mutually exclusive", + }, + { + name: "MissingProvider", + curation: map[string][]curatedModel{ + "google": {{ModelIdentifier: "gemini"}}, + }, + wantErr: "missing from upstream", + }, + { + name: "EmptyModelIdentifier", + curation: map[string][]curatedModel{ + "openai": {{}}, + }, + wantErr: "empty modelIdentifier", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + _, err := buildCatalog(fixtureUpstream(t), tc.curation, fixedNow) + require.Error(t, err) + require.Contains(t, err.Error(), tc.wantErr) + }) + } +} + +// TestEmbeddedCatalogParses guards the checked-in curation file itself: +// valid JSON, required fields present, and the effort/budget exclusivity +// holds without needing upstream data. +func TestEmbeddedCatalogParses(t *testing.T) { + t.Parallel() + + var curation map[string][]curatedModel + require.NoError(t, json.Unmarshal(catalogJSON, &curation)) + require.NotEmpty(t, curation) + for providerID, entries := range curation { + require.NotEmpty(t, entries, providerID) + for _, c := range entries { + require.NotEmpty(t, c.ModelIdentifier, providerID) + require.False(t, c.ReasoningEffort != "" && c.ThinkingBudgetTokens != 0, + "%s/%s sets both reasoningEffort and thinkingBudgetTokens", providerID, c.ModelIdentifier) + } + } +} diff --git a/scripts/aibridgepricesgen/main.go b/scripts/aibridgepricesgen/main.go index 20a26c0f1b2..adfb6fe2266 100644 --- a/scripts/aibridgepricesgen/main.go +++ b/scripts/aibridgepricesgen/main.go @@ -11,6 +11,7 @@ package main import ( "context" "encoding/json" + "flag" "fmt" "io" "math" @@ -42,10 +43,18 @@ type upstreamProvider struct { } type upstreamModel struct { - Cost *upstreamCost `json:"cost"` + Name string `json:"name"` + Limit upstreamLimit `json:"limit"` + Cost *upstreamCost `json:"cost"` + LastUpdated string `json:"last_updated"` } // Pointers distinguish "key absent" (nil) from "key present and zero" (0). +type upstreamLimit struct { + Context *int64 `json:"context"` + Output *int64 `json:"output"` +} + type upstreamCost struct { Input *float64 `json:"input"` Output *float64 `json:"output"` @@ -80,17 +89,30 @@ type priceRow struct { } func main() { - if err := run(); err != nil { + format := flag.String("format", "prices", `output format: "prices" (cost-control seed) or "catalog" (frontend known-models JSON)`) + flag.Parse() + if err := run(*format); err != nil { _, _ = fmt.Fprintf(os.Stderr, "aibridgepricesgen: %v\n", err) os.Exit(1) } } -func run() error { +func run(format string) error { upstream, err := fetch() if err != nil { return xerrors.Errorf("fetch %s: %w", sourceURL, err) } + switch format { + case "prices": + return runPrices(upstream) + case "catalog": + return runCatalog(upstream) + default: + return xerrors.Errorf("unknown -format %q (want \"prices\" or \"catalog\")", format) + } +} + +func runPrices(upstream map[string]upstreamProvider) error { rows, err := convert(upstream, supportedProviders) if err != nil { return err @@ -105,6 +127,26 @@ func run() error { return nil } +func runCatalog(upstream map[string]upstreamProvider) error { + var curation map[string][]curatedModel + if err := json.Unmarshal(catalogJSON, &curation); err != nil { + return xerrors.Errorf("parse embedded catalog.json: %w", err) + } + catalog, err := buildCatalog(upstream, curation, time.Now()) + if err != nil { + return err + } + if err := writeCatalog(os.Stdout, catalog); err != nil { + return err + } + total := 0 + for _, entries := range catalog { + total += len(entries) + } + _, _ = fmt.Fprintf(os.Stderr, "aibridgepricesgen: wrote %d catalog entries for %d provider(s)\n", total, len(catalog)) + return nil +} + func fetch() (map[string]upstreamProvider, error) { ctx, cancel := context.WithTimeout(context.Background(), fetchTimeout) defer cancel() diff --git a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/anthropic.test.ts b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/anthropic.test.ts deleted file mode 100644 index e8bc04b4208..00000000000 --- a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/anthropic.test.ts +++ /dev/null @@ -1,91 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { anthropicKnownModels } from "./anthropic"; -import { getKnownModelsForProvider } from "./index"; -import type { KnownModel } from "./types"; - -const anthropicKnownModelList: readonly KnownModel[] = anthropicKnownModels; - -const requireAnthropicKnownModel = (modelIdentifier: string): KnownModel => { - const knownModel = anthropicKnownModelList.find( - (knownModel) => knownModel.modelIdentifier === modelIdentifier, - ); - if (knownModel === undefined) { - throw new Error(`missing Anthropic Known Model: ${modelIdentifier}`); - } - return knownModel; -}; - -describe("anthropicKnownModels", () => { - it("returns Anthropic canonical IDs in declared order", () => { - expect( - getKnownModelsForProvider("anthropic").map( - (knownModel) => knownModel.modelIdentifier, - ), - ).toEqual([ - "claude-fable-5", - "claude-mythos-5", - "claude-opus-4-8", - "claude-opus-4-7", - "claude-opus-4-6", - "claude-sonnet-4-6", - "claude-haiku-4-5", - "claude-sonnet-4-5", - ]); - }); - - it("declares Anthropic reasoning defaults by API support", () => { - for (const modelIdentifier of [ - "claude-fable-5", - "claude-mythos-5", - "claude-opus-4-8", - "claude-opus-4-7", - "claude-opus-4-6", - ]) { - const knownModel = requireAnthropicKnownModel(modelIdentifier); - - expect(knownModel.reasoningEffort).toBe("high"); - expect(knownModel.thinkingBudgetTokens).toBeUndefined(); - } - - const sonnet46 = requireAnthropicKnownModel("claude-sonnet-4-6"); - expect(sonnet46.reasoningEffort).toBe("medium"); - expect(sonnet46.thinkingBudgetTokens).toBeUndefined(); - - for (const modelIdentifier of ["claude-haiku-4-5", "claude-sonnet-4-5"]) { - const knownModel = requireAnthropicKnownModel(modelIdentifier); - - expect(knownModel.reasoningEffort).toBeUndefined(); - expect(knownModel.thinkingBudgetTokens).toBe(8192); - } - }); - - it("has source metadata, provider equality, and declared order", () => { - expect( - anthropicKnownModels.map((knownModel) => knownModel.modelIdentifier), - ).toEqual([ - "claude-fable-5", - "claude-mythos-5", - "claude-opus-4-8", - "claude-opus-4-7", - "claude-opus-4-6", - "claude-sonnet-4-6", - "claude-haiku-4-5", - "claude-sonnet-4-5", - ]); - - for (const knownModel of anthropicKnownModels) { - expect(knownModel.provider).toBe("anthropic"); - expect(knownModel.sourceMetadata.sourceName).toBe("models.dev"); - expect(knownModel.sourceMetadata.sourceRetrievedAt).not.toBe(""); - expect(knownModel.sourceMetadata.lastUpdated).not.toBe(""); - } - - expect( - requireAnthropicKnownModel("claude-opus-4-8").sourceMetadata, - ).toEqual({ - sourceName: "models.dev", - sourceRetrievedAt: "2026-05-29", - lastUpdated: "2026-05-28", - }); - }); -}); diff --git a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/anthropic.ts b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/anthropic.ts deleted file mode 100644 index 760ee578d6a..00000000000 --- a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/anthropic.ts +++ /dev/null @@ -1,169 +0,0 @@ -import type { KnownModel } from "./types"; - -// Array order controls suggestion order. Keep sourceMetadata.lastUpdated in -// sync with the corresponding models.dev last_updated value for each model. -// Coder currently persists flat pricing only. Tiered models.dev pricing, -// such as context_over_200k, is intentionally omitted. -// -// The `reasoningEffort` value is editorial, not from models.dev. It reflects -// the provider's documented default for reasoning-capable models in this -// catalog and should be reviewed when the catalog is refreshed. -// -// Reasoning configuration is split per model based on Anthropic API support: -// models that support adaptive thinking (Fable 5, Mythos 5, Opus 4.8, -// Opus 4.7, Opus 4.6, Sonnet 4.6) carry `reasoningEffort`, which Coder maps to -// `thinking.type: "adaptive"` with the `effort` parameter. Models that do not -// (Haiku 4.5, Sonnet 4.5) -// carry `thinkingBudgetTokens` instead, which Coder maps to the legacy -// `thinking.type: "enabled"` path with `budget_tokens`. Setting `effort` on -// the legacy path produces an "adaptive thinking is not supported on this -// model" HTTP 400 from Anthropic. -export const anthropicKnownModels = [ - { - provider: "anthropic", - modelIdentifier: "claude-fable-5", - displayName: "Claude Fable 5", - aliases: [], - contextLimit: 1_000_000, - maxOutputTokens: 128_000, - reasoningEffort: "high", - inputCost: 10, - outputCost: 50, - cacheReadCost: 1, - cacheWriteCost: 12.5, - sourceMetadata: { - sourceName: "models.dev", - sourceRetrievedAt: "2026-07-09", - lastUpdated: "2026-06-09", - }, - }, - // models.dev does not list claude-mythos-5. Anthropic documents that - // Mythos 5 shares Fable 5's specs and pricing, so this entry mirrors - // the claude-fable-5 models.dev entry. - { - provider: "anthropic", - modelIdentifier: "claude-mythos-5", - displayName: "Claude Mythos 5", - aliases: [], - contextLimit: 1_000_000, - maxOutputTokens: 128_000, - reasoningEffort: "high", - inputCost: 10, - outputCost: 50, - cacheReadCost: 1, - cacheWriteCost: 12.5, - sourceMetadata: { - sourceName: "models.dev", - sourceRetrievedAt: "2026-07-09", - lastUpdated: "2026-06-09", - }, - }, - { - provider: "anthropic", - modelIdentifier: "claude-opus-4-8", - displayName: "Claude Opus 4.8", - aliases: [], - contextLimit: 1_000_000, - maxOutputTokens: 128_000, - reasoningEffort: "high", - inputCost: 5, - outputCost: 25, - cacheReadCost: 0.5, - cacheWriteCost: 6.25, - sourceMetadata: { - sourceName: "models.dev", - sourceRetrievedAt: "2026-05-29", - lastUpdated: "2026-05-28", - }, - }, - { - provider: "anthropic", - modelIdentifier: "claude-opus-4-7", - displayName: "Claude Opus 4.7", - aliases: [], - contextLimit: 1_000_000, - maxOutputTokens: 128_000, - reasoningEffort: "high", - inputCost: 5, - outputCost: 25, - cacheReadCost: 0.5, - cacheWriteCost: 6.25, - sourceMetadata: { - sourceName: "models.dev", - sourceRetrievedAt: "2026-04-30", - lastUpdated: "2026-04-16", - }, - }, - { - provider: "anthropic", - modelIdentifier: "claude-opus-4-6", - displayName: "Claude Opus 4.6", - aliases: [], - contextLimit: 1_000_000, - maxOutputTokens: 128_000, - reasoningEffort: "high", - inputCost: 5, - outputCost: 25, - cacheReadCost: 0.5, - cacheWriteCost: 6.25, - sourceMetadata: { - sourceName: "models.dev", - sourceRetrievedAt: "2026-04-30", - lastUpdated: "2026-03-13", - }, - }, - { - provider: "anthropic", - modelIdentifier: "claude-sonnet-4-6", - displayName: "Claude Sonnet 4.6", - aliases: [], - contextLimit: 1_000_000, - maxOutputTokens: 64_000, - reasoningEffort: "medium", - inputCost: 3, - outputCost: 15, - cacheReadCost: 0.3, - cacheWriteCost: 3.75, - sourceMetadata: { - sourceName: "models.dev", - sourceRetrievedAt: "2026-04-30", - lastUpdated: "2026-03-13", - }, - }, - { - provider: "anthropic", - modelIdentifier: "claude-haiku-4-5", - displayName: "Claude Haiku 4.5", - aliases: ["claude-haiku-4-5-20251001"], - contextLimit: 200_000, - maxOutputTokens: 64_000, - thinkingBudgetTokens: 8192, - inputCost: 1, - outputCost: 5, - cacheReadCost: 0.1, - cacheWriteCost: 1.25, - sourceMetadata: { - sourceName: "models.dev", - sourceRetrievedAt: "2026-04-30", - lastUpdated: "2025-10-15", - }, - }, - { - provider: "anthropic", - modelIdentifier: "claude-sonnet-4-5", - displayName: "Claude Sonnet 4.5", - aliases: ["claude-sonnet-4-5-20250929"], - contextLimit: 200_000, - maxOutputTokens: 64_000, - thinkingBudgetTokens: 8192, - inputCost: 3, - outputCost: 15, - cacheReadCost: 0.3, - cacheWriteCost: 3.75, - sourceMetadata: { - sourceName: "models.dev", - sourceRetrievedAt: "2026-04-30", - lastUpdated: "2025-09-29", - }, - }, -] as const satisfies readonly KnownModel[]; diff --git a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/openai.test.ts b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/openai.test.ts deleted file mode 100644 index a90d090808b..00000000000 --- a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/openai.test.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { getKnownModelsForProvider, type KnownModel } from "./index"; -import { openAIKnownModels } from "./openai"; - -describe("openAIKnownModels", () => { - it("returns OpenAI canonical IDs in declared order", () => { - expect( - getKnownModelsForProvider("openai").map( - (knownModel) => knownModel.modelIdentifier, - ), - ).toEqual([ - "gpt-5.6-sol", - "gpt-5.6-terra", - "gpt-5.6-luna", - "gpt-5.5", - "gpt-5.5-pro", - "gpt-5.4", - "gpt-5.4-mini", - "gpt-5.4-nano", - "gpt-5.3-codex", - ]); - }); - - it("declares reasoning effort only for reasoning-capable models", () => { - const knownModels: readonly KnownModel[] = openAIKnownModels; - const reasoningEffortByModel = Object.fromEntries( - knownModels.map((knownModel) => [ - knownModel.modelIdentifier, - knownModel.reasoningEffort, - ]), - ); - - expect(reasoningEffortByModel).toEqual({ - "gpt-5.6-sol": "medium", - "gpt-5.6-terra": "medium", - "gpt-5.6-luna": "medium", - "gpt-5.5": "medium", - "gpt-5.5-pro": "high", - "gpt-5.4": undefined, - "gpt-5.4-mini": "medium", - "gpt-5.4-nano": undefined, - "gpt-5.3-codex": "medium", - }); - }); - - it("has source metadata, provider equality, and declared order", () => { - expect( - openAIKnownModels.map((knownModel) => knownModel.modelIdentifier), - ).toEqual([ - "gpt-5.6-sol", - "gpt-5.6-terra", - "gpt-5.6-luna", - "gpt-5.5", - "gpt-5.5-pro", - "gpt-5.4", - "gpt-5.4-mini", - "gpt-5.4-nano", - "gpt-5.3-codex", - ]); - - for (const knownModel of openAIKnownModels) { - expect(knownModel.provider).toBe("openai"); - expect(knownModel.sourceMetadata.sourceName).toBe("models.dev"); - expect(knownModel.sourceMetadata.sourceRetrievedAt).not.toBe(""); - expect(knownModel.sourceMetadata.lastUpdated).not.toBe(""); - } - }); -}); diff --git a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/openai.ts b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/openai.ts deleted file mode 100644 index 3fcfc26883e..00000000000 --- a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/openai.ts +++ /dev/null @@ -1,167 +0,0 @@ -import type { KnownModel } from "./types"; - -// Array order controls suggestion order. Keep sourceMetadata.lastUpdated in -// sync with the corresponding models.dev last_updated value for each model. -// Coder currently persists flat pricing only. Tiered models.dev pricing, -// such as context_over_200k, is intentionally omitted. -// -// The `reasoningEffort` value is editorial, not from models.dev. It reflects -// the provider's documented default for reasoning-capable models in this -// catalog and should be reviewed when the catalog is refreshed. -export const openAIKnownModels = [ - { - provider: "openai", - modelIdentifier: "gpt-5.6-sol", - displayName: "GPT-5.6 Sol", - // models.dev lists gpt-5.6 with specs and pricing identical to - // gpt-5.6-sol, so it resolves to this entry. - aliases: ["gpt-5.6"], - contextLimit: 1_050_000, - maxOutputTokens: 128_000, - reasoningEffort: "medium", - inputCost: 5, - outputCost: 30, - cacheReadCost: 0.5, - cacheWriteCost: 6.25, - sourceMetadata: { - sourceName: "models.dev", - sourceRetrievedAt: "2026-07-10", - lastUpdated: "2026-07-09", - }, - }, - { - provider: "openai", - modelIdentifier: "gpt-5.6-terra", - displayName: "GPT-5.6 Terra", - aliases: [], - contextLimit: 1_050_000, - maxOutputTokens: 128_000, - reasoningEffort: "medium", - inputCost: 2.5, - outputCost: 15, - cacheReadCost: 0.25, - cacheWriteCost: 3.125, - sourceMetadata: { - sourceName: "models.dev", - sourceRetrievedAt: "2026-07-10", - lastUpdated: "2026-07-09", - }, - }, - { - provider: "openai", - modelIdentifier: "gpt-5.6-luna", - displayName: "GPT-5.6 Luna", - aliases: [], - contextLimit: 1_050_000, - maxOutputTokens: 128_000, - reasoningEffort: "medium", - inputCost: 1, - outputCost: 6, - cacheReadCost: 0.1, - cacheWriteCost: 1.25, - sourceMetadata: { - sourceName: "models.dev", - sourceRetrievedAt: "2026-07-10", - lastUpdated: "2026-07-09", - }, - }, - { - provider: "openai", - modelIdentifier: "gpt-5.5", - displayName: "GPT-5.5", - aliases: [], - contextLimit: 1_050_000, - maxOutputTokens: 128_000, - reasoningEffort: "medium", - inputCost: 5, - outputCost: 30, - cacheReadCost: 0.5, - sourceMetadata: { - sourceName: "models.dev", - sourceRetrievedAt: "2026-04-30", - lastUpdated: "2026-04-23", - }, - }, - { - provider: "openai", - modelIdentifier: "gpt-5.5-pro", - displayName: "GPT-5.5 Pro", - aliases: [], - contextLimit: 1_050_000, - maxOutputTokens: 128_000, - reasoningEffort: "high", - inputCost: 30, - outputCost: 180, - sourceMetadata: { - sourceName: "models.dev", - sourceRetrievedAt: "2026-04-30", - lastUpdated: "2026-04-23", - }, - }, - { - provider: "openai", - modelIdentifier: "gpt-5.4", - displayName: "GPT-5.4", - aliases: [], - contextLimit: 1_050_000, - maxOutputTokens: 128_000, - inputCost: 2.5, - outputCost: 15, - cacheReadCost: 0.25, - sourceMetadata: { - sourceName: "models.dev", - sourceRetrievedAt: "2026-04-30", - lastUpdated: "2026-03-05", - }, - }, - { - provider: "openai", - modelIdentifier: "gpt-5.4-mini", - displayName: "GPT-5.4 mini", - aliases: [], - contextLimit: 400_000, - maxOutputTokens: 128_000, - reasoningEffort: "medium", - inputCost: 0.75, - outputCost: 4.5, - cacheReadCost: 0.075, - sourceMetadata: { - sourceName: "models.dev", - sourceRetrievedAt: "2026-04-30", - lastUpdated: "2026-03-17", - }, - }, - { - provider: "openai", - modelIdentifier: "gpt-5.4-nano", - displayName: "GPT-5.4 nano", - aliases: [], - contextLimit: 400_000, - maxOutputTokens: 128_000, - inputCost: 0.2, - outputCost: 1.25, - cacheReadCost: 0.02, - sourceMetadata: { - sourceName: "models.dev", - sourceRetrievedAt: "2026-04-30", - lastUpdated: "2026-03-17", - }, - }, - { - provider: "openai", - modelIdentifier: "gpt-5.3-codex", - displayName: "GPT-5.3 Codex", - aliases: [], - contextLimit: 400_000, - maxOutputTokens: 128_000, - reasoningEffort: "medium", - inputCost: 1.75, - outputCost: 14, - cacheReadCost: 0.175, - sourceMetadata: { - sourceName: "models.dev", - sourceRetrievedAt: "2026-04-30", - lastUpdated: "2026-02-05", - }, - }, -] as const satisfies readonly KnownModel[]; From de469492e104f15c94fde352d31f5a0ce5d0ea52 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 10 Jul 2026 14:24:17 +0000 Subject: [PATCH 04/14] feat: wire knownModelsGenerated.json into gen/aibridge-prices --- Makefile | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index d39020b5f09..0faa0bc8f94 100644 --- a/Makefile +++ b/Makefile @@ -169,7 +169,7 @@ _gen/bin/apikeyscopesgen: $(wildcard scripts/apikeyscopesgen/*.go) $(RBAC_GO_FIL @mkdir -p _gen/bin go build -o $@ ./scripts/apikeyscopesgen -_gen/bin/aibridgepricesgen: $(wildcard scripts/aibridgepricesgen/*.go) | _gen +_gen/bin/aibridgepricesgen: $(wildcard scripts/aibridgepricesgen/*.go) scripts/aibridgepricesgen/catalog.json | _gen @mkdir -p _gen/bin go build -o $@ ./scripts/aibridgepricesgen @@ -1029,7 +1029,16 @@ coderd/aibridge/prices/data/prices.json: _gen/bin/aibridgepricesgen | _gen $(call atomic_write,_gen/bin/aibridgepricesgen) .PHONY: coderd/aibridge/prices/data/prices.json -gen/aibridge-prices: coderd/aibridge/prices/data/prices.json +# Frontend known-models catalog, generated from the same models.dev fetch +# joined with the editorial curation in scripts/aibridgepricesgen/catalog.json. +# Kept out of `make gen` for the same live-upstream-data reason as prices.json. +site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.json: _gen/bin/aibridgepricesgen | _gen + $(call atomic_write,_gen/bin/aibridgepricesgen -format=catalog,./scripts/biome_format.sh) +.PHONY: site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.json + +gen/aibridge-prices: \ + coderd/aibridge/prices/data/prices.json \ + site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.json .PHONY: gen/aibridge-prices gen/golden-files: \ From bc213bf23c7c23b805e5816d021fe3af2d846768 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 10 Jul 2026 14:24:17 +0000 Subject: [PATCH 05/14] feat(site/src/pages/AgentsPage): generate known models catalog from models.dev --- .../ChatModelAdminPanel/knownModels/index.ts | 17 +- .../knownModels/knownModelsGenerated.json | 303 ++++++++++++++++++ .../knownModels/knownModelsGenerated.test.ts | 75 +++++ 3 files changed, 389 insertions(+), 6 deletions(-) create mode 100644 site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.json create mode 100644 site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.test.ts diff --git a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/index.ts b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/index.ts index 76161aaa346..6aadaa9ad11 100644 --- a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/index.ts +++ b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/index.ts @@ -1,14 +1,19 @@ import { normalizeProvider } from "#/modules/aiModels/helpers"; -import { anthropicKnownModels } from "./anthropic"; -import { openAIKnownModels } from "./openai"; +import knownModelsGenerated from "./knownModelsGenerated.json"; import type { KnownModel, KnownModelSourceMetadata } from "./types"; export type { KnownModel, KnownModelSourceMetadata }; -const knownModelsByProvider = { - anthropic: anthropicKnownModels, - openai: openAIKnownModels, -} as const satisfies Record; +// knownModelsGenerated.json is produced by `make gen/aibridge-prices` from +// models.dev joined with the editorial curation in +// scripts/aibridgepricesgen/catalog.json. Do not edit it manually. JSON +// imports widen literal types (e.g. reasoningEffort becomes string), so this +// cast is the single typed boundary; knownModelsGenerated.test.ts validates +// shape and enum values for every entry. +const knownModelsByProvider = knownModelsGenerated as Record< + string, + readonly KnownModel[] +>; type KnownProvider = keyof typeof knownModelsByProvider; diff --git a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.json b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.json new file mode 100644 index 00000000000..83e90a4f44a --- /dev/null +++ b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.json @@ -0,0 +1,303 @@ +{ + "anthropic": [ + { + "provider": "anthropic", + "modelIdentifier": "claude-fable-5", + "displayName": "Claude Fable 5", + "aliases": [], + "contextLimit": 1000000, + "maxOutputTokens": 128000, + "reasoningEffort": "high", + "inputCost": 10, + "outputCost": 50, + "cacheReadCost": 1, + "cacheWriteCost": 12.5, + "sourceMetadata": { + "sourceName": "models.dev", + "sourceRetrievedAt": "2026-07-10", + "lastUpdated": "2026-06-09" + } + }, + { + "provider": "anthropic", + "modelIdentifier": "claude-mythos-5", + "displayName": "Claude Mythos 5", + "aliases": [], + "contextLimit": 1000000, + "maxOutputTokens": 128000, + "reasoningEffort": "high", + "inputCost": 10, + "outputCost": 50, + "cacheReadCost": 1, + "cacheWriteCost": 12.5, + "sourceMetadata": { + "sourceName": "models.dev", + "sourceRetrievedAt": "2026-07-10", + "lastUpdated": "2026-06-09" + } + }, + { + "provider": "anthropic", + "modelIdentifier": "claude-opus-4-8", + "displayName": "Claude Opus 4.8", + "aliases": [], + "contextLimit": 1000000, + "maxOutputTokens": 128000, + "reasoningEffort": "high", + "inputCost": 5, + "outputCost": 25, + "cacheReadCost": 0.5, + "cacheWriteCost": 6.25, + "sourceMetadata": { + "sourceName": "models.dev", + "sourceRetrievedAt": "2026-07-10", + "lastUpdated": "2026-05-28" + } + }, + { + "provider": "anthropic", + "modelIdentifier": "claude-opus-4-7", + "displayName": "Claude Opus 4.7", + "aliases": [], + "contextLimit": 1000000, + "maxOutputTokens": 128000, + "reasoningEffort": "high", + "inputCost": 5, + "outputCost": 25, + "cacheReadCost": 0.5, + "cacheWriteCost": 6.25, + "sourceMetadata": { + "sourceName": "models.dev", + "sourceRetrievedAt": "2026-07-10", + "lastUpdated": "2026-04-16" + } + }, + { + "provider": "anthropic", + "modelIdentifier": "claude-opus-4-6", + "displayName": "Claude Opus 4.6", + "aliases": [], + "contextLimit": 1000000, + "maxOutputTokens": 128000, + "reasoningEffort": "high", + "inputCost": 5, + "outputCost": 25, + "cacheReadCost": 0.5, + "cacheWriteCost": 6.25, + "sourceMetadata": { + "sourceName": "models.dev", + "sourceRetrievedAt": "2026-07-10", + "lastUpdated": "2026-03-13" + } + }, + { + "provider": "anthropic", + "modelIdentifier": "claude-sonnet-4-6", + "displayName": "Claude Sonnet 4.6", + "aliases": [], + "contextLimit": 1000000, + "maxOutputTokens": 128000, + "reasoningEffort": "medium", + "inputCost": 3, + "outputCost": 15, + "cacheReadCost": 0.3, + "cacheWriteCost": 3.75, + "sourceMetadata": { + "sourceName": "models.dev", + "sourceRetrievedAt": "2026-07-10", + "lastUpdated": "2026-03-13" + } + }, + { + "provider": "anthropic", + "modelIdentifier": "claude-haiku-4-5", + "displayName": "Claude Haiku 4.5", + "aliases": ["claude-haiku-4-5-20251001"], + "contextLimit": 200000, + "maxOutputTokens": 64000, + "thinkingBudgetTokens": 8192, + "inputCost": 1, + "outputCost": 5, + "cacheReadCost": 0.1, + "cacheWriteCost": 1.25, + "sourceMetadata": { + "sourceName": "models.dev", + "sourceRetrievedAt": "2026-07-10", + "lastUpdated": "2025-10-15" + } + }, + { + "provider": "anthropic", + "modelIdentifier": "claude-sonnet-4-5", + "displayName": "Claude Sonnet 4.5", + "aliases": ["claude-sonnet-4-5-20250929"], + "contextLimit": 1000000, + "maxOutputTokens": 64000, + "thinkingBudgetTokens": 8192, + "inputCost": 3, + "outputCost": 15, + "cacheReadCost": 0.3, + "cacheWriteCost": 3.75, + "sourceMetadata": { + "sourceName": "models.dev", + "sourceRetrievedAt": "2026-07-10", + "lastUpdated": "2025-09-29" + } + } + ], + "openai": [ + { + "provider": "openai", + "modelIdentifier": "gpt-5.6-sol", + "displayName": "GPT-5.6 Sol", + "aliases": ["gpt-5.6"], + "contextLimit": 1050000, + "maxOutputTokens": 128000, + "reasoningEffort": "medium", + "inputCost": 5, + "outputCost": 30, + "cacheReadCost": 0.5, + "cacheWriteCost": 6.25, + "sourceMetadata": { + "sourceName": "models.dev", + "sourceRetrievedAt": "2026-07-10", + "lastUpdated": "2026-07-09" + } + }, + { + "provider": "openai", + "modelIdentifier": "gpt-5.6-terra", + "displayName": "GPT-5.6 Terra", + "aliases": [], + "contextLimit": 1050000, + "maxOutputTokens": 128000, + "reasoningEffort": "medium", + "inputCost": 2.5, + "outputCost": 15, + "cacheReadCost": 0.25, + "cacheWriteCost": 3.125, + "sourceMetadata": { + "sourceName": "models.dev", + "sourceRetrievedAt": "2026-07-10", + "lastUpdated": "2026-07-09" + } + }, + { + "provider": "openai", + "modelIdentifier": "gpt-5.6-luna", + "displayName": "GPT-5.6 Luna", + "aliases": [], + "contextLimit": 1050000, + "maxOutputTokens": 128000, + "reasoningEffort": "medium", + "inputCost": 1, + "outputCost": 6, + "cacheReadCost": 0.1, + "cacheWriteCost": 1.25, + "sourceMetadata": { + "sourceName": "models.dev", + "sourceRetrievedAt": "2026-07-10", + "lastUpdated": "2026-07-09" + } + }, + { + "provider": "openai", + "modelIdentifier": "gpt-5.5", + "displayName": "GPT-5.5", + "aliases": [], + "contextLimit": 1050000, + "maxOutputTokens": 128000, + "reasoningEffort": "medium", + "inputCost": 5, + "outputCost": 30, + "cacheReadCost": 0.5, + "sourceMetadata": { + "sourceName": "models.dev", + "sourceRetrievedAt": "2026-07-10", + "lastUpdated": "2026-04-23" + } + }, + { + "provider": "openai", + "modelIdentifier": "gpt-5.5-pro", + "displayName": "GPT-5.5 Pro", + "aliases": [], + "contextLimit": 1050000, + "maxOutputTokens": 128000, + "reasoningEffort": "high", + "inputCost": 30, + "outputCost": 180, + "sourceMetadata": { + "sourceName": "models.dev", + "sourceRetrievedAt": "2026-07-10", + "lastUpdated": "2026-04-23" + } + }, + { + "provider": "openai", + "modelIdentifier": "gpt-5.4", + "displayName": "GPT-5.4", + "aliases": [], + "contextLimit": 1050000, + "maxOutputTokens": 128000, + "inputCost": 2.5, + "outputCost": 15, + "cacheReadCost": 0.25, + "sourceMetadata": { + "sourceName": "models.dev", + "sourceRetrievedAt": "2026-07-10", + "lastUpdated": "2026-03-05" + } + }, + { + "provider": "openai", + "modelIdentifier": "gpt-5.4-mini", + "displayName": "GPT-5.4 mini", + "aliases": [], + "contextLimit": 400000, + "maxOutputTokens": 128000, + "reasoningEffort": "medium", + "inputCost": 0.75, + "outputCost": 4.5, + "cacheReadCost": 0.075, + "sourceMetadata": { + "sourceName": "models.dev", + "sourceRetrievedAt": "2026-07-10", + "lastUpdated": "2026-03-17" + } + }, + { + "provider": "openai", + "modelIdentifier": "gpt-5.4-nano", + "displayName": "GPT-5.4 nano", + "aliases": [], + "contextLimit": 400000, + "maxOutputTokens": 128000, + "inputCost": 0.2, + "outputCost": 1.25, + "cacheReadCost": 0.02, + "sourceMetadata": { + "sourceName": "models.dev", + "sourceRetrievedAt": "2026-07-10", + "lastUpdated": "2026-03-17" + } + }, + { + "provider": "openai", + "modelIdentifier": "gpt-5.3-codex", + "displayName": "GPT-5.3 Codex", + "aliases": [], + "contextLimit": 400000, + "maxOutputTokens": 128000, + "reasoningEffort": "medium", + "inputCost": 1.75, + "outputCost": 14, + "cacheReadCost": 0.175, + "sourceMetadata": { + "sourceName": "models.dev", + "sourceRetrievedAt": "2026-07-10", + "lastUpdated": "2026-02-05" + } + } + ] +} diff --git a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.test.ts b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.test.ts new file mode 100644 index 00000000000..b3ed7b0abff --- /dev/null +++ b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it } from "vitest"; +import knownModelsGenerated from "./knownModelsGenerated.json"; + +// knownModelsGenerated.json crosses a typed boundary via a cast in index.ts, +// so this suite validates the shape and enum values of every generated entry. +const providers = Object.entries(knownModelsGenerated); + +describe("knownModelsGenerated", () => { + it("contains the expected canonical model identifiers in display order", () => { + expect( + knownModelsGenerated.openai.map((model) => model.modelIdentifier), + ).toEqual([ + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-5.6-luna", + "gpt-5.5", + "gpt-5.5-pro", + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.4-nano", + "gpt-5.3-codex", + ]); + expect( + knownModelsGenerated.anthropic.map((model) => model.modelIdentifier), + ).toEqual([ + "claude-fable-5", + "claude-mythos-5", + "claude-opus-4-8", + "claude-opus-4-7", + "claude-opus-4-6", + "claude-sonnet-4-6", + "claude-haiku-4-5", + "claude-sonnet-4-5", + ]); + }); + + it.each(providers)("validates every %s entry", (provider, models) => { + expect(models.length).toBeGreaterThan(0); + for (const model of models) { + expect(model.provider).toBe(provider); + expect(model.modelIdentifier).not.toBe(""); + expect(model.displayName).not.toBe(""); + expect(Array.isArray(model.aliases)).toBe(true); + + const record = model as Record; + if (record.reasoningEffort !== undefined) { + expect(["low", "medium", "high"]).toContain(record.reasoningEffort); + } + expect( + record.reasoningEffort !== undefined && + record.thinkingBudgetTokens !== undefined, + ).toBe(false); + + for (const field of [ + "contextLimit", + "maxOutputTokens", + "thinkingBudgetTokens", + "inputCost", + "outputCost", + "cacheReadCost", + "cacheWriteCost", + ]) { + const value = record[field]; + if (value !== undefined) { + expect(typeof value, field).toBe("number"); + expect(value, field).toBeGreaterThan(0); + } + } + + expect(model.sourceMetadata.sourceName).toBe("models.dev"); + expect(model.sourceMetadata.sourceRetrievedAt).not.toBe(""); + expect(model.sourceMetadata.lastUpdated).not.toBe(""); + } + }); +}); From 2f2d117995f3f4fde8d5f429b91f7bcb3a9ba6b2 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 10 Jul 2026 14:24:17 +0000 Subject: [PATCH 06/14] docs(coderd/aibridge/prices): note paired catalog generation --- coderd/aibridge/prices/data/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/coderd/aibridge/prices/data/README.md b/coderd/aibridge/prices/data/README.md index e5d90b34720..a6349513bdf 100644 --- a/coderd/aibridge/prices/data/README.md +++ b/coderd/aibridge/prices/data/README.md @@ -3,3 +3,8 @@ `prices.json` in this directory is generated by `make gen/aibridge-prices` and embedded into the Coder binary at build time. Do not edit it manually; the next regeneration will overwrite any changes. + +The same Make target also regenerates the frontend known-models catalog at +`site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.json` +from the same models.dev fetch, joined with the editorial curation in +`scripts/aibridgepricesgen/catalog.json`. From c4b6c623c75e33442eeb01b97c2ad6e86029b308 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 10 Jul 2026 15:35:07 +0000 Subject: [PATCH 07/14] feat: source model overrides from overrides.jq via a single-fetch pipeline --- Makefile | 30 ++- coderd/aibridge/prices/data/README.md | 9 +- coderd/aibridge/prices/data/prices.json | 8 + scripts/aibridgepricesgen/catalog.go | 93 +++++---- scripts/aibridgepricesgen/catalog_test.go | 191 +++++++++++++++--- .../{catalog.json => curation.json} | 2 - scripts/aibridgepricesgen/main.go | 100 ++++----- scripts/aibridgepricesgen/overrides.jq | 29 +++ .../ChatModelAdminPanel/knownModels/index.ts | 7 +- .../knownModels/knownModelsGenerated.json | 2 +- .../knownModels/knownModelsGenerated.test.ts | 75 ++++--- 11 files changed, 377 insertions(+), 169 deletions(-) rename scripts/aibridgepricesgen/{catalog.json => curation.json} (95%) create mode 100644 scripts/aibridgepricesgen/overrides.jq diff --git a/Makefile b/Makefile index 0faa0bc8f94..3200ae9565c 100644 --- a/Makefile +++ b/Makefile @@ -169,7 +169,7 @@ _gen/bin/apikeyscopesgen: $(wildcard scripts/apikeyscopesgen/*.go) $(RBAC_GO_FIL @mkdir -p _gen/bin go build -o $@ ./scripts/apikeyscopesgen -_gen/bin/aibridgepricesgen: $(wildcard scripts/aibridgepricesgen/*.go) scripts/aibridgepricesgen/catalog.json | _gen +_gen/bin/aibridgepricesgen: $(wildcard scripts/aibridgepricesgen/*.go) scripts/aibridgepricesgen/curation.json | _gen @mkdir -p _gen/bin go build -o $@ ./scripts/aibridgepricesgen @@ -1022,18 +1022,28 @@ gen: gen/db $(if $(GEN_SKIP_GOLDEN),,gen/golden-files) $(GEN_FILES) gen/db: $(DB_GEN_FILES) .PHONY: gen/db -# Refresh the AI Bridge pricing seed file from models.dev. Kept out of -# `make gen`. Phony so each invocation regenerates. -coderd/aibridge/prices/data/prices.json: _gen/bin/aibridgepricesgen | _gen +# Patched snapshot of the models.dev catalog. Fetched once per +# gen/aibridge-prices run, with upstream corrections applied by +# overrides.jq; both prices.json and the frontend known-models catalog are +# generated from this single snapshot. Phony so each invocation refreshes it. +_gen/models-dev.json: | _gen + set -o pipefail; $(call atomic_write,curl -fsSL https://models.dev/api.json | jq -f scripts/aibridgepricesgen/overrides.jq) +.PHONY: _gen/models-dev.json + +# Refresh the AI Bridge pricing seed file from the patched models.dev +# snapshot. Kept out of `make gen` because the output depends on live +# upstream data. Phony so each invocation regenerates. +coderd/aibridge/prices/data/prices.json: _gen/bin/aibridgepricesgen _gen/models-dev.json | _gen @mkdir -p $(dir $@) - $(call atomic_write,_gen/bin/aibridgepricesgen) + $(call atomic_write,_gen/bin/aibridgepricesgen -upstream _gen/models-dev.json) .PHONY: coderd/aibridge/prices/data/prices.json -# Frontend known-models catalog, generated from the same models.dev fetch -# joined with the editorial curation in scripts/aibridgepricesgen/catalog.json. -# Kept out of `make gen` for the same live-upstream-data reason as prices.json. -site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.json: _gen/bin/aibridgepricesgen | _gen - $(call atomic_write,_gen/bin/aibridgepricesgen -format=catalog,./scripts/biome_format.sh) +# Frontend known-models catalog, generated from the same patched models.dev +# snapshot joined with the editorial curation in +# scripts/aibridgepricesgen/curation.json. Kept out of `make gen` for the +# same live-upstream-data reason as prices.json. +site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.json: _gen/bin/aibridgepricesgen _gen/models-dev.json | _gen + $(call atomic_write,_gen/bin/aibridgepricesgen -format=catalog -upstream _gen/models-dev.json,./scripts/biome_format.sh) .PHONY: site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.json gen/aibridge-prices: \ diff --git a/coderd/aibridge/prices/data/README.md b/coderd/aibridge/prices/data/README.md index a6349513bdf..f92025c8f79 100644 --- a/coderd/aibridge/prices/data/README.md +++ b/coderd/aibridge/prices/data/README.md @@ -4,7 +4,10 @@ embedded into the Coder binary at build time. Do not edit it manually; the next regeneration will overwrite any changes. -The same Make target also regenerates the frontend known-models catalog at +The Make target fetches models.dev once into `_gen/models-dev.json`, applying +the upstream corrections in `scripts/aibridgepricesgen/overrides.jq`. Both +`prices.json` and the frontend known-models catalog at `site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.json` -from the same models.dev fetch, joined with the editorial curation in -`scripts/aibridgepricesgen/catalog.json`. +are generated from that single patched snapshot; the catalog is additionally +joined with the editorial curation in +`scripts/aibridgepricesgen/curation.json`. diff --git a/coderd/aibridge/prices/data/prices.json b/coderd/aibridge/prices/data/prices.json index 6851a433bd2..4748236dd50 100644 --- a/coderd/aibridge/prices/data/prices.json +++ b/coderd/aibridge/prices/data/prices.json @@ -23,6 +23,14 @@ "cache_read_price": 100000, "cache_write_price": 1250000 }, + { + "provider": "anthropic", + "model": "claude-mythos-5", + "input_price": 10000000, + "output_price": 50000000, + "cache_read_price": 1000000, + "cache_write_price": 12500000 + }, { "provider": "anthropic", "model": "claude-opus-4-1", diff --git a/scripts/aibridgepricesgen/catalog.go b/scripts/aibridgepricesgen/catalog.go index 146159c7ca3..b0c20b8fe4f 100644 --- a/scripts/aibridgepricesgen/catalog.go +++ b/scripts/aibridgepricesgen/catalog.go @@ -1,6 +1,7 @@ package main import ( + "cmp" _ "embed" "encoding/json" "io" @@ -9,23 +10,23 @@ import ( "golang.org/x/xerrors" ) -// catalogJSON is the checked-in editorial curation input for the frontend +// curationJSON is the checked-in editorial curation input for the frontend // known-models catalog. Entry order within each provider controls suggestion // order in the UI. Everything factual (display name, limits, pricing, // last_updated) is joined from models.dev at generation time; the curation // file only carries editorial choices: which models to suggest, aliases, // reasoning defaults, and overrides. // -//go:embed catalog.json -var catalogJSON []byte +//go:embed curation.json +var curationJSON []byte -// curatedModel is one entry in catalog.json. +// curatedModel is one entry in curation.json. type curatedModel struct { ModelIdentifier string `json:"modelIdentifier"` Aliases []string `json:"aliases"` // DisplayName overrides the upstream `name` when set. Needed where // upstream naming does not match what we want to show (for example - // "Claude Haiku 4.5 (latest)"), and for mirrored models. + // "Claude Haiku 4.5 (latest)"). DisplayName string `json:"displayName"` // ReasoningEffort is editorial, not from models.dev. Mutually // exclusive with ThinkingBudgetTokens. @@ -34,38 +35,36 @@ type curatedModel struct { // support adaptive thinking and use the legacy // `thinking.budget_tokens` API instead. ThinkingBudgetTokens int `json:"thinkingBudgetTokens"` - // MirrorOf names another upstream model whose specs and pricing this - // entry copies. Used for models absent from models.dev (for example - // claude-mythos-5, which Anthropic documents as sharing - // claude-fable-5's specs and pricing). - MirrorOf string `json:"mirrorOf"` } // catalogEntry matches the frontend KnownModel shape (knownModels/types.ts). // Costs are flat USD per million tokens, straight from models.dev; tiered // pricing such as context_over_200k is intentionally omitted. type catalogEntry struct { - Provider string `json:"provider"` - ModelIdentifier string `json:"modelIdentifier"` - DisplayName string `json:"displayName"` - Aliases []string `json:"aliases"` - ContextLimit *int64 `json:"contextLimit,omitempty"` - MaxOutputTokens *int64 `json:"maxOutputTokens,omitempty"` - ReasoningEffort string `json:"reasoningEffort,omitempty"` - ThinkingBudgetTokens int `json:"thinkingBudgetTokens,omitempty"` - InputCost *float64 `json:"inputCost,omitempty"` - OutputCost *float64 `json:"outputCost,omitempty"` - CacheReadCost *float64 `json:"cacheReadCost,omitempty"` - CacheWriteCost *float64 `json:"cacheWriteCost,omitempty"` - SourceMetadata catalogMetadata `json:"sourceMetadata"` + Provider string `json:"provider"` + ModelIdentifier string `json:"modelIdentifier"` + DisplayName string `json:"displayName"` + Aliases []string `json:"aliases"` + ContextLimit *int64 `json:"contextLimit,omitempty"` + MaxOutputTokens *int64 `json:"maxOutputTokens,omitempty"` + ReasoningEffort string `json:"reasoningEffort,omitempty"` + ThinkingBudgetTokens int `json:"thinkingBudgetTokens,omitempty"` + InputCost *float64 `json:"inputCost,omitempty"` + OutputCost *float64 `json:"outputCost,omitempty"` + CacheReadCost *float64 `json:"cacheReadCost,omitempty"` + CacheWriteCost *float64 `json:"cacheWriteCost,omitempty"` + SourceMetadata sourceMetadata `json:"sourceMetadata"` } -type catalogMetadata struct { +type sourceMetadata struct { SourceName string `json:"sourceName"` SourceRetrievedAt string `json:"sourceRetrievedAt"` LastUpdated string `json:"lastUpdated"` } +// validReasoningEfforts are the values accepted for curatedModel.ReasoningEffort. +var validReasoningEfforts = map[string]bool{"low": true, "medium": true, "high": true} + // buildCatalog joins the curation file with the upstream models.dev payload // and returns provider-keyed ordered entry lists. now supplies the // sourceRetrievedAt date so output is deterministic under test. @@ -77,32 +76,43 @@ func buildCatalog(upstream map[string]upstreamProvider, curation map[string][]cu if !ok { return nil, xerrors.Errorf("provider %q missing from upstream", providerID) } + seenIdentifiers := make(map[string]bool, len(curated)) + seenAliases := make(map[string]bool) entries := make([]catalogEntry, 0, len(curated)) for _, c := range curated { if c.ModelIdentifier == "" { return nil, xerrors.Errorf("provider %q: entry with empty modelIdentifier", providerID) } + if seenIdentifiers[c.ModelIdentifier] { + return nil, xerrors.Errorf("provider %q: duplicate modelIdentifier %q", providerID, c.ModelIdentifier) + } + seenIdentifiers[c.ModelIdentifier] = true + if c.ReasoningEffort != "" && !validReasoningEfforts[c.ReasoningEffort] { + return nil, xerrors.Errorf(`%s/%s: reasoningEffort %q is not one of "low", "medium", "high"`, providerID, c.ModelIdentifier, c.ReasoningEffort) + } + if c.ThinkingBudgetTokens < 0 { + return nil, xerrors.Errorf("%s/%s: thinkingBudgetTokens %d is negative", providerID, c.ModelIdentifier, c.ThinkingBudgetTokens) + } if c.ReasoningEffort != "" && c.ThinkingBudgetTokens != 0 { return nil, xerrors.Errorf("%s/%s: reasoningEffort and thinkingBudgetTokens are mutually exclusive", providerID, c.ModelIdentifier) } - sourceID := c.ModelIdentifier - if c.MirrorOf != "" { - sourceID = c.MirrorOf + for _, alias := range c.Aliases { + if alias == "" { + return nil, xerrors.Errorf("%s/%s: empty-string alias", providerID, c.ModelIdentifier) + } + if seenAliases[alias] { + return nil, xerrors.Errorf("%s/%s: alias %q declared more than once in provider", providerID, c.ModelIdentifier, alias) + } + seenAliases[alias] = true } - m, ok := provider.Models[sourceID] + m, ok := provider.Models[c.ModelIdentifier] if !ok { - if c.MirrorOf != "" { - return nil, xerrors.Errorf("%s/%s: mirrorOf target %q missing from upstream", providerID, c.ModelIdentifier, c.MirrorOf) - } - return nil, xerrors.Errorf("%s/%s: model missing from upstream (use mirrorOf if intentional)", providerID, c.ModelIdentifier) + return nil, xerrors.Errorf("%s/%s: model missing from upstream (patch it in via overrides.jq if intentional)", providerID, c.ModelIdentifier) } if !m.Cost.hasPricing() { - return nil, xerrors.Errorf("%s/%s: upstream model %q has no cost block", providerID, c.ModelIdentifier, sourceID) - } - displayName := c.DisplayName - if displayName == "" { - displayName = m.Name + return nil, xerrors.Errorf("%s/%s: upstream model has no cost block", providerID, c.ModelIdentifier) } + displayName := cmp.Or(c.DisplayName, m.Name) if displayName == "" { return nil, xerrors.Errorf("%s/%s: no displayName override and upstream name is empty", providerID, c.ModelIdentifier) } @@ -123,13 +133,20 @@ func buildCatalog(upstream map[string]upstreamProvider, curation map[string][]cu OutputCost: m.Cost.Output, CacheReadCost: m.Cost.CacheRead, CacheWriteCost: m.Cost.CacheWrite, - SourceMetadata: catalogMetadata{ + SourceMetadata: sourceMetadata{ SourceName: "models.dev", SourceRetrievedAt: retrievedAt, LastUpdated: m.LastUpdated, }, }) } + // An alias resolving to a canonical identifier would make exact-alias + // lookup and canonical-id lookup disagree. + for alias := range seenAliases { + if seenIdentifiers[alias] { + return nil, xerrors.Errorf("alias %q duplicates a modelIdentifier in provider %q", alias, providerID) + } + } out[providerID] = entries } return out, nil diff --git a/scripts/aibridgepricesgen/catalog_test.go b/scripts/aibridgepricesgen/catalog_test.go index 9c34960d5ee..5a8dba921cd 100644 --- a/scripts/aibridgepricesgen/catalog_test.go +++ b/scripts/aibridgepricesgen/catalog_test.go @@ -3,6 +3,7 @@ package main import ( "bytes" "encoding/json" + "os" "testing" "time" @@ -10,7 +11,7 @@ import ( ) // fixtureUpstream returns a small upstream payload covering the join cases: -// a fully priced model with limits, a mirror target, and a costless model. +// fully priced models with limits and a costless model. func fixtureUpstream(t *testing.T) map[string]upstreamProvider { t.Helper() const upstreamJSON = `{ @@ -22,6 +23,12 @@ func fixtureUpstream(t *testing.T) map[string]upstreamProvider { "cost": {"input": 10, "output": 50, "cache_read": 1, "cache_write": 12.5}, "last_updated": "2026-06-09" }, + "claude-mythos-5": { + "name": "Claude Mythos 5", + "limit": {"context": 1000000, "output": 128000}, + "cost": {"input": 10, "output": 50, "cache_read": 1, "cache_write": 12.5}, + "last_updated": "2026-06-09" + }, "claude-costless": { "name": "Claude Costless", "limit": {"context": 200000, "output": 64000}, @@ -63,7 +70,7 @@ func TestBuildCatalog(t *testing.T) { }, "anthropic": { {ModelIdentifier: "claude-fable-5", ReasoningEffort: "high"}, - {ModelIdentifier: "claude-mythos-5", MirrorOf: "claude-fable-5", DisplayName: "Claude Mythos 5", ReasoningEffort: "high"}, + {ModelIdentifier: "claude-mythos-5", ReasoningEffort: "high"}, }, } @@ -154,22 +161,18 @@ func TestBuildCatalog(t *testing.T) { require.Equal(t, want, buf.String()) } -func TestBuildCatalogDeterministic(t *testing.T) { +// TestBuildCatalogClock asserts the injected clock feeds sourceRetrievedAt +// and leaves upstream-derived fields alone. +func TestBuildCatalogClock(t *testing.T) { t.Parallel() curation := map[string][]curatedModel{ "openai": {{ModelIdentifier: "gpt-5.6-sol"}}, } - a, err := buildCatalog(fixtureUpstream(t), curation, fixedNow) - require.NoError(t, err) - b, err := buildCatalog(fixtureUpstream(t), curation, fixedNow) - require.NoError(t, err) - require.Equal(t, a, b) - - // A different injected clock changes only sourceRetrievedAt. later, err := buildCatalog(fixtureUpstream(t), curation, fixedNow.AddDate(0, 0, 1)) require.NoError(t, err) require.Equal(t, "2026-07-11", later["openai"][0].SourceMetadata.SourceRetrievedAt) + require.Equal(t, "2026-07-09", later["openai"][0].SourceMetadata.LastUpdated) } func TestBuildCatalogErrors(t *testing.T) { @@ -187,13 +190,6 @@ func TestBuildCatalogErrors(t *testing.T) { }, wantErr: "missing from upstream", }, - { - name: "DanglingMirrorOf", - curation: map[string][]curatedModel{ - "anthropic": {{ModelIdentifier: "claude-mythos-5", MirrorOf: "claude-nonexistent"}}, - }, - wantErr: "mirrorOf target", - }, { name: "NoCostBlock", curation: map[string][]curatedModel{ @@ -208,6 +204,57 @@ func TestBuildCatalogErrors(t *testing.T) { }, wantErr: "mutually exclusive", }, + { + name: "InvalidReasoningEffort", + curation: map[string][]curatedModel{ + "anthropic": {{ModelIdentifier: "claude-fable-5", ReasoningEffort: "maximum"}}, + }, + wantErr: "is not one of", + }, + { + name: "NegativeThinkingBudget", + curation: map[string][]curatedModel{ + "anthropic": {{ModelIdentifier: "claude-fable-5", ThinkingBudgetTokens: -1}}, + }, + wantErr: "is negative", + }, + { + name: "DuplicateModelIdentifier", + curation: map[string][]curatedModel{ + "anthropic": { + {ModelIdentifier: "claude-fable-5"}, + {ModelIdentifier: "claude-fable-5"}, + }, + }, + wantErr: "duplicate modelIdentifier", + }, + { + name: "DuplicateAlias", + curation: map[string][]curatedModel{ + "anthropic": { + {ModelIdentifier: "claude-fable-5", Aliases: []string{"claude-latest"}}, + {ModelIdentifier: "claude-mythos-5", Aliases: []string{"claude-latest"}}, + }, + }, + wantErr: "declared more than once", + }, + { + name: "EmptyAlias", + curation: map[string][]curatedModel{ + "anthropic": {{ModelIdentifier: "claude-fable-5", Aliases: []string{""}}}, + }, + wantErr: "empty-string alias", + }, + { + name: "AliasShadowsModelIdentifier", + curation: map[string][]curatedModel{ + "anthropic": { + {ModelIdentifier: "claude-fable-5", Aliases: []string{"claude-mythos-5"}}, + {ModelIdentifier: "claude-mythos-5"}, + }, + }, + wantErr: "duplicates a modelIdentifier", + }, { name: "MissingProvider", curation: map[string][]curatedModel{ @@ -234,21 +281,113 @@ func TestBuildCatalogErrors(t *testing.T) { } } -// TestEmbeddedCatalogParses guards the checked-in curation file itself: -// valid JSON, required fields present, and the effort/budget exclusivity -// holds without needing upstream data. -func TestEmbeddedCatalogParses(t *testing.T) { +// TestEmbeddedCurationBuilds runs the checked-in curation.json through +// buildCatalog against a synthetic upstream derived from the curation's own +// model identifiers, so every buildCatalog validation (present and future) +// automatically guards the checked-in file. +func TestEmbeddedCurationBuilds(t *testing.T) { t.Parallel() - var curation map[string][]curatedModel - require.NoError(t, json.Unmarshal(catalogJSON, &curation)) + curation := embeddedCuration(t) require.NotEmpty(t, curation) + + price := 1.0 + limit := int64(100000) + upstream := make(map[string]upstreamProvider, len(curation)) for providerID, entries := range curation { + models := make(map[string]upstreamModel, len(entries)) + for _, c := range entries { + models[c.ModelIdentifier] = upstreamModel{ + Name: "Synthetic " + c.ModelIdentifier, + Limit: upstreamLimit{Context: &limit, Output: &limit}, + Cost: &upstreamCost{Input: &price, Output: &price}, + LastUpdated: "2026-01-01", + } + } + upstream[providerID] = upstreamProvider{Models: models} + } + + catalog, err := buildCatalog(upstream, curation, fixedNow) + require.NoError(t, err) + for providerID, entries := range catalog { require.NotEmpty(t, entries, providerID) + } +} + +// TestCurationMatchesGeneratedCatalog is a drift test: the editorial fields +// (per provider, in order) in the embedded curation.json must exactly match +// their projection in the checked-in generated frontend catalog. Fails when +// curation.json changes without running `make gen/aibridge-prices`. +func TestCurationMatchesGeneratedCatalog(t *testing.T) { + t.Parallel() + + curation := embeddedCuration(t) + + data, err := os.ReadFile("../../site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.json") + require.NoError(t, err) + var generated map[string][]catalogEntry + require.NoError(t, json.Unmarshal(data, &generated)) + + // editorial is the curation-owned projection of an entry. displayName is + // only compared when the curation sets an override; otherwise it comes + // from upstream and is not the curation's to pin. + type editorial struct { + ModelIdentifier string + Aliases []string + DisplayName string + ReasoningEffort string + ThinkingBudgetTokens int + } + + curatedProjection := make(map[string][]editorial, len(curation)) + for providerID, entries := range curation { + projected := make([]editorial, 0, len(entries)) for _, c := range entries { - require.NotEmpty(t, c.ModelIdentifier, providerID) - require.False(t, c.ReasoningEffort != "" && c.ThinkingBudgetTokens != 0, - "%s/%s sets both reasoningEffort and thinkingBudgetTokens", providerID, c.ModelIdentifier) + aliases := c.Aliases + if aliases == nil { + aliases = []string{} + } + projected = append(projected, editorial{ + ModelIdentifier: c.ModelIdentifier, + Aliases: aliases, + DisplayName: c.DisplayName, + ReasoningEffort: c.ReasoningEffort, + ThinkingBudgetTokens: c.ThinkingBudgetTokens, + }) } + curatedProjection[providerID] = projected } + + generatedProjection := make(map[string][]editorial, len(generated)) + for providerID, entries := range generated { + curated := map[string]curatedModel{} + for _, c := range curation[providerID] { + curated[c.ModelIdentifier] = c + } + projected := make([]editorial, 0, len(entries)) + for _, e := range entries { + displayName := "" + if curated[e.ModelIdentifier].DisplayName != "" { + displayName = e.DisplayName + } + projected = append(projected, editorial{ + ModelIdentifier: e.ModelIdentifier, + Aliases: e.Aliases, + DisplayName: displayName, + ReasoningEffort: e.ReasoningEffort, + ThinkingBudgetTokens: e.ThinkingBudgetTokens, + }) + } + generatedProjection[providerID] = projected + } + + require.Equal(t, curatedProjection, generatedProjection, + "curation.json and knownModelsGenerated.json disagree; run `make gen/aibridge-prices`") +} + +func embeddedCuration(t *testing.T) map[string][]curatedModel { + t.Helper() + var curation map[string][]curatedModel + require.NoError(t, json.Unmarshal(curationJSON, &curation)) + return curation } diff --git a/scripts/aibridgepricesgen/catalog.json b/scripts/aibridgepricesgen/curation.json similarity index 95% rename from scripts/aibridgepricesgen/catalog.json rename to scripts/aibridgepricesgen/curation.json index cfcf7c8b818..44190120e8c 100644 --- a/scripts/aibridgepricesgen/catalog.json +++ b/scripts/aibridgepricesgen/curation.json @@ -43,8 +43,6 @@ }, { "modelIdentifier": "claude-mythos-5", - "mirrorOf": "claude-fable-5", - "displayName": "Claude Mythos 5", "reasoningEffort": "high" }, { diff --git a/scripts/aibridgepricesgen/main.go b/scripts/aibridgepricesgen/main.go index adfb6fe2266..367141be2a5 100644 --- a/scripts/aibridgepricesgen/main.go +++ b/scripts/aibridgepricesgen/main.go @@ -1,21 +1,19 @@ -// aibridgepricesgen fetches model pricing from models.dev and writes a JSON +// aibridgepricesgen converts a models.dev api.json snapshot into a JSON // seed file consumable by the AI Bridge cost-control loader. Output is sorted // by (provider, model) so regenerations produce minimal diffs. // -// Run via the gen/aibridge-prices Make target. Kept out of `make gen` because -// the output depends on live upstream data; refreshing prices should land in -// dedicated, reviewable commits rather than appearing as drift on unrelated -// gen runs. +// Run via the gen/aibridge-prices Make target, which fetches and patches the +// snapshot (_gen/models-dev.json). Kept out of `make gen` because the output +// depends on live upstream data; refreshing prices should land in dedicated, +// reviewable commits rather than appearing as drift on unrelated gen runs. package main import ( - "context" "encoding/json" "flag" "fmt" "io" "math" - "net/http" "os" "sort" "time" @@ -23,16 +21,6 @@ import ( "golang.org/x/xerrors" ) -const ( - sourceURL = "https://models.dev/api.json" - fetchTimeout = 30 * time.Second - // Cap the upstream body read. The current api.json is ~2 MiB, so 100 - // MiB is pure defense-in-depth against a misbehaving upstream eating - // arbitrary memory on developer or CI machines. An overflow surfaces - // as a JSON parse error (LimitReader truncates silently at the cap). - maxBodyBytes = 100 << 20 -) - // supportedProviders lists the providers we ship prices for. Adding a // provider here is enough to include it on the next regeneration. var supportedProviders = []string{"anthropic", "openai"} @@ -49,7 +37,8 @@ type upstreamModel struct { LastUpdated string `json:"last_updated"` } -// Pointers distinguish "key absent" (nil) from "key present and zero" (0). +// Pointer fields in upstreamLimit and upstreamCost distinguish "key absent" +// (nil) from "key present and zero" (0). type upstreamLimit struct { Context *int64 `json:"context"` Output *int64 `json:"output"` @@ -90,26 +79,47 @@ type priceRow struct { func main() { format := flag.String("format", "prices", `output format: "prices" (cost-control seed) or "catalog" (frontend known-models JSON)`) + upstreamPath := flag.String("upstream", "", "path to a models.dev api.json snapshot (required)") flag.Parse() - if err := run(*format); err != nil { + if err := run(*format, *upstreamPath); err != nil { _, _ = fmt.Fprintf(os.Stderr, "aibridgepricesgen: %v\n", err) os.Exit(1) } } -func run(format string) error { - upstream, err := fetch() +func run(format, upstreamPath string) error { + // Validate flags before touching the filesystem so a typo fails fast. + switch format { + case "prices", "catalog": + default: + return xerrors.Errorf(`unknown -format %q (want "prices" or "catalog")`, format) + } + if upstreamPath == "" { + return xerrors.New("-upstream is required; run via `make gen/aibridge-prices`, which fetches and patches the snapshot") + } + + upstream, err := readUpstream(upstreamPath) if err != nil { - return xerrors.Errorf("fetch %s: %w", sourceURL, err) + return xerrors.Errorf("read %s: %w", upstreamPath, err) } - switch format { - case "prices": - return runPrices(upstream) - case "catalog": + if format == "catalog" { return runCatalog(upstream) - default: - return xerrors.Errorf("unknown -format %q (want \"prices\" or \"catalog\")", format) } + return runPrices(upstream) +} + +// readUpstream loads a models.dev api.json snapshot from disk, typically the +// Makefile's _gen/models-dev.json (fetched once and patched by overrides.jq). +func readUpstream(path string) (map[string]upstreamProvider, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var upstream map[string]upstreamProvider + if err := json.Unmarshal(data, &upstream); err != nil { + return nil, xerrors.Errorf("parse: %w", err) + } + return upstream, nil } func runPrices(upstream map[string]upstreamProvider) error { @@ -129,8 +139,8 @@ func runPrices(upstream map[string]upstreamProvider) error { func runCatalog(upstream map[string]upstreamProvider) error { var curation map[string][]curatedModel - if err := json.Unmarshal(catalogJSON, &curation); err != nil { - return xerrors.Errorf("parse embedded catalog.json: %w", err) + if err := json.Unmarshal(curationJSON, &curation); err != nil { + return xerrors.Errorf("parse embedded curation.json: %w", err) } catalog, err := buildCatalog(upstream, curation, time.Now()) if err != nil { @@ -139,38 +149,10 @@ func runCatalog(upstream map[string]upstreamProvider) error { if err := writeCatalog(os.Stdout, catalog); err != nil { return err } - total := 0 - for _, entries := range catalog { - total += len(entries) - } - _, _ = fmt.Fprintf(os.Stderr, "aibridgepricesgen: wrote %d catalog entries for %d provider(s)\n", total, len(catalog)) + _, _ = fmt.Fprintf(os.Stderr, "aibridgepricesgen: wrote catalog for %d provider(s)\n", len(catalog)) return nil } -func fetch() (map[string]upstreamProvider, error) { - ctx, cancel := context.WithTimeout(context.Background(), fetchTimeout) - defer cancel() - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, sourceURL, nil) - if err != nil { - return nil, err - } - resp, err := http.DefaultClient.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return nil, xerrors.Errorf("status %d", resp.StatusCode) - } - - var data map[string]upstreamProvider - if err := json.NewDecoder(io.LimitReader(resp.Body, maxBodyBytes)).Decode(&data); err != nil { - return nil, xerrors.Errorf("parse: %w", err) - } - return data, nil -} - // convert flattens the upstream map into table-shaped rows for the configured // providers. If any configured provider is absent from the upstream payload, // every missing provider is reported and the function returns an error so the diff --git a/scripts/aibridgepricesgen/overrides.jq b/scripts/aibridgepricesgen/overrides.jq new file mode 100644 index 00000000000..8d8a445f6ec --- /dev/null +++ b/scripts/aibridgepricesgen/overrides.jq @@ -0,0 +1,29 @@ +# Patches applied to the raw models.dev api.json before aibridgepricesgen +# consumes it. The Makefile pipes the fetched payload through this filter +# (jq -f scripts/aibridgepricesgen/overrides.jq) and both generated outputs +# (prices.json and knownModelsGenerated.json) read the patched snapshot. +# +# Every patch guards its assumption about upstream, so a stale override +# fails the pipeline loudly instead of silently patching nothing. + +# claude-sonnet-4-5: upstream advertises a 1M-token context window, which +# implies tiered context_over_200k pricing. Coder persists flat pricing only, +# so pin the context limit to the flat-priced 200k tier. +if .anthropic.models | has("claude-sonnet-4-5") then + .anthropic.models."claude-sonnet-4-5".limit.context = 200000 +else + error("overrides.jq: claude-sonnet-4-5 gone from upstream; drop or update its context pin") +end + +# claude-mythos-5: not listed upstream. Anthropic documents it as sharing +# claude-fable-5's specs and pricing, so inject it as a copy with its own +# id and display name. +| if .anthropic.models | has("claude-fable-5") then + .anthropic.models."claude-mythos-5" = ( + .anthropic.models."claude-fable-5" + | .id = "claude-mythos-5" + | .name = "Claude Mythos 5" + ) + else + error("overrides.jq: claude-fable-5 gone from upstream; the claude-mythos-5 copy has no source") + end diff --git a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/index.ts b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/index.ts index 6aadaa9ad11..e50d663c335 100644 --- a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/index.ts +++ b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/index.ts @@ -6,12 +6,13 @@ export type { KnownModel, KnownModelSourceMetadata }; // knownModelsGenerated.json is produced by `make gen/aibridge-prices` from // models.dev joined with the editorial curation in -// scripts/aibridgepricesgen/catalog.json. Do not edit it manually. JSON +// scripts/aibridgepricesgen/curation.json. Do not edit it manually. JSON // imports widen literal types (e.g. reasoningEffort becomes string), so this // cast is the single typed boundary; knownModelsGenerated.test.ts validates -// shape and enum values for every entry. +// shape and enum values for every entry. The keyof cast preserves the +// literal provider-key union so isKnownProvider narrows usefully. const knownModelsByProvider = knownModelsGenerated as Record< - string, + keyof typeof knownModelsGenerated, readonly KnownModel[] >; diff --git a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.json b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.json index 83e90a4f44a..49304cffd06 100644 --- a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.json +++ b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.json @@ -131,7 +131,7 @@ "modelIdentifier": "claude-sonnet-4-5", "displayName": "Claude Sonnet 4.5", "aliases": ["claude-sonnet-4-5-20250929"], - "contextLimit": 1000000, + "contextLimit": 200000, "maxOutputTokens": 64000, "thinkingBudgetTokens": 8192, "inputCost": 3, diff --git a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.test.ts b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.test.ts index b3ed7b0abff..1986bec16ef 100644 --- a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.test.ts +++ b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.test.ts @@ -6,32 +6,43 @@ import knownModelsGenerated from "./knownModelsGenerated.json"; const providers = Object.entries(knownModelsGenerated); describe("knownModelsGenerated", () => { - it("contains the expected canonical model identifiers in display order", () => { - expect( - knownModelsGenerated.openai.map((model) => model.modelIdentifier), - ).toEqual([ - "gpt-5.6-sol", - "gpt-5.6-terra", - "gpt-5.6-luna", - "gpt-5.5", - "gpt-5.5-pro", - "gpt-5.4", - "gpt-5.4-mini", - "gpt-5.4-nano", - "gpt-5.3-codex", - ]); - expect( - knownModelsGenerated.anthropic.map((model) => model.modelIdentifier), - ).toEqual([ - "claude-fable-5", - "claude-mythos-5", - "claude-opus-4-8", - "claude-opus-4-7", - "claude-opus-4-6", - "claude-sonnet-4-6", - "claude-haiku-4-5", - "claude-sonnet-4-5", - ]); + it("pins the Anthropic thinking-mode split", () => { + // Anthropic has two mutually exclusive thinking APIs. Models on the + // legacy `thinking.budget_tokens` API must not be sent + // reasoningEffort: setting effort on a legacy-thinking model returns + // HTTP 400 from Anthropic. This map pins which side of the split each + // curated model is on; update it deliberately when curating models. + const thinkingMode = Object.fromEntries( + knownModelsGenerated.anthropic.map((model) => { + let mode = "none"; + if ("thinkingBudgetTokens" in model) { + mode = "thinkingBudgetTokens"; + } else if ("reasoningEffort" in model) { + mode = "reasoningEffort"; + } + return [model.modelIdentifier, mode]; + }), + ); + expect(thinkingMode).toEqual({ + "claude-fable-5": "reasoningEffort", + "claude-mythos-5": "reasoningEffort", + "claude-opus-4-8": "reasoningEffort", + "claude-opus-4-7": "reasoningEffort", + "claude-opus-4-6": "reasoningEffort", + "claude-sonnet-4-6": "reasoningEffort", + "claude-haiku-4-5": "thinkingBudgetTokens", + "claude-sonnet-4-5": "thinkingBudgetTokens", + }); + }); + + it("pins the claude-sonnet-4-5 context limit override", () => { + // Pinned to the flat-priced 200k tier by + // scripts/aibridgepricesgen/overrides.jq; guards the override at the + // generated-artifact layer. + const sonnet45 = knownModelsGenerated.anthropic.find( + (model) => model.modelIdentifier === "claude-sonnet-4-5", + ); + expect(sonnet45?.contextLimit).toBe(200000); }); it.each(providers)("validates every %s entry", (provider, models) => { @@ -51,10 +62,20 @@ describe("knownModelsGenerated", () => { record.thinkingBudgetTokens !== undefined, ).toBe(false); + // Token limits and budgets must be strictly positive; costs may + // legitimately be zero upstream (e.g. gpt-3.5-turbo cache_read). for (const field of [ "contextLimit", "maxOutputTokens", "thinkingBudgetTokens", + ]) { + const value = record[field]; + if (value !== undefined) { + expect(typeof value, field).toBe("number"); + expect(value, field).toBeGreaterThan(0); + } + } + for (const field of [ "inputCost", "outputCost", "cacheReadCost", @@ -63,7 +84,7 @@ describe("knownModelsGenerated", () => { const value = record[field]; if (value !== undefined) { expect(typeof value, field).toBe("number"); - expect(value, field).toBeGreaterThan(0); + expect(value, field).toBeGreaterThanOrEqual(0); } } From 6fa68d273ce59bd938f733944a3f057536d311e2 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 10 Jul 2026 15:49:54 +0000 Subject: [PATCH 08/14] feat(scripts/aibridgepricesgen): curate claude-sonnet-5 --- scripts/aibridgepricesgen/curation.json | 4 ++++ .../knownModels/knownModelsGenerated.json | 18 ++++++++++++++++++ .../knownModels/knownModelsGenerated.test.ts | 1 + 3 files changed, 23 insertions(+) diff --git a/scripts/aibridgepricesgen/curation.json b/scripts/aibridgepricesgen/curation.json index 44190120e8c..b528c8d0d33 100644 --- a/scripts/aibridgepricesgen/curation.json +++ b/scripts/aibridgepricesgen/curation.json @@ -57,6 +57,10 @@ "modelIdentifier": "claude-opus-4-6", "reasoningEffort": "high" }, + { + "modelIdentifier": "claude-sonnet-5", + "reasoningEffort": "high" + }, { "modelIdentifier": "claude-sonnet-4-6", "reasoningEffort": "medium" diff --git a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.json b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.json index 49304cffd06..108dc859313 100644 --- a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.json +++ b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.json @@ -90,6 +90,24 @@ "lastUpdated": "2026-03-13" } }, + { + "provider": "anthropic", + "modelIdentifier": "claude-sonnet-5", + "displayName": "Claude Sonnet 5", + "aliases": [], + "contextLimit": 1000000, + "maxOutputTokens": 128000, + "reasoningEffort": "high", + "inputCost": 2, + "outputCost": 10, + "cacheReadCost": 0.2, + "cacheWriteCost": 2.5, + "sourceMetadata": { + "sourceName": "models.dev", + "sourceRetrievedAt": "2026-07-10", + "lastUpdated": "2026-06-30" + } + }, { "provider": "anthropic", "modelIdentifier": "claude-sonnet-4-6", diff --git a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.test.ts b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.test.ts index 1986bec16ef..4a493ba2bed 100644 --- a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.test.ts +++ b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.test.ts @@ -29,6 +29,7 @@ describe("knownModelsGenerated", () => { "claude-opus-4-8": "reasoningEffort", "claude-opus-4-7": "reasoningEffort", "claude-opus-4-6": "reasoningEffort", + "claude-sonnet-5": "reasoningEffort", "claude-sonnet-4-6": "reasoningEffort", "claude-haiku-4-5": "thinkingBudgetTokens", "claude-sonnet-4-5": "thinkingBudgetTokens", From 67f983f63c1495d5835fa7717df01107596699b6 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 10 Jul 2026 16:19:05 +0000 Subject: [PATCH 09/14] refactor: drop unused sourceMetadata and redundant curation test --- scripts/aibridgepricesgen/catalog.go | 43 ++---- scripts/aibridgepricesgen/catalog_test.go | 97 ++------------ scripts/aibridgepricesgen/main.go | 10 +- .../applyKnownModelDefaults.test.ts | 13 +- .../ChatModelAdminPanel/knownModels/index.ts | 4 +- .../knownModels/knownModelsGenerated.json | 126 +++--------------- .../knownModels/knownModelsGenerated.test.ts | 4 - .../ChatModelAdminPanel/knownModels/types.ts | 7 - 8 files changed, 50 insertions(+), 254 deletions(-) diff --git a/scripts/aibridgepricesgen/catalog.go b/scripts/aibridgepricesgen/catalog.go index b0c20b8fe4f..900da4a5331 100644 --- a/scripts/aibridgepricesgen/catalog.go +++ b/scripts/aibridgepricesgen/catalog.go @@ -5,7 +5,6 @@ import ( _ "embed" "encoding/json" "io" - "time" "golang.org/x/xerrors" ) @@ -41,35 +40,26 @@ type curatedModel struct { // Costs are flat USD per million tokens, straight from models.dev; tiered // pricing such as context_over_200k is intentionally omitted. type catalogEntry struct { - Provider string `json:"provider"` - ModelIdentifier string `json:"modelIdentifier"` - DisplayName string `json:"displayName"` - Aliases []string `json:"aliases"` - ContextLimit *int64 `json:"contextLimit,omitempty"` - MaxOutputTokens *int64 `json:"maxOutputTokens,omitempty"` - ReasoningEffort string `json:"reasoningEffort,omitempty"` - ThinkingBudgetTokens int `json:"thinkingBudgetTokens,omitempty"` - InputCost *float64 `json:"inputCost,omitempty"` - OutputCost *float64 `json:"outputCost,omitempty"` - CacheReadCost *float64 `json:"cacheReadCost,omitempty"` - CacheWriteCost *float64 `json:"cacheWriteCost,omitempty"` - SourceMetadata sourceMetadata `json:"sourceMetadata"` -} - -type sourceMetadata struct { - SourceName string `json:"sourceName"` - SourceRetrievedAt string `json:"sourceRetrievedAt"` - LastUpdated string `json:"lastUpdated"` + Provider string `json:"provider"` + ModelIdentifier string `json:"modelIdentifier"` + DisplayName string `json:"displayName"` + Aliases []string `json:"aliases"` + ContextLimit *int64 `json:"contextLimit,omitempty"` + MaxOutputTokens *int64 `json:"maxOutputTokens,omitempty"` + ReasoningEffort string `json:"reasoningEffort,omitempty"` + ThinkingBudgetTokens int `json:"thinkingBudgetTokens,omitempty"` + InputCost *float64 `json:"inputCost,omitempty"` + OutputCost *float64 `json:"outputCost,omitempty"` + CacheReadCost *float64 `json:"cacheReadCost,omitempty"` + CacheWriteCost *float64 `json:"cacheWriteCost,omitempty"` } // validReasoningEfforts are the values accepted for curatedModel.ReasoningEffort. var validReasoningEfforts = map[string]bool{"low": true, "medium": true, "high": true} // buildCatalog joins the curation file with the upstream models.dev payload -// and returns provider-keyed ordered entry lists. now supplies the -// sourceRetrievedAt date so output is deterministic under test. -func buildCatalog(upstream map[string]upstreamProvider, curation map[string][]curatedModel, now time.Time) (map[string][]catalogEntry, error) { - retrievedAt := now.UTC().Format("2006-01-02") +// and returns provider-keyed ordered entry lists. +func buildCatalog(upstream map[string]upstreamProvider, curation map[string][]curatedModel) (map[string][]catalogEntry, error) { out := make(map[string][]catalogEntry, len(curation)) for providerID, curated := range curation { provider, ok := upstream[providerID] @@ -133,11 +123,6 @@ func buildCatalog(upstream map[string]upstreamProvider, curation map[string][]cu OutputCost: m.Cost.Output, CacheReadCost: m.Cost.CacheRead, CacheWriteCost: m.Cost.CacheWrite, - SourceMetadata: sourceMetadata{ - SourceName: "models.dev", - SourceRetrievedAt: retrievedAt, - LastUpdated: m.LastUpdated, - }, }) } // An alias resolving to a canonical identifier would make exact-alias diff --git a/scripts/aibridgepricesgen/catalog_test.go b/scripts/aibridgepricesgen/catalog_test.go index 5a8dba921cd..357db275001 100644 --- a/scripts/aibridgepricesgen/catalog_test.go +++ b/scripts/aibridgepricesgen/catalog_test.go @@ -5,7 +5,6 @@ import ( "encoding/json" "os" "testing" - "time" "github.com/stretchr/testify/require" ) @@ -20,19 +19,16 @@ func fixtureUpstream(t *testing.T) map[string]upstreamProvider { "claude-fable-5": { "name": "Claude Fable 5", "limit": {"context": 1000000, "output": 128000}, - "cost": {"input": 10, "output": 50, "cache_read": 1, "cache_write": 12.5}, - "last_updated": "2026-06-09" + "cost": {"input": 10, "output": 50, "cache_read": 1, "cache_write": 12.5} }, "claude-mythos-5": { "name": "Claude Mythos 5", "limit": {"context": 1000000, "output": 128000}, - "cost": {"input": 10, "output": 50, "cache_read": 1, "cache_write": 12.5}, - "last_updated": "2026-06-09" + "cost": {"input": 10, "output": 50, "cache_read": 1, "cache_write": 12.5} }, "claude-costless": { "name": "Claude Costless", - "limit": {"context": 200000, "output": 64000}, - "last_updated": "2026-01-01" + "limit": {"context": 200000, "output": 64000} } } }, @@ -41,14 +37,12 @@ func fixtureUpstream(t *testing.T) map[string]upstreamProvider { "gpt-5.6-sol": { "name": "GPT-5.6 Sol", "limit": {"context": 1050000, "output": 128000}, - "cost": {"input": 5, "output": 30, "cache_read": 0.5, "cache_write": 6.25}, - "last_updated": "2026-07-09" + "cost": {"input": 5, "output": 30, "cache_read": 0.5, "cache_write": 6.25} }, "gpt-partial": { "name": "GPT Partial", "limit": {"context": 400000, "output": 128000}, - "cost": {"input": 0.2, "output": 1.25}, - "last_updated": "2026-03-17" + "cost": {"input": 0.2, "output": 1.25} } } } @@ -58,8 +52,6 @@ func fixtureUpstream(t *testing.T) map[string]upstreamProvider { return upstream } -var fixedNow = time.Date(2026, 7, 10, 12, 34, 56, 0, time.UTC) - func TestBuildCatalog(t *testing.T) { t.Parallel() @@ -74,7 +66,7 @@ func TestBuildCatalog(t *testing.T) { }, } - catalog, err := buildCatalog(fixtureUpstream(t), curation, fixedNow) + catalog, err := buildCatalog(fixtureUpstream(t), curation) require.NoError(t, err) var buf bytes.Buffer @@ -93,12 +85,7 @@ func TestBuildCatalog(t *testing.T) { "inputCost": 10, "outputCost": 50, "cacheReadCost": 1, - "cacheWriteCost": 12.5, - "sourceMetadata": { - "sourceName": "models.dev", - "sourceRetrievedAt": "2026-07-10", - "lastUpdated": "2026-06-09" - } + "cacheWriteCost": 12.5 }, { "provider": "anthropic", @@ -111,12 +98,7 @@ func TestBuildCatalog(t *testing.T) { "inputCost": 10, "outputCost": 50, "cacheReadCost": 1, - "cacheWriteCost": 12.5, - "sourceMetadata": { - "sourceName": "models.dev", - "sourceRetrievedAt": "2026-07-10", - "lastUpdated": "2026-06-09" - } + "cacheWriteCost": 12.5 } ], "openai": [ @@ -133,12 +115,7 @@ func TestBuildCatalog(t *testing.T) { "inputCost": 5, "outputCost": 30, "cacheReadCost": 0.5, - "cacheWriteCost": 6.25, - "sourceMetadata": { - "sourceName": "models.dev", - "sourceRetrievedAt": "2026-07-10", - "lastUpdated": "2026-07-09" - } + "cacheWriteCost": 6.25 }, { "provider": "openai", @@ -148,12 +125,7 @@ func TestBuildCatalog(t *testing.T) { "contextLimit": 400000, "maxOutputTokens": 128000, "inputCost": 0.2, - "outputCost": 1.25, - "sourceMetadata": { - "sourceName": "models.dev", - "sourceRetrievedAt": "2026-07-10", - "lastUpdated": "2026-03-17" - } + "outputCost": 1.25 } ] } @@ -161,20 +133,6 @@ func TestBuildCatalog(t *testing.T) { require.Equal(t, want, buf.String()) } -// TestBuildCatalogClock asserts the injected clock feeds sourceRetrievedAt -// and leaves upstream-derived fields alone. -func TestBuildCatalogClock(t *testing.T) { - t.Parallel() - - curation := map[string][]curatedModel{ - "openai": {{ModelIdentifier: "gpt-5.6-sol"}}, - } - later, err := buildCatalog(fixtureUpstream(t), curation, fixedNow.AddDate(0, 0, 1)) - require.NoError(t, err) - require.Equal(t, "2026-07-11", later["openai"][0].SourceMetadata.SourceRetrievedAt) - require.Equal(t, "2026-07-09", later["openai"][0].SourceMetadata.LastUpdated) -} - func TestBuildCatalogErrors(t *testing.T) { t.Parallel() @@ -274,46 +232,13 @@ func TestBuildCatalogErrors(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { t.Parallel() - _, err := buildCatalog(fixtureUpstream(t), tc.curation, fixedNow) + _, err := buildCatalog(fixtureUpstream(t), tc.curation) require.Error(t, err) require.Contains(t, err.Error(), tc.wantErr) }) } } -// TestEmbeddedCurationBuilds runs the checked-in curation.json through -// buildCatalog against a synthetic upstream derived from the curation's own -// model identifiers, so every buildCatalog validation (present and future) -// automatically guards the checked-in file. -func TestEmbeddedCurationBuilds(t *testing.T) { - t.Parallel() - - curation := embeddedCuration(t) - require.NotEmpty(t, curation) - - price := 1.0 - limit := int64(100000) - upstream := make(map[string]upstreamProvider, len(curation)) - for providerID, entries := range curation { - models := make(map[string]upstreamModel, len(entries)) - for _, c := range entries { - models[c.ModelIdentifier] = upstreamModel{ - Name: "Synthetic " + c.ModelIdentifier, - Limit: upstreamLimit{Context: &limit, Output: &limit}, - Cost: &upstreamCost{Input: &price, Output: &price}, - LastUpdated: "2026-01-01", - } - } - upstream[providerID] = upstreamProvider{Models: models} - } - - catalog, err := buildCatalog(upstream, curation, fixedNow) - require.NoError(t, err) - for providerID, entries := range catalog { - require.NotEmpty(t, entries, providerID) - } -} - // TestCurationMatchesGeneratedCatalog is a drift test: the editorial fields // (per provider, in order) in the embedded curation.json must exactly match // their projection in the checked-in generated frontend catalog. Fails when diff --git a/scripts/aibridgepricesgen/main.go b/scripts/aibridgepricesgen/main.go index 367141be2a5..32a4c003ee6 100644 --- a/scripts/aibridgepricesgen/main.go +++ b/scripts/aibridgepricesgen/main.go @@ -16,7 +16,6 @@ import ( "math" "os" "sort" - "time" "golang.org/x/xerrors" ) @@ -31,10 +30,9 @@ type upstreamProvider struct { } type upstreamModel struct { - Name string `json:"name"` - Limit upstreamLimit `json:"limit"` - Cost *upstreamCost `json:"cost"` - LastUpdated string `json:"last_updated"` + Name string `json:"name"` + Limit upstreamLimit `json:"limit"` + Cost *upstreamCost `json:"cost"` } // Pointer fields in upstreamLimit and upstreamCost distinguish "key absent" @@ -142,7 +140,7 @@ func runCatalog(upstream map[string]upstreamProvider) error { if err := json.Unmarshal(curationJSON, &curation); err != nil { return xerrors.Errorf("parse embedded curation.json: %w", err) } - catalog, err := buildCatalog(upstream, curation, time.Now()) + catalog, err := buildCatalog(upstream, curation) if err != nil { return err } diff --git a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/applyKnownModelDefaults.test.ts b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/applyKnownModelDefaults.test.ts index e3ad0bd36c7..67178b03e2a 100644 --- a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/applyKnownModelDefaults.test.ts +++ b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/applyKnownModelDefaults.test.ts @@ -6,11 +6,7 @@ import { type ApplyKnownModelDefaultsResult, applyKnownModelDefaults, } from "./applyKnownModelDefaults"; -import { - findKnownModelByCanonicalId, - type KnownModel, - type KnownModelSourceMetadata, -} from "./index"; +import { findKnownModelByCanonicalId, type KnownModel } from "./index"; const requireKnownModel = ( provider: string, @@ -62,18 +58,11 @@ const applyDefaults = ( parameters: ApplyKnownModelDefaultsParameters, ): ApplyKnownModelDefaultsResult => applyKnownModelDefaults(parameters); -const testSourceMetadata = (): KnownModelSourceMetadata => ({ - sourceName: "models.dev", - sourceRetrievedAt: "2026-04-30", - lastUpdated: "2026-04-30", -}); - const customKnownModel = (overrides: Partial): KnownModel => ({ provider: "openai", modelIdentifier: "test-model", displayName: "Test Model", aliases: [], - sourceMetadata: testSourceMetadata(), ...overrides, }); diff --git a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/index.ts b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/index.ts index e50d663c335..164a8810df3 100644 --- a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/index.ts +++ b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/index.ts @@ -1,8 +1,8 @@ import { normalizeProvider } from "#/modules/aiModels/helpers"; import knownModelsGenerated from "./knownModelsGenerated.json"; -import type { KnownModel, KnownModelSourceMetadata } from "./types"; +import type { KnownModel } from "./types"; -export type { KnownModel, KnownModelSourceMetadata }; +export type { KnownModel }; // knownModelsGenerated.json is produced by `make gen/aibridge-prices` from // models.dev joined with the editorial curation in diff --git a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.json b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.json index 108dc859313..bfcb8c0a23a 100644 --- a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.json +++ b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.json @@ -11,12 +11,7 @@ "inputCost": 10, "outputCost": 50, "cacheReadCost": 1, - "cacheWriteCost": 12.5, - "sourceMetadata": { - "sourceName": "models.dev", - "sourceRetrievedAt": "2026-07-10", - "lastUpdated": "2026-06-09" - } + "cacheWriteCost": 12.5 }, { "provider": "anthropic", @@ -29,12 +24,7 @@ "inputCost": 10, "outputCost": 50, "cacheReadCost": 1, - "cacheWriteCost": 12.5, - "sourceMetadata": { - "sourceName": "models.dev", - "sourceRetrievedAt": "2026-07-10", - "lastUpdated": "2026-06-09" - } + "cacheWriteCost": 12.5 }, { "provider": "anthropic", @@ -47,12 +37,7 @@ "inputCost": 5, "outputCost": 25, "cacheReadCost": 0.5, - "cacheWriteCost": 6.25, - "sourceMetadata": { - "sourceName": "models.dev", - "sourceRetrievedAt": "2026-07-10", - "lastUpdated": "2026-05-28" - } + "cacheWriteCost": 6.25 }, { "provider": "anthropic", @@ -65,12 +50,7 @@ "inputCost": 5, "outputCost": 25, "cacheReadCost": 0.5, - "cacheWriteCost": 6.25, - "sourceMetadata": { - "sourceName": "models.dev", - "sourceRetrievedAt": "2026-07-10", - "lastUpdated": "2026-04-16" - } + "cacheWriteCost": 6.25 }, { "provider": "anthropic", @@ -83,12 +63,7 @@ "inputCost": 5, "outputCost": 25, "cacheReadCost": 0.5, - "cacheWriteCost": 6.25, - "sourceMetadata": { - "sourceName": "models.dev", - "sourceRetrievedAt": "2026-07-10", - "lastUpdated": "2026-03-13" - } + "cacheWriteCost": 6.25 }, { "provider": "anthropic", @@ -101,12 +76,7 @@ "inputCost": 2, "outputCost": 10, "cacheReadCost": 0.2, - "cacheWriteCost": 2.5, - "sourceMetadata": { - "sourceName": "models.dev", - "sourceRetrievedAt": "2026-07-10", - "lastUpdated": "2026-06-30" - } + "cacheWriteCost": 2.5 }, { "provider": "anthropic", @@ -119,12 +89,7 @@ "inputCost": 3, "outputCost": 15, "cacheReadCost": 0.3, - "cacheWriteCost": 3.75, - "sourceMetadata": { - "sourceName": "models.dev", - "sourceRetrievedAt": "2026-07-10", - "lastUpdated": "2026-03-13" - } + "cacheWriteCost": 3.75 }, { "provider": "anthropic", @@ -137,12 +102,7 @@ "inputCost": 1, "outputCost": 5, "cacheReadCost": 0.1, - "cacheWriteCost": 1.25, - "sourceMetadata": { - "sourceName": "models.dev", - "sourceRetrievedAt": "2026-07-10", - "lastUpdated": "2025-10-15" - } + "cacheWriteCost": 1.25 }, { "provider": "anthropic", @@ -155,12 +115,7 @@ "inputCost": 3, "outputCost": 15, "cacheReadCost": 0.3, - "cacheWriteCost": 3.75, - "sourceMetadata": { - "sourceName": "models.dev", - "sourceRetrievedAt": "2026-07-10", - "lastUpdated": "2025-09-29" - } + "cacheWriteCost": 3.75 } ], "openai": [ @@ -175,12 +130,7 @@ "inputCost": 5, "outputCost": 30, "cacheReadCost": 0.5, - "cacheWriteCost": 6.25, - "sourceMetadata": { - "sourceName": "models.dev", - "sourceRetrievedAt": "2026-07-10", - "lastUpdated": "2026-07-09" - } + "cacheWriteCost": 6.25 }, { "provider": "openai", @@ -193,12 +143,7 @@ "inputCost": 2.5, "outputCost": 15, "cacheReadCost": 0.25, - "cacheWriteCost": 3.125, - "sourceMetadata": { - "sourceName": "models.dev", - "sourceRetrievedAt": "2026-07-10", - "lastUpdated": "2026-07-09" - } + "cacheWriteCost": 3.125 }, { "provider": "openai", @@ -211,12 +156,7 @@ "inputCost": 1, "outputCost": 6, "cacheReadCost": 0.1, - "cacheWriteCost": 1.25, - "sourceMetadata": { - "sourceName": "models.dev", - "sourceRetrievedAt": "2026-07-10", - "lastUpdated": "2026-07-09" - } + "cacheWriteCost": 1.25 }, { "provider": "openai", @@ -228,12 +168,7 @@ "reasoningEffort": "medium", "inputCost": 5, "outputCost": 30, - "cacheReadCost": 0.5, - "sourceMetadata": { - "sourceName": "models.dev", - "sourceRetrievedAt": "2026-07-10", - "lastUpdated": "2026-04-23" - } + "cacheReadCost": 0.5 }, { "provider": "openai", @@ -244,12 +179,7 @@ "maxOutputTokens": 128000, "reasoningEffort": "high", "inputCost": 30, - "outputCost": 180, - "sourceMetadata": { - "sourceName": "models.dev", - "sourceRetrievedAt": "2026-07-10", - "lastUpdated": "2026-04-23" - } + "outputCost": 180 }, { "provider": "openai", @@ -260,12 +190,7 @@ "maxOutputTokens": 128000, "inputCost": 2.5, "outputCost": 15, - "cacheReadCost": 0.25, - "sourceMetadata": { - "sourceName": "models.dev", - "sourceRetrievedAt": "2026-07-10", - "lastUpdated": "2026-03-05" - } + "cacheReadCost": 0.25 }, { "provider": "openai", @@ -277,12 +202,7 @@ "reasoningEffort": "medium", "inputCost": 0.75, "outputCost": 4.5, - "cacheReadCost": 0.075, - "sourceMetadata": { - "sourceName": "models.dev", - "sourceRetrievedAt": "2026-07-10", - "lastUpdated": "2026-03-17" - } + "cacheReadCost": 0.075 }, { "provider": "openai", @@ -293,12 +213,7 @@ "maxOutputTokens": 128000, "inputCost": 0.2, "outputCost": 1.25, - "cacheReadCost": 0.02, - "sourceMetadata": { - "sourceName": "models.dev", - "sourceRetrievedAt": "2026-07-10", - "lastUpdated": "2026-03-17" - } + "cacheReadCost": 0.02 }, { "provider": "openai", @@ -310,12 +225,7 @@ "reasoningEffort": "medium", "inputCost": 1.75, "outputCost": 14, - "cacheReadCost": 0.175, - "sourceMetadata": { - "sourceName": "models.dev", - "sourceRetrievedAt": "2026-07-10", - "lastUpdated": "2026-02-05" - } + "cacheReadCost": 0.175 } ] } diff --git a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.test.ts b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.test.ts index 4a493ba2bed..c42ccecc851 100644 --- a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.test.ts +++ b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.test.ts @@ -88,10 +88,6 @@ describe("knownModelsGenerated", () => { expect(value, field).toBeGreaterThanOrEqual(0); } } - - expect(model.sourceMetadata.sourceName).toBe("models.dev"); - expect(model.sourceMetadata.sourceRetrievedAt).not.toBe(""); - expect(model.sourceMetadata.lastUpdated).not.toBe(""); } }); }); diff --git a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/types.ts b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/types.ts index 08037f54cd5..2e7c753ce9a 100644 --- a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/types.ts +++ b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/types.ts @@ -1,9 +1,3 @@ -export type KnownModelSourceMetadata = { - sourceName: "models.dev"; - sourceRetrievedAt: string; - lastUpdated: string; -}; - export type KnownModel = { provider: string; modelIdentifier: string; @@ -24,5 +18,4 @@ export type KnownModel = { outputCost?: number; cacheReadCost?: number; cacheWriteCost?: number; - sourceMetadata: KnownModelSourceMetadata; }; From db139131b5259a07908d0b0bb94b25789bd60508 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 10 Jul 2026 17:40:06 +0000 Subject: [PATCH 10/14] test(scripts/aibridgepricesgen): cover uncovered generator paths --- scripts/aibridgepricesgen/catalog.go | 11 +++++--- scripts/aibridgepricesgen/catalog_test.go | 34 +++++++++++++++++++---- scripts/aibridgepricesgen/main.go | 11 ++++++-- scripts/aibridgepricesgen/overrides.jq | 8 ++++-- 4 files changed, 49 insertions(+), 15 deletions(-) diff --git a/scripts/aibridgepricesgen/catalog.go b/scripts/aibridgepricesgen/catalog.go index 900da4a5331..7702c4e4e72 100644 --- a/scripts/aibridgepricesgen/catalog.go +++ b/scripts/aibridgepricesgen/catalog.go @@ -11,10 +11,10 @@ import ( // curationJSON is the checked-in editorial curation input for the frontend // known-models catalog. Entry order within each provider controls suggestion -// order in the UI. Everything factual (display name, limits, pricing, -// last_updated) is joined from models.dev at generation time; the curation -// file only carries editorial choices: which models to suggest, aliases, -// reasoning defaults, and overrides. +// order in the UI. Everything factual (display name, limits, pricing) is +// joined from models.dev at generation time; the curation file only carries +// editorial choices: which models to suggest, aliases, reasoning defaults, +// and overrides. // //go:embed curation.json var curationJSON []byte @@ -102,6 +102,9 @@ func buildCatalog(upstream map[string]upstreamProvider, curation map[string][]cu if !m.Cost.hasPricing() { return nil, xerrors.Errorf("%s/%s: upstream model has no cost block", providerID, c.ModelIdentifier) } + if m.Limit.Context == nil || m.Limit.Output == nil { + return nil, xerrors.Errorf("%s/%s: upstream model missing limit.context or limit.output", providerID, c.ModelIdentifier) + } displayName := cmp.Or(c.DisplayName, m.Name) if displayName == "" { return nil, xerrors.Errorf("%s/%s: no displayName override and upstream name is empty", providerID, c.ModelIdentifier) diff --git a/scripts/aibridgepricesgen/catalog_test.go b/scripts/aibridgepricesgen/catalog_test.go index 357db275001..5553fd18e6f 100644 --- a/scripts/aibridgepricesgen/catalog_test.go +++ b/scripts/aibridgepricesgen/catalog_test.go @@ -29,6 +29,11 @@ func fixtureUpstream(t *testing.T) map[string]upstreamProvider { "claude-costless": { "name": "Claude Costless", "limit": {"context": 200000, "output": 64000} + }, + "claude-nameless": { + "name": "", + "limit": {"context": 200000, "output": 64000}, + "cost": {"input": 1, "output": 5} } } }, @@ -43,6 +48,11 @@ func fixtureUpstream(t *testing.T) map[string]upstreamProvider { "name": "GPT Partial", "limit": {"context": 400000, "output": 128000}, "cost": {"input": 0.2, "output": 1.25} + }, + "gpt-limitless": { + "name": "GPT Limitless", + "limit": {"context": 400000}, + "cost": {"input": 0.2, "output": 1.25} } } } @@ -62,7 +72,7 @@ func TestBuildCatalog(t *testing.T) { }, "anthropic": { {ModelIdentifier: "claude-fable-5", ReasoningEffort: "high"}, - {ModelIdentifier: "claude-mythos-5", ReasoningEffort: "high"}, + {ModelIdentifier: "claude-mythos-5", DisplayName: "Mythos 5 Override", ThinkingBudgetTokens: 8192}, }, } @@ -90,11 +100,11 @@ func TestBuildCatalog(t *testing.T) { { "provider": "anthropic", "modelIdentifier": "claude-mythos-5", - "displayName": "Claude Mythos 5", + "displayName": "Mythos 5 Override", "aliases": [], "contextLimit": 1000000, "maxOutputTokens": 128000, - "reasoningEffort": "high", + "thinkingBudgetTokens": 8192, "inputCost": 10, "outputCost": 50, "cacheReadCost": 1, @@ -146,7 +156,7 @@ func TestBuildCatalogErrors(t *testing.T) { curation: map[string][]curatedModel{ "openai": {{ModelIdentifier: "gpt-nonexistent"}}, }, - wantErr: "missing from upstream", + wantErr: "model missing from upstream", }, { name: "NoCostBlock", @@ -155,6 +165,20 @@ func TestBuildCatalogErrors(t *testing.T) { }, wantErr: "no cost block", }, + { + name: "MissingUpstreamLimit", + curation: map[string][]curatedModel{ + "openai": {{ModelIdentifier: "gpt-limitless"}}, + }, + wantErr: "missing limit.context or limit.output", + }, + { + name: "EmptyUpstreamName", + curation: map[string][]curatedModel{ + "anthropic": {{ModelIdentifier: "claude-nameless"}}, + }, + wantErr: "upstream name is empty", + }, { name: "EffortAndBudgetBothSet", curation: map[string][]curatedModel{ @@ -218,7 +242,7 @@ func TestBuildCatalogErrors(t *testing.T) { curation: map[string][]curatedModel{ "google": {{ModelIdentifier: "gemini"}}, }, - wantErr: "missing from upstream", + wantErr: `provider "google" missing`, }, { name: "EmptyModelIdentifier", diff --git a/scripts/aibridgepricesgen/main.go b/scripts/aibridgepricesgen/main.go index 32a4c003ee6..299473470ae 100644 --- a/scripts/aibridgepricesgen/main.go +++ b/scripts/aibridgepricesgen/main.go @@ -1,6 +1,11 @@ -// aibridgepricesgen converts a models.dev api.json snapshot into a JSON -// seed file consumable by the AI Bridge cost-control loader. Output is sorted -// by (provider, model) so regenerations produce minimal diffs. +// aibridgepricesgen converts a models.dev api.json snapshot into generated +// artifacts, selected by -format: +// +// - "prices": a JSON seed file consumable by the AI Bridge cost-control +// loader, sorted by (provider, model) so regenerations produce minimal +// diffs. +// - "catalog": the frontend known-models JSON, joining the snapshot with +// the editorial curation in curation.json and preserving its entry order. // // Run via the gen/aibridge-prices Make target, which fetches and patches the // snapshot (_gen/models-dev.json). Kept out of `make gen` because the output diff --git a/scripts/aibridgepricesgen/overrides.jq b/scripts/aibridgepricesgen/overrides.jq index 8d8a445f6ec..001b8aa076a 100644 --- a/scripts/aibridgepricesgen/overrides.jq +++ b/scripts/aibridgepricesgen/overrides.jq @@ -18,12 +18,14 @@ end # claude-mythos-5: not listed upstream. Anthropic documents it as sharing # claude-fable-5's specs and pricing, so inject it as a copy with its own # id and display name. -| if .anthropic.models | has("claude-fable-5") then +| if (.anthropic.models | has("claude-fable-5") | not) then + error("overrides.jq: claude-fable-5 gone from upstream; the claude-mythos-5 copy has no source") + elif (.anthropic.models | has("claude-mythos-5")) then + error("overrides.jq: claude-mythos-5 now present upstream; drop the injection") + else .anthropic.models."claude-mythos-5" = ( .anthropic.models."claude-fable-5" | .id = "claude-mythos-5" | .name = "Claude Mythos 5" ) - else - error("overrides.jq: claude-fable-5 gone from upstream; the claude-mythos-5 copy has no source") end From 2e2fed3bd47a8dcf0796fc19d54a2b7fab1f7e58 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 10 Jul 2026 22:28:20 +0100 Subject: [PATCH 11/14] better words from the bot Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- scripts/aibridgepricesgen/catalog.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/aibridgepricesgen/catalog.go b/scripts/aibridgepricesgen/catalog.go index 7702c4e4e72..0a2ff05ea5a 100644 --- a/scripts/aibridgepricesgen/catalog.go +++ b/scripts/aibridgepricesgen/catalog.go @@ -100,7 +100,7 @@ func buildCatalog(upstream map[string]upstreamProvider, curation map[string][]cu return nil, xerrors.Errorf("%s/%s: model missing from upstream (patch it in via overrides.jq if intentional)", providerID, c.ModelIdentifier) } if !m.Cost.hasPricing() { - return nil, xerrors.Errorf("%s/%s: upstream model has no cost block", providerID, c.ModelIdentifier) + return nil, xerrors.Errorf("%s/%s: upstream model has no pricing data", providerID, c.ModelIdentifier) } if m.Limit.Context == nil || m.Limit.Output == nil { return nil, xerrors.Errorf("%s/%s: upstream model missing limit.context or limit.output", providerID, c.ModelIdentifier) From 41221a664dc66e85873a98a9d4cbec718121c010 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 10 Jul 2026 22:34:42 +0100 Subject: [PATCH 12/14] fixup! better words from the bot --- scripts/aibridgepricesgen/catalog_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/aibridgepricesgen/catalog_test.go b/scripts/aibridgepricesgen/catalog_test.go index 5553fd18e6f..cc8c5affa4e 100644 --- a/scripts/aibridgepricesgen/catalog_test.go +++ b/scripts/aibridgepricesgen/catalog_test.go @@ -163,7 +163,7 @@ func TestBuildCatalogErrors(t *testing.T) { curation: map[string][]curatedModel{ "anthropic": {{ModelIdentifier: "claude-costless"}}, }, - wantErr: "no cost block", + wantErr: "no pricing data", }, { name: "MissingUpstreamLimit", From 451d524d806d53e84747a055aa6019110654d80a Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 14 Jul 2026 20:15:28 +0100 Subject: [PATCH 13/14] update comment re sonnet 4.5 1M context --- scripts/aibridgepricesgen/overrides.jq | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/aibridgepricesgen/overrides.jq b/scripts/aibridgepricesgen/overrides.jq index 001b8aa076a..e8e01576a4f 100644 --- a/scripts/aibridgepricesgen/overrides.jq +++ b/scripts/aibridgepricesgen/overrides.jq @@ -6,16 +6,16 @@ # Every patch guards its assumption about upstream, so a stale override # fails the pipeline loudly instead of silently patching nothing. -# claude-sonnet-4-5: upstream advertises a 1M-token context window, which -# implies tiered context_over_200k pricing. Coder persists flat pricing only, -# so pin the context limit to the flat-priced 200k tier. +# claude-sonnet-4-5: models.dev advertises a 1M-token context window, which +# is incorrect. Anthropic retired the 1M context window beta on May 1st, +# 2026. Ref: https://platform.claude.com/docs/en/about-claude/models/overview if .anthropic.models | has("claude-sonnet-4-5") then .anthropic.models."claude-sonnet-4-5".limit.context = 200000 else error("overrides.jq: claude-sonnet-4-5 gone from upstream; drop or update its context pin") end -# claude-mythos-5: not listed upstream. Anthropic documents it as sharing +# claude-mythos-5: not listed on models.dev. Anthropic documents it as sharing # claude-fable-5's specs and pricing, so inject it as a copy with its own # id and display name. | if (.anthropic.models | has("claude-fable-5") | not) then From e6ea5e6bbf5779d0e6e9f910d72aea5d16a36f67 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 14 Jul 2026 20:19:03 +0100 Subject: [PATCH 14/14] fix up nits --- scripts/aibridgepricesgen/main.go | 2 +- scripts/aibridgepricesgen/overrides.jq | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/aibridgepricesgen/main.go b/scripts/aibridgepricesgen/main.go index 299473470ae..19778c357f3 100644 --- a/scripts/aibridgepricesgen/main.go +++ b/scripts/aibridgepricesgen/main.go @@ -1,7 +1,7 @@ // aibridgepricesgen converts a models.dev api.json snapshot into generated // artifacts, selected by -format: // -// - "prices": a JSON seed file consumable by the AI Bridge cost-control +// - "prices": a JSON seed file consumable by the AI Gateway cost-control // loader, sorted by (provider, model) so regenerations produce minimal // diffs. // - "catalog": the frontend known-models JSON, joining the snapshot with diff --git a/scripts/aibridgepricesgen/overrides.jq b/scripts/aibridgepricesgen/overrides.jq index e8e01576a4f..cc1b424a1f8 100644 --- a/scripts/aibridgepricesgen/overrides.jq +++ b/scripts/aibridgepricesgen/overrides.jq @@ -18,6 +18,7 @@ end # claude-mythos-5: not listed on models.dev. Anthropic documents it as sharing # claude-fable-5's specs and pricing, so inject it as a copy with its own # id and display name. +# Ref: https://platform.claude.com/docs/en/about-claude/pricing#model-pricing | if (.anthropic.models | has("claude-fable-5") | not) then error("overrides.jq: claude-fable-5 gone from upstream; the claude-mythos-5 copy has no source") elif (.anthropic.models | has("claude-mythos-5")) then