diff --git a/.server-changes/fix-llm-cost-prefer-provider-reported.md b/.server-changes/fix-llm-cost-prefer-provider-reported.md new file mode 100644 index 0000000000..0648912067 --- /dev/null +++ b/.server-changes/fix-llm-cost-prefer-provider-reported.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +AI generation cost now uses the exact provider-reported cost from OpenRouter/Vercel AI Gateway when present, instead of catalog pricing, so cache-discounted and fallback-routed requests match the amount the provider actually billed. diff --git a/apps/webapp/app/v3/utils/enrichCreatableEvents.server.test.ts b/apps/webapp/app/v3/utils/enrichCreatableEvents.server.test.ts new file mode 100644 index 0000000000..9deaa51307 --- /dev/null +++ b/apps/webapp/app/v3/utils/enrichCreatableEvents.server.test.ts @@ -0,0 +1,214 @@ +import { describe, it, expect } from "vitest"; +import { enrichCreatableEvents, setLlmPricingRegistry } from "./enrichCreatableEvents.server"; +import type { CreateEventInput } from "../eventRepository/eventRepository.types"; + +type Registry = Parameters[0]; +type RegistryCost = NonNullable>; + +function registryReturning(cost: RegistryCost | null, isLoaded = true): Registry { + return { + isLoaded, + calculateCost: () => cost, + }; +} + +// A catalog cost result the registry would produce. The numbers here stand in for a catalog +// price that disagrees with what the provider actually billed. +function catalogCost(overrides: Partial = {}): RegistryCost { + return { + matchedModelId: "llm_model_test", + matchedModelName: "test-model", + pricingTierId: "tier_standard", + pricingTierName: "Standard", + inputCost: 0, + outputCost: 0, + totalCost: 0, + costDetails: {}, + ...overrides, + }; +} + +function makeEvent(properties: Record): CreateEventInput { + return { + message: "ai.generateText.doGenerate", + kind: "INTERNAL", + isPartial: false, + properties: properties as CreateEventInput["properties"], + } as unknown as CreateEventInput; +} + +function enrichOne(event: CreateEventInput): CreateEventInput { + const [out] = enrichCreatableEvents([event]); + return out; +} + +function costPillText(event: CreateEventInput): string | undefined { + const accessory = (event.style as any)?.accessory; + const items: Array<{ text: string; icon: string }> = accessory?.items ?? []; + return items.find((i) => i.icon === "tabler-currency-dollar")?.text; +} + +function modelPillText(event: CreateEventInput): string | undefined { + const accessory = (event.style as any)?.accessory; + const items: Array<{ text: string; icon: string }> = accessory?.items ?? []; + return items.find((i) => i.icon === "tabler-cube")?.text; +} + +describe("enrichLlmMetrics — provider-reported cost", () => { + it("prefers the provider-reported cost over catalog pricing when a cache discount applies", () => { + // OpenRouter served mimo with 25,280 of 34,374 prompt tokens as cache reads. Cache counts + // only reach us via providerMetadata, so the catalog bills the full prompt at the input + // rate while OpenRouter's exact bill reflects the cache discount. + setLlmPricingRegistry(registryReturning(catalogCost({ totalCost: 0.01603584 }))); + + const out = enrichOne( + makeEvent({ + "gen_ai.system": "openrouter", + "gen_ai.request.model": "xiaomi/mimo-v2.5-pro", + "gen_ai.response.model": "xiaomi/mimo-v2.5-pro-20260422", + "gen_ai.usage.input_tokens": 34374, + "gen_ai.usage.output_tokens": 1245, + "ai.response.providerMetadata": JSON.stringify({ + openrouter: { + provider: "Xiaomi", + usage: { cost: 0.005130048, promptTokensDetails: { cachedTokens: 25280 } }, + }, + }), + }) + ); + + expect(out.properties["trigger.llm.total_cost"]).toBe(0.005130048); + expect(out.properties["trigger.llm.cost_source"]).toBe("openrouter"); + // The catalog breakdown must not be written — it priced the wrong (full-rate) total. + expect(out.properties["trigger.llm.input_cost"]).toBeUndefined(); + expect(out.properties["trigger.llm.matched_model"]).toBeUndefined(); + expect(costPillText(out)).toBe("$0.005130"); + + expect(out._llmMetrics?.totalCost).toBe(0.005130048); + expect(out._llmMetrics?.costSource).toBe("openrouter"); + expect(out._llmMetrics?.providerCost).toBe(0.005130048); + expect(out._llmMetrics?.inputCost).toBe(0); + }); + + it("prices the served fallback model's provider cost, not the requested model", () => { + // Requested mimo, OpenRouter routed to a gemini fallback: gen_ai.response.model carries the + // SERVED model, and the provider cost is authoritative. + setLlmPricingRegistry(registryReturning(catalogCost({ totalCost: 0.02 }))); + + const out = enrichOne( + makeEvent({ + "gen_ai.system": "openrouter", + "gen_ai.request.model": "xiaomi/mimo-v2.5-pro", + "gen_ai.response.model": "google/gemini-3.5-flash-20260519", + "gen_ai.usage.input_tokens": 5000, + "gen_ai.usage.output_tokens": 800, + "ai.response.providerMetadata": JSON.stringify({ + openrouter: { provider: "Google", usage: { cost: 0.011058 } }, + }), + }) + ); + + expect(out.properties["trigger.llm.total_cost"]).toBe(0.011058); + expect(out.properties["trigger.llm.cost_source"]).toBe("openrouter"); + // The pill (and stored response model) reflect the served fallback model. + expect(modelPillText(out)).toBe("google/gemini-3.5-flash-20260519"); + expect(out._llmMetrics?.requestModel).toBe("xiaomi/mimo-v2.5-pro"); + expect(out._llmMetrics?.responseModel).toBe("google/gemini-3.5-flash-20260519"); + expect(out._llmMetrics?.totalCost).toBe(0.011058); + }); + + it("prefers the gateway-reported cost over catalog pricing", () => { + setLlmPricingRegistry(registryReturning(catalogCost({ totalCost: 0.02 }))); + + const out = enrichOne( + makeEvent({ + "gen_ai.system": "gateway", + "gen_ai.response.model": "openai/gpt-4o", + "gen_ai.usage.input_tokens": 1000, + "gen_ai.usage.output_tokens": 500, + "ai.response.providerMetadata": JSON.stringify({ + gateway: { cost: "0.0006615" }, + }), + }) + ); + + expect(out.properties["trigger.llm.total_cost"]).toBe(0.0006615); + expect(out.properties["trigger.llm.cost_source"]).toBe("gateway"); + expect(out._llmMetrics?.costSource).toBe("gateway"); + }); + + it("uses provider cost even when the catalog does not match the model", () => { + setLlmPricingRegistry(registryReturning(null)); + + const out = enrichOne( + makeEvent({ + "gen_ai.system": "openrouter", + "gen_ai.response.model": "some/unlisted-model", + "gen_ai.usage.input_tokens": 2000, + "gen_ai.usage.output_tokens": 300, + "ai.response.providerMetadata": JSON.stringify({ + openrouter: { provider: "SomeProvider", usage: { cost: 0.00042 } }, + }), + }) + ); + + expect(out.properties["trigger.llm.total_cost"]).toBe(0.00042); + expect(out.properties["trigger.llm.cost_source"]).toBe("openrouter"); + }); + + it("falls back to catalog pricing when no provider cost is reported", () => { + setLlmPricingRegistry( + registryReturning( + catalogCost({ + matchedModelId: "llm_model_gpt4o", + matchedModelName: "gpt-4o", + inputCost: 0.04, + outputCost: 0.01, + totalCost: 0.05, + costDetails: { input: 0.04, output: 0.01 }, + }) + ) + ); + + const out = enrichOne( + makeEvent({ + "gen_ai.system": "openai", + "gen_ai.response.model": "gpt-4o", + "gen_ai.usage.input_tokens": 1000, + "gen_ai.usage.output_tokens": 500, + }) + ); + + expect(out.properties["trigger.llm.total_cost"]).toBe(0.05); + expect(out.properties["trigger.llm.input_cost"]).toBe(0.04); + expect(out.properties["trigger.llm.output_cost"]).toBe(0.01); + expect(out.properties["trigger.llm.matched_model"]).toBe("gpt-4o"); + // Registry path does not set a cost_source attribute. + expect(out.properties["trigger.llm.cost_source"]).toBeUndefined(); + expect(out._llmMetrics?.costSource).toBe("registry"); + expect(out._llmMetrics?.matchedModelId).toBe("llm_model_gpt4o"); + }); + + it("falls back to catalog pricing when providerMetadata carries no cost field", () => { + // The cheap `"cost"` guard should skip parsing and let the registry price this span. + setLlmPricingRegistry( + registryReturning(catalogCost({ totalCost: 0.03, matchedModelName: "claude" })) + ); + + const out = enrichOne( + makeEvent({ + "gen_ai.system": "anthropic", + "gen_ai.response.model": "claude-sonnet-4-0", + "gen_ai.usage.input_tokens": 1000, + "gen_ai.usage.output_tokens": 500, + "ai.response.providerMetadata": JSON.stringify({ + anthropic: { usage: { service_tier: "standard" } }, + }), + }) + ); + + expect(out.properties["trigger.llm.total_cost"]).toBe(0.03); + expect(out.properties["trigger.llm.cost_source"]).toBeUndefined(); + expect(out._llmMetrics?.costSource).toBe("registry"); + }); +}); diff --git a/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts b/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts index 9173038db0..73bf8bf31b 100644 --- a/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts +++ b/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts @@ -89,20 +89,17 @@ function enrichLlmMetrics(event: CreateEventInput): void { { text: formatTokenCount(totalTokens), icon: "tabler-hash" }, ]; - // Try cost enrichment if the registry is loaded. - // The registry handles prefix stripping (e.g. "mistral/mistral-large-3" → "mistral-large-3") - // for gateway/openrouter models automatically in its match() method. + // Provider-reported cost (gateway/openrouter) is the exact per-request bill and already + // reflects cache-read discounts and the real per-provider rate, so prefer it and only fall + // back to catalog pricing when it is absent. The registry handles prefix stripping (e.g. + // "mistral/mistral-large-3" → "mistral-large-3") for gateway/openrouter models in match(). + const providerCost = extractProviderCost(props); + let cost: ReturnType["calculateCost"]> | null = null; - if (_registry?.isLoaded) { + if (!providerCost && _registry?.isLoaded) { cost = _registry.calculateCost(responseModel, usageDetails); } - // Fallback: extract cost from provider metadata (gateway/openrouter report per-request cost) - let providerCost: { totalCost: number; source: string } | null = null; - if (!cost) { - providerCost = extractProviderCost(props); - } - if (cost) { // Add trigger.llm.* attributes to the span from our pricing registry event.properties = { @@ -342,6 +339,11 @@ function extractProviderCost( const rawMeta = props["ai.response.providerMetadata"]; if (typeof rawMeta !== "string") return null; + // Cheap guard: providerMetadata can be large for reasoning models (it carries the full + // reasoning_details text), and this now runs on every AI span. Skip the JSON parse when + // there is no cost field to find. + if (!rawMeta.includes('"cost"')) return null; + let meta: Record; try { meta = JSON.parse(rawMeta) as Record; diff --git a/apps/webapp/vitest.config.ts b/apps/webapp/vitest.config.ts index f9ec17fec0..18f0be4e6b 100644 --- a/apps/webapp/vitest.config.ts +++ b/apps/webapp/vitest.config.ts @@ -13,6 +13,7 @@ export default defineConfig({ "test/**/*.test.ts", "app/v3/runOpsMigration/**/*.test.ts", "app/v3/runStore.server.test.ts", + "app/v3/utils/**/*.test.ts", "app/v3/services/bulk/**/*.test.ts", "app/runEngine/concerns/**/*.test.ts", "app/runEngine/services/**/*.test.ts",