From 7576b19c0ce2c292a35da3b8be4d294033fd0483 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 5 Aug 2026 16:18:06 -0700 Subject: [PATCH 1/2] feat(agent): surface a normalized finishReason on the agent block --- apps/sim/blocks/blocks/agent.ts | 5 + .../executor/handlers/agent/agent-handler.ts | 25 +++++ apps/sim/providers/finish-reason.test.ts | 78 +++++++++++++++ apps/sim/providers/finish-reason.ts | 95 +++++++++++++++++++ apps/sim/providers/types.ts | 6 ++ 5 files changed, 209 insertions(+) create mode 100644 apps/sim/providers/finish-reason.test.ts create mode 100644 apps/sim/providers/finish-reason.ts diff --git a/apps/sim/blocks/blocks/agent.ts b/apps/sim/blocks/blocks/agent.ts index 363fd39b21c..49640d3c1f8 100644 --- a/apps/sim/blocks/blocks/agent.ts +++ b/apps/sim/blocks/blocks/agent.ts @@ -672,6 +672,11 @@ Return ONLY the JSON array.`, model: { type: 'string', description: 'Model used for generation' }, tokens: { type: 'json', description: 'Token usage statistics' }, toolCalls: { type: 'json', description: 'Tool calls made' }, + finishReason: { + type: 'string', + description: + 'Why generation stopped: stop, length (truncated by the token limit), tool_calls, content_filter, error, or other', + }, providerTiming: { type: 'json', description: 'Provider timing information', diff --git a/apps/sim/executor/handlers/agent/agent-handler.ts b/apps/sim/executor/handlers/agent/agent-handler.ts index 209443bb7c2..11422183768 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.ts @@ -62,7 +62,9 @@ import { canUseProviderLargeFilePath, getInlineHydrationMaxBytes, } from '@/providers/file-attachments.server' +import { normalizeFinishReason } from '@/providers/finish-reason' import { isAutoModel, SIM_AUTO_MODEL_ID } from '@/providers/models' +import type { ProviderResponse } from '@/providers/types' import { getProviderFromModel, transformBlockTool } from '@/providers/utils' import type { SerializedBlock } from '@/serializer/types' import { filterSchemaForLLM, type ToolSchema } from '@/tools/params' @@ -1487,9 +1489,32 @@ export class AgentBlockHandler implements BlockHandler { }, providerTiming: result.timing, cost: result.cost, + /** + * Read from the last model segment rather than threaded through each provider: + * every family already records its raw stop reason there for the trace, so this + * normalizes the value the enrichment layer has already collected. A run whose + * provider reported nothing simply has no reason, which stays distinct from one + * the vocabulary could not place. + */ + finishReason: normalizeFinishReason(this.lastModelSegmentFinishReasonImpl(result.timing)), } } + /** + * The raw stop reason from the most recent `model` segment. Later segments win + * because a tool loop appends one segment per turn and the final turn is the one + * that ended the generation. + */ + private lastModelSegmentFinishReasonImpl(timing: ProviderResponse['timing']): string | undefined { + const segments = timing?.timeSegments + if (!segments) return undefined + for (let i = segments.length - 1; i >= 0; i--) { + const segment = segments[i] + if (segment.type === 'model' && segment.finishReason) return segment.finishReason + } + return undefined + } + private formatToolCall(tc: any) { const toolName = stripCustomToolPrefix(tc.name) diff --git a/apps/sim/providers/finish-reason.test.ts b/apps/sim/providers/finish-reason.test.ts new file mode 100644 index 00000000000..78e07c042bd --- /dev/null +++ b/apps/sim/providers/finish-reason.test.ts @@ -0,0 +1,78 @@ +/** + * @vitest-environment node + * + * The raw values are taken from the SDK enums this repo compiles against: + * `ChatCompletion.finish_reason`, Anthropic `StopReason`, Gemini `FinishReason`, + * and Bedrock `StopReason`. A provider adding a case must not fail a run, so an + * unknown value normalizes rather than throwing. + */ +import { describe, expect, it } from 'vitest' +import { normalizeFinishReason } from '@/providers/finish-reason' + +describe('normalizeFinishReason', () => { + it('reports nothing when the provider reported nothing', () => { + expect(normalizeFinishReason(undefined)).toBeUndefined() + expect(normalizeFinishReason(null)).toBeUndefined() + expect(normalizeFinishReason('')).toBeUndefined() + }) + + /** The case this exists for: one branch catches truncation on every provider. */ + it('maps every provider spelling of truncation to length', () => { + expect(normalizeFinishReason('length')).toBe('length') // OpenAI chat + expect(normalizeFinishReason('max_output_tokens')).toBe('length') // OpenAI Responses + expect(normalizeFinishReason('max_tokens')).toBe('length') // Anthropic, Bedrock + expect(normalizeFinishReason('MAX_TOKENS')).toBe('length') // Gemini + expect(normalizeFinishReason('model_context_window_exceeded')).toBe('length') + }) + + it('maps natural completion to stop', () => { + expect(normalizeFinishReason('stop')).toBe('stop') + expect(normalizeFinishReason('STOP')).toBe('stop') + expect(normalizeFinishReason('end_turn')).toBe('stop') + expect(normalizeFinishReason('stop_sequence')).toBe('stop') + }) + + it('maps tool stops to tool_calls', () => { + expect(normalizeFinishReason('tool_calls')).toBe('tool_calls') + expect(normalizeFinishReason('function_call')).toBe('tool_calls') + expect(normalizeFinishReason('tool_use')).toBe('tool_calls') + }) + + it('maps every safety stop to content_filter', () => { + for (const raw of [ + 'content_filter', + 'content_filtered', + 'guardrail_intervened', + 'refusal', + 'SAFETY', + 'BLOCKLIST', + 'PROHIBITED_CONTENT', + 'SPII', + 'RECITATION', + 'IMAGE_SAFETY', + ]) { + expect(normalizeFinishReason(raw)).toBe('content_filter') + } + }) + + it('maps malformed generations to error', () => { + expect(normalizeFinishReason('MALFORMED_FUNCTION_CALL')).toBe('error') + expect(normalizeFinishReason('malformed_tool_use')).toBe('error') + expect(normalizeFinishReason('malformed_model_output')).toBe('error') + }) + + /** A pause is a continuation point, not an outcome the caller should branch on. */ + it('does not treat a server-tool pause as a stop', () => { + expect(normalizeFinishReason('pause_turn')).toBe('other') + }) + + it('degrades an unrecognized value to other rather than throwing', () => { + expect(normalizeFinishReason('some_future_reason')).toBe('other') + expect(normalizeFinishReason('OTHER')).toBe('other') + expect(normalizeFinishReason('FINISH_REASON_UNSPECIFIED')).toBe('other') + }) + + it('is insensitive to case and surrounding whitespace', () => { + expect(normalizeFinishReason(' Length ')).toBe('length') + }) +}) diff --git a/apps/sim/providers/finish-reason.ts b/apps/sim/providers/finish-reason.ts new file mode 100644 index 00000000000..285a6793fb8 --- /dev/null +++ b/apps/sim/providers/finish-reason.ts @@ -0,0 +1,95 @@ +/** + * Why a model stopped generating, normalized across providers. + * + * Providers disagree on vocabulary for the same event — a truncated generation is + * `length` on OpenAI, `max_tokens` on Anthropic and Bedrock, and `MAX_TOKENS` on + * Gemini. Traces keep each provider's raw string because it is the ground truth for + * debugging; this normalized value exists so a workflow can branch on the outcome + * without enumerating every provider's spelling. + */ +export type AgentFinishReason = + /** Generation completed naturally, or hit a caller-supplied stop sequence. */ + | 'stop' + /** Truncated by a token limit — the model had more to say. */ + | 'length' + /** Stopped in order to call tools. */ + | 'tool_calls' + /** Blocked or refused by a safety system. */ + | 'content_filter' + /** The provider reported the generation itself as malformed. */ + | 'error' + /** Reported, but not a case this vocabulary distinguishes. */ + | 'other' + +/** + * Raw provider value → normalized reason, keyed on the lowercased string. + * + * A single table rather than a per-provider mapper because the vocabularies do not + * collide: no raw value means one thing to one provider and something else to + * another. Sources are the SDK types this repo compiles against — + * `ChatCompletion.finish_reason`, Anthropic's `StopReason`, Gemini's `FinishReason`, + * and Bedrock's `StopReason`. + */ +const NORMALIZED_BY_RAW = new Map([ + // OpenAI Chat Completions, and every OpenAI-compatible provider. + ['stop', 'stop'], + ['length', 'length'], + ['tool_calls', 'tool_calls'], + ['function_call', 'tool_calls'], + ['content_filter', 'content_filter'], + + // OpenAI Responses reports truncation through `incomplete_details.reason`. + ['max_output_tokens', 'length'], + + // Anthropic Messages, shared by Bedrock's Converse API. + ['end_turn', 'stop'], + ['stop_sequence', 'stop'], + ['max_tokens', 'length'], + ['model_context_window_exceeded', 'length'], + ['tool_use', 'tool_calls'], + ['refusal', 'content_filter'], + /** A server-tool pause is a continuation point, not an outcome. */ + ['pause_turn', 'other'], + + // Bedrock Converse additions. + ['content_filtered', 'content_filter'], + ['guardrail_intervened', 'content_filter'], + ['malformed_model_output', 'error'], + ['malformed_tool_use', 'error'], + + /** + * Gemini. `STOP` and `MAX_TOKENS` already lowercase onto the entries above, so + * only the values with no counterpart elsewhere are listed here. Recitation is a + * content restriction, so it groups with the filters. + */ + ['safety', 'content_filter'], + ['blocklist', 'content_filter'], + ['prohibited_content', 'content_filter'], + ['spii', 'content_filter'], + ['recitation', 'content_filter'], + ['image_safety', 'content_filter'], + ['image_prohibited_content', 'content_filter'], + ['image_recitation', 'content_filter'], + ['malformed_function_call', 'error'], + ['unexpected_tool_call', 'error'], + ['language', 'other'], + ['other', 'other'], + ['no_image', 'other'], + ['image_other', 'other'], + ['finish_reason_unspecified', 'other'], +]) + +/** + * Normalizes a provider's raw stop reason. + * + * Returns `undefined` when the provider reported nothing, so an absent reason stays + * distinguishable from one the vocabulary could not place. An unrecognized value maps + * to `'other'` rather than throwing: a provider adding a case must not fail a run + * that otherwise succeeded. + */ +export function normalizeFinishReason( + raw: string | null | undefined +): AgentFinishReason | undefined { + if (!raw) return undefined + return NORMALIZED_BY_RAW.get(raw.trim().toLowerCase()) ?? 'other' +} diff --git a/apps/sim/providers/types.ts b/apps/sim/providers/types.ts index e029f830d2c..d2863cf1330 100644 --- a/apps/sim/providers/types.ts +++ b/apps/sim/providers/types.ts @@ -1,5 +1,6 @@ import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' import type { ProviderTimingSegment, StreamingExecution, UserFile } from '@/executor/types' +import type { AgentFinishReason } from '@/providers/finish-reason' export type ProviderId = | 'openai' @@ -95,6 +96,11 @@ export interface ProviderResponse { } toolCalls?: FunctionCallResponse[] toolResults?: Record[] + /** + * Why generation stopped, normalized across providers. Absent when the provider + * reported nothing; see {@link AgentFinishReason}. + */ + finishReason?: AgentFinishReason timing?: { startTime: string endTime: string From 4f19e41e48d8c3873fdde5587662ca47309e1030 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 5 Aug 2026 16:38:48 -0700 Subject: [PATCH 2/2] fix(agent): map the non-OpenAI finish reasons compatible vendors actually emit --- apps/sim/providers/finish-reason.test.ts | 33 ++++++++++++++++++++ apps/sim/providers/finish-reason.ts | 39 +++++++++++++++++++++++- 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/apps/sim/providers/finish-reason.test.ts b/apps/sim/providers/finish-reason.test.ts index 78e07c042bd..085834ca3f1 100644 --- a/apps/sim/providers/finish-reason.test.ts +++ b/apps/sim/providers/finish-reason.test.ts @@ -66,6 +66,39 @@ describe('normalizeFinishReason', () => { expect(normalizeFinishReason('pause_turn')).toBe('other') }) + /** + * Values confirmed against each vendor's own enum by a per-provider documentation + * sweep. Several are the routine success path on open-weight models, so leaving + * them unclassified would misreport healthy runs. + */ + it('classifies the non-OpenAI vocabularies of compatible vendors', () => { + expect(normalizeFinishReason('model_length')).toBe('length') // Mistral + expect(normalizeFinishReason('eos')).toBe('stop') // Together + expect(normalizeFinishReason('eos_token')).toBe('stop') // LiteLLM/HuggingFace + expect(normalizeFinishReason('error')).toBe('error') // Mistral, OpenRouter, Together + expect(normalizeFinishReason('insufficient_system_resource')).toBe('error') // DeepSeek + expect(normalizeFinishReason('network_error')).toBe('error') // Z.ai + expect(normalizeFinishReason('sensitive')).toBe('content_filter') // Z.ai + }) + + it('classifies the Gemini and Vertex values absent from the TS enum', () => { + expect(normalizeFinishReason('MODEL_ARMOR')).toBe('content_filter') // Vertex only + expect(normalizeFinishReason('ESCALATION')).toBe('content_filter') + expect(normalizeFinishReason('MALFORMED_RESPONSE')).toBe('error') + expect(normalizeFinishReason('MISSING_THOUGHT_SIGNATURE')).toBe('error') + }) + + /** + * A server-aborted tool loop is not a request to execute tools, and a repetition + * cutoff is a normal finish rather than a failure — both would mislead a workflow + * branching on the value. + */ + it('does not overclaim on aborted or degenerate stops', () => { + expect(normalizeFinishReason('too_many_tool_calls')).toBe('other') + expect(normalizeFinishReason('repetition')).toBe('other') + expect(normalizeFinishReason('abort')).toBe('other') + }) + it('degrades an unrecognized value to other rather than throwing', () => { expect(normalizeFinishReason('some_future_reason')).toBe('other') expect(normalizeFinishReason('OTHER')).toBe('other') diff --git a/apps/sim/providers/finish-reason.ts b/apps/sim/providers/finish-reason.ts index 285a6793fb8..a235d475a79 100644 --- a/apps/sim/providers/finish-reason.ts +++ b/apps/sim/providers/finish-reason.ts @@ -16,7 +16,7 @@ export type AgentFinishReason = | 'tool_calls' /** Blocked or refused by a safety system. */ | 'content_filter' - /** The provider reported the generation itself as malformed. */ + /** The provider reported the generation itself as failed or malformed. */ | 'error' /** Reported, but not a case this vocabulary distinguishes. */ | 'other' @@ -41,6 +41,37 @@ const NORMALIZED_BY_RAW = new Map([ // OpenAI Responses reports truncation through `incomplete_details.reason`. ['max_output_tokens', 'length'], + /** Mistral separates the model's own max length from the caller's `max_tokens`. */ + ['model_length', 'length'], + + /** + * Natural end-of-sequence. Together and HuggingFace-backed proxies report the + * model's EOS token separately from a caller-supplied stop sequence, so without + * these the routine success path on open-weight models is unclassified. + */ + ['eos', 'stop'], + ['eos_token', 'stop'], + + /** + * A generation the provider itself reported as failed. Mistral, OpenRouter, + * Together and Fireworks all spell this `error`; the DeepSeek and Z.ai values are + * the same class with a stated cause. + */ + ['error', 'error'], + ['insufficient_system_resource', 'error'], + ['network_error', 'error'], + + /** Z.ai (GLM) sensitive-content block. */ + ['sensitive', 'content_filter'], + + /** + * Cut short with no outcome to report, on vLLM and NIM-hosted models. vLLM treats + * a repetition cutoff as a normal finish rather than a failure, so it is not + * `error`. + */ + ['abort', 'other'], + ['repetition', 'other'], + // Anthropic Messages, shared by Bedrock's Converse API. ['end_turn', 'stop'], ['stop_sequence', 'stop'], @@ -72,6 +103,12 @@ const NORMALIZED_BY_RAW = new Map([ ['image_recitation', 'content_filter'], ['malformed_function_call', 'error'], ['unexpected_tool_call', 'error'], + ['malformed_response', 'error'], + ['missing_thought_signature', 'error'], + ['model_armor', 'content_filter'], + ['escalation', 'content_filter'], + /** A runaway tool loop the server aborted, not a request to execute tools. */ + ['too_many_tool_calls', 'other'], ['language', 'other'], ['other', 'other'], ['no_image', 'other'],