diff --git a/apps/sim/lib/logs/execution/logger.test.ts b/apps/sim/lib/logs/execution/logger.test.ts index 714753bdbbc..0521b0639a8 100644 --- a/apps/sim/lib/logs/execution/logger.test.ts +++ b/apps/sim/lib/logs/execution/logger.test.ts @@ -5,6 +5,7 @@ import { queueTableRows, resetDbChainMock, } from '@sim/testing' +import { isPlainRecord } from '@sim/utils/object' import { afterAll, beforeEach, describe, expect, test, vi } from 'vitest' import { recordUsage } from '@/lib/billing/core/usage-log' import { ExecutionLogger } from '@/lib/logs/execution/logger' @@ -288,6 +289,109 @@ describe('ExecutionLogger', () => { expect(emitExecutionCompletedEvent).not.toHaveBeenCalled() }) + const EMPTY_STATE = { + blockStates: {}, + executedBlocks: [], + blockLogs: [], + decisions: { router: {}, condition: {} }, + completedLoops: [], + activeExecutionPath: [], + } + const RUN_PROVENANCE = { version: 1, complete: true, entries: [] } + + /** + * Drives a real completion and returns the `execution_data` actually written. + * `redactedState` stands in for the PII pass, which either hands back a + * redacted state or none at all. + */ + async function completeAndReadWrite(params: { + executionState?: SerializableExecutionState + redactedState?: SerializableExecutionState + }) { + const startedAt = new Date('2026-08-11T00:00:00.000Z') + queueTableRows(workflowExecutionLogs, [ + { + id: 'log-1', + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + executionId: 'execution-1', + stateSnapshotId: 'snapshot-1', + level: 'info', + status: 'running', + trigger: 'api', + startedAt, + endedAt: null, + totalDurationMs: null, + executionData: {}, + createdAt: startedAt, + }, + ]) + dbChainMockFns.returning.mockResolvedValueOnce([ + { id: 'log-1', executionData: {}, startedAt, createdAt: startedAt }, + ]) + const internals = logger as unknown as { + applyPiiRedaction: (workspaceId: string, payload: Record) => unknown + recordExecutionUsage: () => Promise + } + vi.spyOn(internals, 'applyPiiRedaction').mockImplementation( + async (_workspaceId: string, payload: Record) => + Object.hasOwn(params, 'redactedState') + ? { ...payload, executionState: params.redactedState } + : payload + ) + vi.spyOn(internals, 'recordExecutionUsage').mockResolvedValue(0) + + await logger.completeWorkflowExecution({ + executionId: 'execution-1', + endedAt: '2026-08-11T00:00:02.000Z', + totalDurationMs: 2000, + costSummary: { + totalCost: 0, + totalInputCost: 0, + totalOutputCost: 0, + totalTokens: 0, + totalPromptTokens: 0, + totalCompletionTokens: 0, + baseExecutionCharge: 0, + models: {}, + }, + finalOutput: { completed: true }, + traceSpans: [], + ...(params.executionState ? { executionState: params.executionState } : {}), + }) + + return dbChainMockFns.set.mock.calls + .map(([values]: [{ executionData?: unknown }]) => values?.executionData) + .find((data): data is Record => isPlainRecord(data)) + } + + /** + * The display projection rebuilds its redaction registry from this key. + * Compaction drops `executionState`, so the run provenance has to reach the + * row independently of it or truncated runs render as an empty trace. + */ + test.each([ + ['redaction preserves the state', EMPTY_STATE], + ['redaction drops the state entirely', undefined], + ])('lifts run provenance onto the top-level key when %s', async (_case, redactedState) => { + const written = await completeAndReadWrite({ + executionState: { + ...EMPTY_STATE, + resolvedSecretTraceProvenance: RUN_PROVENANCE, + } as unknown as SerializableExecutionState, + redactedState: redactedState as SerializableExecutionState | undefined, + }) + + expect(written?.resolvedSecretTraceProvenance).toEqual(RUN_PROVENANCE) + }) + + test('omits the provenance key when the run carried none', async () => { + const written = await completeAndReadWrite({}) + + expect(written).toBeDefined() + expect(written).not.toHaveProperty('resolvedSecretTraceProvenance') + }) + test('preserves correlation and diagnostics when execution completes', () => { const loggerInstance = new ExecutionLogger() as any @@ -628,6 +732,84 @@ describe('ExecutionLogger', () => { expect(compacted.traceSpans?.[0]?.toolCalls?.[0]).not.toHaveProperty('output') expect(compacted.traceSpans?.[0]?.toolCalls?.[0]).not.toHaveProperty('error') }) + + const PROVENANCE = { version: 1, complete: true, entries: [] } as const + + function buildSpans(spanCount: number, ioBytes: number) { + const payload = 'x'.repeat(ioBytes) + return Array.from({ length: spanCount }, (_unused, index) => ({ + id: `span-${index}`, + name: `Block ${index}`, + type: 'function', + duration: 1, + startTime: '2025-01-01T00:00:00.000Z', + endTime: '2025-01-01T00:00:01.000Z', + status: 'success' as const, + output: { data: payload }, + })) + } + + function compactWithProvenance(traceSpans: unknown[], finalOutput: unknown) { + const loggerInstance = new ExecutionLogger() as unknown as { + compactExecutionDataForStorage: ( + data: Record, + executionId: string + ) => Record + } + return loggerInstance.compactExecutionDataForStorage( + { + secretProjectionVersion: SECRET_PROJECTION_VERSION, + resolvedSecretTraceProvenance: PROVENANCE, + hasTraceSpans: true, + traceSpanCount: traceSpans.length, + finalOutput, + executionState: { + blockStates: {}, + executedBlocks: [], + blockLogs: [], + decisions: { router: {}, condition: {} }, + completedLoops: [], + activeExecutionPath: [], + resolvedSecretTraceProvenance: PROVENANCE, + }, + traceSpans, + }, + 'execution-provenance' + ) + } + + test('preserves run provenance through the summarized compaction tier', () => { + // One oversized value: summarization alone brings the row under the cap. + const compacted = compactWithProvenance(buildSpans(1, 4 * 1024 * 1024), { + data: 'x'.repeat(4 * 1024 * 1024), + }) + + expect(compacted.executionDataTruncated).toBe(true) + expect(compacted.executionDataTruncationReason).toContain('were summarized') + expect(compacted.executionState).toBeUndefined() + expect(compacted.resolvedSecretTraceProvenance).toEqual(PROVENANCE) + }) + + test('drops run provenance from the metadata-only tier, which stores no spans', () => { + // That tier keeps no traceSpans, so provenance there buys nothing and + // would put an unbounded value in the last-resort size floor. + const compacted = compactWithProvenance(buildSpans(20_000, 8), {}) + + expect(compacted.executionDataTruncationReason).toContain('only execution metadata') + expect(compacted.traceSpans).toBeUndefined() + expect(compacted.resolvedSecretTraceProvenance).toBeUndefined() + }) + + test('preserves run provenance through the minimal compaction tier', () => { + // Many spans whose IO each sits under MAX_TRACE_IO_BYTES survive + // summarization, so only the IO-stripping minimal tier fits the cap. + const compacted = compactWithProvenance(buildSpans(1200, 4 * 1024), {}) + + expect(compacted.executionDataTruncated).toBe(true) + expect(compacted.executionDataTruncationReason).toContain('details were omitted') + expect(compacted.executionState).toBeUndefined() + expect(compacted.resolvedSecretTraceProvenance).toEqual(PROVENANCE) + }) }) describe('file extraction', () => { diff --git a/apps/sim/lib/logs/execution/logger.ts b/apps/sim/lib/logs/execution/logger.ts index 0bde3786d2b..6e5d8c27d3a 100644 --- a/apps/sim/lib/logs/execution/logger.ts +++ b/apps/sim/lib/logs/execution/logger.ts @@ -449,6 +449,9 @@ export class ExecutionLogger implements IExecutionLoggerService { const minimal: ExecutionData = { secretProjectionVersion: SECRET_PROJECTION_VERSION, + ...(executionData.resolvedSecretTraceProvenance !== undefined + ? { resolvedSecretTraceProvenance: executionData.resolvedSecretTraceProvenance } + : {}), ...(executionData.environment ? { environment: executionData.environment } : {}), ...(executionData.trigger ? { trigger: executionData.trigger } : {}), ...(executionData.billingAttribution @@ -1102,8 +1105,18 @@ export class ExecutionLogger implements IExecutionLoggerService { builtExecutionData.executionState ) + /** + * Duplicated top-level so the display projection can still rebuild its + * registry after compaction drops `executionState`. Read from the + * pre-redaction state: `preservePrivateExecutionStateMetadata` copies the + * provenance across verbatim, and this one also survives redaction + * producing no state at all. + */ + const runProvenance = builtExecutionData.executionState?.resolvedSecretTraceProvenance + const cleanExecutionData: ExecutionData = { ...builtExecutionData, + ...(runProvenance !== undefined ? { resolvedSecretTraceProvenance: runProvenance } : {}), traceSpans: copyTraceSpansWithoutCosts(preparedTraceSpans), finalOutput: pii.finalOutput as BlockOutputData, ...(pii.workflowInput !== undefined ? { workflowInput: pii.workflowInput } : {}), diff --git a/apps/sim/lib/logs/execution/trace-store.test.ts b/apps/sim/lib/logs/execution/trace-store.test.ts index 431e7599209..4b9762cb4fc 100644 --- a/apps/sim/lib/logs/execution/trace-store.test.ts +++ b/apps/sim/lib/logs/execution/trace-store.test.ts @@ -22,6 +22,7 @@ import { externalizeExecutionData, materializeExecutionData, projectExecutionDataForDisplay, + RESOLVED_SECRET_PROVENANCE_KEY, SECRET_PROJECTION_VERSION, TRACE_STORE_REF_KEY, } from '@/lib/logs/execution/trace-store' @@ -284,3 +285,89 @@ describe('projectExecutionDataForDisplay', () => { expect(displayData).not.toHaveProperty('traceSpans') }) }) + +describe('projectExecutionDataForDisplay provenance handling', () => { + const PROVENANCE = { version: 1, complete: true, entries: [] } as const + + /** A truncated row: spans and markers survive, `executionState` does not. */ + function truncatedRow(overrides: Record = {}) { + return { + secretProjectionVersion: SECRET_PROJECTION_VERSION, + executionDataTruncated: true, + finalOutput: { result: 'unknown-secret' }, + traceSpans: [ + { + id: 'span-1', + name: 'activeEmails', + type: 'function', + duration: 16, + startTime: '2026-08-11T00:38:53.000Z', + endTime: '2026-08-11T00:38:53.016Z', + status: 'error', + input: { code: 'const activeEmails = rows.length' }, + output: { error: 'nested large values' }, + }, + ], + ...overrides, + } + } + + /** First span of a projected display payload. */ + function firstSpan(displayData: Record): Record { + const [span] = displayData.traceSpans as Record[] + return span + } + + it.each([ + ['a contract row', () => truncatedRow({ [RESOLVED_SECRET_PROVENANCE_KEY]: PROVENANCE })], + ['a legacy row', () => ({ [RESOLVED_SECRET_PROVENANCE_KEY]: PROVENANCE, finalOutput: {} })], + ])('never returns the resolved-secret provenance to the client from %s', async (_case, row) => { + const displayData = await projectExecutionDataForDisplay(row(), CONTEXT) + + expect(displayData).not.toHaveProperty(RESOLVED_SECRET_PROVENANCE_KEY) + }) + + it('rebuilds the registry from the top-level key alone', async () => { + const { secretProjectionVersion: _marker, ...withoutMarker } = truncatedRow() + + const displayData = await projectExecutionDataForDisplay( + { ...withoutMarker, [RESOLVED_SECRET_PROVENANCE_KEY]: PROVENANCE }, + CONTEXT + ) + + expect(displayData.finalOutput).toEqual({ result: 'unknown-secret' }) + expect(firstSpan(displayData)).toHaveProperty('input') + }) + + it('keeps write-time-projected spans on a truncated row with no provenance', async () => { + const displayData = await projectExecutionDataForDisplay(truncatedRow(), CONTEXT) + + expect(firstSpan(displayData)).toMatchObject({ + input: { code: 'const activeEmails = rows.length' }, + output: { error: 'nested large values' }, + }) + // The envelope has no write-time guarantee, so it still fails closed. + expect(displayData).not.toHaveProperty('finalOutput') + }) + + it.each([ + ['the row was never truncated', { executionDataTruncated: undefined }], + ['the provenance key is present but null', { [RESOLVED_SECRET_PROVENANCE_KEY]: null }], + ['the provenance is malformed', { [RESOLVED_SECRET_PROVENANCE_KEY]: { version: 99 } }], + ])('fails closed when %s', async (_case, overrides) => { + const displayData = await projectExecutionDataForDisplay(truncatedRow(overrides), CONTEXT) + + const span = firstSpan(displayData) + expect(span).not.toHaveProperty('input') + expect(span).not.toHaveProperty('output') + }) + + it('leaves an empty span array intact', async () => { + const displayData = await projectExecutionDataForDisplay( + truncatedRow({ traceSpans: [] }), + CONTEXT + ) + + expect(displayData.traceSpans).toEqual([]) + }) +}) diff --git a/apps/sim/lib/logs/execution/trace-store.ts b/apps/sim/lib/logs/execution/trace-store.ts index 9fa5a7b05df..10b8d6eaa1d 100644 --- a/apps/sim/lib/logs/execution/trace-store.ts +++ b/apps/sim/lib/logs/execution/trace-store.ts @@ -28,6 +28,10 @@ export const TRACE_STORE_REF_KEY = 'traceStoreRef' * authenticate terminal Copilot workflow-tool executions. All other fields * (environment, trigger, tokens, models, truncation flags, and of course the * heavy payloads) are recovered from the stored object. + * + * {@link RESOLVED_SECRET_PROVENANCE_KEY} is deliberately absent: it rides in the + * externalized object, and inlining it would put encrypted secret material back + * on the row this slimming exists to keep it off. */ const INLINE_MARKER_KEYS = [ 'secretProjectionVersion', @@ -36,6 +40,27 @@ const INLINE_MARKER_KEYS = [ 'correlation', ] as const +/** + * Top-level `execution_data` key carrying the run's resolved-secret provenance. + * + * Duplicated out of `executionState` because oversized-payload compaction drops + * that field wholesale, leaving a contract-marked row the display projection + * can no longer verify. Server-side only: it holds encrypted secret values and + * their names, so every display projection must omit it. + */ +export const RESOLVED_SECRET_PROVENANCE_KEY = 'resolvedSecretTraceProvenance' + +/** + * Server-only keys stripped from every display projection. Both the contract + * and legacy paths spread this, so a new server-only key is omitted from both + * by construction rather than by review. + */ +const DISPLAY_OMITTED_SERVER_KEYS = [ + 'executionState', + 'secretProjectionVersion', + RESOLVED_SECRET_PROVENANCE_KEY, +] as const + /** * Read-path context. Resolves an externalized payload by storage key, authorized * via the (already-authorized) workspace — no owner needed. @@ -223,8 +248,7 @@ function projectLegacyExecutionDataForDisplay( executionData: Record ): Record { const omittedKeys = [ - 'executionState', - 'secretProjectionVersion', + ...DISPLAY_OMITTED_SERVER_KEYS, ...(!Object.hasOwn(executionData, 'traceSpans') || Array.isArray(executionData.traceSpans) ? [] : ['traceSpans']), @@ -249,8 +273,12 @@ export async function materializeExecutionDataForDisplay( * Projects execution-log content with the encrypted provenance saved by the * trusted executor. Current workflow input and final output values use their * exact sidecars; rows predating those fields retain the run-level fallback. - * Contract-aware rows with missing or malformed provenance deliberately yield - * structural-only content instead of returning data that cannot be proven safe. + * Contract-aware rows whose provenance is missing or malformed yield + * structural-only content rather than data that cannot be proven safe. The one + * carve-out is the trace spans of a truncated row that lost its provenance to + * compaction: those were already projected at write time. Truncation also takes + * the exact per-value sidecars with it, so those rows fall back to the + * run-level registry for `finalOutput` / `workflowInput`. */ export async function projectExecutionDataForDisplay( executionData: Record, @@ -262,10 +290,13 @@ export async function projectExecutionDataForDisplay( !Array.isArray(executionData.executionState) ? (executionData.executionState as Record) : undefined - const provenance = executionState?.resolvedSecretTraceProvenance + const hasTopLevelProvenance = Object.hasOwn(executionData, RESOLVED_SECRET_PROVENANCE_KEY) + const stateProvenance = executionState?.[RESOLVED_SECRET_PROVENANCE_KEY] + const provenance = executionData[RESOLVED_SECRET_PROVENANCE_KEY] ?? stateProvenance const hasProjectionContract = Object.hasOwn(executionData, 'secretProjectionVersion') || - (executionState !== undefined && Object.hasOwn(executionState, 'resolvedSecretTraceProvenance')) + hasTopLevelProvenance || + (executionState !== undefined && Object.hasOwn(executionState, RESOLVED_SECRET_PROVENANCE_KEY)) if (!hasProjectionContract) { return projectLegacyExecutionDataForDisplay(executionData) @@ -281,6 +312,33 @@ export async function projectExecutionDataForDisplay( }) } + /** + * Compaction drops `executionState`, and with it the only copy of the + * provenance on rows written before it was stored top-level. Every write path + * projects spans before persisting them, and that projection yields + * structural-only spans when its registry is incomplete — so a stored tree + * that still carries content was already redacted at write time. + * + * Not a general fallback: scoped to truncated rows whose key is absent + * entirely. A present-but-unusable key (malformed, incomplete, explicit null) + * and the read-time envelope have no such guarantee and keep failing closed. + * + * Self-expiring. New rows carry the key, so this only serves rows truncated + * before that shipped; once the warning below stops firing across a full log + * retention window, delete this branch and its tests. + */ + const retainStoredTraceSpans = + executionData.executionDataTruncated === true && + !hasTopLevelProvenance && + stateProvenance === undefined && + Array.isArray(executionData.traceSpans) && + executionData.traceSpans.length > 0 + if (retainStoredTraceSpans) { + logger.warn('Retaining write-time-projected spans for a truncated row with no provenance', { + executionId: context.executionId, + }) + } + const projectionStore = { workspaceId: context.workspaceId ?? undefined, workflowId: context.workflowId ?? undefined, @@ -356,7 +414,8 @@ export async function projectExecutionDataForDisplay( const sourceTraceSpans = Array.isArray(executionData.traceSpans) ? (executionData.traceSpans as TraceSpan[]) : [] - const projectedSpans = await projectTraceSpansForSecrets([syntheticSpan, ...sourceTraceSpans], { + const spansToProject = retainStoredTraceSpans ? [] : sourceTraceSpans + const projectedSpans = await projectTraceSpansForSecrets([syntheticSpan, ...spansToProject], { registry, allowLargeValueWrites: false, store: projectionStore, @@ -364,8 +423,7 @@ export async function projectExecutionDataForDisplay( const displayData = omit(executionData, [ ...LOG_DISPLAY_CONTENT_KEYS, - 'executionState', - 'secretProjectionVersion', + ...DISPLAY_OMITTED_SERVER_KEYS, 'traceSpans', ]) as Record @@ -382,9 +440,9 @@ export async function projectExecutionDataForDisplay( } if (Array.isArray(executionData.traceSpans)) { - displayData.traceSpans = projectedSpans.filter( - (span) => span.id !== LOG_DISPLAY_PROJECTION_SPAN_ID - ) + displayData.traceSpans = retainStoredTraceSpans + ? sourceTraceSpans + : projectedSpans.filter((span) => span.id !== LOG_DISPLAY_PROJECTION_SPAN_ID) } return displayData diff --git a/apps/sim/lib/logs/types.ts b/apps/sim/lib/logs/types.ts index 6345377ff7c..ebcc5531d20 100644 --- a/apps/sim/lib/logs/types.ts +++ b/apps/sim/lib/logs/types.ts @@ -133,6 +133,13 @@ export interface WorkflowExecutionLog { // Execution details executionData: { secretProjectionVersion?: 1 + /** + * Run-level provenance, stored alongside the contract marker rather than + * only inside `executionState` so it survives both compaction and PII + * redaction dropping the state. The display projection needs it to rebuild + * its registry. + */ + resolvedSecretTraceProvenance?: ResolvedSecretTraceProvenanceV1 environment?: ExecutionEnvironment trigger?: ExecutionTrigger billingAttribution?: BillingAttributionSnapshot