diff --git a/apps/sim/app/api/knowledge/search/route.ts b/apps/sim/app/api/knowledge/search/route.ts index e0530f6b772..ed34313f844 100644 --- a/apps/sim/app/api/knowledge/search/route.ts +++ b/apps/sim/app/api/knowledge/search/route.ts @@ -381,7 +381,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { results, }) if (!resultProvenanceSnapshot.imported) { - resultSecretRegistry.markIncomplete() + resultSecretRegistry.markIncomplete('knowledge-result-provenance-unavailable') if (useReranker) { return NextResponse.json( { error: 'Knowledge result secret provenance is unavailable' }, @@ -608,7 +608,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { 'knowledge' )) ) { - resultSecretRegistry.markIncomplete() + resultSecretRegistry.markIncomplete('knowledge-result-provenance-unavailable') } } diff --git a/apps/sim/app/api/knowledge/secret-provenance.ts b/apps/sim/app/api/knowledge/secret-provenance.ts index 5f3415f1c95..34a83f43e1c 100644 --- a/apps/sim/app/api/knowledge/secret-provenance.ts +++ b/apps/sim/app/api/knowledge/secret-provenance.ts @@ -154,7 +154,7 @@ export async function createKnowledgeProvenanceResponse(options: { }) for (const provenance of options.provenances) { if (provenance.status === 'unknown') { - registry.markIncomplete() + registry.markIncomplete('durable-provenance-unknown') break } const sourceRegistry = await createDurableSecretProvenanceRegistry(provenance, { diff --git a/apps/sim/app/api/memory/secret-provenance.ts b/apps/sim/app/api/memory/secret-provenance.ts index cd14e84b90d..589ede10188 100644 --- a/apps/sim/app/api/memory/secret-provenance.ts +++ b/apps/sim/app/api/memory/secret-provenance.ts @@ -98,7 +98,7 @@ export async function createMemoryResponse(options: { workspaceId: options.workspaceId, }) if (options.memories.length > MAX_PRIVATE_MEMORY_CROSSINGS) { - registry.markIncomplete() + registry.markIncomplete('memory-crossing-capacity-exceeded') } else { const ids = [...new Set(options.memories.map((record) => record.id))] const memoriesById = new Map() @@ -123,7 +123,7 @@ export async function createMemoryResponse(options: { provenanceEntryCount > MAX_PRIVATE_MEMORY_PROVENANCE_ENTRIES || provenanceBytes > MAX_PRIVATE_MEMORY_PROVENANCE_BYTES ) { - registry.markIncomplete() + registry.markIncomplete('memory-crossing-capacity-exceeded') break } const sidecarById = new Map(sidecars.map((sidecar) => [sidecar.memoryId, sidecar])) diff --git a/apps/sim/app/api/workflows/[id]/log/route.ts b/apps/sim/app/api/workflows/[id]/log/route.ts index 93984773999..31c678a245a 100644 --- a/apps/sim/app/api/workflows/[id]/log/route.ts +++ b/apps/sim/app/api/workflows/[id]/log/route.ts @@ -142,7 +142,7 @@ export const POST = withRouteHandler( : undefined const trustedProvenance = trustedExecutionState?.resolvedSecretTraceProvenance if (trustedProvenance === undefined) { - resolvedSecretTraceRegistry.markIncomplete() + resolvedSecretTraceRegistry.markIncomplete('restored-provenance-untrusted') } else { await resolvedSecretTraceRegistry.importProvenance(trustedProvenance, { trusted: true, diff --git a/apps/sim/executor/handlers/agent/memory.test.ts b/apps/sim/executor/handlers/agent/memory.test.ts index e59a4c260ba..60f44761d0a 100644 --- a/apps/sim/executor/handlers/agent/memory.test.ts +++ b/apps/sim/executor/handlers/agent/memory.test.ts @@ -278,7 +278,7 @@ describe('Memory', () => { it('persists raw memory with unknown lineage when provenance is unavailable', async () => { const registry = new ResolvedSecretTraceRegistry() - registry.markIncomplete() + registry.markIncomplete('unspecified') const appendMessage = vi .spyOn(memoryService as any, 'appendMessage') .mockResolvedValue(undefined) @@ -293,7 +293,7 @@ describe('Memory', () => { it('seeds raw memory with unknown lineage when provenance is unavailable', async () => { const registry = new ResolvedSecretTraceRegistry() - registry.markIncomplete() + registry.markIncomplete('unspecified') const seedMemoryRecord = vi .spyOn(memoryService as any, 'seedMemoryRecord') .mockResolvedValue(undefined) diff --git a/apps/sim/executor/handlers/generic/generic-handler.ts b/apps/sim/executor/handlers/generic/generic-handler.ts index b35cc117205..2e713b0fddf 100644 --- a/apps/sim/executor/handlers/generic/generic-handler.ts +++ b/apps/sim/executor/handlers/generic/generic-handler.ts @@ -199,7 +199,8 @@ export class GenericBlockHandler implements BlockHandler { boundary && boundary.paths.length > 0 && registry?.hasResolvedInputProjections() ? registry.projectResolvedInputSelections(inputs) : undefined - if (projectedInputs?.complete === false) registry?.markIncomplete() + if (projectedInputs?.complete === false) + registry?.markIncomplete('structural-input-projection-incomplete') if (projectedInputs?.complete && boundary && tool && registry) { for (const projection of projectedInputs.values) { @@ -233,7 +234,7 @@ export class GenericBlockHandler implements BlockHandler { continue } if (boundary.requiredProjectionRoots.has(projection.path[0])) { - registry.markIncomplete() + registry.markIncomplete('structural-input-root-unprojected') } continue } diff --git a/apps/sim/executor/handlers/mothership/mothership-handler.ts b/apps/sim/executor/handlers/mothership/mothership-handler.ts index 1d285cd5df3..3676f8b8f90 100644 --- a/apps/sim/executor/handlers/mothership/mothership-handler.ts +++ b/apps/sim/executor/handlers/mothership/mothership-handler.ts @@ -372,7 +372,7 @@ async function consumeMothershipProvenance( return false } if (inspection.status === 'invalid') { - registry?.markIncomplete() + registry?.markIncomplete('mothership-provenance-invalid') throw new Error('Mothership response provenance metadata is invalid') } @@ -399,7 +399,7 @@ function inspectMothershipResponseCapability( return false } - registry?.markIncomplete() + registry?.markIncomplete('mothership-provenance-invalid') throw new Error('Mothership response provenance metadata is invalid') } @@ -464,7 +464,7 @@ async function readMothershipExecuteResponse( result = (await response.json()) as MothershipExecuteResult } catch (error) { if (expectsProvenance) { - registry?.markIncomplete() + registry?.markIncomplete('mothership-response-unreadable') throw new Error('Mothership response provenance metadata is invalid') } throw error @@ -528,7 +528,7 @@ async function readMothershipExecuteResponse( return finalResult } finally { if (expectsProvenance && !finalResult && !receivedTerminalProvenance) { - registry?.markIncomplete() + registry?.markIncomplete('mothership-provenance-missing') } reader.releaseLock() } @@ -630,7 +630,7 @@ function createMothershipStreamingExecution( } } finally { if (expectsProvenance && !sawFinal && !receivedTerminalProvenance) { - options.registry?.markIncomplete() + options.registry?.markIncomplete('mothership-provenance-missing') } cleanup() reader?.releaseLock() @@ -948,7 +948,7 @@ export class MothershipBlockHandler implements BlockHandler { try { payload = (await response.clone().json()) as MothershipExecuteResult } catch { - resultRegistry?.markIncomplete() + resultRegistry?.markIncomplete('mothership-response-unreadable') throw new Error('Mothership response provenance metadata is invalid') } await consumeMothershipProvenance(payload, response, resultRegistry) diff --git a/apps/sim/executor/handlers/pi/local/sim-tools.test.ts b/apps/sim/executor/handlers/pi/local/sim-tools.test.ts index 1638f39d3bd..1f21dabc4f1 100644 --- a/apps/sim/executor/handlers/pi/local/sim-tools.test.ts +++ b/apps/sim/executor/handlers/pi/local/sim-tools.test.ts @@ -320,7 +320,7 @@ describe('buildSimToolSpecs', () => { output: { result: 'untrusted output' }, }) const registry = new ResolvedSecretTraceRegistry() - registry.markIncomplete() + registry.markIncomplete('unspecified') const [spec] = await buildSimToolSpecs(executionContext(registry), toolInput) const result = await spec.execute({}) diff --git a/apps/sim/executor/handlers/pi/pi-handler.test.ts b/apps/sim/executor/handlers/pi/pi-handler.test.ts index 3e70eb2893b..063b319c6ff 100644 --- a/apps/sim/executor/handlers/pi/pi-handler.test.ts +++ b/apps/sim/executor/handlers/pi/pi-handler.test.ts @@ -227,7 +227,7 @@ describe('PiBlockHandler', () => { it('fails closed when task provenance is incomplete', async () => { const registry = new ResolvedSecretTraceRegistry() - registry.markIncomplete() + registry.markIncomplete('unspecified') await expect( handler.execute( diff --git a/apps/sim/executor/handlers/pi/search/tool.test.ts b/apps/sim/executor/handlers/pi/search/tool.test.ts index d73175dcf74..a4ba221b84e 100644 --- a/apps/sim/executor/handlers/pi/search/tool.test.ts +++ b/apps/sim/executor/handlers/pi/search/tool.test.ts @@ -288,7 +288,7 @@ describe('buildPiSearchToolSpec', () => { it('fails closed before search when provenance is incomplete', async () => { const registry = new ResolvedSecretTraceRegistry() - registry.markIncomplete() + registry.markIncomplete('unspecified') const result = await buildTool('exa', executionContext(registry)).execute({ query: 'pi' }) @@ -303,7 +303,7 @@ describe('buildPiSearchToolSpec', () => { const registry = new ResolvedSecretTraceRegistry() const mergeSpy = vi.spyOn(registry, 'mergeToolCallRegistry') mockExecuteTool.mockImplementation(async (_toolId, _params, options) => { - options.resolvedSecretTraceRegistry.markIncomplete() + options.resolvedSecretTraceRegistry.markIncomplete('unspecified') return { success: true, output: { diff --git a/apps/sim/executor/utils/resolved-secret-content-projection.test.ts b/apps/sim/executor/utils/resolved-secret-content-projection.test.ts index 88a38cad883..0879a60b166 100644 --- a/apps/sim/executor/utils/resolved-secret-content-projection.test.ts +++ b/apps/sim/executor/utils/resolved-secret-content-projection.test.ts @@ -70,7 +70,7 @@ describe('projectResolvedSecretModelContent', () => { value: '{{TOKEN}}', }) - registry.markIncomplete() + registry.markIncomplete('unspecified') expect(projectResolvedSecretModelContent('secret-value', registry)).toEqual({ safe: false }) expect(projectResolvedSecretModelContent('secret-value', undefined)).toEqual({ safe: false }) }) @@ -368,7 +368,7 @@ describe('projectResolvedSecretModelJsonContent', () => { it('does not invoke JSON serialization when provenance is incomplete', () => { const registry = new ResolvedSecretTraceRegistry() - registry.markIncomplete() + registry.markIncomplete('unspecified') const toJSON = vi.fn(() => ({ value: 'untrusted' })) expect(projectResolvedSecretModelJsonContent({ toJSON }, registry)).toEqual({ safe: false }) @@ -474,7 +474,7 @@ describe('projectResolvedSecretDiagnosticError', () => { it('falls back to text-free structure when provenance is missing or incomplete', () => { const error = new Error('secret __var_API_KEY') const registry = new ResolvedSecretTraceRegistry() - registry.markIncomplete() + registry.markIncomplete('unspecified') expect(projectResolvedSecretDiagnosticError(error, undefined)).toEqual({ errorType: 'error', diff --git a/apps/sim/executor/utils/resolved-secret-trace-registry.test.ts b/apps/sim/executor/utils/resolved-secret-trace-registry.test.ts index e16030c2d88..2c5b8cf77a5 100644 --- a/apps/sim/executor/utils/resolved-secret-trace-registry.test.ts +++ b/apps/sim/executor/utils/resolved-secret-trace-registry.test.ts @@ -141,7 +141,7 @@ describe('ResolvedSecretTraceProvenanceAccumulator', () => { entries: [{ name: 'TOKEN', encryptedValue: 'encrypted-value' }], scope, }) - accumulator.markIncomplete() + accumulator.markIncomplete('unspecified') expect(accumulator.exportProvenance().entries).toEqual([]) }) }) @@ -1443,6 +1443,81 @@ describe('incompleteness diagnostics', () => { expect(reasons).not.toContain('value-provenance-filter-incomplete') }) + it.each([ + 'tool-input-not-enumerable', + 'tool-params-transform-failed', + 'structural-input-projection-incomplete', + 'mothership-provenance-invalid', + 'client-tool-seal-failed', + 'knowledge-row-missing', + 'knowledge-row-content-mismatch', + 'mothership-response-unreadable', + 'structural-input-root-unprojected', + 'backfill-checkpoint-unusable', + ] as const)('reports %s at error, since it cannot trip on a healthy run', (reason) => { + new ResolvedSecretTraceRegistry([], scope).markIncomplete(reason) + + expect(mockLogger.error).toHaveBeenCalledWith( + 'Resolved secret registry marked incomplete', + expect.objectContaining({ reason }) + ) + expect(mockLogger.warn).not.toHaveBeenCalled() + }) + + it.each([ + 'mothership-provenance-missing', + 'client-tool-completion-missing', + 'client-tool-completion-deferred', + 'client-tool-completion-unidentified', + 'client-tool-execution-untrusted', + 'client-tool-content-unavailable', + 'knowledge-result-provenance-unavailable', + 'knowledge-response-capacity-exceeded', + 'memory-crossing-capacity-exceeded', + 'workspace-scope-missing', + 'mounted-file-provenance-unavailable', + 'table-snapshot-unsafe-for-mount', + 'restored-provenance-untrusted', + 'backfill-checkpoint-absent', + 'client-tool-seal-absent', + ] as const)('reports %s at warn, since it is reachable without a fault', (reason) => { + new ResolvedSecretTraceRegistry([], scope).markIncomplete(reason) + + expect(mockLogger.warn).toHaveBeenCalledWith( + 'Resolved secret registry marked incomplete', + expect.objectContaining({ reason }) + ) + expect(mockLogger.error).not.toHaveBeenCalled() + }) + + /** + * The taxonomy's rule is that an error reason cannot trip on a healthy run. A backfill walking + * historical rows hits the no-checkpoint case on essentially every legacy row, so classifying it + * as a fault would put one error line per row into the stream this split exists to protect. + */ + it('separates an absent checkpoint from an unusable one, so a backfill cannot flood errors', () => { + new ResolvedSecretTraceRegistry([], scope).markIncomplete('backfill-checkpoint-absent') + expect(mockLogger.error).not.toHaveBeenCalled() + expect(mockLogger.warn).toHaveBeenCalledWith( + 'Resolved secret registry marked incomplete', + expect.objectContaining({ reason: 'backfill-checkpoint-absent' }) + ) + + vi.clearAllMocks() + new ResolvedSecretTraceRegistry([], scope).markIncomplete('backfill-checkpoint-unusable') + expect(mockLogger.error).toHaveBeenCalledWith( + 'Resolved secret registry marked incomplete', + expect.objectContaining({ reason: 'backfill-checkpoint-unusable' }) + ) + }) + + it('does not report a log-less session at all, since it fires on every such run', () => { + new ResolvedSecretTraceRegistry([], scope).markIncomplete('log-creation-skipped') + + expect(mockLogger.error).not.toHaveBeenCalled() + expect(mockLogger.warn).not.toHaveBeenCalled() + }) + it('reports an incoming incomplete bundle at warn, since no catalog was ever on offer', () => { const registry = new ResolvedSecretTraceRegistry([], scope) @@ -1465,7 +1540,7 @@ describe('incompleteness diagnostics', () => { it('keeps an unaudited caller taking the default reason out of the error stream', () => { const registry = new ResolvedSecretTraceRegistry([], scope) - registry.markIncomplete() + registry.markIncomplete('unspecified') expect(mockLogger.error).not.toHaveBeenCalled() expect(mockLogger.warn).toHaveBeenCalledWith( diff --git a/apps/sim/executor/utils/resolved-secret-trace-registry.ts b/apps/sim/executor/utils/resolved-secret-trace-registry.ts index c979642edfa..2e0c9247da9 100644 --- a/apps/sim/executor/utils/resolved-secret-trace-registry.ts +++ b/apps/sim/executor/utils/resolved-secret-trace-registry.ts @@ -41,6 +41,36 @@ export type ResolvedSecretIncompletenessReason = | 'value-provenance-filter-incomplete' | 'durable-provenance-unknown' | 'durable-provenance-malformed' + | 'tool-input-not-enumerable' + | 'tool-params-transform-failed' + | 'structural-input-projection-incomplete' + | 'structural-input-root-unprojected' + | 'mothership-provenance-invalid' + | 'mothership-response-unreadable' + | 'mothership-provenance-missing' + | 'client-tool-seal-absent' + | 'client-tool-seal-failed' + | 'client-tool-completion-missing' + | 'client-tool-completion-deferred' + | 'client-tool-completion-unidentified' + | 'client-tool-execution-untrusted' + | 'client-tool-content-unavailable' + | 'knowledge-result-provenance-unavailable' + | 'knowledge-response-capacity-exceeded' + | 'knowledge-row-missing' + | 'knowledge-row-content-mismatch' + | 'memory-crossing-capacity-exceeded' + | 'workspace-scope-missing' + | 'mounted-file-provenance-unavailable' + | 'table-snapshot-unsafe-for-mount' + | 'restored-provenance-untrusted' + | 'backfill-checkpoint-absent' + | 'backfill-checkpoint-unusable' + | 'log-creation-skipped' + /** + * Only for a caller that has not been given a reason yet. A refusal reporting this names no + * guard, which is the state that made a production latch untraceable — prefer adding a literal. + */ | 'unspecified' /** @@ -68,6 +98,16 @@ const ORIGINATING_FAULT_REASONS = new Set([ 'value-provenance-untrusted', 'value-provenance-import-failed', 'durable-provenance-malformed', + 'tool-input-not-enumerable', + 'tool-params-transform-failed', + 'structural-input-projection-incomplete', + 'mothership-provenance-invalid', + 'client-tool-seal-failed', + 'knowledge-row-missing', + 'knowledge-row-content-mismatch', + 'mothership-response-unreadable', + 'structural-input-root-unprojected', + 'backfill-checkpoint-unusable', ]) /** @@ -77,6 +117,8 @@ const ORIGINATING_FAULT_REASONS = new Set([ */ const BY_DESIGN_INCOMPLETENESS_REASONS = new Set([ 'constructed-incomplete', + /** A session that will not persist a log has nothing to vouch for; it fires on every such run. */ + 'log-creation-skipped', ]) /** @@ -188,6 +230,16 @@ type PreparedProvenanceFilterResult = /** Extra attribution for a latch: which registry it propagated from, and which importer caused it. */ interface MarkIncompleteContext { source?: ResolvedSecretTraceRegistry + /** + * Which importer accepted an already-incomplete bundle — only meaningful where several callers + * share one guard, as {@link ImportResolvedSecretTraceProvenanceOptions.origin} describes. + * + * It is not a second way to say what `reason` says. A latch that reaches for an origin because no + * reason fits is the signal to add a reason literal instead: `reason` is a closed set that can be + * alerted on and aggregated, and splitting the same fact across two fields leaves neither + * trustworthy. Passing `'unspecified'` alongside an origin is the shape that produced a + * production latch naming no guard at all. + */ origin?: string } @@ -1418,8 +1470,15 @@ export class ResolvedSecretTraceRegistry { return !this.complete || this.incompleteInputPaths.size > 0 } + /** + * `reason` is required. It defaulted to `'unspecified'`, and every caller that took the default + * produced a latch naming no guard — which is exactly the state that made a production incident + * untraceable for a day. Making omission a compile error is what keeps the reason set honest as + * new guards are added; a caller that genuinely has nothing to say passes `'unspecified'` on + * purpose, where a reviewer can see it. + */ markIncomplete( - reason: ResolvedSecretIncompletenessReason = 'unspecified', + reason: ResolvedSecretIncompletenessReason, context: MarkIncompleteContext = {} ): void { if (context.source) this.inheritIncompletenessReasonsFrom(context.source) @@ -1870,7 +1929,7 @@ export class ResolvedSecretTraceRegistry { private markInputPathIncomplete( path: ResolvedSecretInputPath | undefined, - reason: ResolvedSecretIncompletenessReason = 'unspecified', + reason: ResolvedSecretIncompletenessReason, origin?: string ): void { if (!path || path.length === 0) { diff --git a/apps/sim/lib/copilot/request/handlers/tool.ts b/apps/sim/lib/copilot/request/handlers/tool.ts index 9606e8e5261..14c0afb259e 100644 --- a/apps/sim/lib/copilot/request/handlers/tool.ts +++ b/apps/sim/lib/copilot/request/handlers/tool.ts @@ -235,7 +235,7 @@ export async function prePersistClientExecutableToolCall( toolInput: data.arguments, }) } catch (error) { - execContext.resolvedSecretTraceRegistry.markIncomplete() + execContext.resolvedSecretTraceRegistry.markIncomplete('client-tool-seal-failed') logger.warn('Failed to seal client tool provenance', { toolCallId: data.toolCallId, error: getErrorMessage(error), diff --git a/apps/sim/lib/copilot/request/lifecycle/run.test.ts b/apps/sim/lib/copilot/request/lifecycle/run.test.ts index 2fdf7002eb8..94f79e5c6c5 100644 --- a/apps/sim/lib/copilot/request/lifecycle/run.test.ts +++ b/apps/sim/lib/copilot/request/lifecycle/run.test.ts @@ -950,7 +950,7 @@ describe('runCopilotLifecycle', () => { it('does not block ordinary initial Go payloads on unrelated incomplete provenance', async () => { const registry = new ResolvedSecretTraceRegistry() - registry.markIncomplete() + registry.markIncomplete('unspecified') const result = await runCopilotLifecycle( { message: 'possibly secret', messageId: 'stream-incomplete-projection' }, diff --git a/apps/sim/lib/copilot/request/tools/client.ts b/apps/sim/lib/copilot/request/tools/client.ts index c2651bfa114..88cf40cd35e 100644 --- a/apps/sim/lib/copilot/request/tools/client.ts +++ b/apps/sim/lib/copilot/request/tools/client.ts @@ -91,8 +91,14 @@ export async function waitForClientToolCompletion({ const finishPendingActivation = toolRegistry?.beginPendingActivation() let content: Awaited> = null try { + /** + * A tool invoked without a run id has no binding to unseal against, which is a configuration + * rather than a fault. Tracking whether unsealing was even attempted keeps that ordinary case + * out of the error stream while a genuine unseal failure stays in it. + */ + const sealingAttempted = Boolean(binding && registry && toolRegistry && registryCanImport) const [sealedContent, sealedContext] = - binding && registry && toolRegistry && registryCanImport + sealingAttempted && binding && registry ? await Promise.all([ unsealClientToolCompletion(completion.data, binding), unsealClientToolContext(completion.data, binding, registry), @@ -100,7 +106,9 @@ export async function waitForClientToolCompletion({ : [null, null] if (toolRegistry && registryCanImport) { if (!sealedContent || !sealedContext) { - toolRegistry.markIncomplete() + toolRegistry.markIncomplete( + sealingAttempted ? 'client-tool-seal-failed' : 'client-tool-seal-absent' + ) } else { const imported = await toolRegistry.importProvenance(sealedContext.provenance, { origin: 'copilotToolClient.sealedContext', @@ -116,7 +124,9 @@ export async function waitForClientToolCompletion({ } } } catch { - toolRegistry?.markIncomplete('unspecified', { origin: 'copilotToolClient.sealedContext' }) + toolRegistry?.markIncomplete('client-tool-seal-failed', { + origin: 'copilotToolClient.sealedContext', + }) } finally { finishPendingActivation?.() } @@ -243,18 +253,18 @@ export async function waitForWorkflowToolCompletion({ try { completion = await waitForToolCompletion(toolCallId, timeoutMs, abortSignal) if (!completion) { - toolRegistry?.markIncomplete() + toolRegistry?.markIncomplete('client-tool-completion-missing') return null } const executionId = getWorkflowToolCompletionExecutionId(completion.data) const deploymentError = getAsyncWorkflowDeploymentError(completion.data) if (completion.status === ASYNC_TOOL_CONFIRMATION_STATUS.background) { - toolRegistry?.markIncomplete() + toolRegistry?.markIncomplete('client-tool-completion-deferred') return structuralWorkflowCompletion(completion.status, workflowId, executionId) } if (!workflowId || !executionId) { - toolRegistry?.markIncomplete() + toolRegistry?.markIncomplete('client-tool-completion-unidentified') const structuralStatus = completion.status === MothershipStreamV1ToolOutcome.success ? MothershipStreamV1ToolOutcome.error @@ -279,12 +289,12 @@ export async function waitForWorkflowToolCompletion({ } if (!trustedExecution) { - toolRegistry?.markIncomplete() + toolRegistry?.markIncomplete('client-tool-execution-untrusted') return structuralWorkflowCompletion(completion.status, workflowId, executionId) } if (!trustedExecution.contentAvailable) { - toolRegistry?.markIncomplete() + toolRegistry?.markIncomplete('client-tool-content-unavailable') return structuralWorkflowCompletion( getWorkflowToolConfirmationStatus(trustedExecution.status), workflowId, @@ -297,7 +307,8 @@ export async function waitForWorkflowToolCompletion({ toolRegistry.isPermanentlyIncomplete() || !trustedExecution.provenance.complete ) { - if (!trustedExecution.provenance.complete) toolRegistry?.markIncomplete() + if (!trustedExecution.provenance.complete) + toolRegistry?.markIncomplete('source-provenance-incomplete') return structuralWorkflowCompletion( getWorkflowToolConfirmationStatus(trustedExecution.status), workflowId, @@ -317,9 +328,9 @@ export async function waitForWorkflowToolCompletion({ }, { trusted: true } ) - if (!imported) toolRegistry.markIncomplete() + if (!imported) toolRegistry.markIncomplete('value-provenance-import-failed') } catch (error) { - toolRegistry.markIncomplete() + toolRegistry.markIncomplete('value-provenance-import-failed') logger.warn('Failed to import bound workflow provenance', { toolCallId, workflowId, diff --git a/apps/sim/lib/copilot/request/tools/executor.test.ts b/apps/sim/lib/copilot/request/tools/executor.test.ts index b29a92bff5f..4848e891104 100644 --- a/apps/sim/lib/copilot/request/tools/executor.test.ts +++ b/apps/sim/lib/copilot/request/tools/executor.test.ts @@ -237,7 +237,7 @@ describe('executeToolAndReport provenance isolation', () => { _params: Record, toolContext: ExecutionContext ) => { - toolContext.resolvedSecretTraceRegistry?.markIncomplete() + toolContext.resolvedSecretTraceRegistry?.markIncomplete('unspecified') return { success: true, output: { value: 'secret-value' } } } ) @@ -270,7 +270,7 @@ describe('executeToolAndReport provenance isolation', () => { _params: Record, toolContext: ExecutionContext ) => { - toolContext.resolvedSecretTraceRegistry?.markIncomplete() + toolContext.resolvedSecretTraceRegistry?.markIncomplete('unspecified') throw new Error('secret-value') } ) @@ -300,7 +300,7 @@ describe('executeToolAndReport provenance isolation', () => { _params: Record, toolContext: ExecutionContext ) => { - toolContext.resolvedSecretTraceRegistry?.markIncomplete() + toolContext.resolvedSecretTraceRegistry?.markIncomplete('unspecified') abortController.abort() return { success: true, output: { value: 'secret-value' } } } diff --git a/apps/sim/lib/copilot/request/tools/resolved-secret-result.test.ts b/apps/sim/lib/copilot/request/tools/resolved-secret-result.test.ts index 2cb2066c699..3edb21c4718 100644 --- a/apps/sim/lib/copilot/request/tools/resolved-secret-result.test.ts +++ b/apps/sim/lib/copilot/request/tools/resolved-secret-result.test.ts @@ -335,7 +335,7 @@ describe('projectToolResultForCopilot', () => { 'incomplete', (() => { const registry = createRegistry() - registry.markIncomplete() + registry.markIncomplete('unspecified') return registry })(), ], diff --git a/apps/sim/lib/copilot/request/tools/tables.test.ts b/apps/sim/lib/copilot/request/tools/tables.test.ts index cc9faafbff9..f0572cee86c 100644 --- a/apps/sim/lib/copilot/request/tools/tables.test.ts +++ b/apps/sim/lib/copilot/request/tools/tables.test.ts @@ -379,7 +379,7 @@ describe('maybeWriteOutputToTable', () => { it('persists raw rows with unknown provenance when lineage is incomplete', async () => { const registry = new ResolvedSecretTraceRegistry() - registry.markIncomplete() + registry.markIncomplete('unspecified') const result = await maybeWriteOutputToTable( FunctionExecute.id, @@ -678,7 +678,7 @@ describe('maybeWriteReadCsvToTable', () => { it('imports raw CSV rows with unknown provenance when lineage is incomplete', async () => { const registry = new ResolvedSecretTraceRegistry() - registry.markIncomplete() + registry.markIncomplete('unspecified') const result = await maybeWriteReadCsvToTable( ReadTool.id, diff --git a/apps/sim/lib/copilot/tools/handlers/function-execute.ts b/apps/sim/lib/copilot/tools/handlers/function-execute.ts index d6f04a63c33..944c483d7e1 100644 --- a/apps/sim/lib/copilot/tools/handlers/function-execute.ts +++ b/apps/sim/lib/copilot/tools/handlers/function-execute.ts @@ -118,9 +118,9 @@ async function importMountedWorkspaceFileProvenance(args: { }, registry: args.registry, }) - if (!imported) args.registry.markIncomplete() + if (!imported) args.registry.markIncomplete('mounted-file-provenance-unavailable') } catch { - args.registry.markIncomplete() + args.registry.markIncomplete('mounted-file-provenance-unavailable') } } @@ -464,9 +464,10 @@ export async function resolveInputFiles( workspaceId, rowsVersion: snapshot.version, }) - if (!safeForModelMount) resolvedSecretTraceRegistry.markIncomplete() + if (!safeForModelMount) + resolvedSecretTraceRegistry.markIncomplete('table-snapshot-unsafe-for-mount') } catch { - resolvedSecretTraceRegistry.markIncomplete() + resolvedSecretTraceRegistry.markIncomplete('table-snapshot-unsafe-for-mount') } if (hasCloudStorage()) { @@ -538,7 +539,7 @@ export async function resolveInputFiles( }) } } catch { - resolvedSecretTraceRegistry.markIncomplete('unspecified', { + resolvedSecretTraceRegistry.markIncomplete('source-provenance-incomplete', { origin: 'copilotFunctionExecute.result', }) } @@ -572,9 +573,13 @@ async function importMountedProvenance( trusted: true, }) if (!imported) - target.markIncomplete('unspecified', { origin: 'copilotFunctionExecute.crossing' }) + target.markIncomplete('value-provenance-import-failed', { + origin: 'copilotFunctionExecute.crossing', + }) } catch { - target.markIncomplete('unspecified', { origin: 'copilotFunctionExecute.crossing' }) + target.markIncomplete('value-provenance-import-failed', { + origin: 'copilotFunctionExecute.crossing', + }) } } diff --git a/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts b/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts index 843a61e5794..0fbdfc0f4ef 100644 --- a/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts +++ b/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts @@ -296,7 +296,7 @@ export const knowledgeBaseServerTool: BaseServerTool { it('fails before model egress when an active request registry is incomplete', () => { const registry = new ResolvedSecretTraceRegistry([]) - registry.markIncomplete() + registry.markIncomplete('unspecified') expect(() => runWithKnowledgeModelInputProvenance(registry, () => diff --git a/apps/sim/lib/knowledge/secret-provenance.ts b/apps/sim/lib/knowledge/secret-provenance.ts index 139fb0adc44..8f9da22d64e 100644 --- a/apps/sim/lib/knowledge/secret-provenance.ts +++ b/apps/sim/lib/knowledge/secret-provenance.ts @@ -458,7 +458,7 @@ export async function importKnowledgePersistedResponseSecretProvenance(options: documents.length > MAX_KNOWLEDGE_RESPONSE_PROVENANCE_ROWS || chunks.length > MAX_KNOWLEDGE_RESPONSE_PROVENANCE_ROWS ) { - options.registry.markIncomplete() + options.registry.markIncomplete('knowledge-response-capacity-exceeded') return false } @@ -480,14 +480,14 @@ export async function importKnowledgePersistedResponseSecretProvenance(options: const documentById = new Map(documentRows.map((row) => [row.id, row])) const chunkById = new Map(chunkRows.map((row) => [row.id, row])) if (documentById.size !== documentIds.length || chunkById.size !== chunkIds.length) { - options.registry.markIncomplete() + options.registry.markIncomplete('knowledge-row-missing') return false } for (const item of documents) { const row = documentById.get(item.id) if (!row) { - options.registry.markIncomplete() + options.registry.markIncomplete('knowledge-row-missing') return false } const source = createKnowledgeDocumentSourceValue(row) @@ -496,7 +496,7 @@ export async function importKnowledgePersistedResponseSecretProvenance(options: createKnowledgeDocumentSourceValue(item.source) ) if (!actualSourceHash || !expectedSourceHash || actualSourceHash !== expectedSourceHash) { - options.registry.markIncomplete() + options.registry.markIncomplete('knowledge-row-content-mismatch') return false } const provenance = filterKnowledgeDocumentMetadataSecretProvenance( @@ -513,7 +513,7 @@ export async function importKnowledgePersistedResponseSecretProvenance(options: for (const item of chunks) { const row = chunkById.get(item.id) if (!row || row.documentId !== item.documentId || row.content !== item.content) { - options.registry.markIncomplete() + options.registry.markIncomplete('knowledge-row-content-mismatch') return false } const provenance = readBoundKnowledgeEmbeddingSecretProvenance(row) diff --git a/apps/sim/lib/logs/execution/logging-session.ts b/apps/sim/lib/logs/execution/logging-session.ts index ced3101d10c..9102684ee6f 100644 --- a/apps/sim/lib/logs/execution/logging-session.ts +++ b/apps/sim/lib/logs/execution/logging-session.ts @@ -311,7 +311,7 @@ export class LoggingSession { if (!isResolvedSecretTraceProvenanceV1(provenance)) { const incomplete = new ResolvedSecretTraceRegistry() - incomplete.markIncomplete() + incomplete.markIncomplete('restored-provenance-untrusted') return incomplete } @@ -781,7 +781,7 @@ export class LoggingSession { [], scopeUserId ? { userId: scopeUserId, workspaceId } : undefined ) - if (skipLogCreation) this.resolvedSecretTraceRegistry.markIncomplete() + if (skipLogCreation) this.resolvedSecretTraceRegistry.markIncomplete('log-creation-skipped') } try { diff --git a/apps/sim/lib/model-router/resolve.test.ts b/apps/sim/lib/model-router/resolve.test.ts index 6b4ce65483e..b65528152a5 100644 --- a/apps/sim/lib/model-router/resolve.test.ts +++ b/apps/sim/lib/model-router/resolve.test.ts @@ -225,7 +225,7 @@ describe('resolveAutoModel', () => { it('does not gate caller-projected signals on ambient registry completeness', async () => { const registry = new ResolvedSecretTraceRegistry() - registry.markIncomplete() + registry.markIncomplete('unspecified') mockFetchGo.mockResolvedValue(routerResponse({ choice: '1' })) const result = await resolveAutoModel({ diff --git a/apps/sim/lib/table/backfill-runner.ts b/apps/sim/lib/table/backfill-runner.ts index 1d549397ed5..685bd43740b 100644 --- a/apps/sim/lib/table/backfill-runner.ts +++ b/apps/sim/lib/table/backfill-runner.ts @@ -72,14 +72,24 @@ export async function createBackfillExecutionSecretRegistry(options: { ? options.executionData.executionState : undefined const provenance = state?.resolvedSecretTraceProvenance + /** + * A state persisted before the checkpoint contract carries no version at all. That is the bulk of + * any backfill over historical rows, so it is separated from a checkpoint that exists but cannot + * be used — only the latter is worth an error, and conflating them would put one line per legacy + * row into the error stream. + */ + const checkpointPresent = + state?.resolvedSecretTraceCheckpointVersion === RESOLVED_SECRET_TRACE_CHECKPOINT_VERSION const valid = - state?.resolvedSecretTraceCheckpointVersion === RESOLVED_SECRET_TRACE_CHECKPOINT_VERSION && + checkpointPresent && state?.sourceExecutionId === options.executionId && isResolvedSecretTraceProvenanceV1(provenance) && provenance.scope?.workspaceId === options.workspaceId const registry = new ResolvedSecretTraceRegistry([], valid ? provenance.scope : undefined) if (!valid) { - registry.markIncomplete() + registry.markIncomplete( + checkpointPresent ? 'backfill-checkpoint-unusable' : 'backfill-checkpoint-absent' + ) return registry } await registry.importProvenance(provenance, { diff --git a/apps/sim/lib/workflows/executor/execution-core.ts b/apps/sim/lib/workflows/executor/execution-core.ts index aa1fd9094f5..f0fb936cb29 100644 --- a/apps/sim/lib/workflows/executor/execution-core.ts +++ b/apps/sim/lib/workflows/executor/execution-core.ts @@ -539,7 +539,7 @@ async function executeWorkflowCoreImpl( scope: { userId: personalEnvUserId, workspaceId: providedWorkspaceId }, }) if (restoredState && !restoreTrusted) { - resolvedSecretTraceRegistry.markIncomplete() + resolvedSecretTraceRegistry.markIncomplete('restored-provenance-untrusted') } if (options.trustedInitialResolvedSecretTraceProvenance !== undefined) { await resolvedSecretTraceRegistry.importProvenance( diff --git a/apps/sim/providers/index.test.ts b/apps/sim/providers/index.test.ts index 73944552d9d..06fc67d4d03 100644 --- a/apps/sim/providers/index.test.ts +++ b/apps/sim/providers/index.test.ts @@ -1288,7 +1288,7 @@ describe('executeProviderRequest — caller-prepared model input', () => { it('does not make provider execution depend on registry completeness', async () => { const incomplete = new ResolvedSecretTraceRegistry() - incomplete.markIncomplete() + incomplete.markIncomplete('unspecified') await executeProviderRequest( 'anthropic', diff --git a/apps/sim/providers/runtime-context.test.ts b/apps/sim/providers/runtime-context.test.ts index c2919d7f4bc..539bd3580a9 100644 --- a/apps/sim/providers/runtime-context.test.ts +++ b/apps/sim/providers/runtime-context.test.ts @@ -527,7 +527,7 @@ describe('provider runtime context', () => { { name: 'TOKEN', plaintext: 'secret-value', encryptedValue: 'ciphertext' }, ]) mockExecuteTool.mockImplementationOnce(async (_toolId, _params, options) => { - options.resolvedSecretTraceRegistry?.markIncomplete() + options.resolvedSecretTraceRegistry?.markIncomplete('unspecified') return { success: true, output: { value: 'secret-value' } } }) @@ -546,7 +546,7 @@ describe('provider runtime context', () => { { name: 'TOKEN', plaintext: 'secret-value', encryptedValue: 'ciphertext' }, ]) mockExecuteTool.mockImplementationOnce(async (_toolId, _params, options) => { - options.resolvedSecretTraceRegistry?.markIncomplete() + options.resolvedSecretTraceRegistry?.markIncomplete('unspecified') return { success: false, output: { value: 'secret-value' }, @@ -600,7 +600,7 @@ describe('provider runtime context', () => { { name: 'TOKEN', plaintext: 'secret-value', encryptedValue: 'ciphertext' }, ]) mockExecuteTool.mockImplementationOnce(async (_toolId, _params, options) => { - options.resolvedSecretTraceRegistry?.markIncomplete() + options.resolvedSecretTraceRegistry?.markIncomplete('unspecified') throw new Error('secret-value') }) @@ -628,7 +628,7 @@ describe('provider runtime context', () => { const registry = new ResolvedSecretTraceRegistry([ { name: 'TOKEN', plaintext: 'secret-value', encryptedValue: 'ciphertext' }, ]) - registry.markIncomplete() + registry.markIncomplete('unspecified') mockExecuteTool.mockResolvedValueOnce({ success: true, output: { value: 'secret-value' }, diff --git a/apps/sim/providers/utils.ts b/apps/sim/providers/utils.ts index c6017dba864..55d14a41fc9 100644 --- a/apps/sim/providers/utils.ts +++ b/apps/sim/providers/utils.ts @@ -1570,7 +1570,7 @@ export function prepareToolExecution( try { projectedToolParams = tool.paramsTransform(projectedToolParams) } catch { - inputRegistry.markIncomplete() + inputRegistry.markIncomplete('tool-params-transform-failed') projectedToolParams = undefined } } diff --git a/apps/sim/tools/index.test.ts b/apps/sim/tools/index.test.ts index ad87e5608d7..e13434a71db 100644 --- a/apps/sim/tools/index.test.ts +++ b/apps/sim/tools/index.test.ts @@ -1924,7 +1924,7 @@ describe('executeTool Function', () => { it('runs a private-provenance call from an incomplete parent without replacing its result', async () => { const registry = new ResolvedSecretTraceRegistry() - registry.markIncomplete() + registry.markIncomplete('unspecified') const fetchMock = vi.mocked(global.fetch) const result = await executeTool( @@ -1938,10 +1938,26 @@ describe('executeTool Function', () => { expect(fetchMock).toHaveBeenCalled() }) + /** + * The shape that latched in production with `reason: "unspecified"` — a getter or symbol key on + * the params record makes the input lineage unboundable, and the fork is marked before the tool + * runs. Pinned by name so a refusal downstream can be traced back to this guard. + */ + it('names the guard when tool params are not enumerable plain data', async () => { + const registry = new ResolvedSecretTraceRegistry() + const params: Record = { code: 'return "unreachable"' } + Object.defineProperty(params, 'envVars', { enumerable: true, get: () => ({}) }) + + await executeTool('function_execute', params, { resolvedSecretTraceRegistry: registry }) + + expect(registry.isComplete()).toBe(false) + expect(registry.getIncompletenessDiagnostics()?.reasons).toContain('tool-input-not-enumerable') + }) + it('runs a private-provenance call when its input lineage cannot be bounded', async () => { const registry = new ResolvedSecretTraceRegistry() const incompleteToolRegistry = registry.forkForToolCall() - incompleteToolRegistry.markIncomplete() + incompleteToolRegistry.markIncomplete('unspecified') vi.spyOn(registry, 'forkForInputPaths').mockReturnValue(incompleteToolRegistry) const fetchMock = vi.mocked(global.fetch) diff --git a/apps/sim/tools/index.ts b/apps/sim/tools/index.ts index 6e36e73d284..3ec731b69a7 100644 --- a/apps/sim/tools/index.ts +++ b/apps/sim/tools/index.ts @@ -1408,7 +1408,7 @@ export async function executeTool( const toolRegistry = paramEntries ? parentRegistry.forkForInputPaths(paramEntries.map(([key]) => [key] as const)) : parentRegistry.forkForToolCall() - if (!paramEntries) toolRegistry.markIncomplete() + if (!paramEntries) toolRegistry.markIncomplete('tool-input-not-enumerable') const executionContext = options.executionContext ? { ...options.executionContext, resolvedSecretTraceRegistry: toolRegistry } : undefined