From 9c234ad6560e209f02d5717e3dd3e17b6d095732 Mon Sep 17 00:00:00 2001 From: Justin Blumencranz <96924014+j15z@users.noreply.github.com> Date: Sat, 11 Jul 2026 19:17:18 -0700 Subject: [PATCH 1/2] fix(media): validate upload paths before generation Allow media tools to consume chat-scoped uploads directly while keeping outputs under files/. Fail malformed or unresolved declarations before provider work and regenerate the Mothership-derived tool schemas. --- .../lib/copilot/generated/tool-catalog-v1.ts | 275 +++------------- .../lib/copilot/generated/tool-schemas-v1.ts | 299 +++--------------- .../tools/handlers/upload-file-reader.ts | 11 + .../tools/server/image/generate-image.test.ts | 211 ++++++++++++ .../tools/server/image/generate-image.ts | 68 ++-- .../lib/copilot/tools/server/media/ffmpeg.ts | 25 +- .../tools/server/media/file-paths.test.ts | 158 +++++++++ .../copilot/tools/server/media/file-paths.ts | 109 +++++++ .../tools/server/media/generate-audio.ts | 44 ++- .../tools/server/media/generate-video.ts | 34 +- .../media/media-tools-validation.test.ts | 176 +++++++++++ 11 files changed, 859 insertions(+), 551 deletions(-) create mode 100644 apps/sim/lib/copilot/tools/server/image/generate-image.test.ts create mode 100644 apps/sim/lib/copilot/tools/server/media/file-paths.test.ts create mode 100644 apps/sim/lib/copilot/tools/server/media/file-paths.ts create mode 100644 apps/sim/lib/copilot/tools/server/media/media-tools-validation.test.ts diff --git a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts index 9c91d02ef64..6e281299c5d 100644 --- a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts @@ -1208,66 +1208,27 @@ export const Ffmpeg: ToolCatalogEntry = { height: { type: 'number', description: 'Target height in pixels (scale_pad).' }, inputs: { type: 'object', - description: - 'Workspace resources to mount into the sandbox. Copy paths verbatim from glob/read/grep output — they are percent-encoded per segment (spaces are %20, an in-name slash is %2F; parentheses and dots stay literal). Both the encoded path and the plain name resolve, so copy the returned path exactly rather than retyping or decoding it.', + description: 'Existing media files to use as generation or composition inputs.', properties: { - directories: { - type: 'array', - description: - 'Workspace folders to mount recursively into the sandbox, including nested files and empty folders.', - items: { - type: 'object', - properties: { - path: { - type: 'string', - description: - 'Canonical VFS folder path, e.g. "files/Reports". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Optional full sandbox directory path override. Omit to mount at /home/user/{path}.', - }, - }, - required: ['path'], - }, - }, files: { type: 'array', - description: 'Workspace files to mount into the sandbox.', + description: + 'One or more media inputs from files/... or uploads/..., in operation order.', items: { type: 'object', properties: { path: { type: 'string', description: - 'Canonical VFS file path, e.g. "files/Reports/sales.csv". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Full sandbox path to mount at, e.g. /home/user/inputs/data.csv. STRONGLY RECOMMENDED whenever the file name has spaces or special characters: the default mount path is the percent-ENCODED canonical path (e.g. /home/user/files/Q4%20Sales%20(Final).csv), which code using the human-readable name will not find. Set a simple sandboxPath and read exactly that.', + 'Input VFS path copied verbatim from glob/read, using either persistent "files/..." or chat-scoped "uploads/...".', }, }, required: ['path'], }, - }, - tables: { - type: 'array', - description: 'Workspace tables to mount as CSV files.', - items: { - type: 'object', - properties: { - path: { type: 'string', description: 'Canonical VFS table path when available.' }, - sandboxPath: { - type: 'string', - description: 'Optional full sandbox path for the mounted CSV.', - }, - tableId: { type: 'string', description: 'Workspace table ID.' }, - }, - }, + minItems: 1, }, }, + required: ['files'], }, loopToVideo: { type: 'boolean', @@ -1298,20 +1259,15 @@ export const Ffmpeg: ToolCatalogEntry = { outputs: { type: 'object', description: - 'Workspace files to create or overwrite from returned code results or sandbox-created files.', + 'One persistent workspace file to create or overwrite. Chat uploads are read-only inputs and cannot be output targets.', properties: { files: { type: 'array', description: - 'File outputs. Missing parent folders are created automatically for create mode.', + 'Exactly one file output. Parent folders must already exist for create mode.', items: { type: 'object', properties: { - format: { - type: 'string', - description: 'Optional serialization format for returned values.', - enum: ['json', 'csv', 'txt', 'md', 'html'], - }, mimeType: { type: 'string', description: 'Optional MIME type override when inference is not enough.', @@ -1322,19 +1278,18 @@ export const Ffmpeg: ToolCatalogEntry = { enum: ['create', 'overwrite'], }, path: { - type: 'string', - description: 'Canonical destination VFS path, e.g. "files/Reports/chart.png".', - }, - sandboxPath: { type: 'string', description: - 'Optional full path to a file created inside the sandbox. Omit to save the code return value.', + 'Persistent destination VFS path starting with "files/", e.g. "files/Reports/chart.png". "uploads/" paths are invalid output targets.', }, }, required: ['path', 'mode'], }, + minItems: 1, + maxItems: 1, }, }, + required: ['files'], }, position: { type: 'string', @@ -1550,66 +1505,27 @@ export const GenerateAudio: ToolCatalogEntry = { }, inputs: { type: 'object', - description: - 'Workspace resources to mount into the sandbox. Copy paths verbatim from glob/read/grep output — they are percent-encoded per segment (spaces are %20, an in-name slash is %2F; parentheses and dots stay literal). Both the encoded path and the plain name resolve, so copy the returned path exactly rather than retyping or decoding it.', + description: 'Existing media files to use as generation or composition inputs.', properties: { - directories: { - type: 'array', - description: - 'Workspace folders to mount recursively into the sandbox, including nested files and empty folders.', - items: { - type: 'object', - properties: { - path: { - type: 'string', - description: - 'Canonical VFS folder path, e.g. "files/Reports". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Optional full sandbox directory path override. Omit to mount at /home/user/{path}.', - }, - }, - required: ['path'], - }, - }, files: { type: 'array', - description: 'Workspace files to mount into the sandbox.', + description: 'Exactly one reference voice sample from files/... or uploads/....', items: { type: 'object', properties: { path: { type: 'string', description: - 'Canonical VFS file path, e.g. "files/Reports/sales.csv". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Full sandbox path to mount at, e.g. /home/user/inputs/data.csv. STRONGLY RECOMMENDED whenever the file name has spaces or special characters: the default mount path is the percent-ENCODED canonical path (e.g. /home/user/files/Q4%20Sales%20(Final).csv), which code using the human-readable name will not find. Set a simple sandboxPath and read exactly that.', + 'Input VFS path copied verbatim from glob/read, using either persistent "files/..." or chat-scoped "uploads/...".', }, }, required: ['path'], }, - }, - tables: { - type: 'array', - description: 'Workspace tables to mount as CSV files.', - items: { - type: 'object', - properties: { - path: { type: 'string', description: 'Canonical VFS table path when available.' }, - sandboxPath: { - type: 'string', - description: 'Optional full sandbox path for the mounted CSV.', - }, - tableId: { type: 'string', description: 'Workspace table ID.' }, - }, - }, + minItems: 1, + maxItems: 1, }, }, + required: ['files'], }, instrumental: { type: 'boolean', @@ -1629,20 +1545,15 @@ export const GenerateAudio: ToolCatalogEntry = { outputs: { type: 'object', description: - 'Workspace files to create or overwrite from returned code results or sandbox-created files.', + 'One persistent workspace file to create or overwrite. Chat uploads are read-only inputs and cannot be output targets.', properties: { files: { type: 'array', description: - 'File outputs. Missing parent folders are created automatically for create mode.', + 'Exactly one file output. Parent folders must already exist for create mode.', items: { type: 'object', properties: { - format: { - type: 'string', - description: 'Optional serialization format for returned values.', - enum: ['json', 'csv', 'txt', 'md', 'html'], - }, mimeType: { type: 'string', description: 'Optional MIME type override when inference is not enough.', @@ -1653,19 +1564,18 @@ export const GenerateAudio: ToolCatalogEntry = { enum: ['create', 'overwrite'], }, path: { - type: 'string', - description: 'Canonical destination VFS path, e.g. "files/Reports/chart.png".', - }, - sandboxPath: { type: 'string', description: - 'Optional full path to a file created inside the sandbox. Omit to save the code return value.', + 'Persistent destination VFS path starting with "files/", e.g. "files/Reports/chart.png". "uploads/" paths are invalid output targets.', }, }, required: ['path', 'mode'], }, + minItems: 1, + maxItems: 1, }, }, + required: ['files'], }, prompt: { type: 'string', @@ -1700,84 +1610,39 @@ export const GenerateImage: ToolCatalogEntry = { }, inputs: { type: 'object', - description: - 'Workspace resources to mount into the sandbox. Copy paths verbatim from glob/read/grep output — they are percent-encoded per segment (spaces are %20, an in-name slash is %2F; parentheses and dots stay literal). Both the encoded path and the plain name resolve, so copy the returned path exactly rather than retyping or decoding it.', + description: 'Existing media files to use as generation or composition inputs.', properties: { - directories: { - type: 'array', - description: - 'Workspace folders to mount recursively into the sandbox, including nested files and empty folders.', - items: { - type: 'object', - properties: { - path: { - type: 'string', - description: - 'Canonical VFS folder path, e.g. "files/Reports". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Optional full sandbox directory path override. Omit to mount at /home/user/{path}.', - }, - }, - required: ['path'], - }, - }, files: { type: 'array', - description: 'Workspace files to mount into the sandbox.', + description: 'One or more reference images to edit, combine, or restyle.', items: { type: 'object', properties: { path: { type: 'string', description: - 'Canonical VFS file path, e.g. "files/Reports/sales.csv". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Full sandbox path to mount at, e.g. /home/user/inputs/data.csv. STRONGLY RECOMMENDED whenever the file name has spaces or special characters: the default mount path is the percent-ENCODED canonical path (e.g. /home/user/files/Q4%20Sales%20(Final).csv), which code using the human-readable name will not find. Set a simple sandboxPath and read exactly that.', + 'Input VFS path copied verbatim from glob/read, using either persistent "files/..." or chat-scoped "uploads/...".', }, }, required: ['path'], }, - }, - tables: { - type: 'array', - description: 'Workspace tables to mount as CSV files.', - items: { - type: 'object', - properties: { - path: { type: 'string', description: 'Canonical VFS table path when available.' }, - sandboxPath: { - type: 'string', - description: 'Optional full sandbox path for the mounted CSV.', - }, - tableId: { type: 'string', description: 'Workspace table ID.' }, - }, - }, + minItems: 1, }, }, + required: ['files'], }, outputs: { type: 'object', description: - 'Workspace files to create or overwrite from returned code results or sandbox-created files.', + 'One persistent workspace file to create or overwrite. Chat uploads are read-only inputs and cannot be output targets.', properties: { files: { type: 'array', description: - 'File outputs. Missing parent folders are created automatically for create mode.', + 'Exactly one file output. Parent folders must already exist for create mode.', items: { type: 'object', properties: { - format: { - type: 'string', - description: 'Optional serialization format for returned values.', - enum: ['json', 'csv', 'txt', 'md', 'html'], - }, mimeType: { type: 'string', description: 'Optional MIME type override when inference is not enough.', @@ -1788,19 +1653,18 @@ export const GenerateImage: ToolCatalogEntry = { enum: ['create', 'overwrite'], }, path: { - type: 'string', - description: 'Canonical destination VFS path, e.g. "files/Reports/chart.png".', - }, - sandboxPath: { type: 'string', description: - 'Optional full path to a file created inside the sandbox. Omit to save the code return value.', + 'Persistent destination VFS path starting with "files/", e.g. "files/Reports/chart.png". "uploads/" paths are invalid output targets.', }, }, required: ['path', 'mode'], }, + minItems: 1, + maxItems: 1, }, }, + required: ['files'], }, prompt: { type: 'string', @@ -1838,66 +1702,27 @@ export const GenerateVideo: ToolCatalogEntry = { }, inputs: { type: 'object', - description: - 'Workspace resources to mount into the sandbox. Copy paths verbatim from glob/read/grep output — they are percent-encoded per segment (spaces are %20, an in-name slash is %2F; parentheses and dots stay literal). Both the encoded path and the plain name resolve, so copy the returned path exactly rather than retyping or decoding it.', + description: 'Existing media files to use as generation or composition inputs.', properties: { - directories: { - type: 'array', - description: - 'Workspace folders to mount recursively into the sandbox, including nested files and empty folders.', - items: { - type: 'object', - properties: { - path: { - type: 'string', - description: - 'Canonical VFS folder path, e.g. "files/Reports". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Optional full sandbox directory path override. Omit to mount at /home/user/{path}.', - }, - }, - required: ['path'], - }, - }, files: { type: 'array', - description: 'Workspace files to mount into the sandbox.', + description: 'Exactly one start-frame image from files/... or uploads/....', items: { type: 'object', properties: { path: { type: 'string', description: - 'Canonical VFS file path, e.g. "files/Reports/sales.csv". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Full sandbox path to mount at, e.g. /home/user/inputs/data.csv. STRONGLY RECOMMENDED whenever the file name has spaces or special characters: the default mount path is the percent-ENCODED canonical path (e.g. /home/user/files/Q4%20Sales%20(Final).csv), which code using the human-readable name will not find. Set a simple sandboxPath and read exactly that.', + 'Input VFS path copied verbatim from glob/read, using either persistent "files/..." or chat-scoped "uploads/...".', }, }, required: ['path'], }, - }, - tables: { - type: 'array', - description: 'Workspace tables to mount as CSV files.', - items: { - type: 'object', - properties: { - path: { type: 'string', description: 'Canonical VFS table path when available.' }, - sandboxPath: { - type: 'string', - description: 'Optional full sandbox path for the mounted CSV.', - }, - tableId: { type: 'string', description: 'Workspace table ID.' }, - }, - }, + minItems: 1, + maxItems: 1, }, }, + required: ['files'], }, model: { type: 'string', @@ -1923,20 +1748,15 @@ export const GenerateVideo: ToolCatalogEntry = { outputs: { type: 'object', description: - 'Workspace files to create or overwrite from returned code results or sandbox-created files.', + 'One persistent workspace file to create or overwrite. Chat uploads are read-only inputs and cannot be output targets.', properties: { files: { type: 'array', description: - 'File outputs. Missing parent folders are created automatically for create mode.', + 'Exactly one file output. Parent folders must already exist for create mode.', items: { type: 'object', properties: { - format: { - type: 'string', - description: 'Optional serialization format for returned values.', - enum: ['json', 'csv', 'txt', 'md', 'html'], - }, mimeType: { type: 'string', description: 'Optional MIME type override when inference is not enough.', @@ -1947,19 +1767,18 @@ export const GenerateVideo: ToolCatalogEntry = { enum: ['create', 'overwrite'], }, path: { - type: 'string', - description: 'Canonical destination VFS path, e.g. "files/Reports/chart.png".', - }, - sandboxPath: { type: 'string', description: - 'Optional full path to a file created inside the sandbox. Omit to save the code return value.', + 'Persistent destination VFS path starting with "files/", e.g. "files/Reports/chart.png". "uploads/" paths are invalid output targets.', }, }, required: ['path', 'mode'], }, + minItems: 1, + maxItems: 1, }, }, + required: ['files'], }, prompt: { type: 'string', diff --git a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts index d5a90741800..cb5a3092eb1 100644 --- a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts @@ -1033,72 +1033,27 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, inputs: { type: 'object', - description: - 'Workspace resources to mount into the sandbox. Copy paths verbatim from glob/read/grep output — they are percent-encoded per segment (spaces are %20, an in-name slash is %2F; parentheses and dots stay literal). Both the encoded path and the plain name resolve, so copy the returned path exactly rather than retyping or decoding it.', + description: 'Existing media files to use as generation or composition inputs.', properties: { - directories: { - type: 'array', - description: - 'Workspace folders to mount recursively into the sandbox, including nested files and empty folders.', - items: { - type: 'object', - properties: { - path: { - type: 'string', - description: - 'Canonical VFS folder path, e.g. "files/Reports". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Optional full sandbox directory path override. Omit to mount at /home/user/{path}.', - }, - }, - required: ['path'], - }, - }, files: { type: 'array', - description: 'Workspace files to mount into the sandbox.', + description: + 'One or more media inputs from files/... or uploads/..., in operation order.', items: { type: 'object', properties: { path: { type: 'string', description: - 'Canonical VFS file path, e.g. "files/Reports/sales.csv". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Full sandbox path to mount at, e.g. /home/user/inputs/data.csv. STRONGLY RECOMMENDED whenever the file name has spaces or special characters: the default mount path is the percent-ENCODED canonical path (e.g. /home/user/files/Q4%20Sales%20(Final).csv), which code using the human-readable name will not find. Set a simple sandboxPath and read exactly that.', + 'Input VFS path copied verbatim from glob/read, using either persistent "files/..." or chat-scoped "uploads/...".', }, }, required: ['path'], }, - }, - tables: { - type: 'array', - description: 'Workspace tables to mount as CSV files.', - items: { - type: 'object', - properties: { - path: { - type: 'string', - description: 'Canonical VFS table path when available.', - }, - sandboxPath: { - type: 'string', - description: 'Optional full sandbox path for the mounted CSV.', - }, - tableId: { - type: 'string', - description: 'Workspace table ID.', - }, - }, - }, + minItems: 1, }, }, + required: ['files'], }, loopToVideo: { type: 'boolean', @@ -1129,20 +1084,15 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { outputs: { type: 'object', description: - 'Workspace files to create or overwrite from returned code results or sandbox-created files.', + 'One persistent workspace file to create or overwrite. Chat uploads are read-only inputs and cannot be output targets.', properties: { files: { type: 'array', description: - 'File outputs. Missing parent folders are created automatically for create mode.', + 'Exactly one file output. Parent folders must already exist for create mode.', items: { type: 'object', properties: { - format: { - type: 'string', - description: 'Optional serialization format for returned values.', - enum: ['json', 'csv', 'txt', 'md', 'html'], - }, mimeType: { type: 'string', description: 'Optional MIME type override when inference is not enough.', @@ -1153,19 +1103,18 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { enum: ['create', 'overwrite'], }, path: { - type: 'string', - description: 'Canonical destination VFS path, e.g. "files/Reports/chart.png".', - }, - sandboxPath: { type: 'string', description: - 'Optional full path to a file created inside the sandbox. Omit to save the code return value.', + 'Persistent destination VFS path starting with "files/", e.g. "files/Reports/chart.png". "uploads/" paths are invalid output targets.', }, }, required: ['path', 'mode'], }, + minItems: 1, + maxItems: 1, }, }, + required: ['files'], }, position: { type: 'string', @@ -1374,72 +1323,27 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, inputs: { type: 'object', - description: - 'Workspace resources to mount into the sandbox. Copy paths verbatim from glob/read/grep output — they are percent-encoded per segment (spaces are %20, an in-name slash is %2F; parentheses and dots stay literal). Both the encoded path and the plain name resolve, so copy the returned path exactly rather than retyping or decoding it.', + description: 'Existing media files to use as generation or composition inputs.', properties: { - directories: { - type: 'array', - description: - 'Workspace folders to mount recursively into the sandbox, including nested files and empty folders.', - items: { - type: 'object', - properties: { - path: { - type: 'string', - description: - 'Canonical VFS folder path, e.g. "files/Reports". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Optional full sandbox directory path override. Omit to mount at /home/user/{path}.', - }, - }, - required: ['path'], - }, - }, files: { type: 'array', - description: 'Workspace files to mount into the sandbox.', + description: 'Exactly one reference voice sample from files/... or uploads/....', items: { type: 'object', properties: { path: { type: 'string', description: - 'Canonical VFS file path, e.g. "files/Reports/sales.csv". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Full sandbox path to mount at, e.g. /home/user/inputs/data.csv. STRONGLY RECOMMENDED whenever the file name has spaces or special characters: the default mount path is the percent-ENCODED canonical path (e.g. /home/user/files/Q4%20Sales%20(Final).csv), which code using the human-readable name will not find. Set a simple sandboxPath and read exactly that.', + 'Input VFS path copied verbatim from glob/read, using either persistent "files/..." or chat-scoped "uploads/...".', }, }, required: ['path'], }, - }, - tables: { - type: 'array', - description: 'Workspace tables to mount as CSV files.', - items: { - type: 'object', - properties: { - path: { - type: 'string', - description: 'Canonical VFS table path when available.', - }, - sandboxPath: { - type: 'string', - description: 'Optional full sandbox path for the mounted CSV.', - }, - tableId: { - type: 'string', - description: 'Workspace table ID.', - }, - }, - }, + minItems: 1, + maxItems: 1, }, }, + required: ['files'], }, instrumental: { type: 'boolean', @@ -1459,20 +1363,15 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { outputs: { type: 'object', description: - 'Workspace files to create or overwrite from returned code results or sandbox-created files.', + 'One persistent workspace file to create or overwrite. Chat uploads are read-only inputs and cannot be output targets.', properties: { files: { type: 'array', description: - 'File outputs. Missing parent folders are created automatically for create mode.', + 'Exactly one file output. Parent folders must already exist for create mode.', items: { type: 'object', properties: { - format: { - type: 'string', - description: 'Optional serialization format for returned values.', - enum: ['json', 'csv', 'txt', 'md', 'html'], - }, mimeType: { type: 'string', description: 'Optional MIME type override when inference is not enough.', @@ -1483,19 +1382,18 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { enum: ['create', 'overwrite'], }, path: { - type: 'string', - description: 'Canonical destination VFS path, e.g. "files/Reports/chart.png".', - }, - sandboxPath: { type: 'string', description: - 'Optional full path to a file created inside the sandbox. Omit to save the code return value.', + 'Persistent destination VFS path starting with "files/", e.g. "files/Reports/chart.png". "uploads/" paths are invalid output targets.', }, }, required: ['path', 'mode'], }, + minItems: 1, + maxItems: 1, }, }, + required: ['files'], }, prompt: { type: 'string', @@ -1527,90 +1425,39 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, inputs: { type: 'object', - description: - 'Workspace resources to mount into the sandbox. Copy paths verbatim from glob/read/grep output — they are percent-encoded per segment (spaces are %20, an in-name slash is %2F; parentheses and dots stay literal). Both the encoded path and the plain name resolve, so copy the returned path exactly rather than retyping or decoding it.', + description: 'Existing media files to use as generation or composition inputs.', properties: { - directories: { - type: 'array', - description: - 'Workspace folders to mount recursively into the sandbox, including nested files and empty folders.', - items: { - type: 'object', - properties: { - path: { - type: 'string', - description: - 'Canonical VFS folder path, e.g. "files/Reports". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Optional full sandbox directory path override. Omit to mount at /home/user/{path}.', - }, - }, - required: ['path'], - }, - }, files: { type: 'array', - description: 'Workspace files to mount into the sandbox.', + description: 'One or more reference images to edit, combine, or restyle.', items: { type: 'object', properties: { path: { type: 'string', description: - 'Canonical VFS file path, e.g. "files/Reports/sales.csv". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Full sandbox path to mount at, e.g. /home/user/inputs/data.csv. STRONGLY RECOMMENDED whenever the file name has spaces or special characters: the default mount path is the percent-ENCODED canonical path (e.g. /home/user/files/Q4%20Sales%20(Final).csv), which code using the human-readable name will not find. Set a simple sandboxPath and read exactly that.', + 'Input VFS path copied verbatim from glob/read, using either persistent "files/..." or chat-scoped "uploads/...".', }, }, required: ['path'], }, - }, - tables: { - type: 'array', - description: 'Workspace tables to mount as CSV files.', - items: { - type: 'object', - properties: { - path: { - type: 'string', - description: 'Canonical VFS table path when available.', - }, - sandboxPath: { - type: 'string', - description: 'Optional full sandbox path for the mounted CSV.', - }, - tableId: { - type: 'string', - description: 'Workspace table ID.', - }, - }, - }, + minItems: 1, }, }, + required: ['files'], }, outputs: { type: 'object', description: - 'Workspace files to create or overwrite from returned code results or sandbox-created files.', + 'One persistent workspace file to create or overwrite. Chat uploads are read-only inputs and cannot be output targets.', properties: { files: { type: 'array', description: - 'File outputs. Missing parent folders are created automatically for create mode.', + 'Exactly one file output. Parent folders must already exist for create mode.', items: { type: 'object', properties: { - format: { - type: 'string', - description: 'Optional serialization format for returned values.', - enum: ['json', 'csv', 'txt', 'md', 'html'], - }, mimeType: { type: 'string', description: 'Optional MIME type override when inference is not enough.', @@ -1621,19 +1468,18 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { enum: ['create', 'overwrite'], }, path: { - type: 'string', - description: 'Canonical destination VFS path, e.g. "files/Reports/chart.png".', - }, - sandboxPath: { type: 'string', description: - 'Optional full path to a file created inside the sandbox. Omit to save the code return value.', + 'Persistent destination VFS path starting with "files/", e.g. "files/Reports/chart.png". "uploads/" paths are invalid output targets.', }, }, required: ['path', 'mode'], }, + minItems: 1, + maxItems: 1, }, }, + required: ['files'], }, prompt: { type: 'string', @@ -1665,72 +1511,27 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, inputs: { type: 'object', - description: - 'Workspace resources to mount into the sandbox. Copy paths verbatim from glob/read/grep output — they are percent-encoded per segment (spaces are %20, an in-name slash is %2F; parentheses and dots stay literal). Both the encoded path and the plain name resolve, so copy the returned path exactly rather than retyping or decoding it.', + description: 'Existing media files to use as generation or composition inputs.', properties: { - directories: { - type: 'array', - description: - 'Workspace folders to mount recursively into the sandbox, including nested files and empty folders.', - items: { - type: 'object', - properties: { - path: { - type: 'string', - description: - 'Canonical VFS folder path, e.g. "files/Reports". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Optional full sandbox directory path override. Omit to mount at /home/user/{path}.', - }, - }, - required: ['path'], - }, - }, files: { type: 'array', - description: 'Workspace files to mount into the sandbox.', + description: 'Exactly one start-frame image from files/... or uploads/....', items: { type: 'object', properties: { path: { type: 'string', description: - 'Canonical VFS file path, e.g. "files/Reports/sales.csv". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Full sandbox path to mount at, e.g. /home/user/inputs/data.csv. STRONGLY RECOMMENDED whenever the file name has spaces or special characters: the default mount path is the percent-ENCODED canonical path (e.g. /home/user/files/Q4%20Sales%20(Final).csv), which code using the human-readable name will not find. Set a simple sandboxPath and read exactly that.', + 'Input VFS path copied verbatim from glob/read, using either persistent "files/..." or chat-scoped "uploads/...".', }, }, required: ['path'], }, - }, - tables: { - type: 'array', - description: 'Workspace tables to mount as CSV files.', - items: { - type: 'object', - properties: { - path: { - type: 'string', - description: 'Canonical VFS table path when available.', - }, - sandboxPath: { - type: 'string', - description: 'Optional full sandbox path for the mounted CSV.', - }, - tableId: { - type: 'string', - description: 'Workspace table ID.', - }, - }, - }, + minItems: 1, + maxItems: 1, }, }, + required: ['files'], }, model: { type: 'string', @@ -1756,20 +1557,15 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { outputs: { type: 'object', description: - 'Workspace files to create or overwrite from returned code results or sandbox-created files.', + 'One persistent workspace file to create or overwrite. Chat uploads are read-only inputs and cannot be output targets.', properties: { files: { type: 'array', description: - 'File outputs. Missing parent folders are created automatically for create mode.', + 'Exactly one file output. Parent folders must already exist for create mode.', items: { type: 'object', properties: { - format: { - type: 'string', - description: 'Optional serialization format for returned values.', - enum: ['json', 'csv', 'txt', 'md', 'html'], - }, mimeType: { type: 'string', description: 'Optional MIME type override when inference is not enough.', @@ -1780,19 +1576,18 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { enum: ['create', 'overwrite'], }, path: { - type: 'string', - description: 'Canonical destination VFS path, e.g. "files/Reports/chart.png".', - }, - sandboxPath: { type: 'string', description: - 'Optional full path to a file created inside the sandbox. Omit to save the code return value.', + 'Persistent destination VFS path starting with "files/", e.g. "files/Reports/chart.png". "uploads/" paths are invalid output targets.', }, }, required: ['path', 'mode'], }, + minItems: 1, + maxItems: 1, }, }, + required: ['files'], }, prompt: { type: 'string', diff --git a/apps/sim/lib/copilot/tools/handlers/upload-file-reader.ts b/apps/sim/lib/copilot/tools/handlers/upload-file-reader.ts index 0e914229c8a..2fb29783c4f 100644 --- a/apps/sim/lib/copilot/tools/handlers/upload-file-reader.ts +++ b/apps/sim/lib/copilot/tools/handlers/upload-file-reader.ts @@ -61,6 +61,17 @@ function toWorkspaceFileRecord(row: typeof workspaceFiles.$inferSelect): Workspa } } +/** + * Resolve a chat-scoped upload to the same file-record shape used by workspace files. + */ +export async function resolveChatUpload( + filename: string, + chatId: string +): Promise { + const row = await findMothershipUploadRowByChatAndName(chatId, filename) + return row ? toWorkspaceFileRecord(row) : null +} + /** * Resolve a mothership upload row by VFS name (the collision-disambiguated `displayName` * for new rows, or `originalName` for legacy rows that predate the column). Prefers an diff --git a/apps/sim/lib/copilot/tools/server/image/generate-image.test.ts b/apps/sim/lib/copilot/tools/server/image/generate-image.test.ts new file mode 100644 index 00000000000..2082006e735 --- /dev/null +++ b/apps/sim/lib/copilot/tools/server/image/generate-image.test.ts @@ -0,0 +1,211 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + fetchWorkspaceFileBuffer: vi.fn(), + generateContent: vi.fn(), + resolveChatUpload: vi.fn(), + resolveWorkspaceFileReference: vi.fn(), + validateWorkspaceFileWriteTarget: vi.fn(), + writeWorkspaceFileByPath: vi.fn(), +})) + +vi.mock('@google/genai', () => ({ + GoogleGenAI: class { + models = { generateContent: mocks.generateContent } + }, +})) + +vi.mock('@/lib/core/config/api-keys', () => ({ + getRotatingApiKey: vi.fn(() => 'test-api-key'), +})) + +vi.mock('@/lib/copilot/tools/handlers/upload-file-reader', () => ({ + resolveChatUpload: mocks.resolveChatUpload, +})) + +vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({ + fetchWorkspaceFileBuffer: mocks.fetchWorkspaceFileBuffer, + resolveWorkspaceFileReference: mocks.resolveWorkspaceFileReference, +})) + +vi.mock('@/lib/copilot/vfs/resource-writer', () => ({ + validateWorkspaceFileWriteTarget: mocks.validateWorkspaceFileWriteTarget, + writeWorkspaceFileByPath: mocks.writeWorkspaceFileByPath, +})) + +import { generateImageServerTool } from '@/lib/copilot/tools/server/image/generate-image' + +const CONTEXT = { + userId: 'user-1', + workspaceId: 'workspace-1', + chatId: 'chat-1', +} + +const IMAGE_RECORD = { + id: 'wf_image', + workspaceId: 'workspace-1', + name: 'person.png', + key: 'uploads/person.png', + path: '/api/files/serve/uploads%2Fperson.png', + size: 3, + type: 'image/png', + uploadedBy: 'user-1', + uploadedAt: new Date('2026-01-01T00:00:00Z'), + updatedAt: new Date('2026-01-01T00:00:00Z'), + storageContext: 'mothership' as const, +} + +describe('generateImageServerTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.fetchWorkspaceFileBuffer.mockResolvedValue(Buffer.from('image')) + mocks.validateWorkspaceFileWriteTarget.mockResolvedValue({}) + mocks.generateContent.mockResolvedValue({ + candidates: [ + { content: { parts: [{ inlineData: { data: 'aW1hZ2U=', mimeType: 'image/png' } }] } }, + ], + }) + mocks.writeWorkspaceFileByPath.mockResolvedValue({ + id: 'wf_result', + name: 'result.png', + vfsPath: 'files/result.png', + mode: 'create', + downloadUrl: '/download/result.png', + }) + }) + + it('loads a chat upload when inputs.files uses an uploads/ path', async () => { + mocks.resolveChatUpload.mockResolvedValue(IMAGE_RECORD) + + const result = await generateImageServerTool.execute( + { + prompt: 'Preserve this person and put them in a library', + inputs: { files: [{ path: 'uploads/person.png' }] }, + outputs: { files: [{ path: 'files/result.png' }] }, + }, + CONTEXT + ) + + expect(result.success).toBe(true) + expect(mocks.resolveChatUpload).toHaveBeenCalledWith('person.png', 'chat-1') + expect(mocks.resolveWorkspaceFileReference).not.toHaveBeenCalled() + expect(mocks.fetchWorkspaceFileBuffer).toHaveBeenCalledWith(IMAGE_RECORD) + }) + + it('fails before generation when an input path cannot be resolved', async () => { + mocks.resolveWorkspaceFileReference.mockResolvedValue(null) + + const result = await generateImageServerTool.execute( + { + prompt: 'Edit the reference image', + inputs: { files: [{ path: 'files/missing.png' }] }, + outputs: { files: [{ path: 'files/result.png' }] }, + }, + CONTEXT + ) + + expect(result).toEqual({ + success: false, + message: 'Failed to generate image: Input file not found: files/missing.png', + }) + expect(mocks.generateContent).not.toHaveBeenCalled() + expect(mocks.writeWorkspaceFileByPath).not.toHaveBeenCalled() + }) + + it('fails before generation when outputs.files targets uploads/', async () => { + const result = await generateImageServerTool.execute( + { + prompt: 'Create a portrait', + outputs: { files: [{ path: 'uploads/result.png' }] }, + }, + CONTEXT + ) + + expect(result).toEqual({ + success: false, + message: + 'Failed to generate image: Media output paths must start with "files/"; uploads/ paths are read-only: uploads/result.png', + }) + expect(mocks.generateContent).not.toHaveBeenCalled() + expect(mocks.writeWorkspaceFileByPath).not.toHaveBeenCalled() + }) + + it('fails before generation when an explicit output path is empty', async () => { + mocks.validateWorkspaceFileWriteTarget.mockRejectedValue( + new Error('Workspace file paths must start with "files/"') + ) + + const result = await generateImageServerTool.execute( + { + prompt: 'Create a portrait', + outputs: { files: [{ path: '' }] }, + }, + CONTEXT + ) + + expect(result.success).toBe(false) + expect(mocks.generateContent).not.toHaveBeenCalled() + expect(mocks.writeWorkspaceFileByPath).not.toHaveBeenCalled() + }) + + it('rejects additional output declarations before generation', async () => { + const result = await generateImageServerTool.execute( + { + prompt: 'Create a portrait', + outputs: { + files: [{ path: 'files/one.png' }, { path: 'uploads/two.png' }], + }, + }, + CONTEXT + ) + + expect(result).toEqual({ + success: false, + message: 'Failed to generate image: Output requires exactly one file; received 2', + }) + expect(mocks.validateWorkspaceFileWriteTarget).not.toHaveBeenCalled() + expect(mocks.generateContent).not.toHaveBeenCalled() + }) + + it('rejects explicitly empty input and output containers before generation', async () => { + const emptyInputs = await generateImageServerTool.execute( + { prompt: 'Edit the supplied image', inputs: {} }, + CONTEXT + ) + const emptyOutputs = await generateImageServerTool.execute( + { prompt: 'Create a portrait', outputs: {} }, + CONTEXT + ) + + expect(emptyInputs).toEqual({ + success: false, + message: 'Failed to generate image: Input requires at least one file', + }) + expect(emptyOutputs).toEqual({ + success: false, + message: 'Failed to generate image: Output requires exactly one file; received 0', + }) + expect(mocks.generateContent).not.toHaveBeenCalled() + }) + + it('fails before generation when canonical output preflight rejects a files/ target', async () => { + mocks.validateWorkspaceFileWriteTarget.mockRejectedValue( + new Error('File already exists at files/result.png') + ) + + const result = await generateImageServerTool.execute( + { + prompt: 'Create a portrait', + outputs: { files: [{ path: 'files/result.png' }] }, + }, + CONTEXT + ) + + expect(result.success).toBe(false) + expect(mocks.generateContent).not.toHaveBeenCalled() + expect(mocks.writeWorkspaceFileByPath).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/copilot/tools/server/image/generate-image.ts b/apps/sim/lib/copilot/tools/server/image/generate-image.ts index 3a6c25b8240..b6dc6205c76 100644 --- a/apps/sim/lib/copilot/tools/server/image/generate-image.ts +++ b/apps/sim/lib/copilot/tools/server/image/generate-image.ts @@ -1,18 +1,20 @@ import { GoogleGenAI, type Part } from '@google/genai' import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' +import { getErrorMessage } from '@sim/utils/errors' import { GenerateImage } from '@/lib/copilot/generated/tool-catalog-v1' import { assertServerToolNotAborted, type BaseServerTool, type ServerToolContext, } from '@/lib/copilot/tools/server/base-tool' +import { + prepareMediaOutput, + requireMediaFileDeclarations, + resolveMediaInputFile, +} from '@/lib/copilot/tools/server/media/file-paths' import { writeWorkspaceFileByPath } from '@/lib/copilot/vfs/resource-writer' import { getRotatingApiKey } from '@/lib/core/config/api-keys' -import { - fetchWorkspaceFileBuffer, - resolveWorkspaceFileReference, -} from '@/lib/uploads/contexts/workspace/workspace-file-manager' +import { fetchWorkspaceFileBuffer } from '@/lib/uploads/contexts/workspace/workspace-file-manager' const logger = createLogger('GenerateImageTool') @@ -74,6 +76,12 @@ export const generateImageServerTool: BaseServerTool file.path) ?? [] + const referencePaths = params.inputs + ? requireMediaFileDeclarations(params.inputs.files, 'Input').map((file) => file.path) + : [] if (referencePaths.length) { for (const filePath of referencePaths) { - try { - const fileRecord = await resolveWorkspaceFileReference(workspaceId, filePath) - if (fileRecord) { - const buffer = await fetchWorkspaceFileBuffer(fileRecord) - const base64 = buffer.toString('base64') - const mime = fileRecord.type || 'image/png' - parts.push({ - inlineData: { mimeType: mime, data: base64 }, - }) - logger.info('Loaded reference image', { - filePath, - name: fileRecord.name, - size: buffer.length, - mimeType: mime, - }) - } else { - logger.warn('Reference file not found, skipping', { filePath }) - } - } catch (err) { - logger.warn('Failed to load reference image, skipping', { - filePath, - error: toError(err).message, - }) - } + const fileRecord = await resolveMediaInputFile({ + workspaceId, + chatId: context.chatId, + path: filePath, + }) + const buffer = await fetchWorkspaceFileBuffer(fileRecord) + const base64 = buffer.toString('base64') + const mime = fileRecord.type || 'image/png' + parts.push({ + inlineData: { mimeType: mime, data: base64 }, + }) + logger.info('Loaded reference image', { + filePath, + name: fileRecord.name, + size: buffer.length, + mimeType: mime, + }) } } @@ -161,8 +164,7 @@ export const generateImageServerTool: BaseServerTool = { } try { + const outputFile = await prepareMediaOutput({ + output: params.outputs, + workspaceId, + userId: context.userId, + }) + const mediaFiles: MediaFile[] = [] for (const filePath of inputPaths) { - const fileRecord = await resolveWorkspaceFileReference(workspaceId, filePath) - if (!fileRecord) { - return { success: false, message: `Input file not found: ${filePath}` } - } + const fileRecord = await resolveMediaInputFile({ + workspaceId, + chatId: context.chatId, + path: filePath, + }) const buffer = await fetchWorkspaceFileBuffer(fileRecord) mediaFiles.push({ buffer, @@ -123,7 +131,6 @@ export const ffmpegServerTool: BaseServerTool = { return { success: false, message: `ffmpeg ${params.operation} produced no output` } } - const outputFile = params.outputs?.files?.[0] const outputPath = outputFile?.path || `files/ffmpeg-${params.operation}.${result.ext}` const mode = outputFile?.mode ?? 'create' diff --git a/apps/sim/lib/copilot/tools/server/media/file-paths.test.ts b/apps/sim/lib/copilot/tools/server/media/file-paths.test.ts new file mode 100644 index 00000000000..45d3253ca03 --- /dev/null +++ b/apps/sim/lib/copilot/tools/server/media/file-paths.test.ts @@ -0,0 +1,158 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + resolveChatUpload: vi.fn(), + resolveWorkspaceFileReference: vi.fn(), + validateWorkspaceFileWriteTarget: vi.fn(), +})) + +vi.mock('@/lib/copilot/tools/handlers/upload-file-reader', () => ({ + resolveChatUpload: mocks.resolveChatUpload, +})) + +vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({ + resolveWorkspaceFileReference: mocks.resolveWorkspaceFileReference, +})) + +vi.mock('@/lib/copilot/vfs/resource-writer', () => ({ + validateWorkspaceFileWriteTarget: mocks.validateWorkspaceFileWriteTarget, +})) + +import { + getSingleMediaFileDeclaration, + resolveMediaInputFile, + validateMediaOutputFile, +} from '@/lib/copilot/tools/server/media/file-paths' + +const FILE_RECORD = { id: 'wf_file', name: 'portrait.png' } + +describe('media file paths', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.validateWorkspaceFileWriteTarget.mockResolvedValue({}) + }) + + it('resolves uploads/ inputs within the current chat', async () => { + mocks.resolveChatUpload.mockResolvedValue(FILE_RECORD) + + await expect( + resolveMediaInputFile({ + workspaceId: 'workspace-1', + chatId: 'chat-1', + path: 'uploads/My%20Portrait.png', + }) + ).resolves.toBe(FILE_RECORD) + + expect(mocks.resolveChatUpload).toHaveBeenCalledWith('My%20Portrait.png', 'chat-1') + expect(mocks.resolveWorkspaceFileReference).not.toHaveBeenCalled() + }) + + it('rejects unresolved inputs instead of dropping them', async () => { + mocks.resolveWorkspaceFileReference.mockResolvedValue(null) + + await expect( + resolveMediaInputFile({ + workspaceId: 'workspace-1', + chatId: 'chat-1', + path: 'files/missing.png', + }) + ).rejects.toThrow('Input file not found: files/missing.png') + }) + + it('preserves legacy workspace IDs and plain-name references', async () => { + mocks.resolveWorkspaceFileReference.mockResolvedValue(FILE_RECORD) + + await expect( + resolveMediaInputFile({ + workspaceId: 'workspace-1', + chatId: 'chat-1', + path: 'portrait.png', + }) + ).resolves.toBe(FILE_RECORD) + + await resolveMediaInputFile({ + workspaceId: 'workspace-1', + chatId: 'chat-1', + path: 'wf_file', + }) + + expect(mocks.resolveWorkspaceFileReference).toHaveBeenNthCalledWith( + 1, + 'workspace-1', + 'portrait.png' + ) + expect(mocks.resolveWorkspaceFileReference).toHaveBeenNthCalledWith(2, 'workspace-1', 'wf_file') + }) + + it('preflights outputs through the canonical workspace writer policy', async () => { + await validateMediaOutputFile({ + workspaceId: 'workspace-1', + userId: 'user-1', + path: 'files/Campaign/portrait.png', + mode: 'overwrite', + mimeType: 'image/png', + }) + + expect(mocks.validateWorkspaceFileWriteTarget).toHaveBeenCalledWith({ + workspaceId: 'workspace-1', + userId: 'user-1', + target: { + path: 'files/Campaign/portrait.png', + mode: 'overwrite', + mimeType: 'image/png', + }, + }) + }) + + it('rejects uploads/ output paths before canonical resolution', async () => { + await expect( + validateMediaOutputFile({ + workspaceId: 'workspace-1', + userId: 'user-1', + path: 'uploads/portrait.png', + mode: 'create', + }) + ).rejects.toThrow('Media output paths must start with "files/"') + expect(mocks.validateWorkspaceFileWriteTarget).not.toHaveBeenCalled() + }) + + it('propagates canonical writer validation errors for files/ paths', async () => { + mocks.validateWorkspaceFileWriteTarget.mockRejectedValue( + new Error('File already exists at files/portrait.png') + ) + + await expect( + validateMediaOutputFile({ + workspaceId: 'workspace-1', + userId: 'user-1', + path: 'files/portrait.png', + mode: 'create', + }) + ).rejects.toThrow('File already exists at files/portrait.png') + expect(mocks.validateWorkspaceFileWriteTarget).toHaveBeenCalledOnce() + }) + + it('rejects non-file output namespaces before canonical resolution', async () => { + await expect( + validateMediaOutputFile({ + workspaceId: 'workspace-1', + userId: 'user-1', + path: 'workflows/My%20Flow/state.json', + mode: 'overwrite', + }) + ).rejects.toThrow('Media output paths must start with "files/"') + expect(mocks.validateWorkspaceFileWriteTarget).not.toHaveBeenCalled() + }) + + it('rejects extra declarations for single-file media parameters', () => { + expect(() => + getSingleMediaFileDeclaration([{ path: 'files/a.png' }, { path: 'files/b.png' }], 'Output') + ).toThrow('Output requires exactly one file; received 2') + expect(() => getSingleMediaFileDeclaration([], 'Output')).toThrow( + 'Output requires exactly one file; received 0' + ) + }) +}) diff --git a/apps/sim/lib/copilot/tools/server/media/file-paths.ts b/apps/sim/lib/copilot/tools/server/media/file-paths.ts new file mode 100644 index 00000000000..69b661aedc1 --- /dev/null +++ b/apps/sim/lib/copilot/tools/server/media/file-paths.ts @@ -0,0 +1,109 @@ +import { resolveChatUpload } from '@/lib/copilot/tools/handlers/upload-file-reader' +import { + validateWorkspaceFileWriteTarget, + type WorkspaceFileWriteMode, +} from '@/lib/copilot/vfs/resource-writer' +import { + resolveWorkspaceFileReference, + type WorkspaceFileRecord, +} from '@/lib/uploads/contexts/workspace/workspace-file-manager' + +const UPLOADS_PREFIX = 'uploads/' +const FILES_PREFIX = 'files/' + +interface MediaOutputDeclaration { + path: string + mode?: WorkspaceFileWriteMode + mimeType?: string +} + +export function getSingleMediaFileDeclaration( + files: T[] | undefined, + label: 'Input' | 'Output' +): T { + if (files?.length !== 1) { + throw new Error(`${label} requires exactly one file; received ${files?.length ?? 0}`) + } + return files[0] +} + +export function requireMediaFileDeclarations( + files: T[] | undefined, + label: 'Input' | 'Output' +): T[] { + if (!files?.length) { + throw new Error(`${label} requires at least one file`) + } + return files +} + +/** + * Resolve a media input from either persistent workspace files or chat-scoped uploads. + */ +export async function resolveMediaInputFile(args: { + workspaceId: string + chatId?: string + path: string +}): Promise { + const path = args.path.trim() + let file: WorkspaceFileRecord | null + + if (path.startsWith(UPLOADS_PREFIX)) { + const filename = path.slice(UPLOADS_PREFIX.length) + if (!filename || filename.includes('/')) { + throw new Error(`Upload input path must identify a file: ${args.path}`) + } + if (!args.chatId) { + throw new Error(`Chat context is required for upload input: ${path}`) + } + file = await resolveChatUpload(filename, args.chatId) + } else { + file = await resolveWorkspaceFileReference(args.workspaceId, path) + } + + if (!file) { + throw new Error(`Input file not found: ${args.path}`) + } + return file +} + +/** + * Preflight a user-supplied media output through the canonical workspace writer policy. + */ +export async function validateMediaOutputFile(args: { + workspaceId: string + userId: string + path: string + mode: WorkspaceFileWriteMode + mimeType?: string +}): Promise { + if (!args.path.trim().startsWith(FILES_PREFIX)) { + throw new Error( + `Media output paths must start with "files/"; uploads/ paths are read-only: ${args.path}` + ) + } + await validateWorkspaceFileWriteTarget({ + workspaceId: args.workspaceId, + userId: args.userId, + target: { path: args.path, mode: args.mode, mimeType: args.mimeType }, + }) +} + +export async function prepareMediaOutput(args: { + output?: { files?: T[] } + workspaceId: string + userId: string +}): Promise<(T & { mode: WorkspaceFileWriteMode }) | undefined> { + if (!args.output) return undefined + + const file = getSingleMediaFileDeclaration(args.output.files, 'Output') + const output = { ...file, mode: file.mode ?? 'create' } + await validateMediaOutputFile({ + workspaceId: args.workspaceId, + userId: args.userId, + path: output.path, + mode: output.mode, + mimeType: output.mimeType, + }) + return output +} diff --git a/apps/sim/lib/copilot/tools/server/media/generate-audio.ts b/apps/sim/lib/copilot/tools/server/media/generate-audio.ts index 6d224686397..c3f430137c3 100644 --- a/apps/sim/lib/copilot/tools/server/media/generate-audio.ts +++ b/apps/sim/lib/copilot/tools/server/media/generate-audio.ts @@ -6,12 +6,14 @@ import { type BaseServerTool, type ServerToolContext, } from '@/lib/copilot/tools/server/base-tool' +import { + getSingleMediaFileDeclaration, + prepareMediaOutput, + resolveMediaInputFile, +} from '@/lib/copilot/tools/server/media/file-paths' import { writeWorkspaceFileByPath } from '@/lib/copilot/vfs/resource-writer' import { type AudioType, generateFalAudio } from '@/lib/media/falai-audio' -import { - fetchWorkspaceFileBuffer, - resolveWorkspaceFileReference, -} from '@/lib/uploads/contexts/workspace/workspace-file-manager' +import { fetchWorkspaceFileBuffer } from '@/lib/uploads/contexts/workspace/workspace-file-manager' const logger = createLogger('GenerateAudioTool') @@ -80,20 +82,29 @@ export const generateAudioServerTool: BaseServerTool ({ + fetchWorkspaceFileBuffer: vi.fn(), + generateFalAudio: vi.fn(), + generateFalVideo: vi.fn(), + resolveChatUpload: vi.fn(), + resolveWorkspaceFileReference: vi.fn(), + runFfmpegOperation: vi.fn(), + validateWorkspaceFileWriteTarget: vi.fn(), + writeWorkspaceFileByPath: vi.fn(), +})) + +vi.mock('@/lib/media/falai-audio', () => ({ generateFalAudio: mocks.generateFalAudio })) +vi.mock('@/lib/media/falai-video', () => ({ generateFalVideo: mocks.generateFalVideo })) +vi.mock('@/lib/media/ffmpeg', () => ({ runFfmpegOperation: mocks.runFfmpegOperation })) +vi.mock('@/lib/copilot/tools/handlers/upload-file-reader', () => ({ + resolveChatUpload: mocks.resolveChatUpload, +})) +vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({ + fetchWorkspaceFileBuffer: mocks.fetchWorkspaceFileBuffer, + resolveWorkspaceFileReference: mocks.resolveWorkspaceFileReference, +})) +vi.mock('@/lib/copilot/vfs/resource-writer', () => ({ + validateWorkspaceFileWriteTarget: mocks.validateWorkspaceFileWriteTarget, + writeWorkspaceFileByPath: mocks.writeWorkspaceFileByPath, +})) + +import { ffmpegServerTool } from '@/lib/copilot/tools/server/media/ffmpeg' +import { generateAudioServerTool } from '@/lib/copilot/tools/server/media/generate-audio' +import { generateVideoServerTool } from '@/lib/copilot/tools/server/media/generate-video' + +const CONTEXT = { userId: 'user-1', workspaceId: 'workspace-1', chatId: 'chat-1' } +const UPLOAD_RECORD = { + id: 'wf_upload', + name: 'start.png', + type: 'image/png', + storageContext: 'mothership', +} + +describe('media tool path validation', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.validateWorkspaceFileWriteTarget.mockResolvedValue({}) + mocks.fetchWorkspaceFileBuffer.mockResolvedValue(Buffer.from('media')) + mocks.generateFalVideo.mockResolvedValue({ + buffer: Buffer.from('video'), + contentType: 'video/mp4', + model: 'veo-3.1-fast', + cost: { costDollars: 0.1 }, + }) + mocks.generateFalAudio.mockResolvedValue({ + buffer: Buffer.from('audio'), + contentType: 'audio/mpeg', + model: 'fal-ai/f5-tts', + cost: { costDollars: 0.05 }, + }) + mocks.runFfmpegOperation.mockResolvedValue({ + buffer: Buffer.from('converted'), + contentType: 'video/mp4', + ext: 'mp4', + }) + mocks.writeWorkspaceFileByPath.mockResolvedValue({ + id: 'wf_output', + name: 'output.mp4', + vfsPath: 'files/output.mp4', + mode: 'create', + }) + }) + + it('passes uploads/ start frames to video generation', async () => { + mocks.resolveChatUpload.mockResolvedValue(UPLOAD_RECORD) + + const result = await generateVideoServerTool.execute( + { + prompt: 'Animate this portrait', + inputs: { files: [{ path: 'uploads/start.png' }] }, + outputs: { files: [{ path: 'files/output.mp4' }] }, + }, + CONTEXT + ) + + expect(result.success).toBe(true) + expect(mocks.resolveChatUpload).toHaveBeenCalledWith('start.png', 'chat-1') + expect(mocks.generateFalVideo).toHaveBeenCalledWith( + expect.objectContaining({ imageDataUri: 'data:image/png;base64,bWVkaWE=' }) + ) + }) + + it('rejects empty audio reference paths before calling the provider', async () => { + mocks.resolveWorkspaceFileReference.mockResolvedValue(null) + + const result = await generateAudioServerTool.execute( + { prompt: 'Clone this voice', inputs: { files: [{ path: '' }] } }, + CONTEXT + ) + + expect(result.success).toBe(false) + expect(mocks.generateFalAudio).not.toHaveBeenCalled() + }) + + it('passes uploads/ voice samples to audio generation', async () => { + mocks.resolveChatUpload.mockResolvedValue({ + ...UPLOAD_RECORD, + name: 'voice.wav', + type: 'audio/wav', + }) + + const result = await generateAudioServerTool.execute( + { + prompt: 'Read this line in the supplied voice', + inputs: { files: [{ path: 'uploads/voice.wav' }] }, + outputs: { files: [{ path: 'files/voice.mp3' }] }, + }, + CONTEXT + ) + + expect(result.success).toBe(true) + expect(mocks.resolveChatUpload).toHaveBeenCalledWith('voice.wav', 'chat-1') + expect(mocks.generateFalAudio).toHaveBeenCalledWith( + expect.objectContaining({ voiceSampleDataUri: 'data:audio/wav;base64,bWVkaWE=' }) + ) + }) + + it('rejects invalid ffmpeg output paths before processing inputs', async () => { + const result = await ffmpegServerTool.execute( + { + operation: 'convert', + inputs: { files: [{ path: 'files/input.mov' }] }, + outputs: { files: [{ path: 'uploads/output.mp4' }] }, + }, + CONTEXT + ) + + expect(result.success).toBe(false) + expect(mocks.resolveWorkspaceFileReference).not.toHaveBeenCalled() + expect(mocks.runFfmpegOperation).not.toHaveBeenCalled() + }) + + it('passes uploads/ media files to ffmpeg', async () => { + mocks.resolveChatUpload.mockResolvedValue(UPLOAD_RECORD) + + const result = await ffmpegServerTool.execute( + { + operation: 'convert', + inputs: { files: [{ path: 'uploads/start.png' }] }, + outputs: { files: [{ path: 'files/output.mp4' }] }, + }, + CONTEXT + ) + + expect(result.success).toBe(true) + expect(mocks.resolveChatUpload).toHaveBeenCalledWith('start.png', 'chat-1') + expect(mocks.runFfmpegOperation).toHaveBeenCalledWith( + 'convert', + [expect.objectContaining({ buffer: Buffer.from('media'), name: 'start.png' })], + expect.any(Object) + ) + }) + + it('rejects extra video inputs before calling the provider', async () => { + const result = await generateVideoServerTool.execute( + { + prompt: 'Animate these portraits', + inputs: { files: [{ path: 'files/one.png' }, { path: 'files/two.png' }] }, + }, + CONTEXT + ) + + expect(result.success).toBe(false) + expect(mocks.generateFalVideo).not.toHaveBeenCalled() + }) +}) From 7c321ca929599f18b46f5f33af7c45a24fd26fa9 Mon Sep 17 00:00:00 2001 From: Justin Blumencranz <96924014+j15z@users.noreply.github.com> Date: Sat, 11 Jul 2026 19:42:32 -0700 Subject: [PATCH 2/2] fix(media): preflight all writable outputs --- .../lib/copilot/generated/tool-catalog-v1.ts | 6 ++-- .../lib/copilot/generated/tool-schemas-v1.ts | 6 ++-- .../tools/server/image/generate-image.test.ts | 6 +++- .../tools/server/image/generate-image.ts | 7 ++-- .../lib/copilot/tools/server/media/ffmpeg.ts | 24 ++++++++----- .../tools/server/media/file-paths.test.ts | 26 ++++++++++++++ .../copilot/tools/server/media/file-paths.ts | 12 +++---- .../tools/server/media/generate-audio.ts | 17 ++------- .../tools/server/media/generate-video.ts | 6 ++-- .../media/media-tools-validation.test.ts | 35 ++++++++++++++++++- 10 files changed, 100 insertions(+), 45 deletions(-) diff --git a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts index 6e281299c5d..b5a77411d5b 100644 --- a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts @@ -1589,7 +1589,7 @@ export const GenerateAudio: ToolCatalogEntry = { }, voice: { type: 'string', description: 'Optional voice name or id for speech.' }, }, - required: ['prompt'], + required: ['prompt', 'outputs'], }, requiredPermission: 'write', capabilities: ['file_input', 'file_output', 'generated_media'], @@ -1672,7 +1672,7 @@ export const GenerateImage: ToolCatalogEntry = { 'Detailed text description of the image to generate, or editing instructions when editing the image(s) passed in `inputs.files`.', }, }, - required: ['prompt'], + required: ['prompt', 'outputs'], }, requiredPermission: 'write', capabilities: ['file_input', 'file_output', 'generated_media'], @@ -1795,7 +1795,7 @@ export const GenerateVideo: ToolCatalogEntry = { enum: ['720p', '1080p', '4k'], }, }, - required: ['prompt'], + required: ['prompt', 'outputs'], }, requiredPermission: 'write', capabilities: ['file_input', 'file_output', 'generated_media'], diff --git a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts index cb5a3092eb1..d41e0109d5b 100644 --- a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts @@ -1410,7 +1410,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { description: 'Optional voice name or id for speech.', }, }, - required: ['prompt'], + required: ['prompt', 'outputs'], }, resultSchema: undefined, }, @@ -1487,7 +1487,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { 'Detailed text description of the image to generate, or editing instructions when editing the image(s) passed in `inputs.files`.', }, }, - required: ['prompt'], + required: ['prompt', 'outputs'], }, resultSchema: undefined, }, @@ -1604,7 +1604,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { enum: ['720p', '1080p', '4k'], }, }, - required: ['prompt'], + required: ['prompt', 'outputs'], }, resultSchema: undefined, }, diff --git a/apps/sim/lib/copilot/tools/server/image/generate-image.test.ts b/apps/sim/lib/copilot/tools/server/image/generate-image.test.ts index 2082006e735..d5e7e5aed8a 100644 --- a/apps/sim/lib/copilot/tools/server/image/generate-image.test.ts +++ b/apps/sim/lib/copilot/tools/server/image/generate-image.test.ts @@ -172,7 +172,11 @@ describe('generateImageServerTool', () => { it('rejects explicitly empty input and output containers before generation', async () => { const emptyInputs = await generateImageServerTool.execute( - { prompt: 'Edit the supplied image', inputs: {} }, + { + prompt: 'Edit the supplied image', + inputs: {}, + outputs: { files: [{ path: 'files/result.png' }] }, + }, CONTEXT ) const emptyOutputs = await generateImageServerTool.execute( diff --git a/apps/sim/lib/copilot/tools/server/image/generate-image.ts b/apps/sim/lib/copilot/tools/server/image/generate-image.ts index b6dc6205c76..0cda9700a22 100644 --- a/apps/sim/lib/copilot/tools/server/image/generate-image.ts +++ b/apps/sim/lib/copilot/tools/server/image/generate-image.ts @@ -163,10 +163,9 @@ export const generateImageServerTool: BaseServerTool = { } try { - const outputFile = await prepareMediaOutput({ - output: params.outputs, - workspaceId, - userId: context.userId, - }) + const outputFile = + params.operation === 'probe' + ? undefined + : await prepareMediaOutput({ + output: params.outputs, + workspaceId, + userId: context.userId, + }) const mediaFiles: MediaFile[] = [] for (const filePath of inputPaths) { @@ -127,18 +130,21 @@ export const ffmpegServerTool: BaseServerTool = { } } - if (!result.buffer || !result.ext) { + if (!result.buffer) { return { success: false, message: `ffmpeg ${params.operation} produced no output` } } + if (!outputFile) { + return { success: false, message: `ffmpeg ${params.operation} requires an output file` } + } - const outputPath = outputFile?.path || `files/ffmpeg-${params.operation}.${result.ext}` - const mode = outputFile?.mode ?? 'create' + const outputPath = outputFile.path + const mode = outputFile.mode assertServerToolNotAborted(context) const written = await writeWorkspaceFileByPath({ workspaceId, userId: context.userId, - target: { path: outputPath, mode, mimeType: outputFile?.mimeType }, + target: { path: outputPath, mode, mimeType: outputFile.mimeType }, buffer: result.buffer, inferredMimeType: result.contentType || 'application/octet-stream', }) diff --git a/apps/sim/lib/copilot/tools/server/media/file-paths.test.ts b/apps/sim/lib/copilot/tools/server/media/file-paths.test.ts index 45d3253ca03..f757e960f0a 100644 --- a/apps/sim/lib/copilot/tools/server/media/file-paths.test.ts +++ b/apps/sim/lib/copilot/tools/server/media/file-paths.test.ts @@ -23,6 +23,7 @@ vi.mock('@/lib/copilot/vfs/resource-writer', () => ({ import { getSingleMediaFileDeclaration, + prepareMediaOutput, resolveMediaInputFile, validateMediaOutputFile, } from '@/lib/copilot/tools/server/media/file-paths' @@ -50,6 +51,20 @@ describe('media file paths', () => { expect(mocks.resolveWorkspaceFileReference).not.toHaveBeenCalled() }) + it('resolves uploads/ inputs with the read-compatible /content suffix', async () => { + mocks.resolveChatUpload.mockResolvedValue(FILE_RECORD) + + await expect( + resolveMediaInputFile({ + workspaceId: 'workspace-1', + chatId: 'chat-1', + path: 'uploads/My%20Portrait.png/content', + }) + ).resolves.toBe(FILE_RECORD) + + expect(mocks.resolveChatUpload).toHaveBeenCalledWith('My%20Portrait.png', 'chat-1') + }) + it('rejects unresolved inputs instead of dropping them', async () => { mocks.resolveWorkspaceFileReference.mockResolvedValue(null) @@ -155,4 +170,15 @@ describe('media file paths', () => { 'Output requires exactly one file; received 0' ) }) + + it('requires an explicit output before provider work can begin', async () => { + await expect( + prepareMediaOutput({ + workspaceId: 'workspace-1', + userId: 'user-1', + }) + ).rejects.toThrow('Output requires exactly one file; received 0') + + expect(mocks.validateWorkspaceFileWriteTarget).not.toHaveBeenCalled() + }) }) diff --git a/apps/sim/lib/copilot/tools/server/media/file-paths.ts b/apps/sim/lib/copilot/tools/server/media/file-paths.ts index 69b661aedc1..ec07aab70e6 100644 --- a/apps/sim/lib/copilot/tools/server/media/file-paths.ts +++ b/apps/sim/lib/copilot/tools/server/media/file-paths.ts @@ -49,8 +49,8 @@ export async function resolveMediaInputFile(args: { let file: WorkspaceFileRecord | null if (path.startsWith(UPLOADS_PREFIX)) { - const filename = path.slice(UPLOADS_PREFIX.length) - if (!filename || filename.includes('/')) { + const filename = path.slice(UPLOADS_PREFIX.length).split('/')[0] + if (!filename) { throw new Error(`Upload input path must identify a file: ${args.path}`) } if (!args.chatId) { @@ -68,7 +68,7 @@ export async function resolveMediaInputFile(args: { } /** - * Preflight a user-supplied media output through the canonical workspace writer policy. + * Preflight a media output through the canonical workspace writer policy. */ export async function validateMediaOutputFile(args: { workspaceId: string @@ -93,10 +93,8 @@ export async function prepareMediaOutput(args: output?: { files?: T[] } workspaceId: string userId: string -}): Promise<(T & { mode: WorkspaceFileWriteMode }) | undefined> { - if (!args.output) return undefined - - const file = getSingleMediaFileDeclaration(args.output.files, 'Output') +}): Promise { + const file = getSingleMediaFileDeclaration(args.output?.files, 'Output') const output = { ...file, mode: file.mode ?? 'create' } await validateMediaOutputFile({ workspaceId: args.workspaceId, diff --git a/apps/sim/lib/copilot/tools/server/media/generate-audio.ts b/apps/sim/lib/copilot/tools/server/media/generate-audio.ts index c3f430137c3..5b000d67c16 100644 --- a/apps/sim/lib/copilot/tools/server/media/generate-audio.ts +++ b/apps/sim/lib/copilot/tools/server/media/generate-audio.ts @@ -46,16 +46,6 @@ interface GenerateAudioResult { _serviceCost?: { service: string; cost: number } } -function audioExtFromContentType(contentType: string): string { - if (contentType.includes('wav')) return 'wav' - if (contentType.includes('mp4') || contentType.includes('m4a')) return 'm4a' - if (contentType.includes('ogg')) return 'ogg' - if (contentType.includes('flac')) return 'flac' - if (contentType.includes('aac')) return 'aac' - if (contentType.includes('opus')) return 'opus' - return 'mp3' -} - export const generateAudioServerTool: BaseServerTool = { name: GenerateAudio.id, @@ -123,15 +113,14 @@ export const generateAudioServerTool: BaseServerTool { mocks.resolveWorkspaceFileReference.mockResolvedValue(null) const result = await generateAudioServerTool.execute( - { prompt: 'Clone this voice', inputs: { files: [{ path: '' }] } }, + { + prompt: 'Clone this voice', + inputs: { files: [{ path: '' }] }, + outputs: { files: [{ path: 'files/voice.mp3' }] }, + }, CONTEXT ) @@ -140,6 +144,34 @@ describe('media tool path validation', () => { expect(mocks.runFfmpegOperation).not.toHaveBeenCalled() }) + it('ignores unused output declarations for ffmpeg probes', async () => { + mocks.resolveWorkspaceFileReference.mockResolvedValue(UPLOAD_RECORD) + mocks.runFfmpegOperation.mockResolvedValue({ probe: { duration: 3 } }) + + const result = await ffmpegServerTool.execute( + { + operation: 'probe', + inputs: { files: [{ path: 'files/input.mov' }] }, + outputs: { files: [{ path: 'uploads/unused.json' }] }, + }, + CONTEXT + ) + + expect(result.success).toBe(true) + expect(mocks.validateWorkspaceFileWriteTarget).not.toHaveBeenCalled() + expect(mocks.writeWorkspaceFileByPath).not.toHaveBeenCalled() + }) + + it('rejects missing generated-media outputs before calling the provider', async () => { + const result = await generateVideoServerTool.execute( + { prompt: 'Create a launch video' }, + CONTEXT + ) + + expect(result.success).toBe(false) + expect(mocks.generateFalVideo).not.toHaveBeenCalled() + }) + it('passes uploads/ media files to ffmpeg', async () => { mocks.resolveChatUpload.mockResolvedValue(UPLOAD_RECORD) @@ -166,6 +198,7 @@ describe('media tool path validation', () => { { prompt: 'Animate these portraits', inputs: { files: [{ path: 'files/one.png' }, { path: 'files/two.png' }] }, + outputs: { files: [{ path: 'files/output.mp4' }] }, }, CONTEXT )