diff --git a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts index 9c91d02ef64..b5a77411d5b 100644 --- a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts @@ -1208,66 +1208,27 @@ export const Ffmpeg: ToolCatalogEntry = { height: { type: 'number', description: 'Target height in pixels (scale_pad).' }, inputs: { type: 'object', - description: - 'Workspace resources to mount into the sandbox. Copy paths verbatim from glob/read/grep output — they are percent-encoded per segment (spaces are %20, an in-name slash is %2F; parentheses and dots stay literal). Both the encoded path and the plain name resolve, so copy the returned path exactly rather than retyping or decoding it.', + description: 'Existing media files to use as generation or composition inputs.', properties: { - directories: { - type: 'array', - description: - 'Workspace folders to mount recursively into the sandbox, including nested files and empty folders.', - items: { - type: 'object', - properties: { - path: { - type: 'string', - description: - 'Canonical VFS folder path, e.g. "files/Reports". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Optional full sandbox directory path override. Omit to mount at /home/user/{path}.', - }, - }, - required: ['path'], - }, - }, files: { type: 'array', - description: 'Workspace files to mount into the sandbox.', + description: + 'One or more media inputs from files/... or uploads/..., in operation order.', items: { type: 'object', properties: { path: { type: 'string', description: - 'Canonical VFS file path, e.g. "files/Reports/sales.csv". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Full sandbox path to mount at, e.g. /home/user/inputs/data.csv. STRONGLY RECOMMENDED whenever the file name has spaces or special characters: the default mount path is the percent-ENCODED canonical path (e.g. /home/user/files/Q4%20Sales%20(Final).csv), which code using the human-readable name will not find. Set a simple sandboxPath and read exactly that.', + 'Input VFS path copied verbatim from glob/read, using either persistent "files/..." or chat-scoped "uploads/...".', }, }, required: ['path'], }, - }, - tables: { - type: 'array', - description: 'Workspace tables to mount as CSV files.', - items: { - type: 'object', - properties: { - path: { type: 'string', description: 'Canonical VFS table path when available.' }, - sandboxPath: { - type: 'string', - description: 'Optional full sandbox path for the mounted CSV.', - }, - tableId: { type: 'string', description: 'Workspace table ID.' }, - }, - }, + minItems: 1, }, }, + required: ['files'], }, loopToVideo: { type: 'boolean', @@ -1298,20 +1259,15 @@ export const Ffmpeg: ToolCatalogEntry = { outputs: { type: 'object', description: - 'Workspace files to create or overwrite from returned code results or sandbox-created files.', + 'One persistent workspace file to create or overwrite. Chat uploads are read-only inputs and cannot be output targets.', properties: { files: { type: 'array', description: - 'File outputs. Missing parent folders are created automatically for create mode.', + 'Exactly one file output. Parent folders must already exist for create mode.', items: { type: 'object', properties: { - format: { - type: 'string', - description: 'Optional serialization format for returned values.', - enum: ['json', 'csv', 'txt', 'md', 'html'], - }, mimeType: { type: 'string', description: 'Optional MIME type override when inference is not enough.', @@ -1322,19 +1278,18 @@ export const Ffmpeg: ToolCatalogEntry = { enum: ['create', 'overwrite'], }, path: { - type: 'string', - description: 'Canonical destination VFS path, e.g. "files/Reports/chart.png".', - }, - sandboxPath: { type: 'string', description: - 'Optional full path to a file created inside the sandbox. Omit to save the code return value.', + 'Persistent destination VFS path starting with "files/", e.g. "files/Reports/chart.png". "uploads/" paths are invalid output targets.', }, }, required: ['path', 'mode'], }, + minItems: 1, + maxItems: 1, }, }, + required: ['files'], }, position: { type: 'string', @@ -1550,66 +1505,27 @@ export const GenerateAudio: ToolCatalogEntry = { }, inputs: { type: 'object', - description: - 'Workspace resources to mount into the sandbox. Copy paths verbatim from glob/read/grep output — they are percent-encoded per segment (spaces are %20, an in-name slash is %2F; parentheses and dots stay literal). Both the encoded path and the plain name resolve, so copy the returned path exactly rather than retyping or decoding it.', + description: 'Existing media files to use as generation or composition inputs.', properties: { - directories: { - type: 'array', - description: - 'Workspace folders to mount recursively into the sandbox, including nested files and empty folders.', - items: { - type: 'object', - properties: { - path: { - type: 'string', - description: - 'Canonical VFS folder path, e.g. "files/Reports". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Optional full sandbox directory path override. Omit to mount at /home/user/{path}.', - }, - }, - required: ['path'], - }, - }, files: { type: 'array', - description: 'Workspace files to mount into the sandbox.', + description: 'Exactly one reference voice sample from files/... or uploads/....', items: { type: 'object', properties: { path: { type: 'string', description: - 'Canonical VFS file path, e.g. "files/Reports/sales.csv". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Full sandbox path to mount at, e.g. /home/user/inputs/data.csv. STRONGLY RECOMMENDED whenever the file name has spaces or special characters: the default mount path is the percent-ENCODED canonical path (e.g. /home/user/files/Q4%20Sales%20(Final).csv), which code using the human-readable name will not find. Set a simple sandboxPath and read exactly that.', + 'Input VFS path copied verbatim from glob/read, using either persistent "files/..." or chat-scoped "uploads/...".', }, }, required: ['path'], }, - }, - tables: { - type: 'array', - description: 'Workspace tables to mount as CSV files.', - items: { - type: 'object', - properties: { - path: { type: 'string', description: 'Canonical VFS table path when available.' }, - sandboxPath: { - type: 'string', - description: 'Optional full sandbox path for the mounted CSV.', - }, - tableId: { type: 'string', description: 'Workspace table ID.' }, - }, - }, + minItems: 1, + maxItems: 1, }, }, + required: ['files'], }, instrumental: { type: 'boolean', @@ -1629,20 +1545,15 @@ export const GenerateAudio: ToolCatalogEntry = { outputs: { type: 'object', description: - 'Workspace files to create or overwrite from returned code results or sandbox-created files.', + 'One persistent workspace file to create or overwrite. Chat uploads are read-only inputs and cannot be output targets.', properties: { files: { type: 'array', description: - 'File outputs. Missing parent folders are created automatically for create mode.', + 'Exactly one file output. Parent folders must already exist for create mode.', items: { type: 'object', properties: { - format: { - type: 'string', - description: 'Optional serialization format for returned values.', - enum: ['json', 'csv', 'txt', 'md', 'html'], - }, mimeType: { type: 'string', description: 'Optional MIME type override when inference is not enough.', @@ -1653,19 +1564,18 @@ export const GenerateAudio: ToolCatalogEntry = { enum: ['create', 'overwrite'], }, path: { - type: 'string', - description: 'Canonical destination VFS path, e.g. "files/Reports/chart.png".', - }, - sandboxPath: { type: 'string', description: - 'Optional full path to a file created inside the sandbox. Omit to save the code return value.', + 'Persistent destination VFS path starting with "files/", e.g. "files/Reports/chart.png". "uploads/" paths are invalid output targets.', }, }, required: ['path', 'mode'], }, + minItems: 1, + maxItems: 1, }, }, + required: ['files'], }, prompt: { type: 'string', @@ -1679,7 +1589,7 @@ export const GenerateAudio: ToolCatalogEntry = { }, voice: { type: 'string', description: 'Optional voice name or id for speech.' }, }, - required: ['prompt'], + required: ['prompt', 'outputs'], }, requiredPermission: 'write', capabilities: ['file_input', 'file_output', 'generated_media'], @@ -1700,84 +1610,39 @@ export const GenerateImage: ToolCatalogEntry = { }, inputs: { type: 'object', - description: - 'Workspace resources to mount into the sandbox. Copy paths verbatim from glob/read/grep output — they are percent-encoded per segment (spaces are %20, an in-name slash is %2F; parentheses and dots stay literal). Both the encoded path and the plain name resolve, so copy the returned path exactly rather than retyping or decoding it.', + description: 'Existing media files to use as generation or composition inputs.', properties: { - directories: { - type: 'array', - description: - 'Workspace folders to mount recursively into the sandbox, including nested files and empty folders.', - items: { - type: 'object', - properties: { - path: { - type: 'string', - description: - 'Canonical VFS folder path, e.g. "files/Reports". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Optional full sandbox directory path override. Omit to mount at /home/user/{path}.', - }, - }, - required: ['path'], - }, - }, files: { type: 'array', - description: 'Workspace files to mount into the sandbox.', + description: 'One or more reference images to edit, combine, or restyle.', items: { type: 'object', properties: { path: { type: 'string', description: - 'Canonical VFS file path, e.g. "files/Reports/sales.csv". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Full sandbox path to mount at, e.g. /home/user/inputs/data.csv. STRONGLY RECOMMENDED whenever the file name has spaces or special characters: the default mount path is the percent-ENCODED canonical path (e.g. /home/user/files/Q4%20Sales%20(Final).csv), which code using the human-readable name will not find. Set a simple sandboxPath and read exactly that.', + 'Input VFS path copied verbatim from glob/read, using either persistent "files/..." or chat-scoped "uploads/...".', }, }, required: ['path'], }, - }, - tables: { - type: 'array', - description: 'Workspace tables to mount as CSV files.', - items: { - type: 'object', - properties: { - path: { type: 'string', description: 'Canonical VFS table path when available.' }, - sandboxPath: { - type: 'string', - description: 'Optional full sandbox path for the mounted CSV.', - }, - tableId: { type: 'string', description: 'Workspace table ID.' }, - }, - }, + minItems: 1, }, }, + required: ['files'], }, outputs: { type: 'object', description: - 'Workspace files to create or overwrite from returned code results or sandbox-created files.', + 'One persistent workspace file to create or overwrite. Chat uploads are read-only inputs and cannot be output targets.', properties: { files: { type: 'array', description: - 'File outputs. Missing parent folders are created automatically for create mode.', + 'Exactly one file output. Parent folders must already exist for create mode.', items: { type: 'object', properties: { - format: { - type: 'string', - description: 'Optional serialization format for returned values.', - enum: ['json', 'csv', 'txt', 'md', 'html'], - }, mimeType: { type: 'string', description: 'Optional MIME type override when inference is not enough.', @@ -1788,19 +1653,18 @@ export const GenerateImage: ToolCatalogEntry = { enum: ['create', 'overwrite'], }, path: { - type: 'string', - description: 'Canonical destination VFS path, e.g. "files/Reports/chart.png".', - }, - sandboxPath: { type: 'string', description: - 'Optional full path to a file created inside the sandbox. Omit to save the code return value.', + 'Persistent destination VFS path starting with "files/", e.g. "files/Reports/chart.png". "uploads/" paths are invalid output targets.', }, }, required: ['path', 'mode'], }, + minItems: 1, + maxItems: 1, }, }, + required: ['files'], }, prompt: { type: 'string', @@ -1808,7 +1672,7 @@ export const GenerateImage: ToolCatalogEntry = { 'Detailed text description of the image to generate, or editing instructions when editing the image(s) passed in `inputs.files`.', }, }, - required: ['prompt'], + required: ['prompt', 'outputs'], }, requiredPermission: 'write', capabilities: ['file_input', 'file_output', 'generated_media'], @@ -1838,66 +1702,27 @@ export const GenerateVideo: ToolCatalogEntry = { }, inputs: { type: 'object', - description: - 'Workspace resources to mount into the sandbox. Copy paths verbatim from glob/read/grep output — they are percent-encoded per segment (spaces are %20, an in-name slash is %2F; parentheses and dots stay literal). Both the encoded path and the plain name resolve, so copy the returned path exactly rather than retyping or decoding it.', + description: 'Existing media files to use as generation or composition inputs.', properties: { - directories: { - type: 'array', - description: - 'Workspace folders to mount recursively into the sandbox, including nested files and empty folders.', - items: { - type: 'object', - properties: { - path: { - type: 'string', - description: - 'Canonical VFS folder path, e.g. "files/Reports". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Optional full sandbox directory path override. Omit to mount at /home/user/{path}.', - }, - }, - required: ['path'], - }, - }, files: { type: 'array', - description: 'Workspace files to mount into the sandbox.', + description: 'Exactly one start-frame image from files/... or uploads/....', items: { type: 'object', properties: { path: { type: 'string', description: - 'Canonical VFS file path, e.g. "files/Reports/sales.csv". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Full sandbox path to mount at, e.g. /home/user/inputs/data.csv. STRONGLY RECOMMENDED whenever the file name has spaces or special characters: the default mount path is the percent-ENCODED canonical path (e.g. /home/user/files/Q4%20Sales%20(Final).csv), which code using the human-readable name will not find. Set a simple sandboxPath and read exactly that.', + 'Input VFS path copied verbatim from glob/read, using either persistent "files/..." or chat-scoped "uploads/...".', }, }, required: ['path'], }, - }, - tables: { - type: 'array', - description: 'Workspace tables to mount as CSV files.', - items: { - type: 'object', - properties: { - path: { type: 'string', description: 'Canonical VFS table path when available.' }, - sandboxPath: { - type: 'string', - description: 'Optional full sandbox path for the mounted CSV.', - }, - tableId: { type: 'string', description: 'Workspace table ID.' }, - }, - }, + minItems: 1, + maxItems: 1, }, }, + required: ['files'], }, model: { type: 'string', @@ -1923,20 +1748,15 @@ export const GenerateVideo: ToolCatalogEntry = { outputs: { type: 'object', description: - 'Workspace files to create or overwrite from returned code results or sandbox-created files.', + 'One persistent workspace file to create or overwrite. Chat uploads are read-only inputs and cannot be output targets.', properties: { files: { type: 'array', description: - 'File outputs. Missing parent folders are created automatically for create mode.', + 'Exactly one file output. Parent folders must already exist for create mode.', items: { type: 'object', properties: { - format: { - type: 'string', - description: 'Optional serialization format for returned values.', - enum: ['json', 'csv', 'txt', 'md', 'html'], - }, mimeType: { type: 'string', description: 'Optional MIME type override when inference is not enough.', @@ -1947,19 +1767,18 @@ export const GenerateVideo: ToolCatalogEntry = { enum: ['create', 'overwrite'], }, path: { - type: 'string', - description: 'Canonical destination VFS path, e.g. "files/Reports/chart.png".', - }, - sandboxPath: { type: 'string', description: - 'Optional full path to a file created inside the sandbox. Omit to save the code return value.', + 'Persistent destination VFS path starting with "files/", e.g. "files/Reports/chart.png". "uploads/" paths are invalid output targets.', }, }, required: ['path', 'mode'], }, + minItems: 1, + maxItems: 1, }, }, + required: ['files'], }, prompt: { type: 'string', @@ -1976,7 +1795,7 @@ export const GenerateVideo: ToolCatalogEntry = { enum: ['720p', '1080p', '4k'], }, }, - required: ['prompt'], + required: ['prompt', 'outputs'], }, requiredPermission: 'write', capabilities: ['file_input', 'file_output', 'generated_media'], diff --git a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts index d5a90741800..d41e0109d5b 100644 --- a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts @@ -1033,72 +1033,27 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, inputs: { type: 'object', - description: - 'Workspace resources to mount into the sandbox. Copy paths verbatim from glob/read/grep output — they are percent-encoded per segment (spaces are %20, an in-name slash is %2F; parentheses and dots stay literal). Both the encoded path and the plain name resolve, so copy the returned path exactly rather than retyping or decoding it.', + description: 'Existing media files to use as generation or composition inputs.', properties: { - directories: { - type: 'array', - description: - 'Workspace folders to mount recursively into the sandbox, including nested files and empty folders.', - items: { - type: 'object', - properties: { - path: { - type: 'string', - description: - 'Canonical VFS folder path, e.g. "files/Reports". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Optional full sandbox directory path override. Omit to mount at /home/user/{path}.', - }, - }, - required: ['path'], - }, - }, files: { type: 'array', - description: 'Workspace files to mount into the sandbox.', + description: + 'One or more media inputs from files/... or uploads/..., in operation order.', items: { type: 'object', properties: { path: { type: 'string', description: - 'Canonical VFS file path, e.g. "files/Reports/sales.csv". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Full sandbox path to mount at, e.g. /home/user/inputs/data.csv. STRONGLY RECOMMENDED whenever the file name has spaces or special characters: the default mount path is the percent-ENCODED canonical path (e.g. /home/user/files/Q4%20Sales%20(Final).csv), which code using the human-readable name will not find. Set a simple sandboxPath and read exactly that.', + 'Input VFS path copied verbatim from glob/read, using either persistent "files/..." or chat-scoped "uploads/...".', }, }, required: ['path'], }, - }, - tables: { - type: 'array', - description: 'Workspace tables to mount as CSV files.', - items: { - type: 'object', - properties: { - path: { - type: 'string', - description: 'Canonical VFS table path when available.', - }, - sandboxPath: { - type: 'string', - description: 'Optional full sandbox path for the mounted CSV.', - }, - tableId: { - type: 'string', - description: 'Workspace table ID.', - }, - }, - }, + minItems: 1, }, }, + required: ['files'], }, loopToVideo: { type: 'boolean', @@ -1129,20 +1084,15 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { outputs: { type: 'object', description: - 'Workspace files to create or overwrite from returned code results or sandbox-created files.', + 'One persistent workspace file to create or overwrite. Chat uploads are read-only inputs and cannot be output targets.', properties: { files: { type: 'array', description: - 'File outputs. Missing parent folders are created automatically for create mode.', + 'Exactly one file output. Parent folders must already exist for create mode.', items: { type: 'object', properties: { - format: { - type: 'string', - description: 'Optional serialization format for returned values.', - enum: ['json', 'csv', 'txt', 'md', 'html'], - }, mimeType: { type: 'string', description: 'Optional MIME type override when inference is not enough.', @@ -1153,19 +1103,18 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { enum: ['create', 'overwrite'], }, path: { - type: 'string', - description: 'Canonical destination VFS path, e.g. "files/Reports/chart.png".', - }, - sandboxPath: { type: 'string', description: - 'Optional full path to a file created inside the sandbox. Omit to save the code return value.', + 'Persistent destination VFS path starting with "files/", e.g. "files/Reports/chart.png". "uploads/" paths are invalid output targets.', }, }, required: ['path', 'mode'], }, + minItems: 1, + maxItems: 1, }, }, + required: ['files'], }, position: { type: 'string', @@ -1374,72 +1323,27 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, inputs: { type: 'object', - description: - 'Workspace resources to mount into the sandbox. Copy paths verbatim from glob/read/grep output — they are percent-encoded per segment (spaces are %20, an in-name slash is %2F; parentheses and dots stay literal). Both the encoded path and the plain name resolve, so copy the returned path exactly rather than retyping or decoding it.', + description: 'Existing media files to use as generation or composition inputs.', properties: { - directories: { - type: 'array', - description: - 'Workspace folders to mount recursively into the sandbox, including nested files and empty folders.', - items: { - type: 'object', - properties: { - path: { - type: 'string', - description: - 'Canonical VFS folder path, e.g. "files/Reports". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Optional full sandbox directory path override. Omit to mount at /home/user/{path}.', - }, - }, - required: ['path'], - }, - }, files: { type: 'array', - description: 'Workspace files to mount into the sandbox.', + description: 'Exactly one reference voice sample from files/... or uploads/....', items: { type: 'object', properties: { path: { type: 'string', description: - 'Canonical VFS file path, e.g. "files/Reports/sales.csv". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Full sandbox path to mount at, e.g. /home/user/inputs/data.csv. STRONGLY RECOMMENDED whenever the file name has spaces or special characters: the default mount path is the percent-ENCODED canonical path (e.g. /home/user/files/Q4%20Sales%20(Final).csv), which code using the human-readable name will not find. Set a simple sandboxPath and read exactly that.', + 'Input VFS path copied verbatim from glob/read, using either persistent "files/..." or chat-scoped "uploads/...".', }, }, required: ['path'], }, - }, - tables: { - type: 'array', - description: 'Workspace tables to mount as CSV files.', - items: { - type: 'object', - properties: { - path: { - type: 'string', - description: 'Canonical VFS table path when available.', - }, - sandboxPath: { - type: 'string', - description: 'Optional full sandbox path for the mounted CSV.', - }, - tableId: { - type: 'string', - description: 'Workspace table ID.', - }, - }, - }, + minItems: 1, + maxItems: 1, }, }, + required: ['files'], }, instrumental: { type: 'boolean', @@ -1459,20 +1363,15 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { outputs: { type: 'object', description: - 'Workspace files to create or overwrite from returned code results or sandbox-created files.', + 'One persistent workspace file to create or overwrite. Chat uploads are read-only inputs and cannot be output targets.', properties: { files: { type: 'array', description: - 'File outputs. Missing parent folders are created automatically for create mode.', + 'Exactly one file output. Parent folders must already exist for create mode.', items: { type: 'object', properties: { - format: { - type: 'string', - description: 'Optional serialization format for returned values.', - enum: ['json', 'csv', 'txt', 'md', 'html'], - }, mimeType: { type: 'string', description: 'Optional MIME type override when inference is not enough.', @@ -1483,19 +1382,18 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { enum: ['create', 'overwrite'], }, path: { - type: 'string', - description: 'Canonical destination VFS path, e.g. "files/Reports/chart.png".', - }, - sandboxPath: { type: 'string', description: - 'Optional full path to a file created inside the sandbox. Omit to save the code return value.', + 'Persistent destination VFS path starting with "files/", e.g. "files/Reports/chart.png". "uploads/" paths are invalid output targets.', }, }, required: ['path', 'mode'], }, + minItems: 1, + maxItems: 1, }, }, + required: ['files'], }, prompt: { type: 'string', @@ -1512,7 +1410,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { description: 'Optional voice name or id for speech.', }, }, - required: ['prompt'], + required: ['prompt', 'outputs'], }, resultSchema: undefined, }, @@ -1527,90 +1425,39 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, inputs: { type: 'object', - description: - 'Workspace resources to mount into the sandbox. Copy paths verbatim from glob/read/grep output — they are percent-encoded per segment (spaces are %20, an in-name slash is %2F; parentheses and dots stay literal). Both the encoded path and the plain name resolve, so copy the returned path exactly rather than retyping or decoding it.', + description: 'Existing media files to use as generation or composition inputs.', properties: { - directories: { - type: 'array', - description: - 'Workspace folders to mount recursively into the sandbox, including nested files and empty folders.', - items: { - type: 'object', - properties: { - path: { - type: 'string', - description: - 'Canonical VFS folder path, e.g. "files/Reports". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Optional full sandbox directory path override. Omit to mount at /home/user/{path}.', - }, - }, - required: ['path'], - }, - }, files: { type: 'array', - description: 'Workspace files to mount into the sandbox.', + description: 'One or more reference images to edit, combine, or restyle.', items: { type: 'object', properties: { path: { type: 'string', description: - 'Canonical VFS file path, e.g. "files/Reports/sales.csv". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Full sandbox path to mount at, e.g. /home/user/inputs/data.csv. STRONGLY RECOMMENDED whenever the file name has spaces or special characters: the default mount path is the percent-ENCODED canonical path (e.g. /home/user/files/Q4%20Sales%20(Final).csv), which code using the human-readable name will not find. Set a simple sandboxPath and read exactly that.', + 'Input VFS path copied verbatim from glob/read, using either persistent "files/..." or chat-scoped "uploads/...".', }, }, required: ['path'], }, - }, - tables: { - type: 'array', - description: 'Workspace tables to mount as CSV files.', - items: { - type: 'object', - properties: { - path: { - type: 'string', - description: 'Canonical VFS table path when available.', - }, - sandboxPath: { - type: 'string', - description: 'Optional full sandbox path for the mounted CSV.', - }, - tableId: { - type: 'string', - description: 'Workspace table ID.', - }, - }, - }, + minItems: 1, }, }, + required: ['files'], }, outputs: { type: 'object', description: - 'Workspace files to create or overwrite from returned code results or sandbox-created files.', + 'One persistent workspace file to create or overwrite. Chat uploads are read-only inputs and cannot be output targets.', properties: { files: { type: 'array', description: - 'File outputs. Missing parent folders are created automatically for create mode.', + 'Exactly one file output. Parent folders must already exist for create mode.', items: { type: 'object', properties: { - format: { - type: 'string', - description: 'Optional serialization format for returned values.', - enum: ['json', 'csv', 'txt', 'md', 'html'], - }, mimeType: { type: 'string', description: 'Optional MIME type override when inference is not enough.', @@ -1621,19 +1468,18 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { enum: ['create', 'overwrite'], }, path: { - type: 'string', - description: 'Canonical destination VFS path, e.g. "files/Reports/chart.png".', - }, - sandboxPath: { type: 'string', description: - 'Optional full path to a file created inside the sandbox. Omit to save the code return value.', + 'Persistent destination VFS path starting with "files/", e.g. "files/Reports/chart.png". "uploads/" paths are invalid output targets.', }, }, required: ['path', 'mode'], }, + minItems: 1, + maxItems: 1, }, }, + required: ['files'], }, prompt: { type: 'string', @@ -1641,7 +1487,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { 'Detailed text description of the image to generate, or editing instructions when editing the image(s) passed in `inputs.files`.', }, }, - required: ['prompt'], + required: ['prompt', 'outputs'], }, resultSchema: undefined, }, @@ -1665,72 +1511,27 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, inputs: { type: 'object', - description: - 'Workspace resources to mount into the sandbox. Copy paths verbatim from glob/read/grep output — they are percent-encoded per segment (spaces are %20, an in-name slash is %2F; parentheses and dots stay literal). Both the encoded path and the plain name resolve, so copy the returned path exactly rather than retyping or decoding it.', + description: 'Existing media files to use as generation or composition inputs.', properties: { - directories: { - type: 'array', - description: - 'Workspace folders to mount recursively into the sandbox, including nested files and empty folders.', - items: { - type: 'object', - properties: { - path: { - type: 'string', - description: - 'Canonical VFS folder path, e.g. "files/Reports". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Optional full sandbox directory path override. Omit to mount at /home/user/{path}.', - }, - }, - required: ['path'], - }, - }, files: { type: 'array', - description: 'Workspace files to mount into the sandbox.', + description: 'Exactly one start-frame image from files/... or uploads/....', items: { type: 'object', properties: { path: { type: 'string', description: - 'Canonical VFS file path, e.g. "files/Reports/sales.csv". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Full sandbox path to mount at, e.g. /home/user/inputs/data.csv. STRONGLY RECOMMENDED whenever the file name has spaces or special characters: the default mount path is the percent-ENCODED canonical path (e.g. /home/user/files/Q4%20Sales%20(Final).csv), which code using the human-readable name will not find. Set a simple sandboxPath and read exactly that.', + 'Input VFS path copied verbatim from glob/read, using either persistent "files/..." or chat-scoped "uploads/...".', }, }, required: ['path'], }, - }, - tables: { - type: 'array', - description: 'Workspace tables to mount as CSV files.', - items: { - type: 'object', - properties: { - path: { - type: 'string', - description: 'Canonical VFS table path when available.', - }, - sandboxPath: { - type: 'string', - description: 'Optional full sandbox path for the mounted CSV.', - }, - tableId: { - type: 'string', - description: 'Workspace table ID.', - }, - }, - }, + minItems: 1, + maxItems: 1, }, }, + required: ['files'], }, model: { type: 'string', @@ -1756,20 +1557,15 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { outputs: { type: 'object', description: - 'Workspace files to create or overwrite from returned code results or sandbox-created files.', + 'One persistent workspace file to create or overwrite. Chat uploads are read-only inputs and cannot be output targets.', properties: { files: { type: 'array', description: - 'File outputs. Missing parent folders are created automatically for create mode.', + 'Exactly one file output. Parent folders must already exist for create mode.', items: { type: 'object', properties: { - format: { - type: 'string', - description: 'Optional serialization format for returned values.', - enum: ['json', 'csv', 'txt', 'md', 'html'], - }, mimeType: { type: 'string', description: 'Optional MIME type override when inference is not enough.', @@ -1780,19 +1576,18 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { enum: ['create', 'overwrite'], }, path: { - type: 'string', - description: 'Canonical destination VFS path, e.g. "files/Reports/chart.png".', - }, - sandboxPath: { type: 'string', description: - 'Optional full path to a file created inside the sandbox. Omit to save the code return value.', + 'Persistent destination VFS path starting with "files/", e.g. "files/Reports/chart.png". "uploads/" paths are invalid output targets.', }, }, required: ['path', 'mode'], }, + minItems: 1, + maxItems: 1, }, }, + required: ['files'], }, prompt: { type: 'string', @@ -1809,7 +1604,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { enum: ['720p', '1080p', '4k'], }, }, - required: ['prompt'], + required: ['prompt', 'outputs'], }, resultSchema: undefined, }, diff --git a/apps/sim/lib/copilot/tools/handlers/upload-file-reader.ts b/apps/sim/lib/copilot/tools/handlers/upload-file-reader.ts index 0e914229c8a..2fb29783c4f 100644 --- a/apps/sim/lib/copilot/tools/handlers/upload-file-reader.ts +++ b/apps/sim/lib/copilot/tools/handlers/upload-file-reader.ts @@ -61,6 +61,17 @@ function toWorkspaceFileRecord(row: typeof workspaceFiles.$inferSelect): Workspa } } +/** + * Resolve a chat-scoped upload to the same file-record shape used by workspace files. + */ +export async function resolveChatUpload( + filename: string, + chatId: string +): Promise { + const row = await findMothershipUploadRowByChatAndName(chatId, filename) + return row ? toWorkspaceFileRecord(row) : null +} + /** * Resolve a mothership upload row by VFS name (the collision-disambiguated `displayName` * for new rows, or `originalName` for legacy rows that predate the column). Prefers an diff --git a/apps/sim/lib/copilot/tools/server/image/generate-image.test.ts b/apps/sim/lib/copilot/tools/server/image/generate-image.test.ts new file mode 100644 index 00000000000..d5e7e5aed8a --- /dev/null +++ b/apps/sim/lib/copilot/tools/server/image/generate-image.test.ts @@ -0,0 +1,215 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + fetchWorkspaceFileBuffer: vi.fn(), + generateContent: vi.fn(), + resolveChatUpload: vi.fn(), + resolveWorkspaceFileReference: vi.fn(), + validateWorkspaceFileWriteTarget: vi.fn(), + writeWorkspaceFileByPath: vi.fn(), +})) + +vi.mock('@google/genai', () => ({ + GoogleGenAI: class { + models = { generateContent: mocks.generateContent } + }, +})) + +vi.mock('@/lib/core/config/api-keys', () => ({ + getRotatingApiKey: vi.fn(() => 'test-api-key'), +})) + +vi.mock('@/lib/copilot/tools/handlers/upload-file-reader', () => ({ + resolveChatUpload: mocks.resolveChatUpload, +})) + +vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({ + fetchWorkspaceFileBuffer: mocks.fetchWorkspaceFileBuffer, + resolveWorkspaceFileReference: mocks.resolveWorkspaceFileReference, +})) + +vi.mock('@/lib/copilot/vfs/resource-writer', () => ({ + validateWorkspaceFileWriteTarget: mocks.validateWorkspaceFileWriteTarget, + writeWorkspaceFileByPath: mocks.writeWorkspaceFileByPath, +})) + +import { generateImageServerTool } from '@/lib/copilot/tools/server/image/generate-image' + +const CONTEXT = { + userId: 'user-1', + workspaceId: 'workspace-1', + chatId: 'chat-1', +} + +const IMAGE_RECORD = { + id: 'wf_image', + workspaceId: 'workspace-1', + name: 'person.png', + key: 'uploads/person.png', + path: '/api/files/serve/uploads%2Fperson.png', + size: 3, + type: 'image/png', + uploadedBy: 'user-1', + uploadedAt: new Date('2026-01-01T00:00:00Z'), + updatedAt: new Date('2026-01-01T00:00:00Z'), + storageContext: 'mothership' as const, +} + +describe('generateImageServerTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.fetchWorkspaceFileBuffer.mockResolvedValue(Buffer.from('image')) + mocks.validateWorkspaceFileWriteTarget.mockResolvedValue({}) + mocks.generateContent.mockResolvedValue({ + candidates: [ + { content: { parts: [{ inlineData: { data: 'aW1hZ2U=', mimeType: 'image/png' } }] } }, + ], + }) + mocks.writeWorkspaceFileByPath.mockResolvedValue({ + id: 'wf_result', + name: 'result.png', + vfsPath: 'files/result.png', + mode: 'create', + downloadUrl: '/download/result.png', + }) + }) + + it('loads a chat upload when inputs.files uses an uploads/ path', async () => { + mocks.resolveChatUpload.mockResolvedValue(IMAGE_RECORD) + + const result = await generateImageServerTool.execute( + { + prompt: 'Preserve this person and put them in a library', + inputs: { files: [{ path: 'uploads/person.png' }] }, + outputs: { files: [{ path: 'files/result.png' }] }, + }, + CONTEXT + ) + + expect(result.success).toBe(true) + expect(mocks.resolveChatUpload).toHaveBeenCalledWith('person.png', 'chat-1') + expect(mocks.resolveWorkspaceFileReference).not.toHaveBeenCalled() + expect(mocks.fetchWorkspaceFileBuffer).toHaveBeenCalledWith(IMAGE_RECORD) + }) + + it('fails before generation when an input path cannot be resolved', async () => { + mocks.resolveWorkspaceFileReference.mockResolvedValue(null) + + const result = await generateImageServerTool.execute( + { + prompt: 'Edit the reference image', + inputs: { files: [{ path: 'files/missing.png' }] }, + outputs: { files: [{ path: 'files/result.png' }] }, + }, + CONTEXT + ) + + expect(result).toEqual({ + success: false, + message: 'Failed to generate image: Input file not found: files/missing.png', + }) + expect(mocks.generateContent).not.toHaveBeenCalled() + expect(mocks.writeWorkspaceFileByPath).not.toHaveBeenCalled() + }) + + it('fails before generation when outputs.files targets uploads/', async () => { + const result = await generateImageServerTool.execute( + { + prompt: 'Create a portrait', + outputs: { files: [{ path: 'uploads/result.png' }] }, + }, + CONTEXT + ) + + expect(result).toEqual({ + success: false, + message: + 'Failed to generate image: Media output paths must start with "files/"; uploads/ paths are read-only: uploads/result.png', + }) + expect(mocks.generateContent).not.toHaveBeenCalled() + expect(mocks.writeWorkspaceFileByPath).not.toHaveBeenCalled() + }) + + it('fails before generation when an explicit output path is empty', async () => { + mocks.validateWorkspaceFileWriteTarget.mockRejectedValue( + new Error('Workspace file paths must start with "files/"') + ) + + const result = await generateImageServerTool.execute( + { + prompt: 'Create a portrait', + outputs: { files: [{ path: '' }] }, + }, + CONTEXT + ) + + expect(result.success).toBe(false) + expect(mocks.generateContent).not.toHaveBeenCalled() + expect(mocks.writeWorkspaceFileByPath).not.toHaveBeenCalled() + }) + + it('rejects additional output declarations before generation', async () => { + const result = await generateImageServerTool.execute( + { + prompt: 'Create a portrait', + outputs: { + files: [{ path: 'files/one.png' }, { path: 'uploads/two.png' }], + }, + }, + CONTEXT + ) + + expect(result).toEqual({ + success: false, + message: 'Failed to generate image: Output requires exactly one file; received 2', + }) + expect(mocks.validateWorkspaceFileWriteTarget).not.toHaveBeenCalled() + expect(mocks.generateContent).not.toHaveBeenCalled() + }) + + it('rejects explicitly empty input and output containers before generation', async () => { + const emptyInputs = await generateImageServerTool.execute( + { + prompt: 'Edit the supplied image', + inputs: {}, + outputs: { files: [{ path: 'files/result.png' }] }, + }, + CONTEXT + ) + const emptyOutputs = await generateImageServerTool.execute( + { prompt: 'Create a portrait', outputs: {} }, + CONTEXT + ) + + expect(emptyInputs).toEqual({ + success: false, + message: 'Failed to generate image: Input requires at least one file', + }) + expect(emptyOutputs).toEqual({ + success: false, + message: 'Failed to generate image: Output requires exactly one file; received 0', + }) + expect(mocks.generateContent).not.toHaveBeenCalled() + }) + + it('fails before generation when canonical output preflight rejects a files/ target', async () => { + mocks.validateWorkspaceFileWriteTarget.mockRejectedValue( + new Error('File already exists at files/result.png') + ) + + const result = await generateImageServerTool.execute( + { + prompt: 'Create a portrait', + outputs: { files: [{ path: 'files/result.png' }] }, + }, + CONTEXT + ) + + expect(result.success).toBe(false) + expect(mocks.generateContent).not.toHaveBeenCalled() + expect(mocks.writeWorkspaceFileByPath).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/copilot/tools/server/image/generate-image.ts b/apps/sim/lib/copilot/tools/server/image/generate-image.ts index 3a6c25b8240..0cda9700a22 100644 --- a/apps/sim/lib/copilot/tools/server/image/generate-image.ts +++ b/apps/sim/lib/copilot/tools/server/image/generate-image.ts @@ -1,18 +1,20 @@ import { GoogleGenAI, type Part } from '@google/genai' import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' +import { getErrorMessage } from '@sim/utils/errors' import { GenerateImage } from '@/lib/copilot/generated/tool-catalog-v1' import { assertServerToolNotAborted, type BaseServerTool, type ServerToolContext, } from '@/lib/copilot/tools/server/base-tool' +import { + prepareMediaOutput, + requireMediaFileDeclarations, + resolveMediaInputFile, +} from '@/lib/copilot/tools/server/media/file-paths' import { writeWorkspaceFileByPath } from '@/lib/copilot/vfs/resource-writer' import { getRotatingApiKey } from '@/lib/core/config/api-keys' -import { - fetchWorkspaceFileBuffer, - resolveWorkspaceFileReference, -} from '@/lib/uploads/contexts/workspace/workspace-file-manager' +import { fetchWorkspaceFileBuffer } from '@/lib/uploads/contexts/workspace/workspace-file-manager' const logger = createLogger('GenerateImageTool') @@ -74,6 +76,12 @@ export const generateImageServerTool: BaseServerTool file.path) ?? [] + const referencePaths = params.inputs + ? requireMediaFileDeclarations(params.inputs.files, 'Input').map((file) => file.path) + : [] if (referencePaths.length) { for (const filePath of referencePaths) { - try { - const fileRecord = await resolveWorkspaceFileReference(workspaceId, filePath) - if (fileRecord) { - const buffer = await fetchWorkspaceFileBuffer(fileRecord) - const base64 = buffer.toString('base64') - const mime = fileRecord.type || 'image/png' - parts.push({ - inlineData: { mimeType: mime, data: base64 }, - }) - logger.info('Loaded reference image', { - filePath, - name: fileRecord.name, - size: buffer.length, - mimeType: mime, - }) - } else { - logger.warn('Reference file not found, skipping', { filePath }) - } - } catch (err) { - logger.warn('Failed to load reference image, skipping', { - filePath, - error: toError(err).message, - }) - } + const fileRecord = await resolveMediaInputFile({ + workspaceId, + chatId: context.chatId, + path: filePath, + }) + const buffer = await fetchWorkspaceFileBuffer(fileRecord) + const base64 = buffer.toString('base64') + const mime = fileRecord.type || 'image/png' + parts.push({ + inlineData: { mimeType: mime, data: base64 }, + }) + logger.info('Loaded reference image', { + filePath, + name: fileRecord.name, + size: buffer.length, + mimeType: mime, + }) } } @@ -160,20 +163,18 @@ export const generateImageServerTool: BaseServerTool = { } try { + const outputFile = + params.operation === 'probe' + ? undefined + : await prepareMediaOutput({ + output: params.outputs, + workspaceId, + userId: context.userId, + }) + const mediaFiles: MediaFile[] = [] for (const filePath of inputPaths) { - const fileRecord = await resolveWorkspaceFileReference(workspaceId, filePath) - if (!fileRecord) { - return { success: false, message: `Input file not found: ${filePath}` } - } + const fileRecord = await resolveMediaInputFile({ + workspaceId, + chatId: context.chatId, + path: filePath, + }) const buffer = await fetchWorkspaceFileBuffer(fileRecord) mediaFiles.push({ buffer, @@ -119,19 +130,21 @@ export const ffmpegServerTool: BaseServerTool = { } } - if (!result.buffer || !result.ext) { + if (!result.buffer) { return { success: false, message: `ffmpeg ${params.operation} produced no output` } } + if (!outputFile) { + return { success: false, message: `ffmpeg ${params.operation} requires an output file` } + } - const outputFile = params.outputs?.files?.[0] - const outputPath = outputFile?.path || `files/ffmpeg-${params.operation}.${result.ext}` - const mode = outputFile?.mode ?? 'create' + const outputPath = outputFile.path + const mode = outputFile.mode assertServerToolNotAborted(context) const written = await writeWorkspaceFileByPath({ workspaceId, userId: context.userId, - target: { path: outputPath, mode, mimeType: outputFile?.mimeType }, + target: { path: outputPath, mode, mimeType: outputFile.mimeType }, buffer: result.buffer, inferredMimeType: result.contentType || 'application/octet-stream', }) diff --git a/apps/sim/lib/copilot/tools/server/media/file-paths.test.ts b/apps/sim/lib/copilot/tools/server/media/file-paths.test.ts new file mode 100644 index 00000000000..f757e960f0a --- /dev/null +++ b/apps/sim/lib/copilot/tools/server/media/file-paths.test.ts @@ -0,0 +1,184 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + resolveChatUpload: vi.fn(), + resolveWorkspaceFileReference: vi.fn(), + validateWorkspaceFileWriteTarget: vi.fn(), +})) + +vi.mock('@/lib/copilot/tools/handlers/upload-file-reader', () => ({ + resolveChatUpload: mocks.resolveChatUpload, +})) + +vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({ + resolveWorkspaceFileReference: mocks.resolveWorkspaceFileReference, +})) + +vi.mock('@/lib/copilot/vfs/resource-writer', () => ({ + validateWorkspaceFileWriteTarget: mocks.validateWorkspaceFileWriteTarget, +})) + +import { + getSingleMediaFileDeclaration, + prepareMediaOutput, + resolveMediaInputFile, + validateMediaOutputFile, +} from '@/lib/copilot/tools/server/media/file-paths' + +const FILE_RECORD = { id: 'wf_file', name: 'portrait.png' } + +describe('media file paths', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.validateWorkspaceFileWriteTarget.mockResolvedValue({}) + }) + + it('resolves uploads/ inputs within the current chat', async () => { + mocks.resolveChatUpload.mockResolvedValue(FILE_RECORD) + + await expect( + resolveMediaInputFile({ + workspaceId: 'workspace-1', + chatId: 'chat-1', + path: 'uploads/My%20Portrait.png', + }) + ).resolves.toBe(FILE_RECORD) + + expect(mocks.resolveChatUpload).toHaveBeenCalledWith('My%20Portrait.png', 'chat-1') + expect(mocks.resolveWorkspaceFileReference).not.toHaveBeenCalled() + }) + + it('resolves uploads/ inputs with the read-compatible /content suffix', async () => { + mocks.resolveChatUpload.mockResolvedValue(FILE_RECORD) + + await expect( + resolveMediaInputFile({ + workspaceId: 'workspace-1', + chatId: 'chat-1', + path: 'uploads/My%20Portrait.png/content', + }) + ).resolves.toBe(FILE_RECORD) + + expect(mocks.resolveChatUpload).toHaveBeenCalledWith('My%20Portrait.png', 'chat-1') + }) + + it('rejects unresolved inputs instead of dropping them', async () => { + mocks.resolveWorkspaceFileReference.mockResolvedValue(null) + + await expect( + resolveMediaInputFile({ + workspaceId: 'workspace-1', + chatId: 'chat-1', + path: 'files/missing.png', + }) + ).rejects.toThrow('Input file not found: files/missing.png') + }) + + it('preserves legacy workspace IDs and plain-name references', async () => { + mocks.resolveWorkspaceFileReference.mockResolvedValue(FILE_RECORD) + + await expect( + resolveMediaInputFile({ + workspaceId: 'workspace-1', + chatId: 'chat-1', + path: 'portrait.png', + }) + ).resolves.toBe(FILE_RECORD) + + await resolveMediaInputFile({ + workspaceId: 'workspace-1', + chatId: 'chat-1', + path: 'wf_file', + }) + + expect(mocks.resolveWorkspaceFileReference).toHaveBeenNthCalledWith( + 1, + 'workspace-1', + 'portrait.png' + ) + expect(mocks.resolveWorkspaceFileReference).toHaveBeenNthCalledWith(2, 'workspace-1', 'wf_file') + }) + + it('preflights outputs through the canonical workspace writer policy', async () => { + await validateMediaOutputFile({ + workspaceId: 'workspace-1', + userId: 'user-1', + path: 'files/Campaign/portrait.png', + mode: 'overwrite', + mimeType: 'image/png', + }) + + expect(mocks.validateWorkspaceFileWriteTarget).toHaveBeenCalledWith({ + workspaceId: 'workspace-1', + userId: 'user-1', + target: { + path: 'files/Campaign/portrait.png', + mode: 'overwrite', + mimeType: 'image/png', + }, + }) + }) + + it('rejects uploads/ output paths before canonical resolution', async () => { + await expect( + validateMediaOutputFile({ + workspaceId: 'workspace-1', + userId: 'user-1', + path: 'uploads/portrait.png', + mode: 'create', + }) + ).rejects.toThrow('Media output paths must start with "files/"') + expect(mocks.validateWorkspaceFileWriteTarget).not.toHaveBeenCalled() + }) + + it('propagates canonical writer validation errors for files/ paths', async () => { + mocks.validateWorkspaceFileWriteTarget.mockRejectedValue( + new Error('File already exists at files/portrait.png') + ) + + await expect( + validateMediaOutputFile({ + workspaceId: 'workspace-1', + userId: 'user-1', + path: 'files/portrait.png', + mode: 'create', + }) + ).rejects.toThrow('File already exists at files/portrait.png') + expect(mocks.validateWorkspaceFileWriteTarget).toHaveBeenCalledOnce() + }) + + it('rejects non-file output namespaces before canonical resolution', async () => { + await expect( + validateMediaOutputFile({ + workspaceId: 'workspace-1', + userId: 'user-1', + path: 'workflows/My%20Flow/state.json', + mode: 'overwrite', + }) + ).rejects.toThrow('Media output paths must start with "files/"') + expect(mocks.validateWorkspaceFileWriteTarget).not.toHaveBeenCalled() + }) + + it('rejects extra declarations for single-file media parameters', () => { + expect(() => + getSingleMediaFileDeclaration([{ path: 'files/a.png' }, { path: 'files/b.png' }], 'Output') + ).toThrow('Output requires exactly one file; received 2') + expect(() => getSingleMediaFileDeclaration([], 'Output')).toThrow( + 'Output requires exactly one file; received 0' + ) + }) + + it('requires an explicit output before provider work can begin', async () => { + await expect( + prepareMediaOutput({ + workspaceId: 'workspace-1', + userId: 'user-1', + }) + ).rejects.toThrow('Output requires exactly one file; received 0') + + expect(mocks.validateWorkspaceFileWriteTarget).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/copilot/tools/server/media/file-paths.ts b/apps/sim/lib/copilot/tools/server/media/file-paths.ts new file mode 100644 index 00000000000..ec07aab70e6 --- /dev/null +++ b/apps/sim/lib/copilot/tools/server/media/file-paths.ts @@ -0,0 +1,107 @@ +import { resolveChatUpload } from '@/lib/copilot/tools/handlers/upload-file-reader' +import { + validateWorkspaceFileWriteTarget, + type WorkspaceFileWriteMode, +} from '@/lib/copilot/vfs/resource-writer' +import { + resolveWorkspaceFileReference, + type WorkspaceFileRecord, +} from '@/lib/uploads/contexts/workspace/workspace-file-manager' + +const UPLOADS_PREFIX = 'uploads/' +const FILES_PREFIX = 'files/' + +interface MediaOutputDeclaration { + path: string + mode?: WorkspaceFileWriteMode + mimeType?: string +} + +export function getSingleMediaFileDeclaration( + files: T[] | undefined, + label: 'Input' | 'Output' +): T { + if (files?.length !== 1) { + throw new Error(`${label} requires exactly one file; received ${files?.length ?? 0}`) + } + return files[0] +} + +export function requireMediaFileDeclarations( + files: T[] | undefined, + label: 'Input' | 'Output' +): T[] { + if (!files?.length) { + throw new Error(`${label} requires at least one file`) + } + return files +} + +/** + * Resolve a media input from either persistent workspace files or chat-scoped uploads. + */ +export async function resolveMediaInputFile(args: { + workspaceId: string + chatId?: string + path: string +}): Promise { + const path = args.path.trim() + let file: WorkspaceFileRecord | null + + if (path.startsWith(UPLOADS_PREFIX)) { + const filename = path.slice(UPLOADS_PREFIX.length).split('/')[0] + if (!filename) { + throw new Error(`Upload input path must identify a file: ${args.path}`) + } + if (!args.chatId) { + throw new Error(`Chat context is required for upload input: ${path}`) + } + file = await resolveChatUpload(filename, args.chatId) + } else { + file = await resolveWorkspaceFileReference(args.workspaceId, path) + } + + if (!file) { + throw new Error(`Input file not found: ${args.path}`) + } + return file +} + +/** + * Preflight a media output through the canonical workspace writer policy. + */ +export async function validateMediaOutputFile(args: { + workspaceId: string + userId: string + path: string + mode: WorkspaceFileWriteMode + mimeType?: string +}): Promise { + if (!args.path.trim().startsWith(FILES_PREFIX)) { + throw new Error( + `Media output paths must start with "files/"; uploads/ paths are read-only: ${args.path}` + ) + } + await validateWorkspaceFileWriteTarget({ + workspaceId: args.workspaceId, + userId: args.userId, + target: { path: args.path, mode: args.mode, mimeType: args.mimeType }, + }) +} + +export async function prepareMediaOutput(args: { + output?: { files?: T[] } + workspaceId: string + userId: string +}): Promise { + const file = getSingleMediaFileDeclaration(args.output?.files, 'Output') + const output = { ...file, mode: file.mode ?? 'create' } + await validateMediaOutputFile({ + workspaceId: args.workspaceId, + userId: args.userId, + path: output.path, + mode: output.mode, + mimeType: output.mimeType, + }) + return output +} diff --git a/apps/sim/lib/copilot/tools/server/media/generate-audio.ts b/apps/sim/lib/copilot/tools/server/media/generate-audio.ts index 6d224686397..5b000d67c16 100644 --- a/apps/sim/lib/copilot/tools/server/media/generate-audio.ts +++ b/apps/sim/lib/copilot/tools/server/media/generate-audio.ts @@ -6,12 +6,14 @@ import { type BaseServerTool, type ServerToolContext, } from '@/lib/copilot/tools/server/base-tool' +import { + getSingleMediaFileDeclaration, + prepareMediaOutput, + resolveMediaInputFile, +} from '@/lib/copilot/tools/server/media/file-paths' import { writeWorkspaceFileByPath } from '@/lib/copilot/vfs/resource-writer' import { type AudioType, generateFalAudio } from '@/lib/media/falai-audio' -import { - fetchWorkspaceFileBuffer, - resolveWorkspaceFileReference, -} from '@/lib/uploads/contexts/workspace/workspace-file-manager' +import { fetchWorkspaceFileBuffer } from '@/lib/uploads/contexts/workspace/workspace-file-manager' const logger = createLogger('GenerateAudioTool') @@ -44,16 +46,6 @@ interface GenerateAudioResult { _serviceCost?: { service: string; cost: number } } -function audioExtFromContentType(contentType: string): string { - if (contentType.includes('wav')) return 'wav' - if (contentType.includes('mp4') || contentType.includes('m4a')) return 'm4a' - if (contentType.includes('ogg')) return 'ogg' - if (contentType.includes('flac')) return 'flac' - if (contentType.includes('aac')) return 'aac' - if (contentType.includes('opus')) return 'opus' - return 'mp3' -} - export const generateAudioServerTool: BaseServerTool = { name: GenerateAudio.id, @@ -80,20 +72,29 @@ export const generateAudioServerTool: BaseServerTool ({ + fetchWorkspaceFileBuffer: vi.fn(), + generateFalAudio: vi.fn(), + generateFalVideo: vi.fn(), + resolveChatUpload: vi.fn(), + resolveWorkspaceFileReference: vi.fn(), + runFfmpegOperation: vi.fn(), + validateWorkspaceFileWriteTarget: vi.fn(), + writeWorkspaceFileByPath: vi.fn(), +})) + +vi.mock('@/lib/media/falai-audio', () => ({ generateFalAudio: mocks.generateFalAudio })) +vi.mock('@/lib/media/falai-video', () => ({ generateFalVideo: mocks.generateFalVideo })) +vi.mock('@/lib/media/ffmpeg', () => ({ runFfmpegOperation: mocks.runFfmpegOperation })) +vi.mock('@/lib/copilot/tools/handlers/upload-file-reader', () => ({ + resolveChatUpload: mocks.resolveChatUpload, +})) +vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({ + fetchWorkspaceFileBuffer: mocks.fetchWorkspaceFileBuffer, + resolveWorkspaceFileReference: mocks.resolveWorkspaceFileReference, +})) +vi.mock('@/lib/copilot/vfs/resource-writer', () => ({ + validateWorkspaceFileWriteTarget: mocks.validateWorkspaceFileWriteTarget, + writeWorkspaceFileByPath: mocks.writeWorkspaceFileByPath, +})) + +import { ffmpegServerTool } from '@/lib/copilot/tools/server/media/ffmpeg' +import { generateAudioServerTool } from '@/lib/copilot/tools/server/media/generate-audio' +import { generateVideoServerTool } from '@/lib/copilot/tools/server/media/generate-video' + +const CONTEXT = { userId: 'user-1', workspaceId: 'workspace-1', chatId: 'chat-1' } +const UPLOAD_RECORD = { + id: 'wf_upload', + name: 'start.png', + type: 'image/png', + storageContext: 'mothership', +} + +describe('media tool path validation', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.validateWorkspaceFileWriteTarget.mockResolvedValue({}) + mocks.fetchWorkspaceFileBuffer.mockResolvedValue(Buffer.from('media')) + mocks.generateFalVideo.mockResolvedValue({ + buffer: Buffer.from('video'), + contentType: 'video/mp4', + model: 'veo-3.1-fast', + cost: { costDollars: 0.1 }, + }) + mocks.generateFalAudio.mockResolvedValue({ + buffer: Buffer.from('audio'), + contentType: 'audio/mpeg', + model: 'fal-ai/f5-tts', + cost: { costDollars: 0.05 }, + }) + mocks.runFfmpegOperation.mockResolvedValue({ + buffer: Buffer.from('converted'), + contentType: 'video/mp4', + ext: 'mp4', + }) + mocks.writeWorkspaceFileByPath.mockResolvedValue({ + id: 'wf_output', + name: 'output.mp4', + vfsPath: 'files/output.mp4', + mode: 'create', + }) + }) + + it('passes uploads/ start frames to video generation', async () => { + mocks.resolveChatUpload.mockResolvedValue(UPLOAD_RECORD) + + const result = await generateVideoServerTool.execute( + { + prompt: 'Animate this portrait', + inputs: { files: [{ path: 'uploads/start.png' }] }, + outputs: { files: [{ path: 'files/output.mp4' }] }, + }, + CONTEXT + ) + + expect(result.success).toBe(true) + expect(mocks.resolveChatUpload).toHaveBeenCalledWith('start.png', 'chat-1') + expect(mocks.generateFalVideo).toHaveBeenCalledWith( + expect.objectContaining({ imageDataUri: 'data:image/png;base64,bWVkaWE=' }) + ) + }) + + it('rejects empty audio reference paths before calling the provider', async () => { + mocks.resolveWorkspaceFileReference.mockResolvedValue(null) + + const result = await generateAudioServerTool.execute( + { + prompt: 'Clone this voice', + inputs: { files: [{ path: '' }] }, + outputs: { files: [{ path: 'files/voice.mp3' }] }, + }, + CONTEXT + ) + + expect(result.success).toBe(false) + expect(mocks.generateFalAudio).not.toHaveBeenCalled() + }) + + it('passes uploads/ voice samples to audio generation', async () => { + mocks.resolveChatUpload.mockResolvedValue({ + ...UPLOAD_RECORD, + name: 'voice.wav', + type: 'audio/wav', + }) + + const result = await generateAudioServerTool.execute( + { + prompt: 'Read this line in the supplied voice', + inputs: { files: [{ path: 'uploads/voice.wav' }] }, + outputs: { files: [{ path: 'files/voice.mp3' }] }, + }, + CONTEXT + ) + + expect(result.success).toBe(true) + expect(mocks.resolveChatUpload).toHaveBeenCalledWith('voice.wav', 'chat-1') + expect(mocks.generateFalAudio).toHaveBeenCalledWith( + expect.objectContaining({ voiceSampleDataUri: 'data:audio/wav;base64,bWVkaWE=' }) + ) + }) + + it('rejects invalid ffmpeg output paths before processing inputs', async () => { + const result = await ffmpegServerTool.execute( + { + operation: 'convert', + inputs: { files: [{ path: 'files/input.mov' }] }, + outputs: { files: [{ path: 'uploads/output.mp4' }] }, + }, + CONTEXT + ) + + expect(result.success).toBe(false) + expect(mocks.resolveWorkspaceFileReference).not.toHaveBeenCalled() + expect(mocks.runFfmpegOperation).not.toHaveBeenCalled() + }) + + it('ignores unused output declarations for ffmpeg probes', async () => { + mocks.resolveWorkspaceFileReference.mockResolvedValue(UPLOAD_RECORD) + mocks.runFfmpegOperation.mockResolvedValue({ probe: { duration: 3 } }) + + const result = await ffmpegServerTool.execute( + { + operation: 'probe', + inputs: { files: [{ path: 'files/input.mov' }] }, + outputs: { files: [{ path: 'uploads/unused.json' }] }, + }, + CONTEXT + ) + + expect(result.success).toBe(true) + expect(mocks.validateWorkspaceFileWriteTarget).not.toHaveBeenCalled() + expect(mocks.writeWorkspaceFileByPath).not.toHaveBeenCalled() + }) + + it('rejects missing generated-media outputs before calling the provider', async () => { + const result = await generateVideoServerTool.execute( + { prompt: 'Create a launch video' }, + CONTEXT + ) + + expect(result.success).toBe(false) + expect(mocks.generateFalVideo).not.toHaveBeenCalled() + }) + + it('passes uploads/ media files to ffmpeg', async () => { + mocks.resolveChatUpload.mockResolvedValue(UPLOAD_RECORD) + + const result = await ffmpegServerTool.execute( + { + operation: 'convert', + inputs: { files: [{ path: 'uploads/start.png' }] }, + outputs: { files: [{ path: 'files/output.mp4' }] }, + }, + CONTEXT + ) + + expect(result.success).toBe(true) + expect(mocks.resolveChatUpload).toHaveBeenCalledWith('start.png', 'chat-1') + expect(mocks.runFfmpegOperation).toHaveBeenCalledWith( + 'convert', + [expect.objectContaining({ buffer: Buffer.from('media'), name: 'start.png' })], + expect.any(Object) + ) + }) + + it('rejects extra video inputs before calling the provider', async () => { + const result = await generateVideoServerTool.execute( + { + prompt: 'Animate these portraits', + inputs: { files: [{ path: 'files/one.png' }, { path: 'files/two.png' }] }, + outputs: { files: [{ path: 'files/output.mp4' }] }, + }, + CONTEXT + ) + + expect(result.success).toBe(false) + expect(mocks.generateFalVideo).not.toHaveBeenCalled() + }) +})