Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
281 changes: 50 additions & 231 deletions apps/sim/lib/copilot/generated/tool-catalog-v1.ts

Large diffs are not rendered by default.

305 changes: 50 additions & 255 deletions apps/sim/lib/copilot/generated/tool-schemas-v1.ts

Large diffs are not rendered by default.

11 changes: 11 additions & 0 deletions apps/sim/lib/copilot/tools/handlers/upload-file-reader.ts
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,17 @@ function toWorkspaceFileRecord(row: typeof workspaceFiles.$inferSelect): Workspa
}
}

/**
* Resolve a chat-scoped upload to the same file-record shape used by workspace files.
*/
export async function resolveChatUpload(
filename: string,
chatId: string
): Promise<WorkspaceFileRecord | null> {
const row = await findMothershipUploadRowByChatAndName(chatId, filename)
return row ? toWorkspaceFileRecord(row) : null
}

/**
* Resolve a mothership upload row by VFS name (the collision-disambiguated `displayName`
* for new rows, or `originalName` for legacy rows that predate the column). Prefers an
Expand Down
215 changes: 215 additions & 0 deletions apps/sim/lib/copilot/tools/server/image/generate-image.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,215 @@
/**
* @vitest-environment node
*/
import { beforeEach, describe, expect, it, vi } from 'vitest'

const mocks = vi.hoisted(() => ({
fetchWorkspaceFileBuffer: vi.fn(),
generateContent: vi.fn(),
resolveChatUpload: vi.fn(),
resolveWorkspaceFileReference: vi.fn(),
validateWorkspaceFileWriteTarget: vi.fn(),
writeWorkspaceFileByPath: vi.fn(),
}))

vi.mock('@google/genai', () => ({
GoogleGenAI: class {
models = { generateContent: mocks.generateContent }
},
}))

vi.mock('@/lib/core/config/api-keys', () => ({
getRotatingApiKey: vi.fn(() => 'test-api-key'),
}))

vi.mock('@/lib/copilot/tools/handlers/upload-file-reader', () => ({
resolveChatUpload: mocks.resolveChatUpload,
}))

vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({
fetchWorkspaceFileBuffer: mocks.fetchWorkspaceFileBuffer,
resolveWorkspaceFileReference: mocks.resolveWorkspaceFileReference,
}))

vi.mock('@/lib/copilot/vfs/resource-writer', () => ({
validateWorkspaceFileWriteTarget: mocks.validateWorkspaceFileWriteTarget,
writeWorkspaceFileByPath: mocks.writeWorkspaceFileByPath,
}))

import { generateImageServerTool } from '@/lib/copilot/tools/server/image/generate-image'

const CONTEXT = {
userId: 'user-1',
workspaceId: 'workspace-1',
chatId: 'chat-1',
}

const IMAGE_RECORD = {
id: 'wf_image',
workspaceId: 'workspace-1',
name: 'person.png',
key: 'uploads/person.png',
path: '/api/files/serve/uploads%2Fperson.png',
size: 3,
type: 'image/png',
uploadedBy: 'user-1',
uploadedAt: new Date('2026-01-01T00:00:00Z'),
updatedAt: new Date('2026-01-01T00:00:00Z'),
storageContext: 'mothership' as const,
}

describe('generateImageServerTool', () => {
beforeEach(() => {
vi.clearAllMocks()
mocks.fetchWorkspaceFileBuffer.mockResolvedValue(Buffer.from('image'))
mocks.validateWorkspaceFileWriteTarget.mockResolvedValue({})
mocks.generateContent.mockResolvedValue({
candidates: [
{ content: { parts: [{ inlineData: { data: 'aW1hZ2U=', mimeType: 'image/png' } }] } },
],
})
mocks.writeWorkspaceFileByPath.mockResolvedValue({
id: 'wf_result',
name: 'result.png',
vfsPath: 'files/result.png',
mode: 'create',
downloadUrl: '/download/result.png',
})
})

it('loads a chat upload when inputs.files uses an uploads/ path', async () => {
mocks.resolveChatUpload.mockResolvedValue(IMAGE_RECORD)

const result = await generateImageServerTool.execute(
{
prompt: 'Preserve this person and put them in a library',
inputs: { files: [{ path: 'uploads/person.png' }] },
outputs: { files: [{ path: 'files/result.png' }] },
},
CONTEXT
)

expect(result.success).toBe(true)
expect(mocks.resolveChatUpload).toHaveBeenCalledWith('person.png', 'chat-1')
expect(mocks.resolveWorkspaceFileReference).not.toHaveBeenCalled()
expect(mocks.fetchWorkspaceFileBuffer).toHaveBeenCalledWith(IMAGE_RECORD)
})

it('fails before generation when an input path cannot be resolved', async () => {
mocks.resolveWorkspaceFileReference.mockResolvedValue(null)

const result = await generateImageServerTool.execute(
{
prompt: 'Edit the reference image',
inputs: { files: [{ path: 'files/missing.png' }] },
outputs: { files: [{ path: 'files/result.png' }] },
},
CONTEXT
)

expect(result).toEqual({
success: false,
message: 'Failed to generate image: Input file not found: files/missing.png',
})
expect(mocks.generateContent).not.toHaveBeenCalled()
expect(mocks.writeWorkspaceFileByPath).not.toHaveBeenCalled()
})

it('fails before generation when outputs.files targets uploads/', async () => {
const result = await generateImageServerTool.execute(
{
prompt: 'Create a portrait',
outputs: { files: [{ path: 'uploads/result.png' }] },
},
CONTEXT
)

expect(result).toEqual({
success: false,
message:
'Failed to generate image: Media output paths must start with "files/"; uploads/ paths are read-only: uploads/result.png',
})
expect(mocks.generateContent).not.toHaveBeenCalled()
expect(mocks.writeWorkspaceFileByPath).not.toHaveBeenCalled()
})

it('fails before generation when an explicit output path is empty', async () => {
mocks.validateWorkspaceFileWriteTarget.mockRejectedValue(
new Error('Workspace file paths must start with "files/"')
)

const result = await generateImageServerTool.execute(
{
prompt: 'Create a portrait',
outputs: { files: [{ path: '' }] },
},
CONTEXT
)

expect(result.success).toBe(false)
expect(mocks.generateContent).not.toHaveBeenCalled()
expect(mocks.writeWorkspaceFileByPath).not.toHaveBeenCalled()
})

it('rejects additional output declarations before generation', async () => {
const result = await generateImageServerTool.execute(
{
prompt: 'Create a portrait',
outputs: {
files: [{ path: 'files/one.png' }, { path: 'uploads/two.png' }],
},
},
CONTEXT
)

expect(result).toEqual({
success: false,
message: 'Failed to generate image: Output requires exactly one file; received 2',
})
expect(mocks.validateWorkspaceFileWriteTarget).not.toHaveBeenCalled()
expect(mocks.generateContent).not.toHaveBeenCalled()
})

it('rejects explicitly empty input and output containers before generation', async () => {
const emptyInputs = await generateImageServerTool.execute(
{
prompt: 'Edit the supplied image',
inputs: {},
outputs: { files: [{ path: 'files/result.png' }] },
},
CONTEXT
)
const emptyOutputs = await generateImageServerTool.execute(
{ prompt: 'Create a portrait', outputs: {} },
CONTEXT
)

expect(emptyInputs).toEqual({
success: false,
message: 'Failed to generate image: Input requires at least one file',
})
expect(emptyOutputs).toEqual({
success: false,
message: 'Failed to generate image: Output requires exactly one file; received 0',
})
expect(mocks.generateContent).not.toHaveBeenCalled()
})

it('fails before generation when canonical output preflight rejects a files/ target', async () => {
mocks.validateWorkspaceFileWriteTarget.mockRejectedValue(
new Error('File already exists at files/result.png')
)

const result = await generateImageServerTool.execute(
{
prompt: 'Create a portrait',
outputs: { files: [{ path: 'files/result.png' }] },
},
CONTEXT
)

expect(result.success).toBe(false)
expect(mocks.generateContent).not.toHaveBeenCalled()
expect(mocks.writeWorkspaceFileByPath).not.toHaveBeenCalled()
})
})
73 changes: 37 additions & 36 deletions apps/sim/lib/copilot/tools/server/image/generate-image.ts
Original file line number Diff line number Diff line change
@@ -1,18 +1,20 @@
import { GoogleGenAI, type Part } from '@google/genai'
import { createLogger } from '@sim/logger'
import { getErrorMessage, toError } from '@sim/utils/errors'
import { getErrorMessage } from '@sim/utils/errors'
import { GenerateImage } from '@/lib/copilot/generated/tool-catalog-v1'
import {
assertServerToolNotAborted,
type BaseServerTool,
type ServerToolContext,
} from '@/lib/copilot/tools/server/base-tool'
import {
prepareMediaOutput,
requireMediaFileDeclarations,
resolveMediaInputFile,
} from '@/lib/copilot/tools/server/media/file-paths'
import { writeWorkspaceFileByPath } from '@/lib/copilot/vfs/resource-writer'
import { getRotatingApiKey } from '@/lib/core/config/api-keys'
import {
fetchWorkspaceFileBuffer,
resolveWorkspaceFileReference,
} from '@/lib/uploads/contexts/workspace/workspace-file-manager'
import { fetchWorkspaceFileBuffer } from '@/lib/uploads/contexts/workspace/workspace-file-manager'

const logger = createLogger('GenerateImageTool')

Expand Down Expand Up @@ -74,6 +76,12 @@ export const generateImageServerTool: BaseServerTool<GenerateImageArgs, Generate
}

try {
const outputFile = await prepareMediaOutput({
output: params.outputs,
workspaceId,
userId: context.userId,
})

const apiKey = getRotatingApiKey('gemini')
const ai = new GoogleGenAI({ apiKey })

Expand All @@ -82,34 +90,29 @@ export const generateImageServerTool: BaseServerTool<GenerateImageArgs, Generate

const parts: Part[] = []

const referencePaths = params.inputs?.files?.map((file) => file.path) ?? []
const referencePaths = params.inputs
? requireMediaFileDeclarations(params.inputs.files, 'Input').map((file) => file.path)
: []

if (referencePaths.length) {
for (const filePath of referencePaths) {
try {
const fileRecord = await resolveWorkspaceFileReference(workspaceId, filePath)
if (fileRecord) {
const buffer = await fetchWorkspaceFileBuffer(fileRecord)
const base64 = buffer.toString('base64')
const mime = fileRecord.type || 'image/png'
parts.push({
inlineData: { mimeType: mime, data: base64 },
})
logger.info('Loaded reference image', {
filePath,
name: fileRecord.name,
size: buffer.length,
mimeType: mime,
})
} else {
logger.warn('Reference file not found, skipping', { filePath })
}
} catch (err) {
logger.warn('Failed to load reference image, skipping', {
filePath,
error: toError(err).message,
})
}
const fileRecord = await resolveMediaInputFile({
workspaceId,
chatId: context.chatId,
path: filePath,
})
const buffer = await fetchWorkspaceFileBuffer(fileRecord)
const base64 = buffer.toString('base64')
const mime = fileRecord.type || 'image/png'
parts.push({
inlineData: { mimeType: mime, data: base64 },
})
logger.info('Loaded reference image', {
filePath,
name: fileRecord.name,
size: buffer.length,
mimeType: mime,
})
}
}

Expand Down Expand Up @@ -160,20 +163,18 @@ export const generateImageServerTool: BaseServerTool<GenerateImageArgs, Generate
}
}

const ext = mimeType.includes('jpeg') || mimeType.includes('jpg') ? '.jpg' : '.png'
const outputFile = params.outputs?.files?.[0]
const outputPath = outputFile?.path || `files/generated-image${ext}`
const resolvedOutputPath = outputFile.path
const imageBuffer = Buffer.from(imageBase64, 'base64')
const mode = outputFile?.mode ?? 'create'
const mode = outputFile.mode

assertServerToolNotAborted(context)
const written = await writeWorkspaceFileByPath({
workspaceId,
userId: context.userId,
target: {
path: outputPath,
path: resolvedOutputPath,
mode,
mimeType: outputFile?.mimeType,
mimeType: outputFile.mimeType,
},
buffer: imageBuffer,
inferredMimeType: mimeType,
Expand Down
Loading
Loading