diff --git a/packages/core/src/aisdk.ts b/packages/core/src/aisdk.ts index 51765799248f..ab01b79648a4 100644 --- a/packages/core/src/aisdk.ts +++ b/packages/core/src/aisdk.ts @@ -33,6 +33,7 @@ import { } from "@opencode-ai/ai" import { Auth, Endpoint, RequestExecutor, type AnyRoute } from "@opencode-ai/ai/route" import { ProviderShared } from "@opencode-ai/ai/protocols/shared" +import type { Content } from "@opencode-ai/schema/tool" import { Cause, Context, Effect, Layer, Option, Schema, Scope, Stream } from "effect" import type { ID, Info } from "./model" import { Provider } from "./provider" @@ -40,9 +41,15 @@ import { State } from "./state" type SDK = any type UserContent = Extract["content"] +type UserFileContent = Extract type AssistantContent = Extract["content"] type ToolResultContent = Extract +const SYNTHETIC_ATTACHMENT_PROMPT = "Attached media from tool result:" +const TOOL_RESULT_ATTACHMENT_TEXT = "Media attached in following user message." +const isToolResultMedia = (item: Content): item is Extract => + item.type === "file" && (item.mime.toLowerCase().startsWith("image/") || item.mime === "application/pdf") + export interface SDKEvent { readonly model: Info readonly package: string @@ -435,21 +442,34 @@ function prompt(request: LLMRequest): LanguageModelV3Prompt { .map((part) => part.text) .filter(Boolean) .join("\n\n") - const messages = request.messages.flatMap(message) + const messages: LanguageModelV3Message[] = [] + const media: UserFileContent[] = [] + const flushMedia = () => { + if (media.length === 0) return + messages.push({ + role: "user", + content: [{ type: "text", text: SYNTHETIC_ATTACHMENT_PROMPT }, ...media.splice(0)], + }) + } + for (const input of request.messages) { + if (input.role !== "tool") flushMedia() + messages.push(...message(input, media)) + } + flushMedia() if (!system.length) return messages return [{ role: "system", content: system }, ...messages] } -function message(input: LLMRequest["messages"][number]): LanguageModelV3Message[] { +function message(input: LLMRequest["messages"][number], media: UserFileContent[]): LanguageModelV3Message[] { switch (input.role) { case "system": return [{ role: "system", content: input.content.flatMap(text).join("\n\n") }] case "user": return [{ role: "user", content: input.content.flatMap(userPart) }] case "assistant": - return [{ role: "assistant", content: input.content.flatMap(assistantPart) }] + return [{ role: "assistant", content: input.content.flatMap((part) => assistantPart(part, media)) }] case "tool": { - const content = input.content.flatMap(toolResultPart) + const content = input.content.flatMap((part) => toolResultPart(part, media)) return content.length ? [{ role: "tool", content }] : [] } } @@ -466,7 +486,7 @@ function userPart(part: ContentPart): UserContent { return [] } -function assistantPart(part: ContentPart): AssistantContent { +function assistantPart(part: ContentPart, media: UserFileContent[]): AssistantContent { switch (part.type) { case "text": return [{ type: "text", text: part.text }] @@ -486,18 +506,34 @@ function assistantPart(part: ContentPart): AssistantContent { }, ] case "tool-result": - return toolResultPart(part) + return toolResultPart(part, media) } } -function toolResultPart(part: ContentPart): ToolResultContent[] { +function toolResultPart(part: ContentPart, media: UserFileContent[]): ToolResultContent[] { if (part.type !== "tool-result") return [] + const result = (() => { + if (part.result.type !== "content") return part.result + const extracted = part.result.value.filter(isToolResultMedia) + if (extracted.length === 0) return part.result + media.push( + ...extracted.map((item) => ({ + type: "file" as const, + mediaType: item.mime, + data: item.uri, + filename: item.name, + })), + ) + const content = part.result.value.filter((item) => !isToolResultMedia(item)) + if (content.length === 0) return { type: "text" as const, value: TOOL_RESULT_ATTACHMENT_TEXT } + return { type: "content" as const, value: content } + })() return [ { type: "tool-result", toolCallId: part.id, toolName: part.name, - output: toolOutput(part.result), + output: toolOutput(result), providerOptions: providerOptions(part.providerMetadata), }, ] diff --git a/packages/core/test/aisdk.test.ts b/packages/core/test/aisdk.test.ts index 9decce67b813..fddfa83037e6 100644 --- a/packages/core/test/aisdk.test.ts +++ b/packages/core/test/aisdk.test.ts @@ -275,7 +275,7 @@ it.effect("projects replay metadata onto AI SDK prompt parts", () => }), ) -it.effect("preserves tool result content in AI SDK prompts", () => +it.effect("moves tool result images and PDFs into an AI SDK user message", () => Effect.gen(function* () { const aisdk = yield* AISDK.Service yield* aisdk.hook.sdk((event) => { @@ -323,21 +323,96 @@ it.effect("preserves tool result content in AI SDK prompts", () => type: "content", value: [ { type: "text", text: "attachments" }, - { type: "image-data", data: "AAAA", mediaType: "image/png" }, - { - type: "file-data", - data: "JVBERg==", - mediaType: "application/pdf", - filename: "document.pdf", - }, { type: "file-data", data: "SUQz", mediaType: "audio/mpeg", filename: "clip.mp3" }, - { type: "image-url", url: "https://example.com/pixel.png" }, - { type: "file-url", url: "https://example.com/document.pdf" }, ], }, }, ], }, + { + role: "user", + content: [ + { type: "text", text: "Attached media from tool result:" }, + { type: "file", mediaType: "image/png", data: "data:image/png;base64,AAAA", filename: "pixel.png" }, + { + type: "file", + mediaType: "application/pdf", + data: "data:application/pdf;charset=utf-8;base64,JVBERg==", + filename: "document.pdf", + }, + { type: "file", mediaType: "image/png", data: "https://example.com/pixel.png" }, + { type: "file", mediaType: "application/pdf", data: "https://example.com/document.pdf" }, + ], + }, + ]) + }), +) + +it.effect("groups consecutive AI SDK tool media and keeps file-only results non-empty", () => + Effect.gen(function* () { + const aisdk = yield* AISDK.Service + yield* aisdk.hook.sdk((event) => { + event.sdk = { languageModel: () => ({ provider: event.model.providerID }) } + }) + + const resolved = yield* aisdk.model(model("test-ai-sdk")) + const prepared = yield* compileRequest( + LLM.request({ + model: resolved, + messages: [ + Message.tool({ + id: "call_1", + name: "read", + result: { + type: "content", + value: [{ type: "file", uri: "data:image/png;base64,AAAA", mime: "image/png", name: "one.png" }], + }, + }), + Message.tool({ + id: "call_2", + name: "read", + result: { + type: "content", + value: [{ type: "file", uri: "data:image/png;base64,BBBB", mime: "image/png", name: "two.png" }], + }, + }), + Message.assistant("Images received"), + ], + }), + ) + + expect(prepared.body.prompt).toEqual([ + { + role: "tool", + content: [ + { + type: "tool-result", + toolCallId: "call_1", + toolName: "read", + output: { type: "text", value: "Media attached in following user message." }, + }, + ], + }, + { + role: "tool", + content: [ + { + type: "tool-result", + toolCallId: "call_2", + toolName: "read", + output: { type: "text", value: "Media attached in following user message." }, + }, + ], + }, + { + role: "user", + content: [ + { type: "text", text: "Attached media from tool result:" }, + { type: "file", mediaType: "image/png", data: "data:image/png;base64,AAAA", filename: "one.png" }, + { type: "file", mediaType: "image/png", data: "data:image/png;base64,BBBB", filename: "two.png" }, + ], + }, + { role: "assistant", content: [{ type: "text", text: "Images received" }] }, ]) }), )