Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 44 additions & 8 deletions packages/core/src/aisdk.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,16 +33,23 @@ import {
} from "@opencode-ai/ai"
import { Auth, Endpoint, RequestExecutor, type AnyRoute } from "@opencode-ai/ai/route"
import { ProviderShared } from "@opencode-ai/ai/protocols/shared"
import type { Content } from "@opencode-ai/schema/tool"
import { Cause, Context, Effect, Layer, Option, Schema, Scope, Stream } from "effect"
import type { ID, Info } from "./model"
import { Provider } from "./provider"
import { State } from "./state"

type SDK = any
type UserContent = Extract<LanguageModelV3Message, { role: "user" }>["content"]
type UserFileContent = Extract<UserContent[number], { type: "file" }>
type AssistantContent = Extract<LanguageModelV3Message, { role: "assistant" }>["content"]
type ToolResultContent = Extract<AssistantContent[number], { type: "tool-result" }>

const SYNTHETIC_ATTACHMENT_PROMPT = "Attached media from tool result:"
const TOOL_RESULT_ATTACHMENT_TEXT = "Media attached in following user message."
const isToolResultMedia = (item: Content): item is Extract<Content, { type: "file" }> =>
item.type === "file" && (item.mime.toLowerCase().startsWith("image/") || item.mime === "application/pdf")

export interface SDKEvent {
readonly model: Info
readonly package: string
Expand Down Expand Up @@ -435,21 +442,34 @@ function prompt(request: LLMRequest): LanguageModelV3Prompt {
.map((part) => part.text)
.filter(Boolean)
.join("\n\n")
const messages = request.messages.flatMap(message)
const messages: LanguageModelV3Message[] = []
const media: UserFileContent[] = []
const flushMedia = () => {
if (media.length === 0) return
messages.push({
role: "user",
content: [{ type: "text", text: SYNTHETIC_ATTACHMENT_PROMPT }, ...media.splice(0)],
})
}
for (const input of request.messages) {
if (input.role !== "tool") flushMedia()
messages.push(...message(input, media))
}
flushMedia()
if (!system.length) return messages
return [{ role: "system", content: system }, ...messages]
}

function message(input: LLMRequest["messages"][number]): LanguageModelV3Message[] {
function message(input: LLMRequest["messages"][number], media: UserFileContent[]): LanguageModelV3Message[] {
switch (input.role) {
case "system":
return [{ role: "system", content: input.content.flatMap(text).join("\n\n") }]
case "user":
return [{ role: "user", content: input.content.flatMap(userPart) }]
case "assistant":
return [{ role: "assistant", content: input.content.flatMap(assistantPart) }]
return [{ role: "assistant", content: input.content.flatMap((part) => assistantPart(part, media)) }]
case "tool": {
const content = input.content.flatMap(toolResultPart)
const content = input.content.flatMap((part) => toolResultPart(part, media))
return content.length ? [{ role: "tool", content }] : []
}
}
Expand All @@ -466,7 +486,7 @@ function userPart(part: ContentPart): UserContent {
return []
}

function assistantPart(part: ContentPart): AssistantContent {
function assistantPart(part: ContentPart, media: UserFileContent[]): AssistantContent {
switch (part.type) {
case "text":
return [{ type: "text", text: part.text }]
Expand All @@ -486,18 +506,34 @@ function assistantPart(part: ContentPart): AssistantContent {
},
]
case "tool-result":
return toolResultPart(part)
return toolResultPart(part, media)
}
}

function toolResultPart(part: ContentPart): ToolResultContent[] {
function toolResultPart(part: ContentPart, media: UserFileContent[]): ToolResultContent[] {
if (part.type !== "tool-result") return []
const result = (() => {
if (part.result.type !== "content") return part.result
const extracted = part.result.value.filter(isToolResultMedia)
if (extracted.length === 0) return part.result
media.push(
...extracted.map((item) => ({
type: "file" as const,
mediaType: item.mime,
data: item.uri,
filename: item.name,
})),
)
const content = part.result.value.filter((item) => !isToolResultMedia(item))
if (content.length === 0) return { type: "text" as const, value: TOOL_RESULT_ATTACHMENT_TEXT }
return { type: "content" as const, value: content }
})()
return [
{
type: "tool-result",
toolCallId: part.id,
toolName: part.name,
output: toolOutput(part.result),
output: toolOutput(result),
providerOptions: providerOptions(part.providerMetadata),
},
]
Expand Down
95 changes: 85 additions & 10 deletions packages/core/test/aisdk.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -275,7 +275,7 @@ it.effect("projects replay metadata onto AI SDK prompt parts", () =>
}),
)

it.effect("preserves tool result content in AI SDK prompts", () =>
it.effect("moves tool result images and PDFs into an AI SDK user message", () =>
Effect.gen(function* () {
const aisdk = yield* AISDK.Service
yield* aisdk.hook.sdk((event) => {
Expand Down Expand Up @@ -323,21 +323,96 @@ it.effect("preserves tool result content in AI SDK prompts", () =>
type: "content",
value: [
{ type: "text", text: "attachments" },
{ type: "image-data", data: "AAAA", mediaType: "image/png" },
{
type: "file-data",
data: "JVBERg==",
mediaType: "application/pdf",
filename: "document.pdf",
},
{ type: "file-data", data: "SUQz", mediaType: "audio/mpeg", filename: "clip.mp3" },
{ type: "image-url", url: "https://example.com/pixel.png" },
{ type: "file-url", url: "https://example.com/document.pdf" },
],
},
},
],
},
{
role: "user",
content: [
{ type: "text", text: "Attached media from tool result:" },
{ type: "file", mediaType: "image/png", data: "data:image/png;base64,AAAA", filename: "pixel.png" },
{
type: "file",
mediaType: "application/pdf",
data: "data:application/pdf;charset=utf-8;base64,JVBERg==",
filename: "document.pdf",
},
{ type: "file", mediaType: "image/png", data: "https://example.com/pixel.png" },
{ type: "file", mediaType: "application/pdf", data: "https://example.com/document.pdf" },
],
},
])
}),
)

it.effect("groups consecutive AI SDK tool media and keeps file-only results non-empty", () =>
Effect.gen(function* () {
const aisdk = yield* AISDK.Service
yield* aisdk.hook.sdk((event) => {
event.sdk = { languageModel: () => ({ provider: event.model.providerID }) }
})

const resolved = yield* aisdk.model(model("test-ai-sdk"))
const prepared = yield* compileRequest(
LLM.request({
model: resolved,
messages: [
Message.tool({
id: "call_1",
name: "read",
result: {
type: "content",
value: [{ type: "file", uri: "data:image/png;base64,AAAA", mime: "image/png", name: "one.png" }],
},
}),
Message.tool({
id: "call_2",
name: "read",
result: {
type: "content",
value: [{ type: "file", uri: "data:image/png;base64,BBBB", mime: "image/png", name: "two.png" }],
},
}),
Message.assistant("Images received"),
],
}),
)

expect(prepared.body.prompt).toEqual([
{
role: "tool",
content: [
{
type: "tool-result",
toolCallId: "call_1",
toolName: "read",
output: { type: "text", value: "Media attached in following user message." },
},
],
},
{
role: "tool",
content: [
{
type: "tool-result",
toolCallId: "call_2",
toolName: "read",
output: { type: "text", value: "Media attached in following user message." },
},
],
},
{
role: "user",
content: [
{ type: "text", text: "Attached media from tool result:" },
{ type: "file", mediaType: "image/png", data: "data:image/png;base64,AAAA", filename: "one.png" },
{ type: "file", mediaType: "image/png", data: "data:image/png;base64,BBBB", filename: "two.png" },
],
},
{ role: "assistant", content: [{ type: "text", text: "Images received" }] },
])
}),
)
Expand Down
Loading