feat: support claude agent SDK-style structured outputs in the OpenCode SDK (#8161)
Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com> Co-authored-by: Dax Raad <d@ironbay.co>
This commit is contained in:
Kyle Mistele
@@ -0,0 +1,233 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import path from "path"
|
||||
import { Session } from "../../src/session"
|
||||
import { SessionPrompt } from "../../src/session/prompt"
|
||||
import { Log } from "../../src/util/log"
|
||||
import { Instance } from "../../src/project/instance"
|
||||
import { MessageV2 } from "../../src/session/message-v2"
|
||||
|
||||
const projectRoot = path.join(__dirname, "../..")
|
||||
Log.init({ print: false })
|
||||
|
||||
// Skip tests if no API key is available
|
||||
const hasApiKey = !!process.env.ANTHROPIC_API_KEY
|
||||
|
||||
// Helper to run test within Instance context
|
||||
async function withInstance<T>(fn: () => Promise<T>): Promise<T> {
|
||||
return Instance.provide({
|
||||
directory: projectRoot,
|
||||
fn,
|
||||
})
|
||||
}
|
||||
|
||||
describe("StructuredOutput Integration", () => {
|
||||
test.skipIf(!hasApiKey)(
|
||||
"produces structured output with simple schema",
|
||||
async () => {
|
||||
await withInstance(async () => {
|
||||
const session = await Session.create({ title: "Structured Output Test" })
|
||||
|
||||
const result = await SessionPrompt.prompt({
|
||||
sessionID: session.id,
|
||||
parts: [
|
||||
{
|
||||
type: "text",
|
||||
text: "What is 2 + 2? Provide a simple answer.",
|
||||
},
|
||||
],
|
||||
format: {
|
||||
type: "json_schema",
|
||||
schema: {
|
||||
type: "object",
|
||||
properties: {
|
||||
answer: { type: "number", description: "The numerical answer" },
|
||||
explanation: { type: "string", description: "Brief explanation" },
|
||||
},
|
||||
required: ["answer"],
|
||||
},
|
||||
retryCount: 0,
|
||||
},
|
||||
})
|
||||
|
||||
// Verify structured output was captured (only on assistant messages)
|
||||
expect(result.info.role).toBe("assistant")
|
||||
if (result.info.role === "assistant") {
|
||||
expect(result.info.structured).toBeDefined()
|
||||
expect(typeof result.info.structured).toBe("object")
|
||||
|
||||
const output = result.info.structured as any
|
||||
expect(output.answer).toBe(4)
|
||||
|
||||
// Verify no error was set
|
||||
expect(result.info.error).toBeUndefined()
|
||||
}
|
||||
|
||||
// Clean up
|
||||
// Note: Not removing session to avoid race with background SessionSummary.summarize
|
||||
})
|
||||
},
|
||||
60000,
|
||||
)
|
||||
|
||||
test.skipIf(!hasApiKey)(
|
||||
"produces structured output with nested objects",
|
||||
async () => {
|
||||
await withInstance(async () => {
|
||||
const session = await Session.create({ title: "Nested Schema Test" })
|
||||
|
||||
const result = await SessionPrompt.prompt({
|
||||
sessionID: session.id,
|
||||
parts: [
|
||||
{
|
||||
type: "text",
|
||||
text: "Tell me about Anthropic company in a structured format.",
|
||||
},
|
||||
],
|
||||
format: {
|
||||
type: "json_schema",
|
||||
schema: {
|
||||
type: "object",
|
||||
properties: {
|
||||
company: {
|
||||
type: "object",
|
||||
properties: {
|
||||
name: { type: "string" },
|
||||
founded: { type: "number" },
|
||||
},
|
||||
required: ["name", "founded"],
|
||||
},
|
||||
products: {
|
||||
type: "array",
|
||||
items: { type: "string" },
|
||||
},
|
||||
},
|
||||
required: ["company"],
|
||||
},
|
||||
retryCount: 0,
|
||||
},
|
||||
})
|
||||
|
||||
// Verify structured output was captured (only on assistant messages)
|
||||
expect(result.info.role).toBe("assistant")
|
||||
if (result.info.role === "assistant") {
|
||||
expect(result.info.structured).toBeDefined()
|
||||
const output = result.info.structured as any
|
||||
|
||||
expect(output.company).toBeDefined()
|
||||
expect(output.company.name).toBe("Anthropic")
|
||||
expect(typeof output.company.founded).toBe("number")
|
||||
|
||||
if (output.products) {
|
||||
expect(Array.isArray(output.products)).toBe(true)
|
||||
}
|
||||
|
||||
// Verify no error was set
|
||||
expect(result.info.error).toBeUndefined()
|
||||
}
|
||||
|
||||
// Clean up
|
||||
// Note: Not removing session to avoid race with background SessionSummary.summarize
|
||||
})
|
||||
},
|
||||
60000,
|
||||
)
|
||||
|
||||
test.skipIf(!hasApiKey)(
|
||||
"works with text outputFormat (default)",
|
||||
async () => {
|
||||
await withInstance(async () => {
|
||||
const session = await Session.create({ title: "Text Output Test" })
|
||||
|
||||
const result = await SessionPrompt.prompt({
|
||||
sessionID: session.id,
|
||||
parts: [
|
||||
{
|
||||
type: "text",
|
||||
text: "Say hello.",
|
||||
},
|
||||
],
|
||||
format: {
|
||||
type: "text",
|
||||
},
|
||||
})
|
||||
|
||||
// Verify no structured output (text mode) and no error
|
||||
expect(result.info.role).toBe("assistant")
|
||||
if (result.info.role === "assistant") {
|
||||
expect(result.info.structured).toBeUndefined()
|
||||
expect(result.info.error).toBeUndefined()
|
||||
}
|
||||
|
||||
// Verify we got a response with parts
|
||||
expect(result.parts.length).toBeGreaterThan(0)
|
||||
|
||||
// Clean up
|
||||
// Note: Not removing session to avoid race with background SessionSummary.summarize
|
||||
})
|
||||
},
|
||||
60000,
|
||||
)
|
||||
|
||||
test.skipIf(!hasApiKey)(
|
||||
"stores outputFormat on user message",
|
||||
async () => {
|
||||
await withInstance(async () => {
|
||||
const session = await Session.create({ title: "OutputFormat Storage Test" })
|
||||
|
||||
await SessionPrompt.prompt({
|
||||
sessionID: session.id,
|
||||
parts: [
|
||||
{
|
||||
type: "text",
|
||||
text: "What is 1 + 1?",
|
||||
},
|
||||
],
|
||||
format: {
|
||||
type: "json_schema",
|
||||
schema: {
|
||||
type: "object",
|
||||
properties: {
|
||||
result: { type: "number" },
|
||||
},
|
||||
required: ["result"],
|
||||
},
|
||||
retryCount: 3,
|
||||
},
|
||||
})
|
||||
|
||||
// Get all messages from session
|
||||
const messages = await Session.messages({ sessionID: session.id })
|
||||
const userMessage = messages.find((m) => m.info.role === "user")
|
||||
|
||||
// Verify outputFormat was stored on user message
|
||||
expect(userMessage).toBeDefined()
|
||||
if (userMessage?.info.role === "user") {
|
||||
expect(userMessage.info.format).toBeDefined()
|
||||
expect(userMessage.info.format?.type).toBe("json_schema")
|
||||
if (userMessage.info.format?.type === "json_schema") {
|
||||
expect(userMessage.info.format.retryCount).toBe(3)
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up
|
||||
// Note: Not removing session to avoid race with background SessionSummary.summarize
|
||||
})
|
||||
},
|
||||
60000,
|
||||
)
|
||||
|
||||
test("unit test: StructuredOutputError is properly structured", () => {
|
||||
const error = new MessageV2.StructuredOutputError({
|
||||
message: "Failed to produce valid structured output after 3 attempts",
|
||||
retries: 3,
|
||||
})
|
||||
|
||||
expect(error.name).toBe("StructuredOutputError")
|
||||
expect(error.data.message).toContain("3 attempts")
|
||||
expect(error.data.retries).toBe(3)
|
||||
|
||||
const obj = error.toObject()
|
||||
expect(obj.name).toBe("StructuredOutputError")
|
||||
expect(obj.data.retries).toBe(3)
|
||||
})
|
||||
})
|
||||
385
packages/opencode/test/session/structured-output.test.ts
Normal file
385
packages/opencode/test/session/structured-output.test.ts
Normal file
@@ -0,0 +1,385 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import { MessageV2 } from "../../src/session/message-v2"
|
||||
import { SessionPrompt } from "../../src/session/prompt"
|
||||
|
||||
describe("structured-output.OutputFormat", () => {
|
||||
test("parses text format", () => {
|
||||
const result = MessageV2.Format.safeParse({ type: "text" })
|
||||
expect(result.success).toBe(true)
|
||||
if (result.success) {
|
||||
expect(result.data.type).toBe("text")
|
||||
}
|
||||
})
|
||||
|
||||
test("parses json_schema format with defaults", () => {
|
||||
const result = MessageV2.Format.safeParse({
|
||||
type: "json_schema",
|
||||
schema: { type: "object", properties: { name: { type: "string" } } },
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
if (result.success) {
|
||||
expect(result.data.type).toBe("json_schema")
|
||||
if (result.data.type === "json_schema") {
|
||||
expect(result.data.retryCount).toBe(2) // default value
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
test("parses json_schema format with custom retryCount", () => {
|
||||
const result = MessageV2.Format.safeParse({
|
||||
type: "json_schema",
|
||||
schema: { type: "object" },
|
||||
retryCount: 5,
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
if (result.success && result.data.type === "json_schema") {
|
||||
expect(result.data.retryCount).toBe(5)
|
||||
}
|
||||
})
|
||||
|
||||
test("rejects invalid type", () => {
|
||||
const result = MessageV2.Format.safeParse({ type: "invalid" })
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
test("rejects json_schema without schema", () => {
|
||||
const result = MessageV2.Format.safeParse({ type: "json_schema" })
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
test("rejects negative retryCount", () => {
|
||||
const result = MessageV2.Format.safeParse({
|
||||
type: "json_schema",
|
||||
schema: { type: "object" },
|
||||
retryCount: -1,
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe("structured-output.StructuredOutputError", () => {
|
||||
test("creates error with message and retries", () => {
|
||||
const error = new MessageV2.StructuredOutputError({
|
||||
message: "Failed to validate",
|
||||
retries: 3,
|
||||
})
|
||||
|
||||
expect(error.name).toBe("StructuredOutputError")
|
||||
expect(error.data.message).toBe("Failed to validate")
|
||||
expect(error.data.retries).toBe(3)
|
||||
})
|
||||
|
||||
test("converts to object correctly", () => {
|
||||
const error = new MessageV2.StructuredOutputError({
|
||||
message: "Test error",
|
||||
retries: 2,
|
||||
})
|
||||
|
||||
const obj = error.toObject()
|
||||
expect(obj.name).toBe("StructuredOutputError")
|
||||
expect(obj.data.message).toBe("Test error")
|
||||
expect(obj.data.retries).toBe(2)
|
||||
})
|
||||
|
||||
test("isInstance correctly identifies error", () => {
|
||||
const error = new MessageV2.StructuredOutputError({
|
||||
message: "Test",
|
||||
retries: 1,
|
||||
})
|
||||
|
||||
expect(MessageV2.StructuredOutputError.isInstance(error)).toBe(true)
|
||||
expect(MessageV2.StructuredOutputError.isInstance({ name: "other" })).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe("structured-output.UserMessage", () => {
|
||||
test("user message accepts outputFormat", () => {
|
||||
const result = MessageV2.User.safeParse({
|
||||
id: "test-id",
|
||||
sessionID: "test-session",
|
||||
role: "user",
|
||||
time: { created: Date.now() },
|
||||
agent: "default",
|
||||
model: { providerID: "anthropic", modelID: "claude-3" },
|
||||
outputFormat: {
|
||||
type: "json_schema",
|
||||
schema: { type: "object" },
|
||||
},
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
test("user message works without outputFormat (optional)", () => {
|
||||
const result = MessageV2.User.safeParse({
|
||||
id: "test-id",
|
||||
sessionID: "test-session",
|
||||
role: "user",
|
||||
time: { created: Date.now() },
|
||||
agent: "default",
|
||||
model: { providerID: "anthropic", modelID: "claude-3" },
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe("structured-output.AssistantMessage", () => {
|
||||
const baseAssistantMessage = {
|
||||
id: "test-id",
|
||||
sessionID: "test-session",
|
||||
role: "assistant" as const,
|
||||
parentID: "parent-id",
|
||||
modelID: "claude-3",
|
||||
providerID: "anthropic",
|
||||
mode: "default",
|
||||
agent: "default",
|
||||
path: { cwd: "/test", root: "/test" },
|
||||
cost: 0.001,
|
||||
tokens: { input: 100, output: 50, reasoning: 0, cache: { read: 0, write: 0 } },
|
||||
time: { created: Date.now() },
|
||||
}
|
||||
|
||||
test("assistant message accepts structured", () => {
|
||||
const result = MessageV2.Assistant.safeParse({
|
||||
...baseAssistantMessage,
|
||||
structured: { company: "Anthropic", founded: 2021 },
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
if (result.success) {
|
||||
expect(result.data.structured).toEqual({ company: "Anthropic", founded: 2021 })
|
||||
}
|
||||
})
|
||||
|
||||
test("assistant message works without structured_output (optional)", () => {
|
||||
const result = MessageV2.Assistant.safeParse(baseAssistantMessage)
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe("structured-output.createStructuredOutputTool", () => {
|
||||
test("creates tool with correct id", () => {
|
||||
const tool = SessionPrompt.createStructuredOutputTool({
|
||||
schema: { type: "object", properties: { name: { type: "string" } } },
|
||||
onSuccess: () => {},
|
||||
})
|
||||
|
||||
// AI SDK tool type doesn't expose id, but we set it internally
|
||||
expect((tool as any).id).toBe("StructuredOutput")
|
||||
})
|
||||
|
||||
test("creates tool with description", () => {
|
||||
const tool = SessionPrompt.createStructuredOutputTool({
|
||||
schema: { type: "object" },
|
||||
onSuccess: () => {},
|
||||
})
|
||||
|
||||
expect(tool.description).toContain("structured format")
|
||||
})
|
||||
|
||||
test("creates tool with schema as inputSchema", () => {
|
||||
const schema = {
|
||||
type: "object",
|
||||
properties: {
|
||||
company: { type: "string" },
|
||||
founded: { type: "number" },
|
||||
},
|
||||
required: ["company"],
|
||||
}
|
||||
|
||||
const tool = SessionPrompt.createStructuredOutputTool({
|
||||
schema,
|
||||
onSuccess: () => {},
|
||||
})
|
||||
|
||||
// AI SDK wraps schema in { jsonSchema: {...} }
|
||||
expect(tool.inputSchema).toBeDefined()
|
||||
const inputSchema = tool.inputSchema as any
|
||||
expect(inputSchema.jsonSchema?.properties?.company).toBeDefined()
|
||||
expect(inputSchema.jsonSchema?.properties?.founded).toBeDefined()
|
||||
})
|
||||
|
||||
test("strips $schema property from inputSchema", () => {
|
||||
const schema = {
|
||||
$schema: "http://json-schema.org/draft-07/schema#",
|
||||
type: "object",
|
||||
properties: { name: { type: "string" } },
|
||||
}
|
||||
|
||||
const tool = SessionPrompt.createStructuredOutputTool({
|
||||
schema,
|
||||
onSuccess: () => {},
|
||||
})
|
||||
|
||||
// AI SDK wraps schema in { jsonSchema: {...} }
|
||||
const inputSchema = tool.inputSchema as any
|
||||
expect(inputSchema.jsonSchema?.$schema).toBeUndefined()
|
||||
})
|
||||
|
||||
test("execute calls onSuccess with valid args", async () => {
|
||||
let capturedOutput: unknown
|
||||
|
||||
const tool = SessionPrompt.createStructuredOutputTool({
|
||||
schema: { type: "object", properties: { name: { type: "string" } } },
|
||||
onSuccess: (output) => {
|
||||
capturedOutput = output
|
||||
},
|
||||
})
|
||||
|
||||
expect(tool.execute).toBeDefined()
|
||||
const testArgs = { name: "Test Company" }
|
||||
const result = await tool.execute!(testArgs, {
|
||||
toolCallId: "test-call-id",
|
||||
messages: [],
|
||||
abortSignal: undefined as any,
|
||||
})
|
||||
|
||||
expect(capturedOutput).toEqual(testArgs)
|
||||
expect(result.output).toBe("Structured output captured successfully.")
|
||||
expect(result.metadata.valid).toBe(true)
|
||||
})
|
||||
|
||||
test("AI SDK validates schema before execute - missing required field", async () => {
|
||||
// Note: The AI SDK validates the input against the schema BEFORE calling execute()
|
||||
// So invalid inputs never reach the tool's execute function
|
||||
// This test documents the expected schema behavior
|
||||
const tool = SessionPrompt.createStructuredOutputTool({
|
||||
schema: {
|
||||
type: "object",
|
||||
properties: {
|
||||
name: { type: "string" },
|
||||
age: { type: "number" },
|
||||
},
|
||||
required: ["name", "age"],
|
||||
},
|
||||
onSuccess: () => {},
|
||||
})
|
||||
|
||||
// The schema requires both 'name' and 'age'
|
||||
expect(tool.inputSchema).toBeDefined()
|
||||
const inputSchema = tool.inputSchema as any
|
||||
expect(inputSchema.jsonSchema?.required).toContain("name")
|
||||
expect(inputSchema.jsonSchema?.required).toContain("age")
|
||||
})
|
||||
|
||||
test("AI SDK validates schema types before execute - wrong type", async () => {
|
||||
// Note: The AI SDK validates the input against the schema BEFORE calling execute()
|
||||
// So invalid inputs never reach the tool's execute function
|
||||
// This test documents the expected schema behavior
|
||||
const tool = SessionPrompt.createStructuredOutputTool({
|
||||
schema: {
|
||||
type: "object",
|
||||
properties: {
|
||||
count: { type: "number" },
|
||||
},
|
||||
required: ["count"],
|
||||
},
|
||||
onSuccess: () => {},
|
||||
})
|
||||
|
||||
// The schema defines 'count' as a number
|
||||
expect(tool.inputSchema).toBeDefined()
|
||||
const inputSchema = tool.inputSchema as any
|
||||
expect(inputSchema.jsonSchema?.properties?.count?.type).toBe("number")
|
||||
})
|
||||
|
||||
test("execute handles nested objects", async () => {
|
||||
let capturedOutput: unknown
|
||||
|
||||
const tool = SessionPrompt.createStructuredOutputTool({
|
||||
schema: {
|
||||
type: "object",
|
||||
properties: {
|
||||
user: {
|
||||
type: "object",
|
||||
properties: {
|
||||
name: { type: "string" },
|
||||
email: { type: "string" },
|
||||
},
|
||||
required: ["name"],
|
||||
},
|
||||
},
|
||||
required: ["user"],
|
||||
},
|
||||
onSuccess: (output) => {
|
||||
capturedOutput = output
|
||||
},
|
||||
})
|
||||
|
||||
// Valid nested object - AI SDK validates before calling execute()
|
||||
const validResult = await tool.execute!(
|
||||
{ user: { name: "John", email: "john@test.com" } },
|
||||
{
|
||||
toolCallId: "test-call-id",
|
||||
messages: [],
|
||||
abortSignal: undefined as any,
|
||||
},
|
||||
)
|
||||
|
||||
expect(capturedOutput).toEqual({ user: { name: "John", email: "john@test.com" } })
|
||||
expect(validResult.metadata.valid).toBe(true)
|
||||
|
||||
// Verify schema has correct nested structure
|
||||
const inputSchema = tool.inputSchema as any
|
||||
expect(inputSchema.jsonSchema?.properties?.user?.type).toBe("object")
|
||||
expect(inputSchema.jsonSchema?.properties?.user?.properties?.name?.type).toBe("string")
|
||||
expect(inputSchema.jsonSchema?.properties?.user?.required).toContain("name")
|
||||
})
|
||||
|
||||
test("execute handles arrays", async () => {
|
||||
let capturedOutput: unknown
|
||||
|
||||
const tool = SessionPrompt.createStructuredOutputTool({
|
||||
schema: {
|
||||
type: "object",
|
||||
properties: {
|
||||
tags: {
|
||||
type: "array",
|
||||
items: { type: "string" },
|
||||
},
|
||||
},
|
||||
required: ["tags"],
|
||||
},
|
||||
onSuccess: (output) => {
|
||||
capturedOutput = output
|
||||
},
|
||||
})
|
||||
|
||||
// Valid array - AI SDK validates before calling execute()
|
||||
const validResult = await tool.execute!(
|
||||
{ tags: ["a", "b", "c"] },
|
||||
{
|
||||
toolCallId: "test-call-id",
|
||||
messages: [],
|
||||
abortSignal: undefined as any,
|
||||
},
|
||||
)
|
||||
|
||||
expect(capturedOutput).toEqual({ tags: ["a", "b", "c"] })
|
||||
expect(validResult.metadata.valid).toBe(true)
|
||||
|
||||
// Verify schema has correct array structure
|
||||
const inputSchema = tool.inputSchema as any
|
||||
expect(inputSchema.jsonSchema?.properties?.tags?.type).toBe("array")
|
||||
expect(inputSchema.jsonSchema?.properties?.tags?.items?.type).toBe("string")
|
||||
})
|
||||
|
||||
test("toModelOutput returns text value", () => {
|
||||
const tool = SessionPrompt.createStructuredOutputTool({
|
||||
schema: { type: "object" },
|
||||
onSuccess: () => {},
|
||||
})
|
||||
|
||||
expect(tool.toModelOutput).toBeDefined()
|
||||
const modelOutput = tool.toModelOutput!({
|
||||
output: "Test output",
|
||||
title: "Test",
|
||||
metadata: { valid: true },
|
||||
})
|
||||
|
||||
expect(modelOutput.type).toBe("text")
|
||||
expect(modelOutput.value).toBe("Test output")
|
||||
})
|
||||
|
||||
// Note: Retry behavior is handled by the AI SDK and the prompt loop, not the tool itself
|
||||
// The tool simply calls onSuccess when execute() is called with valid args
|
||||
// See prompt.ts loop() for actual retry logic
|
||||
})
|
||||
Reference in New Issue
Block a user