fix: variant logic for anthropic models through openai compat endpoint (#11665)
This commit is contained in:
@@ -395,31 +395,6 @@ export namespace ProviderTransform {
|
|||||||
case "@ai-sdk/deepinfra":
|
case "@ai-sdk/deepinfra":
|
||||||
// https://v5.ai-sdk.dev/providers/ai-sdk-providers/deepinfra
|
// https://v5.ai-sdk.dev/providers/ai-sdk-providers/deepinfra
|
||||||
case "@ai-sdk/openai-compatible":
|
case "@ai-sdk/openai-compatible":
|
||||||
// When using openai-compatible SDK with Claude/Anthropic models,
|
|
||||||
// we must use snake_case (budget_tokens) as the SDK doesn't convert parameter names
|
|
||||||
// and the OpenAI-compatible API spec uses snake_case
|
|
||||||
if (
|
|
||||||
model.providerID === "anthropic" ||
|
|
||||||
model.api.id.includes("anthropic") ||
|
|
||||||
model.api.id.includes("claude") ||
|
|
||||||
model.id.includes("anthropic") ||
|
|
||||||
model.id.includes("claude")
|
|
||||||
) {
|
|
||||||
return {
|
|
||||||
high: {
|
|
||||||
thinking: {
|
|
||||||
type: "enabled",
|
|
||||||
budget_tokens: 16000,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
max: {
|
|
||||||
thinking: {
|
|
||||||
type: "enabled",
|
|
||||||
budget_tokens: 31999,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return Object.fromEntries(WIDELY_SUPPORTED_EFFORTS.map((effort) => [effort, { reasoningEffort: effort }]))
|
return Object.fromEntries(WIDELY_SUPPORTED_EFFORTS.map((effort) => [effort, { reasoningEffort: effort }]))
|
||||||
|
|
||||||
case "@ai-sdk/azure":
|
case "@ai-sdk/azure":
|
||||||
@@ -719,21 +694,9 @@ export namespace ProviderTransform {
|
|||||||
const modelCap = modelLimit || globalLimit
|
const modelCap = modelLimit || globalLimit
|
||||||
const standardLimit = Math.min(modelCap, globalLimit)
|
const standardLimit = Math.min(modelCap, globalLimit)
|
||||||
|
|
||||||
// Handle thinking mode for @ai-sdk/anthropic, @ai-sdk/google-vertex/anthropic (budgetTokens)
|
if (npm === "@ai-sdk/anthropic" || npm === "@ai-sdk/google-vertex/anthropic") {
|
||||||
// and @ai-sdk/openai-compatible with Claude (budget_tokens)
|
|
||||||
if (
|
|
||||||
npm === "@ai-sdk/anthropic" ||
|
|
||||||
npm === "@ai-sdk/google-vertex/anthropic" ||
|
|
||||||
npm === "@ai-sdk/openai-compatible"
|
|
||||||
) {
|
|
||||||
const thinking = options?.["thinking"]
|
const thinking = options?.["thinking"]
|
||||||
// Support both camelCase (for @ai-sdk/anthropic) and snake_case (for openai-compatible)
|
const budgetTokens = typeof thinking?.["budgetTokens"] === "number" ? thinking["budgetTokens"] : 0
|
||||||
const budgetTokens =
|
|
||||||
typeof thinking?.["budgetTokens"] === "number"
|
|
||||||
? thinking["budgetTokens"]
|
|
||||||
: typeof thinking?.["budget_tokens"] === "number"
|
|
||||||
? thinking["budget_tokens"]
|
|
||||||
: 0
|
|
||||||
const enabled = thinking?.["type"] === "enabled"
|
const enabled = thinking?.["type"] === "enabled"
|
||||||
if (enabled && budgetTokens > 0) {
|
if (enabled && budgetTokens > 0) {
|
||||||
// Return text tokens so that text + thinking <= model cap, preferring 32k text when possible.
|
// Return text tokens so that text + thinking <= model cap, preferring 32k text when possible.
|
||||||
|
|||||||
@@ -267,76 +267,6 @@ describe("ProviderTransform.maxOutputTokens", () => {
|
|||||||
expect(result).toBe(OUTPUT_TOKEN_MAX)
|
expect(result).toBe(OUTPUT_TOKEN_MAX)
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
describe("openai-compatible with thinking options (snake_case)", () => {
|
|
||||||
test("returns 32k when budget_tokens + 32k <= modelLimit", () => {
|
|
||||||
const modelLimit = 100000
|
|
||||||
const options = {
|
|
||||||
thinking: {
|
|
||||||
type: "enabled",
|
|
||||||
budget_tokens: 10000,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
const result = ProviderTransform.maxOutputTokens(
|
|
||||||
"@ai-sdk/openai-compatible",
|
|
||||||
options,
|
|
||||||
modelLimit,
|
|
||||||
OUTPUT_TOKEN_MAX,
|
|
||||||
)
|
|
||||||
expect(result).toBe(OUTPUT_TOKEN_MAX)
|
|
||||||
})
|
|
||||||
|
|
||||||
test("returns modelLimit - budget_tokens when budget_tokens + 32k > modelLimit", () => {
|
|
||||||
const modelLimit = 50000
|
|
||||||
const options = {
|
|
||||||
thinking: {
|
|
||||||
type: "enabled",
|
|
||||||
budget_tokens: 30000,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
const result = ProviderTransform.maxOutputTokens(
|
|
||||||
"@ai-sdk/openai-compatible",
|
|
||||||
options,
|
|
||||||
modelLimit,
|
|
||||||
OUTPUT_TOKEN_MAX,
|
|
||||||
)
|
|
||||||
expect(result).toBe(20000)
|
|
||||||
})
|
|
||||||
|
|
||||||
test("returns 32k when thinking type is not enabled", () => {
|
|
||||||
const modelLimit = 100000
|
|
||||||
const options = {
|
|
||||||
thinking: {
|
|
||||||
type: "disabled",
|
|
||||||
budget_tokens: 10000,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
const result = ProviderTransform.maxOutputTokens(
|
|
||||||
"@ai-sdk/openai-compatible",
|
|
||||||
options,
|
|
||||||
modelLimit,
|
|
||||||
OUTPUT_TOKEN_MAX,
|
|
||||||
)
|
|
||||||
expect(result).toBe(OUTPUT_TOKEN_MAX)
|
|
||||||
})
|
|
||||||
|
|
||||||
test("returns 32k when budget_tokens is 0", () => {
|
|
||||||
const modelLimit = 100000
|
|
||||||
const options = {
|
|
||||||
thinking: {
|
|
||||||
type: "enabled",
|
|
||||||
budget_tokens: 0,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
const result = ProviderTransform.maxOutputTokens(
|
|
||||||
"@ai-sdk/openai-compatible",
|
|
||||||
options,
|
|
||||||
modelLimit,
|
|
||||||
OUTPUT_TOKEN_MAX,
|
|
||||||
)
|
|
||||||
expect(result).toBe(OUTPUT_TOKEN_MAX)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
})
|
})
|
||||||
|
|
||||||
describe("ProviderTransform.schema - gemini array items", () => {
|
describe("ProviderTransform.schema - gemini array items", () => {
|
||||||
@@ -1564,67 +1494,6 @@ describe("ProviderTransform.variants", () => {
|
|||||||
expect(result.low).toEqual({ reasoningEffort: "low" })
|
expect(result.low).toEqual({ reasoningEffort: "low" })
|
||||||
expect(result.high).toEqual({ reasoningEffort: "high" })
|
expect(result.high).toEqual({ reasoningEffort: "high" })
|
||||||
})
|
})
|
||||||
|
|
||||||
test("Claude via LiteLLM returns thinking with snake_case budget_tokens", () => {
|
|
||||||
const model = createMockModel({
|
|
||||||
id: "anthropic/claude-sonnet-4-5",
|
|
||||||
providerID: "anthropic",
|
|
||||||
api: {
|
|
||||||
id: "claude-sonnet-4-5-20250929",
|
|
||||||
url: "http://localhost:4000",
|
|
||||||
npm: "@ai-sdk/openai-compatible",
|
|
||||||
},
|
|
||||||
})
|
|
||||||
const result = ProviderTransform.variants(model)
|
|
||||||
expect(Object.keys(result)).toEqual(["high", "max"])
|
|
||||||
expect(result.high).toEqual({
|
|
||||||
thinking: {
|
|
||||||
type: "enabled",
|
|
||||||
budget_tokens: 16000,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
expect(result.max).toEqual({
|
|
||||||
thinking: {
|
|
||||||
type: "enabled",
|
|
||||||
budget_tokens: 31999,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
test("Claude model (by model.id) via openai-compatible uses snake_case", () => {
|
|
||||||
const model = createMockModel({
|
|
||||||
id: "litellm/claude-3-opus",
|
|
||||||
providerID: "litellm",
|
|
||||||
api: {
|
|
||||||
id: "claude-3-opus-20240229",
|
|
||||||
url: "http://localhost:4000",
|
|
||||||
npm: "@ai-sdk/openai-compatible",
|
|
||||||
},
|
|
||||||
})
|
|
||||||
const result = ProviderTransform.variants(model)
|
|
||||||
expect(Object.keys(result)).toEqual(["high", "max"])
|
|
||||||
expect(result.high).toEqual({
|
|
||||||
thinking: {
|
|
||||||
type: "enabled",
|
|
||||||
budget_tokens: 16000,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
test("Anthropic model (by model.api.id) via openai-compatible uses snake_case", () => {
|
|
||||||
const model = createMockModel({
|
|
||||||
id: "custom/my-model",
|
|
||||||
providerID: "custom",
|
|
||||||
api: {
|
|
||||||
id: "anthropic.claude-sonnet",
|
|
||||||
url: "http://localhost:4000",
|
|
||||||
npm: "@ai-sdk/openai-compatible",
|
|
||||||
},
|
|
||||||
})
|
|
||||||
const result = ProviderTransform.variants(model)
|
|
||||||
expect(Object.keys(result)).toEqual(["high", "max"])
|
|
||||||
expect(result.high.thinking.budget_tokens).toBe(16000)
|
|
||||||
})
|
|
||||||
})
|
})
|
||||||
|
|
||||||
describe("@ai-sdk/azure", () => {
|
describe("@ai-sdk/azure", () => {
|
||||||
|
|||||||
Reference in New Issue
Block a user