From 46e53e3f2be79cf271d51410b87f979ce2b96cc3 Mon Sep 17 00:00:00 2001 From: James Long Date: Mon, 28 Sep 2026 15:24:21 -0400 Subject: [PATCH] fix(ai): expose evaluation confidence (#51930) --- packages/ai/README.md | 5 +++-- packages/ai/src/experimental/evaluation.ts | 3 +++ packages/ai/src/experimental/system-one.ts | 15 ++++++++++----- packages/ai/test/evaluation.test.ts | 3 ++- packages/ai/test/evaluation.types.ts | 2 ++ .../ai/test/provider/evaluation.recorded.test.ts | 3 ++- 6 files changed, 22 insertions(+), 9 deletions(-) diff --git a/packages/ai/README.md b/packages/ai/README.md index a1545cd81d7..081fe3e48c6 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -129,8 +129,9 @@ VercelAIGateway.configure().experimental.evaluation("typesafe-ai/jev") OpenRouter reads `OPENROUTER_API_KEY`. Vercel reads `AI_GATEWAY_API_KEY`, then `VERCEL_OIDC_TOKEN`. The common API uses `boolean`; System One routes lower it to native `noul`. -Choice and score confidence plus score legends remain available in provider metadata, and the -provider's rounded probabilities are returned unchanged. +Choice and score answers include `confidence` when the provider returns it, such as +`response.answers.department.confidence`. Score legends remain available in provider metadata, and +the provider's rounded probabilities are returned unchanged. ## Alibaba Cloud Model Studio diff --git a/packages/ai/src/experimental/evaluation.ts b/packages/ai/src/experimental/evaluation.ts index 105a4e56dab..7c56d1ceac9 100644 --- a/packages/ai/src/experimental/evaluation.ts +++ b/packages/ai/src/experimental/evaluation.ts @@ -63,6 +63,7 @@ export const ChoiceAnswer = Schema.Struct({ type: Schema.Literal("choice"), choice: Schema.String, probabilities: Schema.optional(Schema.Record(Schema.String, Probability)), + confidence: Schema.optional(Probability), }) export type ChoiceAnswer = Schema.Schema.Type @@ -70,6 +71,7 @@ export const ScoreAnswer = Schema.Struct({ type: Schema.Literal("score"), score: Schema.Number, probabilities: Schema.optional(Schema.Record(Schema.String, Probability)), + confidence: Schema.optional(Probability), }) export type ScoreAnswer = Schema.Schema.Type @@ -92,6 +94,7 @@ export type AnswerFor = Question extends { readonly type: "choice" readonly choice: Extract readonly probabilities?: Readonly, number>> + readonly confidence?: number } : Question extends { readonly type: "score" } ? ScoreAnswer diff --git a/packages/ai/src/experimental/system-one.ts b/packages/ai/src/experimental/system-one.ts index 591d6cd3117..55a386b79dd 100644 --- a/packages/ai/src/experimental/system-one.ts +++ b/packages/ai/src/experimental/system-one.ts @@ -142,32 +142,37 @@ export const model = (cfg Effect.mapError((cause) => fail("System One returned an invalid response", cause, text)), ) - const confidence: Record = {} const legend: Record> = {} const answers = Object.fromEntries( Object.entries(data.answers).map(([id, answer]): [string, EvaluationAnswer] => { if (answer.type === "noul") return [id, { type: "boolean", probability: answer.noul }] if (answer.type === "choice") { - if (answer.confidence !== undefined) confidence[id] = answer.confidence return [ id, { type: "choice", choice: answer.choice, probabilities: answer.probabilities, + ...(answer.confidence === undefined ? {} : { confidence: answer.confidence }), }, ] } - if (answer.confidence !== undefined) confidence[id] = answer.confidence if (answer.legend !== undefined) legend[id] = answer.legend - return [id, { type: "score", score: answer.score, probabilities: answer.probabilities }] + return [ + id, + { + type: "score", + score: answer.score, + probabilities: answer.probabilities, + ...(answer.confidence === undefined ? {} : { confidence: answer.confidence }), + }, + ] }), ) const meta = { ...(data.id === undefined ? {} : { responseId: data.id }), ...(data.provider === undefined ? {} : { provider: data.provider }), ...data.provider_metadata?.[cfg.providerMetadataKey], - ...(Object.keys(confidence).length === 0 ? {} : { confidence }), ...(Object.keys(legend).length === 0 ? {} : { legend }), } return new EvaluationResponse({ diff --git a/packages/ai/test/evaluation.test.ts b/packages/ai/test/evaluation.test.ts index a228e7465e9..8f821e202ca 100644 --- a/packages/ai/test/evaluation.test.ts +++ b/packages/ai/test/evaluation.test.ts @@ -39,17 +39,18 @@ describe("experimental Evaluation", () => { type: "choice", choice: "billing", probabilities: { billing: 0.9, technical: 0.1 }, + confidence: 0.8, }) expect(response.answers.urgency).toEqual({ type: "score", score: 1.2, probabilities: { "0": 0, "1": 0.8, "2": 0.2 }, + confidence: 0.6, }) expect(response.answers.refund).toEqual({ type: "boolean", probability: 0.97 }) expect(response.usage?.totalTokens).toBe(36) expect(response.providerMetadata).toEqual({ typesafe: { - confidence: { department: 0.8, urgency: 0.6 }, legend: { urgency: { "0": "Can wait", "1": "Needs attention", "2": "Blocking" } }, }, }) diff --git a/packages/ai/test/evaluation.types.ts b/packages/ai/test/evaluation.types.ts index 9479cbc5c3d..bddecae693f 100644 --- a/packages/ai/test/evaluation.types.ts +++ b/packages/ai/test/evaluation.types.ts @@ -26,8 +26,10 @@ const request = Evaluation.request({ const result = EvaluationClient.evaluate(request) type Result = Success type Choice = Assert> +type Confidence = Assert> type ClientRequirements = Assert, Service>> void (true satisfies Choice) +void (true satisfies Confidence) void (true satisfies ClientRequirements) Effect.gen(function* () { diff --git a/packages/ai/test/provider/evaluation.recorded.test.ts b/packages/ai/test/provider/evaluation.recorded.test.ts index 94bf216e394..fbf3e25a9c6 100644 --- a/packages/ai/test/provider/evaluation.recorded.test.ts +++ b/packages/ai/test/provider/evaluation.recorded.test.ts @@ -92,5 +92,6 @@ const assertEvaluation = ( expect(response.answers.refund.probability).toBeGreaterThan(0.5) expect(response.usage?.inputTokens).toBeGreaterThan(0) expect(response.usage?.outputTokens).toBeGreaterThan(0) - expect(response.providerMetadata?.[metadataKey]?.confidence).toBeDefined() + expect(response.answers.department.confidence).toBeGreaterThan(0) + expect(response.answers.urgency.confidence).toBeGreaterThan(0) })