typia.llm.evaluation — typed questions for evaluation models

export namespace llm {
  export function evaluation<T extends Record<string, any>>(): ILlmEvaluation<T>;
}

import { IValidation } from "./IValidation";
 
/**
 * LLM evaluation questions with a checked answer decoder.
 *
 * `ILlmEvaluation<T>` is generated by `typia.llm.evaluation<T>()`. Each leaf
 * property of `T` becomes one question for an evaluation model, such as
 * TypeSafe's Jev, or OpenAI, Anthropic, and Google through Vercel AI SDK's
 * `experimental_evaluate()`:
 *
 * - `boolean` becomes an {@link ILlmEvaluation.IBoolean} question
 * - A string enum or string literal union becomes an
 *   {@link ILlmEvaluation.IChoice} question
 * - A numeric enum or numeric literal union becomes an
 *   {@link ILlmEvaluation.IScore} question
 * - An array of a string literal union or string enum becomes one
 *   {@link ILlmEvaluation.IBoolean} question per member, asking in English
 *   whether that member applies
 * - A nested object is flattened, one question per leaf
 *
 * Workflow:
 *
 * 1. Pass {@link questions} to the evaluation model with a shared state
 * 2. Receive the answer map keyed by the same question IDs
 * 3. Call {@link decode} to check the answers and fold them back into `T`
 *
 * Probabilities are calibrated only on native evaluation models such as Jev.
 * The AI SDK adapters for OpenAI, Anthropic, and Google ask a language model to
 * write each number itself, and return no choice or score distribution.
 *
 * This interface is experimental. It follows Vercel AI SDK's evaluation model
 * specification, which is itself experimental and may change in patch releases.
 * Refinements that the answer decoder cannot enforce are compile errors, so a
 * successful {@link decode} does not require a second `typia.validate<T>()`.
 *
 * @author Jeongho Nam - https://github.com/samchon
 *
 * @template T The decision type
 *
 * @evidence contracts/common.md#principled-implementation The record pairs the questions keyed by readable property path with the configuration used and a `decode` function that checks a model's answer map and returns the converted T, which is why its result type is IValidation of T and not a plain value. The comment lists the conversion rules the decoder applies.
 * @evidence contracts/common.md#clear-and-simple-design Three members; question variants and the configuration are namespaced beneath it.
 * @evidence contracts/common.md#prohibited-implementation-shortcuts Decoding is part of the contract and the interface does not reconstruct answers by guessing; the experimental status is stated.
 * @evidence contracts/common.md#meaningful-documentation The comment maps property types to question kinds, shows the workflow and states the calibration and stability limits.
 */
export interface ILlmEvaluation<T = unknown> {
  /**
   * Question map, keyed by readable property paths.
   *
   * Pass it as the `questions` of Vercel AI SDK's `experimental_evaluate()` as
   * it is. Jev's own wire format, shared by TypeSafe's API and OpenRouter's
   * Decisions API, spells the boolean question type `"noul"`;
   * `toJevQuestions()` from `@typia/jev` converts the questions.
   *
   * Jev evaluates each question independently, while the AI SDK adapters for
   * language models only instruct the model to. A key is not a part of the
   * question text. The `instructions` come from the property's JSDoc
   * description, and choice or score criteria from the enum members' JSDoc
   * descriptions.
   */
  questions: Record<string, ILlmEvaluation.IQuestion>;
 
  /**
   * Configuration this evaluation was generated with.
   *
   * It is the second generic argument of `typia.llm.evaluation<T, Config>()`
   * with the defaults filled in, so {@link decode} and a caller can both tell
   * which precision the questions and the decoder were built for.
   */
  config: ILlmEvaluation.IConfig;
 
  /**
   * Check an answer map and decode it into `T`.
   *
   * Unlike `typia.validate<T>()` or `ILlmStructuredOutput.validate()`, this
   * method transforms a model answer map into a different shape. It checks the
   * answers keyed by {@link questions} and returns the converted `T` on success.
   * It accepts both the neutral boolean answer (`{ type: "boolean", probability
   * }`) and TypeSafe's native one (`{ type: "noul", noul }`).
   *
   * Conversion rules:
   *
   * - A boolean is `true` when its probability reaches the threshold of
   *   `tags.Probability<N>` or `@probability N`, `0.5` by default
   * - A choice is the returned option
   * - A score is the level with the highest probability when the answer has
   *   `probabilities`, otherwise the level nearest to `score`
   * - A choice option or score level with a probability requirement fails
   *   validation when it is selected below that requirement, or when the answer
   *   has no `probabilities` to prove it
   * - Once one choice, score, or set member declares a probability requirement,
   *   every member must declare one or inherit a property `@probability`
   *   default
   * - An optional choice or score `probabilities` map must be complete, sum to
   *   one, and agree with the selected choice or score, following AI SDK's
   *   evaluation answer contract and the configured decimals
   *
   * @param answers Answer map from the evaluation model
   *
   * @returns Validation result with the converted `T` on success
   *
   * @evidence contracts/common.md#principled-implementation The function accepts unknown answers and returns a validation result, so malformed or missing answers become errors rather than exceptions; the documented rules specify thresholds, selected choices, score selection and the probability-map consistency checks.
   * @evidence contracts/common.md#clear-and-simple-design A function-valued property whose body is generated by the transform, not defined here.
   * @evidence contracts/common.md#prohibited-implementation-shortcuts It does not claim to verify answers beyond the rules the comment lists.
   * @evidence contracts/common.md#meaningful-documentation The comment gives each conversion rule, including the boolean threshold default and the requirement that every member declare a probability once one does.
   */
  decode: (answers: unknown) => IValidation<T>;
}
export namespace ILlmEvaluation {
  /**
   * Configuration of `typia.llm.evaluation<T, Config>()`.
   *
   * @evidence contracts/common.md#principled-implementation Decimals is the only setting and bounds probability and score precision, which determines the finest requirement the questions can express and the rounding tolerance used when checking answer sums. The comment says it is an integer in the range zero to fifteen, but the `number` type does not enforce this.
   * @evidence contracts/common.md#clear-and-simple-design One field with a stated default of two.
   * @evidence contracts/common.md#prohibited-implementation-shortcuts A plain configuration record.
   * @evidence contracts/common.md#meaningful-documentation The comment states the default, the range and how the setting affects compile errors and decoding tolerance.
   */
  export interface IConfig {
    /**
     * Decimal places of the evaluation model's probabilities and scores.
     *
     * A probability requirement (`@probability`, `tags.Probability`) finer than
     * this is a compile error, because the model's answers never resolve it.
     * `decode()` tolerates the rounding of that many decimals in the answers'
     * distribution sums and weighted scores. An integer in `[0, 15]`, like AI
     * SDK's rounding declaration.
     *
     * @default 2
     */
    decimals: number;
  }
 
  /**
   * Any evaluation question.
   *
   * @evidence contracts/common.md#principled-implementation A union of the three question kinds, discriminated by their `type` literal, which exhausts the question forms the generator produces.
   * @evidence contracts/common.md#clear-and-simple-design A single alias of the three records.
   * @evidence contracts/common.md#prohibited-implementation-shortcuts A representation only.
   * @evidence contracts/common.md#meaningful-documentation The one-line comment names it as any evaluation question.
   */
  export type IQuestion = IChoice | IScore | IBoolean;
 
  /**
   * Question selecting exactly one of the declared options.
   *
   * @evidence contracts/common.md#principled-implementation The record has a "choice" discriminator, instructions and a map from each permitted string literal to an optional explanation, null when the enum member has no description, so the model always sees every option key.
   * @evidence contracts/common.md#clear-and-simple-design Three fields, one of which is a record keyed by option.
   * @evidence contracts/common.md#prohibited-implementation-shortcuts A plain data record.
   * @evidence contracts/common.md#meaningful-documentation The comment explains where each field comes from and gives an example of the criteria map.
   */
  export interface IChoice {
    /** Discriminator. */
    type: "choice";
 
    /** What to decide, from the property's JSDoc description. */
    instructions: string;
 
    /**
     * The alternatives the evaluation model must distinguish.
     *
     * Each key is one string literal value that the model may select, and its
     * value explains when that option applies. The explanation comes from the
     * enum member's JSDoc description. It is `null` when the member has no
     * description, in which case the model sees only the literal key.
     *
     * For example, an enum member `billing = "billing"` documented as `Payments
     * and refunds` becomes `{ billing: "Payments and refunds" }`.
     */
    criteria: Record<string, string | null>;
  }
 
  /**
   * Question placing the state on ordered levels.
   *
   * @evidence contracts/common.md#principled-implementation The criteria is a tuple of at least two strings ordered by the members' numeric values, which expresses the fact that a single level is a compile error, and the wire score is a position on that index scale.
   * @evidence contracts/common.md#clear-and-simple-design Three fields, with the tuple type carrying the minimum-length rule.
   * @evidence contracts/common.md#prohibited-implementation-shortcuts A plain data record.
   * @evidence contracts/common.md#meaningful-documentation The comment explains ordering, the fractional score position, the text for undescribed levels and gives an example.
   */
  export interface IScore {
    /** Discriminator. */
    type: "score";
 
    /** What to decide, from the property's JSDoc description. */
    instructions: string;
 
    /**
     * The ordered levels the evaluation model uses to place the state.
     *
     * Each entry explains when its level applies and comes from that numeric
     * enum member's JSDoc description. The entries are ordered by the members'
     * numeric values, not by declaration order. The wire-level score is a
     * possibly fractional position on the array's index scale;
     * {@link ILlmEvaluation.decode} selects a level and maps it back to the
     * corresponding numeric value of `T`.
     *
     * For example, members `calm = 10` and `angry = 30`, documented as `No
     * frustration` and `Clearly frustrated`, produce `["No frustration",
     * "Clearly frustrated"]`. A level without a description uses its numeric
     * value as text. There are always at least two levels, because a single
     * numeric literal is a compile error.
     */
    criteria: [string, string, ...string[]];
  }
 
  /**
   * Question estimating the probability of "yes".
   *
   * @evidence contracts/common.md#principled-implementation A yes or no question with a "boolean" discriminator and instructions, whose answer is a probability of true.
   * @evidence contracts/common.md#clear-and-simple-design Two fields only.
   * @evidence contracts/common.md#prohibited-implementation-shortcuts A plain data record.
   * @evidence contracts/common.md#meaningful-documentation The one-line comment describes it as a probability estimate of yes.
   */
  export interface IBoolean {
    /** Discriminator. */
    type: "boolean";
 
    /** What to decide, from the property's JSDoc description. */
    instructions: string;
  }
}

import { TagBase } from "./TagBase";
 
/**
 * Probability requirement for an LLM evaluation decision.
 *
 * `Probability<N>` attaches a probability requirement to a decision that
 * `typia.llm.evaluation<T>()` asks an evaluation model to make. `N` must be in
 * `[0, 1]`:
 *
 * - On a `boolean` property, it is the decision threshold: the value becomes
 *   `true` only when the model's P(true) is at least `N`. Without the tag the
 *   threshold is `0.5`.
 * - On every member of a literal union, it is that member's acceptance minimum:
 *   `decode()` fails when the model selects the member with a probability below
 *   `N`, instead of falling back to a less likely member.
 * - On every member of a literal union used as an array element, where every
 *   member is its own yes/no decision, it is that member's inclusion threshold,
 *   as on a `boolean`.
 *
 * TypeScript enum members cannot carry type tags, so write `@probability N` in
 * the member's JSDoc comment instead. Once one member has a probability
 * requirement, every member must have one. The same comment tag on a property
 * sets the boolean threshold, or supplies the default for every member that
 * carries none: a minimum for a choice or score, a threshold for an array set.
 *
 * The tag carries metadata only. It adds no runtime check to `typia.is()` or
 * `typia.validate()` and no JSON Schema keyword, because it constrains how an
 * evaluation answer is converted, not the value itself.
 *
 * @author Jeongho Nam - https://github.com/samchon
 *
 * @example
 *   interface ITicketTriage {
 *     // true only when P(true) >= 0.8
 *     refund: boolean & Probability<0.8>;
 *   }
 *
 * @template Value The probability requirement, in `[0, 1]`, with at most
 *   `ILlmEvaluation.IConfig.decimals` (two by default) decimal places
 *
 * @evidence contracts/common.md#principled-implementation The tag has only metadata, a numeric `value`, and no `validate` or `schema`, because it parameterizes how an evaluation answer is decoded rather than constraining the property value. The `[0, 1]` range and the decimal-place limit are stated in the comment but are not enforced by the `number` parameter, so out-of-range values are caught elsewhere, if at all.
 * @evidence contracts/common.md#clear-and-simple-design A single TagBase record that applies to booleans and literal unions; the comment describes the threshold and acceptance roles in one place.
 * @evidence contracts/common.md#prohibited-implementation-shortcuts It adds no runtime check or schema keyword, and states that it does not.
 * @evidence contracts/common.md#meaningful-documentation The comment is detailed about the three roles, the enum-member comment-tag alternative, the rule that every member needs a requirement and the absence of a runtime or schema effect.
 */
export type Probability<Value extends number> = TagBase<{
  target: "boolean" | "string" | "number";
  kind: "probability";
  value: Value;
  exclusive: true;
}>;

import { TypeSafeClient } from "@typesafe-ai/sdk";
import { toJevQuestions } from "@typia/jev";
import typia, { tags } from "typia";
 
enum Department {
  /**
   * Payments, invoicing, refunds
   *
   * @probability 0.5
   */
  billing = "billing",
  /**
   * Bugs, outages, integrations
   *
   * @probability 0.75
   */
  technical = "technical",
  /**
   * Pricing, upgrades, new accounts
   *
   * @probability 0.5
   */
  sales = "sales",
}
 
enum Frustration {
  /** Calm or neutral */
  calm = 0,
  /** Annoyed but cooperative */
  annoyed = 1,
  /** Angry or threatening to leave */
  angry = 2,
}
 
interface ITicketTriage {
  /** Does the customer convey urgency? */
  urgent: boolean;
 
  /** Which team should handle this ticket? */
  department: Department;
 
  /** How frustrated is the customer? */
  frustration: Frustration;
 
  /** Which products does the customer mention? */
  products: Array<"card" | "loan" | "deposit">;
 
  /** Does the customer ask for a refund? */
  refund: boolean & tags.Probability<0.8>;
}
 
const main = async (): Promise<void> => {
  // Generate the questions and checked answer decoder.
  const triage = typia.llm.evaluation<ITicketTriage>();
 
  // Ask TypeSafe's Jev in its own wire format
  const client = new TypeSafeClient(); // reads TYPESAFE_API_KEY
  const { answers } = await client.systemOne({
    model: "jev-1.13.0", // pinned: thresholds are tuned per model version
    state: "I was charged twice this morning. Refund it now, or I leave.",
    questions: toJevQuestions(triage.questions),
  });
 
  // Check and decode the answers into ITicketTriage.
  const result = triage.decode(answers);
  if (result.success === false) {
    console.error("Evaluation failed:", result.errors);
    return;
  }
  console.log("Triage:", result.data);
  console.log("Department probabilities:", answers.department);
};
main().catch(console.error);

import { TypeSafeClient } from "@typesafe-ai/sdk";
import { toJevQuestions } from "@typia/jev";
import typia from "typia";
import * as _createLlmEvaluation_1 from "typia/lib/internal/_createLlmEvaluation";
 
var Department;
(function (Department) {
  /**
   * Payments, invoicing, refunds
   *
   * @probability 0.5
   */
  Department["billing"] = "billing";
  /**
   * Bugs, outages, integrations
   *
   * @probability 0.75
   */
  Department["technical"] = "technical";
  /**
   * Pricing, upgrades, new accounts
   *
   * @probability 0.5
   */
  Department["sales"] = "sales";
})(Department || (Department = {}));
var Frustration;
(function (Frustration) {
  /** Calm or neutral */
  Frustration[(Frustration["calm"] = 0)] = "calm";
  /** Annoyed but cooperative */
  Frustration[(Frustration["annoyed"] = 1)] = "annoyed";
  /** Angry or threatening to leave */
  Frustration[(Frustration["angry"] = 2)] = "angry";
})(Frustration || (Frustration = {}));
const main = async () => {
  // Generate the questions and checked answer decoder.
  const triage = _createLlmEvaluation_1._createLlmEvaluation(
    [
      {
        instructions: "Does the customer convey urgency?",
        kind: "boolean",
        path: ["urgent"],
        threshold: 0.5,
      },
      {
        instructions: "Which team should handle this ticket?",
        kind: "choice",
        options: [
          {
            minimum: 0.5,
            description: "Payments, invoicing, refunds",
            value: "billing",
          },
          {
            minimum: 0.5,
            description: "Pricing, upgrades, new accounts",
            value: "sales",
          },
          {
            minimum: 0.75,
            description: "Bugs, outages, integrations",
            value: "technical",
          },
        ],
        path: ["department"],
      },
      {
        instructions: "How frustrated is the customer?",
        kind: "score",
        levels: [
          {
            description: "Calm or neutral",
            value: 0,
          },
          {
            description: "Annoyed but cooperative",
            value: 1,
          },
          {
            description: "Angry or threatening to leave",
            value: 2,
          },
        ],
        path: ["frustration"],
      },
      {
        instructions: "Which products does the customer mention?",
        kind: "set",
        members: [
          {
            threshold: 0.5,
            value: "card",
          },
          {
            threshold: 0.5,
            value: "deposit",
          },
          {
            threshold: 0.5,
            value: "loan",
          },
        ],
        path: ["products"],
      },
      {
        instructions: "Does the customer ask for a refund?",
        kind: "boolean",
        path: ["refund"],
        threshold: 0.8,
      },
    ],
    2,
  );
  // Ask TypeSafe's Jev in its own wire format
  const client = new TypeSafeClient(); // reads TYPESAFE_API_KEY
  const { answers } = await client.systemOne({
    model: "jev-1.13.0", // pinned: thresholds are tuned per model version
    state: "I was charged twice this morning. Refund it now, or I leave.",
    questions: toJevQuestions(triage.questions),
  });
  // Check and decode the answers into ITicketTriage.
  const result = triage.decode(answers);
  if (result.success === false) {
    console.error("Evaluation failed:", result.errors);
    return;
  }
  console.log("Triage:", result.data);
  console.log("Department probabilities:", answers.department);
};
main().catch(console.error);