typia.llm.evaluation — typed questions for evaluation models

export namespace llm {
  export function evaluation<T extends Record<string, any>>(): ILlmEvaluation<T>;
}

import { IValidation } from "./IValidation";
 
/**
 * LLM evaluation questions with a converting validator.
 *
 * `ILlmEvaluation<T>` is generated by `typia.llm.evaluation<T>()`. Each leaf
 * property of `T` becomes one question for an evaluation model, such as
 * TypeSafe's Jev, or OpenAI, Anthropic, and Google through Vercel AI SDK's
 * `experimental_evaluate()`:
 *
 * - `boolean` becomes an {@link ILlmEvaluation.IBoolean} question
 * - A string enum or string literal union becomes an
 *   {@link ILlmEvaluation.IChoice} question
 * - A numeric enum or numeric literal union becomes an
 *   {@link ILlmEvaluation.IScore} question
 * - An array of a string literal union or string enum becomes one
 *   {@link ILlmEvaluation.IBoolean} question per member, asking in English
 *   whether that member applies
 * - A nested object is flattened, one question per leaf
 *
 * Workflow:
 *
 * 1. Pass {@link questions} to the evaluation model with a shared state
 * 2. Receive the answer map keyed by the same question IDs
 * 3. Call {@link validate} to check the answers and fold them back into `T`
 *
 * Probabilities are calibrated only on native evaluation models such as Jev.
 * The AI SDK adapters for OpenAI, Anthropic, and Google ask a language model to
 * write each number itself, and return no choice or score distribution.
 *
 * This interface is experimental. It follows Vercel AI SDK's evaluation model
 * specification, which is itself experimental and may change in patch
 * releases.
 *
 * @author Jeongho Nam - https://github.com/samchon
 * @template T The decision type
 */
export interface ILlmEvaluation<T = unknown> {
  /**
   * Question map, keyed by readable property paths.
   *
   * Pass it as the `questions` of Vercel AI SDK's `experimental_evaluate()` as
   * it is. For TypeSafe's own SDK or HTTP API, convert it with
   * `LlmEvaluation.toTypeSafe()` from `@typia/utils` first, because the native
   * API spells the boolean question type `"noul"`.
   *
   * Jev evaluates each question independently, while the AI SDK adapters for
   * language models only instruct the model to. A key is not a part of the
   * question text. The `instructions` come from the property's JSDoc
   * description, and choice or score criteria from the enum members' JSDoc
   * descriptions or `tags.Constant` descriptions.
   */
  questions: Record<string, ILlmEvaluation.IQuestion>;
 
  /**
   * Validate an answer map and fold it back into `T`.
   *
   * Unlike `ILlmStructuredOutput.validate()`, the input is not a `T` shaped
   * value but the model's answer map, keyed by the {@link questions} keys. It
   * accepts both the neutral boolean answer (`{ type: "boolean", probability
   * }`) and TypeSafe's native one (`{ type: "noul", noul }`).
   *
   * Conversion rules:
   *
   * - A boolean is `true` when its probability reaches the threshold of
   *   `tags.Probability<N>` or `@probability N`, `0.5` by default
   * - A choice is the returned option
   * - A score is the level with the highest probability when the answer has
   *   `probabilities`, otherwise the level nearest to `score`
   * - A choice option or score level with a probability requirement fails
   *   validation when it is selected below that requirement, or when the answer
   *   has no `probabilities` to prove it
   *
   * @param answers Answer map from the evaluation model
   * @returns Validation result with the converted `T` on success
   */
  validate: (answers: unknown) => IValidation<T>;
}
export namespace ILlmEvaluation {
  /** Any evaluation question. */
  export type IQuestion = IChoice | IScore | IBoolean;
 
  /** Question selecting exactly one of the declared options. */
  export interface IChoice {
    /** Discriminator. */
    type: "choice";
 
    /** What to decide, from the property's JSDoc description. */
    instructions: string;
 
    /**
     * Options mapped to their descriptions.
     *
     * The keys are the literal values. A value is `null` when the option has no
     * description.
     */
    criteria: Record<string, string | null>;
  }
 
  /** Question placing the state on ordered levels. */
  export interface IScore {
    /** Discriminator. */
    type: "score";
 
    /** What to decide, from the property's JSDoc description. */
    instructions: string;
 
    /**
     * Level descriptions, from the lowest to the highest.
     *
     * Level `i` is the `i`-th smallest numeric value. A level without a
     * description is described by its value. There are always at least two
     * levels, because a single numeric literal is a compile error.
     */
    criteria: [string, string, ...string[]];
  }
 
  /** Question estimating the probability of "yes". */
  export interface IBoolean {
    /** Discriminator. */
    type: "boolean";
 
    /** What to decide, from the property's JSDoc description. */
    instructions: string;
  }
}

import { TagBase } from "./TagBase";
 
/**
 * Probability requirement for an LLM evaluation decision.
 *
 * `Probability<N>` attaches a probability requirement to a decision that
 * `typia.llm.evaluation<T>()` asks an evaluation model to make. `N` must be in
 * `[0, 1]`:
 *
 * - On a `boolean` property, it is the decision threshold: the value becomes
 *   `true` only when the model's P(true) is at least `N`. Without the tag the
 *   threshold is `0.5`.
 * - On one member of a literal union, it is that member's acceptance minimum:
 *   `validate()` fails when the model selects the member with a probability
 *   below `N`, instead of falling back to a less likely member.
 * - On one member of a literal union used as an array element, where every member
 *   is its own yes/no decision, it is that member's inclusion threshold, as on
 *   a `boolean`.
 *
 * TypeScript enum members cannot carry type tags, so write `@probability N` in
 * the member's JSDoc comment instead. The same comment tag on a property sets
 * the boolean threshold, or the default requirement of every member that
 * carries none: a minimum for a choice or score, a threshold for an array set.
 *
 * The tag carries metadata only. It adds no runtime check to `typia.is()` or
 * `typia.validate()` and no JSON Schema keyword, because it constrains how an
 * evaluation answer is converted, not the value itself.
 *
 * @author Jeongho Nam - https://github.com/samchon
 * @example
 *   interface ITicketTriage {
 *     // true only when P(true) >= 0.8
 *     refund: boolean & Probability<0.8>;
 *
 *     // "escalate" is accepted only when its probability is >= 0.9
 *     action:
 *       | (Constant<"escalate", { description: "Page the on-call" }> &
 *           Probability<0.9>)
 *       | Constant<"reply", { description: "Answer the customer" }>;
 *   }
 *
 * @template Value The probability requirement, in `[0, 1]`
 */
export type Probability<Value extends number> = TagBase<{
  target: "boolean" | "string" | "number";
  kind: "probability";
  value: Value;
  exclusive: true;
}>;

import { LlmEvaluation } from "@typia/utils";
import typia, { tags } from "typia";
 
enum Department {
  /** Payments, invoicing, refunds */
  billing = "billing",
  /**
   * Bugs, outages, integrations
   *
   * @probability 0.75
   */
  technical = "technical",
  /** Pricing, upgrades, new accounts */
  sales = "sales",
}
 
interface ITicketTriage {
  /** Does the customer convey urgency? */
  urgent: boolean;
 
  /** Which team should handle this ticket? */
  department: Department;
 
  /** How frustrated is the customer? */
  frustration: 0 | 1 | 2;
 
  /** Which products does the customer mention? */
  products: Array<"card" | "loan" | "deposit">;
 
  refund: {
    /** Does the customer ask for a refund? */
    requested: boolean & tags.Probability<0.8>;
  };
}
 
const main = async (): Promise<void> => {
  // Generate the questions and the converting validator
  const triage = typia.llm.evaluation<ITicketTriage>();
 
  // Ask TypeSafe's Jev directly, in its native wire format
  const response: Response = await fetch(
    "https://api.typesafe.ai/v1/systemone",
    {
      method: "POST",
      headers: {
        Authorization: "Bearer <YOUR_TYPESAFE_API_KEY>",
        "Content-Type": "application/json",
      },
      body: JSON.stringify({
        model: "jev-1.13.0", // pinned: thresholds are tuned per model version
        state: "I was charged twice this morning. Refund it now, or I leave.",
        questions: LlmEvaluation.toTypeSafe(triage.questions),
      }),
    },
  );
  const { answers } = (await response.json()) as { answers: unknown };
 
  // Validate the answers and fold them back into ITicketTriage
  const result = triage.validate(answers);
  if (result.success === false) {
    console.error("Evaluation failed:", result.errors);
    return;
  }
  console.log("Triage:", result.data);
};
main().catch(console.error);

import { LlmEvaluation } from "@typia/utils";
import typia from "typia";
import * as _createLlmEvaluation_1 from "typia/lib/internal/_createLlmEvaluation";
 
var Department;
(function (Department) {
  /** Payments, invoicing, refunds */
  Department["billing"] = "billing";
  /**
   * Bugs, outages, integrations
   *
   * @probability 0.75
   */
  Department["technical"] = "technical";
  /** Pricing, upgrades, new accounts */
  Department["sales"] = "sales";
})(Department || (Department = {}));
const main = async () => {
  // Generate the questions and the converting validator
  const triage = _createLlmEvaluation_1._createLlmEvaluation([
    {
      instructions: "Does the customer convey urgency?",
      kind: "boolean",
      path: ["urgent"],
      threshold: 0.5,
    },
    {
      instructions: "Which team should handle this ticket?",
      kind: "choice",
      options: [
        {
          description: "Payments, invoicing, refunds",
          value: "billing",
        },
        {
          description: "Pricing, upgrades, new accounts",
          value: "sales",
        },
        {
          minimum: 0.75,
          description: "Bugs, outages, integrations",
          value: "technical",
        },
      ],
      path: ["department"],
    },
    {
      instructions: "How frustrated is the customer?",
      kind: "score",
      levels: [
        {
          value: 0,
        },
        {
          value: 1,
        },
        {
          value: 2,
        },
      ],
      path: ["frustration"],
    },
    {
      instructions: "Which products does the customer mention?",
      kind: "set",
      members: [
        {
          threshold: 0.5,
          value: "card",
        },
        {
          threshold: 0.5,
          value: "deposit",
        },
        {
          threshold: 0.5,
          value: "loan",
        },
      ],
      path: ["products"],
    },
    {
      instructions: "Does the customer ask for a refund?",
      kind: "boolean",
      path: ["refund", "requested"],
      threshold: 0.8,
    },
  ]);
  // Ask TypeSafe's Jev directly, in its native wire format
  const response = await fetch("https://api.typesafe.ai/v1/systemone", {
    method: "POST",
    headers: {
      Authorization: "Bearer <YOUR_TYPESAFE_API_KEY>",
      "Content-Type": "application/json",
    },
    body: JSON.stringify({
      model: "jev-1.13.0", // pinned: thresholds are tuned per model version
      state: "I was charged twice this morning. Refund it now, or I leave.",
      questions: LlmEvaluation.toTypeSafe(triage.questions),
    }),
  });
  const { answers } = await response.json();
  // Validate the answers and fold them back into ITicketTriage
  const result = triage.validate(answers);
  if (result.success === false) {
    console.error("Evaluation failed:", result.errors);
    return;
  }
  console.log("Triage:", result.data);
};
main().catch(console.error);