Function ScoreDiscoveryRun

Parameters

• expected:
      | { agentId: string; label: "agent"; labelSource: string }
      | {
          kind: "chat" | "direct" | "workflow";
          label: "none";
          labelSource: string;
      }

  The run's label.

• actual: {
      Answers: Record<
          string,
          {
              Confidence?: number;
              Kind: "Likelihood"
              | "Choice"
              | "Score";
              Probabilities?: Record<string, number>;
              Probability?: number;
              Value?: string | number;
          },
      >;
      AnyApplies: number
      | null;
      Arm: "decision" | "semantic-search";
      Baseline:
          | {
              Floor: number;
              Results: number;
              TopK: number;
              TopMatch: | {
                  AgentId: string;
                  AgentName: string
                  | null;
                  Lexical: number | null;
                  PassesFloor: boolean;
                  Rank: number;
                  Score: number;
                  Semantic: number | null;
              }
              | null;
              TopRanked: | {
                  AgentId: string;
                  AgentName: string
                  | null;
                  Lexical: number | null;
                  PassesFloor: boolean;
                  Rank: number;
                  Score: number;
                  Semantic: number | null;
              }
              | null;
          }
          | null;
      ChosenAgentId: string
      | null;
      ChosenAgentName: string | null;
      Confidence: number | null;
      CostUSD: number | null;
      Decision: "agent-discovery";
      DiscoveryLatencyMs?: number | null;
      Error: string | null;
      LabelledAgentOffered: boolean | null;
      LatencyMs: number | null;
      MinConfidence: number | null;
      Model:
          | {
              AnsweredByPinned: boolean
              | null;
              AnsweredModelId: string | null;
              AnsweredModelName: string | null;
              AnsweredVendorId: string | null;
              AnsweredVendorName: string | null;
              DriverClass: string | null;
              FailedOver: boolean;
              FailoverAllowed: boolean;
              PinnedModelId: string | null;
              PinnedVendorId: string | null;
              ResolvedModel: string | null;
          }
          | null;
      Options: | {
          CatalogSize: number;
          Count: number;
          DeclaredCap: number
          | null;
          Limit: number;
          NarrowedFrom: number | null;
          WithoutDescription: number;
      }
      | null;
      PromptName: string
      | null;
      PromptRunId: string | null;
      Sampling: {
          Applied: boolean;
          Note: string | null;
          RequestedSeed: number | null;
          RequestedTemperature: number | null;
      };
      VerdictReason: string
      | null;
      WithinProductionTimeout: boolean | null;
      WouldInject: boolean | null;
  }

  The run's actual output.

  • Answers: Record<
        string,
        {
            Confidence?: number;
            Kind: "Likelihood"
            | "Choice"
            | "Score";
            Probabilities?: Record<string, number>;
            Probability?: number;
            Value?: string | number;
        },
    >

    Every answer, summarized. The Choice keeps its full distribution, by agent ID.

  • AnyApplies: number | null

    The Likelihood that a specialist agent should handle the request, or null.

  • Arm: "decision" | "semantic-search"

  • Baseline:
        | {
            Floor: number;
            Results: number;
            TopK: number;
            TopMatch: | {
                AgentId: string;
                AgentName: string
                | null;
                Lexical: number | null;
                PassesFloor: boolean;
                Rank: number;
                Score: number;
                Semantic: number | null;
            }
            | null;
            TopRanked: | {
                AgentId: string;
                AgentName: string
                | null;
                Lexical: number | null;
                PassesFloor: boolean;
                Rank: number;
                Score: number;
                Semantic: number | null;
            }
            | null;
        }
        | null

    The baseline's ranking, or null for the decision arm.

  • ChosenAgentId: string | null

  • ChosenAgentName: string | null

  • Confidence: number | null

    The Choice's confidence, or null.

  • CostUSD: number | null

  • Decision: "agent-discovery"

  • OptionalDiscoveryLatencyMs?: number | null

    Decision arm: wall-clock time from building the options, the semantic search included, through the decision call, in milliseconds. That is the span production's discovery timeout bounds. Null for the baseline. Absent from runs recorded before it was measured.

  • Error: string | null

    Why the run produced no answer, or null.

  • LabelledAgentOffered: boolean | null

    For an agent label: whether the labelled agent was among the options (decision) or the catalog (baseline). Null otherwise.

  • LatencyMs: number | null

    Wall-clock time of the decision call alone, or of the search for the baseline, in milliseconds.

  • MinConfidence: number | null

    The calibrated threshold WouldInject was judged at (decision arm), or null.

  • Model:
        | {
            AnsweredByPinned: boolean
            | null;
            AnsweredModelId: string | null;
            AnsweredModelName: string | null;
            AnsweredVendorId: string | null;
            AnsweredVendorName: string | null;
            DriverClass: string | null;
            FailedOver: boolean;
            FailoverAllowed: boolean;
            PinnedModelId: string | null;
            PinnedVendorId: string | null;
            ResolvedModel: string | null;
        }
        | null

    The model record (decision arm), or null.

  • Options:
        | {
            CatalogSize: number;
            Count: number;
            DeclaredCap: number
            | null;
            Limit: number;
            NarrowedFrom: number | null;
            WithoutDescription: number;
        }
        | null

    The decision arm's options, or null.

  • PromptName: string | null

    The decision prompt, or null for the baseline.

  • PromptRunId: string | null

  • Sampling: {
        Applied: boolean;
        Note: string | null;
        RequestedSeed: number | null;
        RequestedTemperature: number | null;
    }

  • VerdictReason: string | null

    Why nothing would be injected, or null.

  • WithinProductionTimeout: boolean | null

    Whether the discovery (DiscoveryLatencyMs) finished within production's timeout, or null without an answer. Production gives up at the timeout and injects nothing, so a late answer is never WouldInject. In runs recorded before DiscoveryLatencyMs, it timed the call alone.

  • WouldInject: boolean | null

    Whether production would suggest the chosen agent, or null without a usable answer.