fable-agent/src/fable5/eval-trace.ts

import * as fs from "node:fs";
import * as path from "node:path";

export interface EvalTraceCommand {
  command: string;
  status: number | null;
}

export interface EvalTraceReceipt {
  task: string;
  repo: string;
  status: "passed" | "failed";
  commands: EvalTraceCommand[];
  gitHead?: string;
  createdAt: string;
}

export interface AgentRunTraceBudget {
  tokens?: number;
  wallClockMs?: number;
  toolCalls?: number;
}

export interface AgentRunTraceEnvironment {
  internet: boolean;
  repoAccess: boolean;
  secretsAccess: false;
  deployAuthority: "none" | "gated";
}

export interface AgentRunTraceSafetyPosture {
  cyber: "not-run" | "read-only" | "approved-active";
  securityScan?: "not-run" | "passed" | "failed";
  approval?: "not-required" | "required" | "granted";
}

export interface AgentRunTraceReceipt {
  schema: "fable.agent_run.trace.v1";
  runId: string;
  task: string;
  startedAt: string;
  endedAt: string;
  toolsCalled: string[];
  receiptsRead: string[];
  receiptsWritten: string[];
  tokensIn?: number;
  tokensOut?: number;
  latencyMs: number;
  budget: AgentRunTraceBudget;
  environment: AgentRunTraceEnvironment;
  safetyPosture: AgentRunTraceSafetyPosture;
  decision: "ok" | "needs-human" | "blocked" | "failed";
  reasons: string[];
  deployAttempted: false;
}

export interface AgentRunTraceOptions {
  runId: string;
  task: string;
  startedAt: Date;
  endedAt?: Date;
  toolsCalled?: string[];
  receiptsRead?: string[];
  receiptsWritten?: string[];
  tokensIn?: number;
  tokensOut?: number;
  budget?: Partial<AgentRunTraceBudget>;
  environment?: Partial<AgentRunTraceEnvironment>;
  safetyPosture?: Partial<AgentRunTraceSafetyPosture>;
  decision?: AgentRunTraceReceipt["decision"];
  reasons?: string[];
}

export interface DuelEvalReceipt {
  schema: "fable.eval.duel.v1";
  task: string;
  contenders: { a: string; b: string };
  winner: "a" | "b" | "tie";
  reason: string;
  reviewer: "human" | "agent";
  createdAt: string;
}

export interface DuelTally {
  total: number;
  wins: Record<string, number>;
  ties: number;
}

export interface SetGradeReceipt {
  schema: "fable.benchmark.set_grade.v1";
  expected: string[];
  actual: string[];
  intersection: string[];
  jaccard: number;
  passed: boolean;
  threshold: number;
}

export interface VerifiableRewardReceipt {
  schema: "fable.eval.verifiable_reward.v1";
  prompt: string;
  expected: string;
  actual: string;
  verifier: "exact-normalized";
  reward: 0 | 1;
  passed: boolean;
  createdAt: string;
}

export function createEvalTraceReceipt(task: string, repo: string, commands: EvalTraceCommand[], gitHead?: string, now = new Date()): EvalTraceReceipt {
  return {
    task,
    repo,
    status: commands.every((cmd) => cmd.status === 0) ? "passed" : "failed",
    commands,
    gitHead,
    createdAt: now.toISOString(),
  };
}

export function createAgentRunTraceReceipt(opts: AgentRunTraceOptions): AgentRunTraceReceipt {
  const endedAt = opts.endedAt ?? new Date();
  const latencyMs = Math.max(0, endedAt.getTime() - opts.startedAt.getTime());
  return {
    schema: "fable.agent_run.trace.v1",
    runId: opts.runId,
    task: opts.task,
    startedAt: opts.startedAt.toISOString(),
    endedAt: endedAt.toISOString(),
    toolsCalled: opts.toolsCalled ?? [],
    receiptsRead: opts.receiptsRead ?? [],
    receiptsWritten: opts.receiptsWritten ?? [],
    tokensIn: opts.tokensIn,
    tokensOut: opts.tokensOut,
    latencyMs,
    budget: {
      tokens: opts.budget?.tokens,
      wallClockMs: opts.budget?.wallClockMs,
      toolCalls: opts.budget?.toolCalls,
    },
    environment: {
      internet: opts.environment?.internet ?? false,
      repoAccess: opts.environment?.repoAccess ?? true,
      secretsAccess: false,
      deployAuthority: opts.environment?.deployAuthority ?? "none",
    },
    safetyPosture: {
      cyber: opts.safetyPosture?.cyber ?? "not-run",
      securityScan: opts.safetyPosture?.securityScan ?? "not-run",
      approval: opts.safetyPosture?.approval ?? "not-required",
    },
    decision: opts.decision ?? "ok",
    reasons: opts.reasons ?? [],
    deployAttempted: false,
  };
}

export function writeAgentRunTraceReceipt(file: string, receipt: AgentRunTraceReceipt): string {
  fs.mkdirSync(path.dirname(file), { recursive: true });
  fs.writeFileSync(file, `${JSON.stringify(receipt, null, 2)}
`);
  return file;
}

export function writeEvalTraceReceipt(file: string, receipt: EvalTraceReceipt): string {
  fs.mkdirSync(path.dirname(file), { recursive: true });
  fs.writeFileSync(file, `${JSON.stringify(receipt, null, 2)}\n`);
  return file;
}

export function createDuelEvalReceipt(opts: {
  task: string;
  a: string;
  b: string;
  winner: "a" | "b" | "tie";
  reason: string;
  reviewer?: "human" | "agent";
  now?: Date;
}): DuelEvalReceipt {
  return {
    schema: "fable.eval.duel.v1",
    task: opts.task,
    contenders: { a: opts.a, b: opts.b },
    winner: opts.winner,
    reason: opts.reason,
    reviewer: opts.reviewer ?? "human",
    createdAt: (opts.now ?? new Date()).toISOString(),
  };
}

export function writeDuelEvalReceipt(file: string, receipt: DuelEvalReceipt): string {
  fs.mkdirSync(path.dirname(file), { recursive: true });
  fs.writeFileSync(file, `${JSON.stringify(receipt, null, 2)}\n`);
  return file;
}

export function tallyDuelReceipts(dir: string): DuelTally {
  const tally: DuelTally = { total: 0, wins: {}, ties: 0 };
  if (!fs.existsSync(dir)) return tally;
  for (const file of fs.readdirSync(dir)) {
    if (!file.endsWith(".json")) continue;
    const receipt = JSON.parse(fs.readFileSync(path.join(dir, file), "utf-8")) as Partial<DuelEvalReceipt>;
    if (receipt.schema !== "fable.eval.duel.v1" || !receipt.contenders || !receipt.winner) continue;
    tally.total += 1;
    if (receipt.winner === "tie") {
      tally.ties += 1;
      continue;
    }
    const label = receipt.contenders[receipt.winner];
    tally.wins[label] = (tally.wins[label] ?? 0) + 1;
  }
  return tally;
}

export function gradeSetRecovery(expected: string[], actual: string[], threshold = 0.8): SetGradeReceipt {
  const norm = (xs: string[]) => [...new Set(xs.map((x) => x.trim().toLowerCase()).filter(Boolean))].sort();
  const e = norm(expected);
  const a = norm(actual);
  const aSet = new Set(a);
  const intersection = e.filter((x) => aSet.has(x));
  const union = new Set([...e, ...a]);
  const jaccard = union.size === 0 ? 1 : intersection.length / union.size;
  return { schema: "fable.benchmark.set_grade.v1", expected: e, actual: a, intersection, jaccard, passed: jaccard >= threshold, threshold };
}

export function gradeVerifiableReward(prompt: string, expected: string, actual: string, now = new Date()): VerifiableRewardReceipt {
  const normalize = (s: string) => s.trim().replace(/\s+/g, " ").toLowerCase();
  const passed = normalize(expected) === normalize(actual);
  return {
    schema: "fable.eval.verifiable_reward.v1",
    prompt,
    expected,
    actual,
    verifier: "exact-normalized",
    reward: passed ? 1 : 0,
    passed,
    createdAt: now.toISOString(),
  };
}