From 25c89cccbb892e72e7ba36cad42dd9baf2e73fed Mon Sep 17 00:00:00 2001 From: artale Date: Mon, 29 Jun 2026 12:14:31 +0200 Subject: [PATCH] feat: add agent run trace receipts --- src/fable5/eval-trace.test.ts | 27 +++++++++++++++- src/fable5/eval-trace.ts | 59 +++++++++++++++++++++++++++++++++++ src/fable5/index.ts | 4 +-- 3 files changed, 87 insertions(+), 3 deletions(-) diff --git a/src/fable5/eval-trace.test.ts b/src/fable5/eval-trace.test.ts index 790f8ae..c69caa3 100644 --- a/src/fable5/eval-trace.test.ts +++ b/src/fable5/eval-trace.test.ts @@ -2,7 +2,7 @@ import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; -import { createDuelEvalReceipt, createEvalTraceReceipt, gradeSetRecovery, gradeVerifiableReward, tallyDuelReceipts, writeDuelEvalReceipt, writeEvalTraceReceipt } from "./eval-trace.js"; +import { createAgentRunTraceReceipt, createDuelEvalReceipt, createEvalTraceReceipt, gradeSetRecovery, gradeVerifiableReward, tallyDuelReceipts, writeAgentRunTraceReceipt, writeDuelEvalReceipt, writeEvalTraceReceipt } from "./eval-trace.js"; const roots: string[] = []; @@ -31,6 +31,31 @@ describe("eval trace receipt", () => { expect(JSON.parse(fs.readFileSync(out, "utf-8"))).toMatchObject({ task: "task", status: "passed", gitHead: "abc123" }); }); + it("writes agent run trace receipts without deploy authority", () => { + const out = path.join(tmpRoot(), "agent-run.json"); + const receipt = createAgentRunTraceReceipt({ + runId: "run-1", + task: "summarize video", + startedAt: new Date("2026-06-29T00:00:00.000Z"), + endedAt: new Date("2026-06-29T00:00:02.500Z"), + toolsCalled: ["yt-dlp", "read"], + receiptsRead: ["fable.model_access.receipt.v1"], + receiptsWritten: ["fable.agent_run.trace.v1"], + tokensIn: 100, + tokensOut: 20, + decision: "ok", + }); + + writeAgentRunTraceReceipt(out, receipt); + + expect(JSON.parse(fs.readFileSync(out, "utf-8"))).toMatchObject({ + schema: "fable.agent_run.trace.v1", + runId: "run-1", + latencyMs: 2500, + deployAttempted: false, + }); + }); + it("writes and tallies two-implementer duel receipts", () => { const dir = tmpRoot(); writeDuelEvalReceipt(path.join(dir, "one.json"), createDuelEvalReceipt({ task: "fix bug", a: "pi", b: "codex", winner: "a", reason: "tests passed", now: new Date("2026-06-21T00:00:00.000Z") })); diff --git a/src/fable5/eval-trace.ts b/src/fable5/eval-trace.ts index 7f8a2ce..26726ed 100644 --- a/src/fable5/eval-trace.ts +++ b/src/fable5/eval-trace.ts @@ -15,6 +15,37 @@ export interface EvalTraceReceipt { createdAt: string; } +export interface AgentRunTraceReceipt { + schema: "fable.agent_run.trace.v1"; + runId: string; + task: string; + startedAt: string; + endedAt: string; + toolsCalled: string[]; + receiptsRead: string[]; + receiptsWritten: string[]; + tokensIn?: number; + tokensOut?: number; + latencyMs: number; + decision: "ok" | "needs-human" | "blocked" | "failed"; + reasons: string[]; + deployAttempted: false; +} + +export interface AgentRunTraceOptions { + runId: string; + task: string; + startedAt: Date; + endedAt?: Date; + toolsCalled?: string[]; + receiptsRead?: string[]; + receiptsWritten?: string[]; + tokensIn?: number; + tokensOut?: number; + decision?: AgentRunTraceReceipt["decision"]; + reasons?: string[]; +} + export interface DuelEvalReceipt { schema: "fable.eval.duel.v1"; task: string; @@ -63,6 +94,34 @@ export function createEvalTraceReceipt(task: string, repo: string, commands: Eva }; } +export function createAgentRunTraceReceipt(opts: AgentRunTraceOptions): AgentRunTraceReceipt { + const endedAt = opts.endedAt ?? new Date(); + const latencyMs = Math.max(0, endedAt.getTime() - opts.startedAt.getTime()); + return { + schema: "fable.agent_run.trace.v1", + runId: opts.runId, + task: opts.task, + startedAt: opts.startedAt.toISOString(), + endedAt: endedAt.toISOString(), + toolsCalled: opts.toolsCalled ?? [], + receiptsRead: opts.receiptsRead ?? [], + receiptsWritten: opts.receiptsWritten ?? [], + tokensIn: opts.tokensIn, + tokensOut: opts.tokensOut, + latencyMs, + decision: opts.decision ?? "ok", + reasons: opts.reasons ?? [], + deployAttempted: false, + }; +} + +export function writeAgentRunTraceReceipt(file: string, receipt: AgentRunTraceReceipt): string { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, `${JSON.stringify(receipt, null, 2)} +`); + return file; +} + export function writeEvalTraceReceipt(file: string, receipt: EvalTraceReceipt): string { fs.mkdirSync(path.dirname(file), { recursive: true }); fs.writeFileSync(file, `${JSON.stringify(receipt, null, 2)}\n`); diff --git a/src/fable5/index.ts b/src/fable5/index.ts index 86c0cd8..6de5510 100644 --- a/src/fable5/index.ts +++ b/src/fable5/index.ts @@ -75,8 +75,8 @@ export type { MemoryBCellEntry, MemoryBCellResult, MemoryBCellStats } from "./me export { checkReceiptHealth } from "./receipt-consumer.js"; export type { ReceiptHealth } from "./receipt-consumer.js"; -export { createDuelEvalReceipt, createEvalTraceReceipt, gradeSetRecovery, gradeVerifiableReward, tallyDuelReceipts, writeDuelEvalReceipt, writeEvalTraceReceipt } from "./eval-trace.js"; -export type { DuelEvalReceipt, DuelTally, EvalTraceCommand, EvalTraceReceipt, SetGradeReceipt, VerifiableRewardReceipt } from "./eval-trace.js"; +export { createAgentRunTraceReceipt, createDuelEvalReceipt, createEvalTraceReceipt, gradeSetRecovery, gradeVerifiableReward, tallyDuelReceipts, writeAgentRunTraceReceipt, writeDuelEvalReceipt, writeEvalTraceReceipt } from "./eval-trace.js"; +export type { AgentRunTraceOptions, AgentRunTraceReceipt, DuelEvalReceipt, DuelTally, EvalTraceCommand, EvalTraceReceipt, SetGradeReceipt, VerifiableRewardReceipt } from "./eval-trace.js"; export { loadBenchHarnesses, runAgentBench, writeAgentBenchReceipt } from "./agent-bench.js"; export type { AgentBenchOptions, AgentBenchReceipt, BenchContenderId, BenchContenderResult, BenchHarnessConfig, BenchmarkHygiene } from "./agent-bench.js";