feat: add agent run trace receipts

This commit is contained in:
artale 2026-06-29 12:14:31 +02:00
parent debf6b82af
commit 25c89cccbb
3 changed files with 87 additions and 3 deletions

View File

@ -2,7 +2,7 @@ import * as fs from "node:fs";
import * as os from "node:os"; import * as os from "node:os";
import * as path from "node:path"; import * as path from "node:path";
import { afterEach, describe, expect, it } from "vitest"; import { afterEach, describe, expect, it } from "vitest";
import { createDuelEvalReceipt, createEvalTraceReceipt, gradeSetRecovery, gradeVerifiableReward, tallyDuelReceipts, writeDuelEvalReceipt, writeEvalTraceReceipt } from "./eval-trace.js"; import { createAgentRunTraceReceipt, createDuelEvalReceipt, createEvalTraceReceipt, gradeSetRecovery, gradeVerifiableReward, tallyDuelReceipts, writeAgentRunTraceReceipt, writeDuelEvalReceipt, writeEvalTraceReceipt } from "./eval-trace.js";
const roots: string[] = []; const roots: string[] = [];
@ -31,6 +31,31 @@ describe("eval trace receipt", () => {
expect(JSON.parse(fs.readFileSync(out, "utf-8"))).toMatchObject({ task: "task", status: "passed", gitHead: "abc123" }); expect(JSON.parse(fs.readFileSync(out, "utf-8"))).toMatchObject({ task: "task", status: "passed", gitHead: "abc123" });
}); });
it("writes agent run trace receipts without deploy authority", () => {
const out = path.join(tmpRoot(), "agent-run.json");
const receipt = createAgentRunTraceReceipt({
runId: "run-1",
task: "summarize video",
startedAt: new Date("2026-06-29T00:00:00.000Z"),
endedAt: new Date("2026-06-29T00:00:02.500Z"),
toolsCalled: ["yt-dlp", "read"],
receiptsRead: ["fable.model_access.receipt.v1"],
receiptsWritten: ["fable.agent_run.trace.v1"],
tokensIn: 100,
tokensOut: 20,
decision: "ok",
});
writeAgentRunTraceReceipt(out, receipt);
expect(JSON.parse(fs.readFileSync(out, "utf-8"))).toMatchObject({
schema: "fable.agent_run.trace.v1",
runId: "run-1",
latencyMs: 2500,
deployAttempted: false,
});
});
it("writes and tallies two-implementer duel receipts", () => { it("writes and tallies two-implementer duel receipts", () => {
const dir = tmpRoot(); const dir = tmpRoot();
writeDuelEvalReceipt(path.join(dir, "one.json"), createDuelEvalReceipt({ task: "fix bug", a: "pi", b: "codex", winner: "a", reason: "tests passed", now: new Date("2026-06-21T00:00:00.000Z") })); writeDuelEvalReceipt(path.join(dir, "one.json"), createDuelEvalReceipt({ task: "fix bug", a: "pi", b: "codex", winner: "a", reason: "tests passed", now: new Date("2026-06-21T00:00:00.000Z") }));

View File

@ -15,6 +15,37 @@ export interface EvalTraceReceipt {
createdAt: string; createdAt: string;
} }
export interface AgentRunTraceReceipt {
schema: "fable.agent_run.trace.v1";
runId: string;
task: string;
startedAt: string;
endedAt: string;
toolsCalled: string[];
receiptsRead: string[];
receiptsWritten: string[];
tokensIn?: number;
tokensOut?: number;
latencyMs: number;
decision: "ok" | "needs-human" | "blocked" | "failed";
reasons: string[];
deployAttempted: false;
}
export interface AgentRunTraceOptions {
runId: string;
task: string;
startedAt: Date;
endedAt?: Date;
toolsCalled?: string[];
receiptsRead?: string[];
receiptsWritten?: string[];
tokensIn?: number;
tokensOut?: number;
decision?: AgentRunTraceReceipt["decision"];
reasons?: string[];
}
export interface DuelEvalReceipt { export interface DuelEvalReceipt {
schema: "fable.eval.duel.v1"; schema: "fable.eval.duel.v1";
task: string; task: string;
@ -63,6 +94,34 @@ export function createEvalTraceReceipt(task: string, repo: string, commands: Eva
}; };
} }
export function createAgentRunTraceReceipt(opts: AgentRunTraceOptions): AgentRunTraceReceipt {
const endedAt = opts.endedAt ?? new Date();
const latencyMs = Math.max(0, endedAt.getTime() - opts.startedAt.getTime());
return {
schema: "fable.agent_run.trace.v1",
runId: opts.runId,
task: opts.task,
startedAt: opts.startedAt.toISOString(),
endedAt: endedAt.toISOString(),
toolsCalled: opts.toolsCalled ?? [],
receiptsRead: opts.receiptsRead ?? [],
receiptsWritten: opts.receiptsWritten ?? [],
tokensIn: opts.tokensIn,
tokensOut: opts.tokensOut,
latencyMs,
decision: opts.decision ?? "ok",
reasons: opts.reasons ?? [],
deployAttempted: false,
};
}
export function writeAgentRunTraceReceipt(file: string, receipt: AgentRunTraceReceipt): string {
fs.mkdirSync(path.dirname(file), { recursive: true });
fs.writeFileSync(file, `${JSON.stringify(receipt, null, 2)}
`);
return file;
}
export function writeEvalTraceReceipt(file: string, receipt: EvalTraceReceipt): string { export function writeEvalTraceReceipt(file: string, receipt: EvalTraceReceipt): string {
fs.mkdirSync(path.dirname(file), { recursive: true }); fs.mkdirSync(path.dirname(file), { recursive: true });
fs.writeFileSync(file, `${JSON.stringify(receipt, null, 2)}\n`); fs.writeFileSync(file, `${JSON.stringify(receipt, null, 2)}\n`);

View File

@ -75,8 +75,8 @@ export type { MemoryBCellEntry, MemoryBCellResult, MemoryBCellStats } from "./me
export { checkReceiptHealth } from "./receipt-consumer.js"; export { checkReceiptHealth } from "./receipt-consumer.js";
export type { ReceiptHealth } from "./receipt-consumer.js"; export type { ReceiptHealth } from "./receipt-consumer.js";
export { createDuelEvalReceipt, createEvalTraceReceipt, gradeSetRecovery, gradeVerifiableReward, tallyDuelReceipts, writeDuelEvalReceipt, writeEvalTraceReceipt } from "./eval-trace.js"; export { createAgentRunTraceReceipt, createDuelEvalReceipt, createEvalTraceReceipt, gradeSetRecovery, gradeVerifiableReward, tallyDuelReceipts, writeAgentRunTraceReceipt, writeDuelEvalReceipt, writeEvalTraceReceipt } from "./eval-trace.js";
export type { DuelEvalReceipt, DuelTally, EvalTraceCommand, EvalTraceReceipt, SetGradeReceipt, VerifiableRewardReceipt } from "./eval-trace.js"; export type { AgentRunTraceOptions, AgentRunTraceReceipt, DuelEvalReceipt, DuelTally, EvalTraceCommand, EvalTraceReceipt, SetGradeReceipt, VerifiableRewardReceipt } from "./eval-trace.js";
export { loadBenchHarnesses, runAgentBench, writeAgentBenchReceipt } from "./agent-bench.js"; export { loadBenchHarnesses, runAgentBench, writeAgentBenchReceipt } from "./agent-bench.js";
export type { AgentBenchOptions, AgentBenchReceipt, BenchContenderId, BenchContenderResult, BenchHarnessConfig, BenchmarkHygiene } from "./agent-bench.js"; export type { AgentBenchOptions, AgentBenchReceipt, BenchContenderId, BenchContenderResult, BenchHarnessConfig, BenchmarkHygiene } from "./agent-bench.js";