feat: add agent run trace receipts
This commit is contained in:
parent
debf6b82af
commit
25c89cccbb
|
|
@ -2,7 +2,7 @@ import * as fs from "node:fs";
|
|||
import * as os from "node:os";
|
||||
import * as path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { createDuelEvalReceipt, createEvalTraceReceipt, gradeSetRecovery, gradeVerifiableReward, tallyDuelReceipts, writeDuelEvalReceipt, writeEvalTraceReceipt } from "./eval-trace.js";
|
||||
import { createAgentRunTraceReceipt, createDuelEvalReceipt, createEvalTraceReceipt, gradeSetRecovery, gradeVerifiableReward, tallyDuelReceipts, writeAgentRunTraceReceipt, writeDuelEvalReceipt, writeEvalTraceReceipt } from "./eval-trace.js";
|
||||
|
||||
const roots: string[] = [];
|
||||
|
||||
|
|
@ -31,6 +31,31 @@ describe("eval trace receipt", () => {
|
|||
expect(JSON.parse(fs.readFileSync(out, "utf-8"))).toMatchObject({ task: "task", status: "passed", gitHead: "abc123" });
|
||||
});
|
||||
|
||||
it("writes agent run trace receipts without deploy authority", () => {
|
||||
const out = path.join(tmpRoot(), "agent-run.json");
|
||||
const receipt = createAgentRunTraceReceipt({
|
||||
runId: "run-1",
|
||||
task: "summarize video",
|
||||
startedAt: new Date("2026-06-29T00:00:00.000Z"),
|
||||
endedAt: new Date("2026-06-29T00:00:02.500Z"),
|
||||
toolsCalled: ["yt-dlp", "read"],
|
||||
receiptsRead: ["fable.model_access.receipt.v1"],
|
||||
receiptsWritten: ["fable.agent_run.trace.v1"],
|
||||
tokensIn: 100,
|
||||
tokensOut: 20,
|
||||
decision: "ok",
|
||||
});
|
||||
|
||||
writeAgentRunTraceReceipt(out, receipt);
|
||||
|
||||
expect(JSON.parse(fs.readFileSync(out, "utf-8"))).toMatchObject({
|
||||
schema: "fable.agent_run.trace.v1",
|
||||
runId: "run-1",
|
||||
latencyMs: 2500,
|
||||
deployAttempted: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("writes and tallies two-implementer duel receipts", () => {
|
||||
const dir = tmpRoot();
|
||||
writeDuelEvalReceipt(path.join(dir, "one.json"), createDuelEvalReceipt({ task: "fix bug", a: "pi", b: "codex", winner: "a", reason: "tests passed", now: new Date("2026-06-21T00:00:00.000Z") }));
|
||||
|
|
|
|||
|
|
@ -15,6 +15,37 @@ export interface EvalTraceReceipt {
|
|||
createdAt: string;
|
||||
}
|
||||
|
||||
export interface AgentRunTraceReceipt {
|
||||
schema: "fable.agent_run.trace.v1";
|
||||
runId: string;
|
||||
task: string;
|
||||
startedAt: string;
|
||||
endedAt: string;
|
||||
toolsCalled: string[];
|
||||
receiptsRead: string[];
|
||||
receiptsWritten: string[];
|
||||
tokensIn?: number;
|
||||
tokensOut?: number;
|
||||
latencyMs: number;
|
||||
decision: "ok" | "needs-human" | "blocked" | "failed";
|
||||
reasons: string[];
|
||||
deployAttempted: false;
|
||||
}
|
||||
|
||||
export interface AgentRunTraceOptions {
|
||||
runId: string;
|
||||
task: string;
|
||||
startedAt: Date;
|
||||
endedAt?: Date;
|
||||
toolsCalled?: string[];
|
||||
receiptsRead?: string[];
|
||||
receiptsWritten?: string[];
|
||||
tokensIn?: number;
|
||||
tokensOut?: number;
|
||||
decision?: AgentRunTraceReceipt["decision"];
|
||||
reasons?: string[];
|
||||
}
|
||||
|
||||
export interface DuelEvalReceipt {
|
||||
schema: "fable.eval.duel.v1";
|
||||
task: string;
|
||||
|
|
@ -63,6 +94,34 @@ export function createEvalTraceReceipt(task: string, repo: string, commands: Eva
|
|||
};
|
||||
}
|
||||
|
||||
export function createAgentRunTraceReceipt(opts: AgentRunTraceOptions): AgentRunTraceReceipt {
|
||||
const endedAt = opts.endedAt ?? new Date();
|
||||
const latencyMs = Math.max(0, endedAt.getTime() - opts.startedAt.getTime());
|
||||
return {
|
||||
schema: "fable.agent_run.trace.v1",
|
||||
runId: opts.runId,
|
||||
task: opts.task,
|
||||
startedAt: opts.startedAt.toISOString(),
|
||||
endedAt: endedAt.toISOString(),
|
||||
toolsCalled: opts.toolsCalled ?? [],
|
||||
receiptsRead: opts.receiptsRead ?? [],
|
||||
receiptsWritten: opts.receiptsWritten ?? [],
|
||||
tokensIn: opts.tokensIn,
|
||||
tokensOut: opts.tokensOut,
|
||||
latencyMs,
|
||||
decision: opts.decision ?? "ok",
|
||||
reasons: opts.reasons ?? [],
|
||||
deployAttempted: false,
|
||||
};
|
||||
}
|
||||
|
||||
export function writeAgentRunTraceReceipt(file: string, receipt: AgentRunTraceReceipt): string {
|
||||
fs.mkdirSync(path.dirname(file), { recursive: true });
|
||||
fs.writeFileSync(file, `${JSON.stringify(receipt, null, 2)}
|
||||
`);
|
||||
return file;
|
||||
}
|
||||
|
||||
export function writeEvalTraceReceipt(file: string, receipt: EvalTraceReceipt): string {
|
||||
fs.mkdirSync(path.dirname(file), { recursive: true });
|
||||
fs.writeFileSync(file, `${JSON.stringify(receipt, null, 2)}\n`);
|
||||
|
|
|
|||
|
|
@ -75,8 +75,8 @@ export type { MemoryBCellEntry, MemoryBCellResult, MemoryBCellStats } from "./me
|
|||
export { checkReceiptHealth } from "./receipt-consumer.js";
|
||||
export type { ReceiptHealth } from "./receipt-consumer.js";
|
||||
|
||||
export { createDuelEvalReceipt, createEvalTraceReceipt, gradeSetRecovery, gradeVerifiableReward, tallyDuelReceipts, writeDuelEvalReceipt, writeEvalTraceReceipt } from "./eval-trace.js";
|
||||
export type { DuelEvalReceipt, DuelTally, EvalTraceCommand, EvalTraceReceipt, SetGradeReceipt, VerifiableRewardReceipt } from "./eval-trace.js";
|
||||
export { createAgentRunTraceReceipt, createDuelEvalReceipt, createEvalTraceReceipt, gradeSetRecovery, gradeVerifiableReward, tallyDuelReceipts, writeAgentRunTraceReceipt, writeDuelEvalReceipt, writeEvalTraceReceipt } from "./eval-trace.js";
|
||||
export type { AgentRunTraceOptions, AgentRunTraceReceipt, DuelEvalReceipt, DuelTally, EvalTraceCommand, EvalTraceReceipt, SetGradeReceipt, VerifiableRewardReceipt } from "./eval-trace.js";
|
||||
|
||||
export { loadBenchHarnesses, runAgentBench, writeAgentBenchReceipt } from "./agent-bench.js";
|
||||
export type { AgentBenchOptions, AgentBenchReceipt, BenchContenderId, BenchContenderResult, BenchHarnessConfig, BenchmarkHygiene } from "./agent-bench.js";
|
||||
|
|
|
|||
Loading…
Reference in New Issue