feat: add agent run trace receipts
This commit is contained in:
parent
debf6b82af
commit
25c89cccbb
|
|
@ -2,7 +2,7 @@ import * as fs from "node:fs";
|
||||||
import * as os from "node:os";
|
import * as os from "node:os";
|
||||||
import * as path from "node:path";
|
import * as path from "node:path";
|
||||||
import { afterEach, describe, expect, it } from "vitest";
|
import { afterEach, describe, expect, it } from "vitest";
|
||||||
import { createDuelEvalReceipt, createEvalTraceReceipt, gradeSetRecovery, gradeVerifiableReward, tallyDuelReceipts, writeDuelEvalReceipt, writeEvalTraceReceipt } from "./eval-trace.js";
|
import { createAgentRunTraceReceipt, createDuelEvalReceipt, createEvalTraceReceipt, gradeSetRecovery, gradeVerifiableReward, tallyDuelReceipts, writeAgentRunTraceReceipt, writeDuelEvalReceipt, writeEvalTraceReceipt } from "./eval-trace.js";
|
||||||
|
|
||||||
const roots: string[] = [];
|
const roots: string[] = [];
|
||||||
|
|
||||||
|
|
@ -31,6 +31,31 @@ describe("eval trace receipt", () => {
|
||||||
expect(JSON.parse(fs.readFileSync(out, "utf-8"))).toMatchObject({ task: "task", status: "passed", gitHead: "abc123" });
|
expect(JSON.parse(fs.readFileSync(out, "utf-8"))).toMatchObject({ task: "task", status: "passed", gitHead: "abc123" });
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("writes agent run trace receipts without deploy authority", () => {
|
||||||
|
const out = path.join(tmpRoot(), "agent-run.json");
|
||||||
|
const receipt = createAgentRunTraceReceipt({
|
||||||
|
runId: "run-1",
|
||||||
|
task: "summarize video",
|
||||||
|
startedAt: new Date("2026-06-29T00:00:00.000Z"),
|
||||||
|
endedAt: new Date("2026-06-29T00:00:02.500Z"),
|
||||||
|
toolsCalled: ["yt-dlp", "read"],
|
||||||
|
receiptsRead: ["fable.model_access.receipt.v1"],
|
||||||
|
receiptsWritten: ["fable.agent_run.trace.v1"],
|
||||||
|
tokensIn: 100,
|
||||||
|
tokensOut: 20,
|
||||||
|
decision: "ok",
|
||||||
|
});
|
||||||
|
|
||||||
|
writeAgentRunTraceReceipt(out, receipt);
|
||||||
|
|
||||||
|
expect(JSON.parse(fs.readFileSync(out, "utf-8"))).toMatchObject({
|
||||||
|
schema: "fable.agent_run.trace.v1",
|
||||||
|
runId: "run-1",
|
||||||
|
latencyMs: 2500,
|
||||||
|
deployAttempted: false,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it("writes and tallies two-implementer duel receipts", () => {
|
it("writes and tallies two-implementer duel receipts", () => {
|
||||||
const dir = tmpRoot();
|
const dir = tmpRoot();
|
||||||
writeDuelEvalReceipt(path.join(dir, "one.json"), createDuelEvalReceipt({ task: "fix bug", a: "pi", b: "codex", winner: "a", reason: "tests passed", now: new Date("2026-06-21T00:00:00.000Z") }));
|
writeDuelEvalReceipt(path.join(dir, "one.json"), createDuelEvalReceipt({ task: "fix bug", a: "pi", b: "codex", winner: "a", reason: "tests passed", now: new Date("2026-06-21T00:00:00.000Z") }));
|
||||||
|
|
|
||||||
|
|
@ -15,6 +15,37 @@ export interface EvalTraceReceipt {
|
||||||
createdAt: string;
|
createdAt: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface AgentRunTraceReceipt {
|
||||||
|
schema: "fable.agent_run.trace.v1";
|
||||||
|
runId: string;
|
||||||
|
task: string;
|
||||||
|
startedAt: string;
|
||||||
|
endedAt: string;
|
||||||
|
toolsCalled: string[];
|
||||||
|
receiptsRead: string[];
|
||||||
|
receiptsWritten: string[];
|
||||||
|
tokensIn?: number;
|
||||||
|
tokensOut?: number;
|
||||||
|
latencyMs: number;
|
||||||
|
decision: "ok" | "needs-human" | "blocked" | "failed";
|
||||||
|
reasons: string[];
|
||||||
|
deployAttempted: false;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface AgentRunTraceOptions {
|
||||||
|
runId: string;
|
||||||
|
task: string;
|
||||||
|
startedAt: Date;
|
||||||
|
endedAt?: Date;
|
||||||
|
toolsCalled?: string[];
|
||||||
|
receiptsRead?: string[];
|
||||||
|
receiptsWritten?: string[];
|
||||||
|
tokensIn?: number;
|
||||||
|
tokensOut?: number;
|
||||||
|
decision?: AgentRunTraceReceipt["decision"];
|
||||||
|
reasons?: string[];
|
||||||
|
}
|
||||||
|
|
||||||
export interface DuelEvalReceipt {
|
export interface DuelEvalReceipt {
|
||||||
schema: "fable.eval.duel.v1";
|
schema: "fable.eval.duel.v1";
|
||||||
task: string;
|
task: string;
|
||||||
|
|
@ -63,6 +94,34 @@ export function createEvalTraceReceipt(task: string, repo: string, commands: Eva
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function createAgentRunTraceReceipt(opts: AgentRunTraceOptions): AgentRunTraceReceipt {
|
||||||
|
const endedAt = opts.endedAt ?? new Date();
|
||||||
|
const latencyMs = Math.max(0, endedAt.getTime() - opts.startedAt.getTime());
|
||||||
|
return {
|
||||||
|
schema: "fable.agent_run.trace.v1",
|
||||||
|
runId: opts.runId,
|
||||||
|
task: opts.task,
|
||||||
|
startedAt: opts.startedAt.toISOString(),
|
||||||
|
endedAt: endedAt.toISOString(),
|
||||||
|
toolsCalled: opts.toolsCalled ?? [],
|
||||||
|
receiptsRead: opts.receiptsRead ?? [],
|
||||||
|
receiptsWritten: opts.receiptsWritten ?? [],
|
||||||
|
tokensIn: opts.tokensIn,
|
||||||
|
tokensOut: opts.tokensOut,
|
||||||
|
latencyMs,
|
||||||
|
decision: opts.decision ?? "ok",
|
||||||
|
reasons: opts.reasons ?? [],
|
||||||
|
deployAttempted: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function writeAgentRunTraceReceipt(file: string, receipt: AgentRunTraceReceipt): string {
|
||||||
|
fs.mkdirSync(path.dirname(file), { recursive: true });
|
||||||
|
fs.writeFileSync(file, `${JSON.stringify(receipt, null, 2)}
|
||||||
|
`);
|
||||||
|
return file;
|
||||||
|
}
|
||||||
|
|
||||||
export function writeEvalTraceReceipt(file: string, receipt: EvalTraceReceipt): string {
|
export function writeEvalTraceReceipt(file: string, receipt: EvalTraceReceipt): string {
|
||||||
fs.mkdirSync(path.dirname(file), { recursive: true });
|
fs.mkdirSync(path.dirname(file), { recursive: true });
|
||||||
fs.writeFileSync(file, `${JSON.stringify(receipt, null, 2)}\n`);
|
fs.writeFileSync(file, `${JSON.stringify(receipt, null, 2)}\n`);
|
||||||
|
|
|
||||||
|
|
@ -75,8 +75,8 @@ export type { MemoryBCellEntry, MemoryBCellResult, MemoryBCellStats } from "./me
|
||||||
export { checkReceiptHealth } from "./receipt-consumer.js";
|
export { checkReceiptHealth } from "./receipt-consumer.js";
|
||||||
export type { ReceiptHealth } from "./receipt-consumer.js";
|
export type { ReceiptHealth } from "./receipt-consumer.js";
|
||||||
|
|
||||||
export { createDuelEvalReceipt, createEvalTraceReceipt, gradeSetRecovery, gradeVerifiableReward, tallyDuelReceipts, writeDuelEvalReceipt, writeEvalTraceReceipt } from "./eval-trace.js";
|
export { createAgentRunTraceReceipt, createDuelEvalReceipt, createEvalTraceReceipt, gradeSetRecovery, gradeVerifiableReward, tallyDuelReceipts, writeAgentRunTraceReceipt, writeDuelEvalReceipt, writeEvalTraceReceipt } from "./eval-trace.js";
|
||||||
export type { DuelEvalReceipt, DuelTally, EvalTraceCommand, EvalTraceReceipt, SetGradeReceipt, VerifiableRewardReceipt } from "./eval-trace.js";
|
export type { AgentRunTraceOptions, AgentRunTraceReceipt, DuelEvalReceipt, DuelTally, EvalTraceCommand, EvalTraceReceipt, SetGradeReceipt, VerifiableRewardReceipt } from "./eval-trace.js";
|
||||||
|
|
||||||
export { loadBenchHarnesses, runAgentBench, writeAgentBenchReceipt } from "./agent-bench.js";
|
export { loadBenchHarnesses, runAgentBench, writeAgentBenchReceipt } from "./agent-bench.js";
|
||||||
export type { AgentBenchOptions, AgentBenchReceipt, BenchContenderId, BenchContenderResult, BenchHarnessConfig, BenchmarkHygiene } from "./agent-bench.js";
|
export type { AgentBenchOptions, AgentBenchReceipt, BenchContenderId, BenchContenderResult, BenchHarnessConfig, BenchmarkHygiene } from "./agent-bench.js";
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue