96 lines
4.4 KiB
TypeScript
96 lines
4.4 KiB
TypeScript
import * as fs from "node:fs";
|
|
import * as os from "node:os";
|
|
import * as path from "node:path";
|
|
import { afterEach, describe, expect, it } from "vitest";
|
|
import { createAgentRunTraceReceipt, createDuelEvalReceipt, createEvalTraceReceipt, gradeSetRecovery, gradeVerifiableReward, tallyDuelReceipts, writeAgentRunTraceReceipt, writeDuelEvalReceipt, writeEvalTraceReceipt } from "./eval-trace.js";
|
|
|
|
const roots: string[] = [];
|
|
|
|
afterEach(() => {
|
|
for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true });
|
|
});
|
|
|
|
function tmpRoot(): string {
|
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), "fable-eval-trace-"));
|
|
roots.push(root);
|
|
return root;
|
|
}
|
|
|
|
describe("eval trace receipt", () => {
|
|
it("passes only when every command exits zero", () => {
|
|
expect(createEvalTraceReceipt("task", ".", [{ command: "test", status: 0 }]).status).toBe("passed");
|
|
expect(createEvalTraceReceipt("task", ".", [{ command: "test", status: 1 }]).status).toBe("failed");
|
|
});
|
|
|
|
it("writes a JSON receipt", () => {
|
|
const out = path.join(tmpRoot(), "trace.json");
|
|
const receipt = createEvalTraceReceipt("task", ".", [{ command: "test", status: 0 }], "abc123", new Date("2026-06-18T00:00:00.000Z"));
|
|
|
|
writeEvalTraceReceipt(out, receipt);
|
|
|
|
expect(JSON.parse(fs.readFileSync(out, "utf-8"))).toMatchObject({ task: "task", status: "passed", gitHead: "abc123" });
|
|
});
|
|
|
|
it("writes agent run trace receipts without deploy authority", () => {
|
|
const out = path.join(tmpRoot(), "agent-run.json");
|
|
const receipt = createAgentRunTraceReceipt({
|
|
runId: "run-1",
|
|
task: "summarize video",
|
|
startedAt: new Date("2026-06-29T00:00:00.000Z"),
|
|
endedAt: new Date("2026-06-29T00:00:02.500Z"),
|
|
toolsCalled: ["yt-dlp", "read"],
|
|
receiptsRead: ["fable.model_access.receipt.v1"],
|
|
receiptsWritten: ["fable.agent_run.trace.v1"],
|
|
tokensIn: 100,
|
|
tokensOut: 20,
|
|
budget: { tokens: 500, wallClockMs: 10_000, toolCalls: 8 },
|
|
environment: { internet: false, repoAccess: true, deployAuthority: "gated" },
|
|
safetyPosture: { cyber: "read-only", securityScan: "passed", approval: "required" },
|
|
decision: "ok",
|
|
});
|
|
|
|
writeAgentRunTraceReceipt(out, receipt);
|
|
|
|
expect(JSON.parse(fs.readFileSync(out, "utf-8"))).toMatchObject({
|
|
schema: "fable.agent_run.trace.v1",
|
|
runId: "run-1",
|
|
latencyMs: 2500,
|
|
budget: { tokens: 500, wallClockMs: 10000, toolCalls: 8 },
|
|
environment: { internet: false, repoAccess: true, secretsAccess: false, deployAuthority: "gated" },
|
|
safetyPosture: { cyber: "read-only", securityScan: "passed", approval: "required" },
|
|
deployAttempted: false,
|
|
});
|
|
});
|
|
|
|
it("defaults agent run traces to no internet, no secrets, and no deploy authority", () => {
|
|
expect(createAgentRunTraceReceipt({
|
|
runId: "run-2",
|
|
task: "inspect",
|
|
startedAt: new Date("2026-06-29T00:00:00.000Z"),
|
|
endedAt: new Date("2026-06-29T00:00:00.001Z"),
|
|
})).toMatchObject({
|
|
environment: { internet: false, repoAccess: true, secretsAccess: false, deployAuthority: "none" },
|
|
safetyPosture: { cyber: "not-run", securityScan: "not-run", approval: "not-required" },
|
|
deployAttempted: false,
|
|
});
|
|
});
|
|
|
|
it("writes and tallies two-implementer duel receipts", () => {
|
|
const dir = tmpRoot();
|
|
writeDuelEvalReceipt(path.join(dir, "one.json"), createDuelEvalReceipt({ task: "fix bug", a: "pi", b: "codex", winner: "a", reason: "tests passed", now: new Date("2026-06-21T00:00:00.000Z") }));
|
|
writeDuelEvalReceipt(path.join(dir, "two.json"), createDuelEvalReceipt({ task: "refactor", a: "pi", b: "codex", winner: "b", reason: "smaller diff", reviewer: "agent" }));
|
|
writeDuelEvalReceipt(path.join(dir, "three.json"), createDuelEvalReceipt({ task: "docs", a: "pi", b: "codex", winner: "tie", reason: "equivalent" }));
|
|
|
|
expect(tallyDuelReceipts(dir)).toEqual({ total: 3, wins: { pi: 1, codex: 1 }, ties: 1 });
|
|
});
|
|
|
|
it("grades target set recovery with Jaccard similarity", () => {
|
|
expect(gradeSetRecovery(["A", "B", "C"], ["a", "c", "d"], 0.5)).toMatchObject({ intersection: ["a", "c"], jaccard: 0.5, passed: true });
|
|
});
|
|
|
|
it("emits deterministic verifiable reward receipts", () => {
|
|
expect(gradeVerifiableReward("2+2", "4", " 4 ")).toMatchObject({ schema: "fable.eval.verifiable_reward.v1", reward: 1, passed: true });
|
|
expect(gradeVerifiableReward("2+2", "4", "5")).toMatchObject({ reward: 0, passed: false });
|
|
});
|
|
});
|