feat: add benchmark hygiene metadata

This commit is contained in:
artale 2026-06-27 13:22:18 +02:00
parent 80d899716d
commit 114831a669
3 changed files with 28 additions and 1 deletions

View File

@ -158,6 +158,7 @@ fable-agent plinius godmode "improve explanation quality"
- `--timeout-ms <n>`
- `--harnesses <path>`
- `--out <path>`
- receipts include benchmark hygiene metadata: split, contamination risk, and local-only claim scope
- `benchmark duel <task>`: record a two-implementer eval winner
- `--a <label>`
- `--b <label>`

View File

@ -25,6 +25,8 @@ describe("agent bench", () => {
expect(seen).toEqual(["pi", "hermes", "opencode"]);
expect(receipt.schema).toBe("fable.benchmark.agent.v1");
expect(receipt.winner).toBe("opencode");
expect(receipt.hygiene).toMatchObject({ split: "local", contaminationRisk: "medium" });
expect(receipt.hygiene.claimScope).toContain("local harness result");
});
it("runs custom harness configs", async () => {
@ -46,7 +48,14 @@ describe("agent bench", () => {
it("writes a receipt", () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "agent-bench-"));
const file = path.join(dir, "bench.json");
writeAgentBenchReceipt(file, { schema: "fable.benchmark.agent.v1", task: "x", contenders: [], winner: "none", createdAt: "now" });
writeAgentBenchReceipt(file, {
schema: "fable.benchmark.agent.v1",
task: "x",
contenders: [],
winner: "none",
hygiene: { split: "private", contaminationRisk: "low", claimScope: "private benchmark" },
createdAt: "now",
});
expect(fs.readFileSync(file, "utf-8")).toContain("fable.benchmark.agent.v1");
});
});

View File

@ -25,11 +25,18 @@ export interface BenchContenderResult {
stderr: string;
}
export interface BenchmarkHygiene {
split: "private" | "public" | "local";
contaminationRisk: "low" | "medium" | "high";
claimScope: string;
}
export interface AgentBenchReceipt {
schema: "fable.benchmark.agent.v1";
task: string;
contenders: BenchContenderResult[];
winner: BenchContenderId | "tie" | "none";
hygiene: BenchmarkHygiene;
createdAt: string;
}
@ -40,6 +47,7 @@ export interface AgentBenchOptions {
timeoutMs?: number;
now?: Date;
runner?: (cmd: BenchCommand) => Promise<BenchContenderResult>;
hygiene?: Partial<BenchmarkHygiene>;
}
export interface BenchCommand {
@ -67,6 +75,7 @@ export async function runAgentBench(opts: AgentBenchOptions): Promise<AgentBench
task: opts.task,
contenders: results,
winner: pickWinner(results),
hygiene: benchmarkHygiene(opts.hygiene),
createdAt: (opts.now ?? new Date()).toISOString(),
};
}
@ -84,6 +93,14 @@ export function writeAgentBenchReceipt(file: string, receipt: AgentBenchReceipt)
return file;
}
function benchmarkHygiene(overrides: Partial<BenchmarkHygiene> = {}): BenchmarkHygiene {
return {
split: overrides.split ?? "local",
contaminationRisk: overrides.contaminationRisk ?? "medium",
claimScope: overrides.claimScope ?? "local harness result; not a frontier benchmark or proof of general capability",
};
}
function commandFor(id: BenchContenderId, task: string, timeoutMs: number, harness?: BenchHarnessConfig): BenchCommand {
if (harness) return commandFromHarness(harness, task, timeoutMs);
if (id === "pi") {