feat: add benchmark hygiene metadata
This commit is contained in:
parent
80d899716d
commit
114831a669
|
|
@ -158,6 +158,7 @@ fable-agent plinius godmode "improve explanation quality"
|
|||
- `--timeout-ms <n>`
|
||||
- `--harnesses <path>`
|
||||
- `--out <path>`
|
||||
- receipts include benchmark hygiene metadata: split, contamination risk, and local-only claim scope
|
||||
- `benchmark duel <task>`: record a two-implementer eval winner
|
||||
- `--a <label>`
|
||||
- `--b <label>`
|
||||
|
|
|
|||
|
|
@ -25,6 +25,8 @@ describe("agent bench", () => {
|
|||
expect(seen).toEqual(["pi", "hermes", "opencode"]);
|
||||
expect(receipt.schema).toBe("fable.benchmark.agent.v1");
|
||||
expect(receipt.winner).toBe("opencode");
|
||||
expect(receipt.hygiene).toMatchObject({ split: "local", contaminationRisk: "medium" });
|
||||
expect(receipt.hygiene.claimScope).toContain("local harness result");
|
||||
});
|
||||
|
||||
it("runs custom harness configs", async () => {
|
||||
|
|
@ -46,7 +48,14 @@ describe("agent bench", () => {
|
|||
it("writes a receipt", () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "agent-bench-"));
|
||||
const file = path.join(dir, "bench.json");
|
||||
writeAgentBenchReceipt(file, { schema: "fable.benchmark.agent.v1", task: "x", contenders: [], winner: "none", createdAt: "now" });
|
||||
writeAgentBenchReceipt(file, {
|
||||
schema: "fable.benchmark.agent.v1",
|
||||
task: "x",
|
||||
contenders: [],
|
||||
winner: "none",
|
||||
hygiene: { split: "private", contaminationRisk: "low", claimScope: "private benchmark" },
|
||||
createdAt: "now",
|
||||
});
|
||||
expect(fs.readFileSync(file, "utf-8")).toContain("fable.benchmark.agent.v1");
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -25,11 +25,18 @@ export interface BenchContenderResult {
|
|||
stderr: string;
|
||||
}
|
||||
|
||||
export interface BenchmarkHygiene {
|
||||
split: "private" | "public" | "local";
|
||||
contaminationRisk: "low" | "medium" | "high";
|
||||
claimScope: string;
|
||||
}
|
||||
|
||||
export interface AgentBenchReceipt {
|
||||
schema: "fable.benchmark.agent.v1";
|
||||
task: string;
|
||||
contenders: BenchContenderResult[];
|
||||
winner: BenchContenderId | "tie" | "none";
|
||||
hygiene: BenchmarkHygiene;
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
|
|
@ -40,6 +47,7 @@ export interface AgentBenchOptions {
|
|||
timeoutMs?: number;
|
||||
now?: Date;
|
||||
runner?: (cmd: BenchCommand) => Promise<BenchContenderResult>;
|
||||
hygiene?: Partial<BenchmarkHygiene>;
|
||||
}
|
||||
|
||||
export interface BenchCommand {
|
||||
|
|
@ -67,6 +75,7 @@ export async function runAgentBench(opts: AgentBenchOptions): Promise<AgentBench
|
|||
task: opts.task,
|
||||
contenders: results,
|
||||
winner: pickWinner(results),
|
||||
hygiene: benchmarkHygiene(opts.hygiene),
|
||||
createdAt: (opts.now ?? new Date()).toISOString(),
|
||||
};
|
||||
}
|
||||
|
|
@ -84,6 +93,14 @@ export function writeAgentBenchReceipt(file: string, receipt: AgentBenchReceipt)
|
|||
return file;
|
||||
}
|
||||
|
||||
function benchmarkHygiene(overrides: Partial<BenchmarkHygiene> = {}): BenchmarkHygiene {
|
||||
return {
|
||||
split: overrides.split ?? "local",
|
||||
contaminationRisk: overrides.contaminationRisk ?? "medium",
|
||||
claimScope: overrides.claimScope ?? "local harness result; not a frontier benchmark or proof of general capability",
|
||||
};
|
||||
}
|
||||
|
||||
function commandFor(id: BenchContenderId, task: string, timeoutMs: number, harness?: BenchHarnessConfig): BenchCommand {
|
||||
if (harness) return commandFromHarness(harness, task, timeoutMs);
|
||||
if (id === "pi") {
|
||||
|
|
|
|||
Loading…
Reference in New Issue