feat: add benchmark hygiene metadata
This commit is contained in:
parent
80d899716d
commit
114831a669
|
|
@ -158,6 +158,7 @@ fable-agent plinius godmode "improve explanation quality"
|
||||||
- `--timeout-ms <n>`
|
- `--timeout-ms <n>`
|
||||||
- `--harnesses <path>`
|
- `--harnesses <path>`
|
||||||
- `--out <path>`
|
- `--out <path>`
|
||||||
|
- receipts include benchmark hygiene metadata: split, contamination risk, and local-only claim scope
|
||||||
- `benchmark duel <task>`: record a two-implementer eval winner
|
- `benchmark duel <task>`: record a two-implementer eval winner
|
||||||
- `--a <label>`
|
- `--a <label>`
|
||||||
- `--b <label>`
|
- `--b <label>`
|
||||||
|
|
|
||||||
|
|
@ -25,6 +25,8 @@ describe("agent bench", () => {
|
||||||
expect(seen).toEqual(["pi", "hermes", "opencode"]);
|
expect(seen).toEqual(["pi", "hermes", "opencode"]);
|
||||||
expect(receipt.schema).toBe("fable.benchmark.agent.v1");
|
expect(receipt.schema).toBe("fable.benchmark.agent.v1");
|
||||||
expect(receipt.winner).toBe("opencode");
|
expect(receipt.winner).toBe("opencode");
|
||||||
|
expect(receipt.hygiene).toMatchObject({ split: "local", contaminationRisk: "medium" });
|
||||||
|
expect(receipt.hygiene.claimScope).toContain("local harness result");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("runs custom harness configs", async () => {
|
it("runs custom harness configs", async () => {
|
||||||
|
|
@ -46,7 +48,14 @@ describe("agent bench", () => {
|
||||||
it("writes a receipt", () => {
|
it("writes a receipt", () => {
|
||||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "agent-bench-"));
|
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "agent-bench-"));
|
||||||
const file = path.join(dir, "bench.json");
|
const file = path.join(dir, "bench.json");
|
||||||
writeAgentBenchReceipt(file, { schema: "fable.benchmark.agent.v1", task: "x", contenders: [], winner: "none", createdAt: "now" });
|
writeAgentBenchReceipt(file, {
|
||||||
|
schema: "fable.benchmark.agent.v1",
|
||||||
|
task: "x",
|
||||||
|
contenders: [],
|
||||||
|
winner: "none",
|
||||||
|
hygiene: { split: "private", contaminationRisk: "low", claimScope: "private benchmark" },
|
||||||
|
createdAt: "now",
|
||||||
|
});
|
||||||
expect(fs.readFileSync(file, "utf-8")).toContain("fable.benchmark.agent.v1");
|
expect(fs.readFileSync(file, "utf-8")).toContain("fable.benchmark.agent.v1");
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -25,11 +25,18 @@ export interface BenchContenderResult {
|
||||||
stderr: string;
|
stderr: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface BenchmarkHygiene {
|
||||||
|
split: "private" | "public" | "local";
|
||||||
|
contaminationRisk: "low" | "medium" | "high";
|
||||||
|
claimScope: string;
|
||||||
|
}
|
||||||
|
|
||||||
export interface AgentBenchReceipt {
|
export interface AgentBenchReceipt {
|
||||||
schema: "fable.benchmark.agent.v1";
|
schema: "fable.benchmark.agent.v1";
|
||||||
task: string;
|
task: string;
|
||||||
contenders: BenchContenderResult[];
|
contenders: BenchContenderResult[];
|
||||||
winner: BenchContenderId | "tie" | "none";
|
winner: BenchContenderId | "tie" | "none";
|
||||||
|
hygiene: BenchmarkHygiene;
|
||||||
createdAt: string;
|
createdAt: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -40,6 +47,7 @@ export interface AgentBenchOptions {
|
||||||
timeoutMs?: number;
|
timeoutMs?: number;
|
||||||
now?: Date;
|
now?: Date;
|
||||||
runner?: (cmd: BenchCommand) => Promise<BenchContenderResult>;
|
runner?: (cmd: BenchCommand) => Promise<BenchContenderResult>;
|
||||||
|
hygiene?: Partial<BenchmarkHygiene>;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface BenchCommand {
|
export interface BenchCommand {
|
||||||
|
|
@ -67,6 +75,7 @@ export async function runAgentBench(opts: AgentBenchOptions): Promise<AgentBench
|
||||||
task: opts.task,
|
task: opts.task,
|
||||||
contenders: results,
|
contenders: results,
|
||||||
winner: pickWinner(results),
|
winner: pickWinner(results),
|
||||||
|
hygiene: benchmarkHygiene(opts.hygiene),
|
||||||
createdAt: (opts.now ?? new Date()).toISOString(),
|
createdAt: (opts.now ?? new Date()).toISOString(),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
@ -84,6 +93,14 @@ export function writeAgentBenchReceipt(file: string, receipt: AgentBenchReceipt)
|
||||||
return file;
|
return file;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function benchmarkHygiene(overrides: Partial<BenchmarkHygiene> = {}): BenchmarkHygiene {
|
||||||
|
return {
|
||||||
|
split: overrides.split ?? "local",
|
||||||
|
contaminationRisk: overrides.contaminationRisk ?? "medium",
|
||||||
|
claimScope: overrides.claimScope ?? "local harness result; not a frontier benchmark or proof of general capability",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
function commandFor(id: BenchContenderId, task: string, timeoutMs: number, harness?: BenchHarnessConfig): BenchCommand {
|
function commandFor(id: BenchContenderId, task: string, timeoutMs: number, harness?: BenchHarnessConfig): BenchCommand {
|
||||||
if (harness) return commandFromHarness(harness, task, timeoutMs);
|
if (harness) return commandFromHarness(harness, task, timeoutMs);
|
||||||
if (id === "pi") {
|
if (id === "pi") {
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue