From 114831a6698ae449245ffd233befc53ef619ac37 Mon Sep 17 00:00:00 2001
From: artale <arosstale1@gmail.com>
Date: Sat, 27 Jun 2026 13:22:18 +0200
Subject: [PATCH] feat: add benchmark hygiene metadata

---
 COMMANDS.md                    |  1 +
 src/fable5/agent-bench.test.ts | 11 ++++++++++-
 src/fable5/agent-bench.ts      | 17 +++++++++++++++++
 3 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/COMMANDS.md b/COMMANDS.md
index 66d57dc..4918dcd 100644
--- a/COMMANDS.md
+++ b/COMMANDS.md
@@ -158,6 +158,7 @@ fable-agent plinius godmode "improve explanation quality"
   - `--timeout-ms <n>`
   - `--harnesses <path>`
   - `--out <path>`
+  - receipts include benchmark hygiene metadata: split, contamination risk, and local-only claim scope
 - `benchmark duel <task>`: record a two-implementer eval winner
   - `--a <label>`
   - `--b <label>`
diff --git a/src/fable5/agent-bench.test.ts b/src/fable5/agent-bench.test.ts
index 76cce80..6091478 100644
--- a/src/fable5/agent-bench.test.ts
+++ b/src/fable5/agent-bench.test.ts
@@ -25,6 +25,8 @@ describe("agent bench", () => {
     expect(seen).toEqual(["pi", "hermes", "opencode"]);
     expect(receipt.schema).toBe("fable.benchmark.agent.v1");
     expect(receipt.winner).toBe("opencode");
+    expect(receipt.hygiene).toMatchObject({ split: "local", contaminationRisk: "medium" });
+    expect(receipt.hygiene.claimScope).toContain("local harness result");
   });
 
   it("runs custom harness configs", async () => {
@@ -46,7 +48,14 @@ describe("agent bench", () => {
   it("writes a receipt", () => {
     const dir = fs.mkdtempSync(path.join(os.tmpdir(), "agent-bench-"));
     const file = path.join(dir, "bench.json");
-    writeAgentBenchReceipt(file, { schema: "fable.benchmark.agent.v1", task: "x", contenders: [], winner: "none", createdAt: "now" });
+    writeAgentBenchReceipt(file, {
+      schema: "fable.benchmark.agent.v1",
+      task: "x",
+      contenders: [],
+      winner: "none",
+      hygiene: { split: "private", contaminationRisk: "low", claimScope: "private benchmark" },
+      createdAt: "now",
+    });
     expect(fs.readFileSync(file, "utf-8")).toContain("fable.benchmark.agent.v1");
   });
 });
diff --git a/src/fable5/agent-bench.ts b/src/fable5/agent-bench.ts
index f19f2ae..f8f5a81 100644
--- a/src/fable5/agent-bench.ts
+++ b/src/fable5/agent-bench.ts
@@ -25,11 +25,18 @@ export interface BenchContenderResult {
   stderr: string;
 }
 
+export interface BenchmarkHygiene {
+  split: "private" | "public" | "local";
+  contaminationRisk: "low" | "medium" | "high";
+  claimScope: string;
+}
+
 export interface AgentBenchReceipt {
   schema: "fable.benchmark.agent.v1";
   task: string;
   contenders: BenchContenderResult[];
   winner: BenchContenderId | "tie" | "none";
+  hygiene: BenchmarkHygiene;
   createdAt: string;
 }
 
@@ -40,6 +47,7 @@ export interface AgentBenchOptions {
   timeoutMs?: number;
   now?: Date;
   runner?: (cmd: BenchCommand) => Promise<BenchContenderResult>;
+  hygiene?: Partial<BenchmarkHygiene>;
 }
 
 export interface BenchCommand {
@@ -67,6 +75,7 @@ export async function runAgentBench(opts: AgentBenchOptions): Promise<AgentBench
     task: opts.task,
     contenders: results,
     winner: pickWinner(results),
+    hygiene: benchmarkHygiene(opts.hygiene),
     createdAt: (opts.now ?? new Date()).toISOString(),
   };
 }
@@ -84,6 +93,14 @@ export function writeAgentBenchReceipt(file: string, receipt: AgentBenchReceipt)
   return file;
 }
 
+function benchmarkHygiene(overrides: Partial<BenchmarkHygiene> = {}): BenchmarkHygiene {
+  return {
+    split: overrides.split ?? "local",
+    contaminationRisk: overrides.contaminationRisk ?? "medium",
+    claimScope: overrides.claimScope ?? "local harness result; not a frontier benchmark or proof of general capability",
+  };
+}
+
 function commandFor(id: BenchContenderId, task: string, timeoutMs: number, harness?: BenchHarnessConfig): BenchCommand {
   if (harness) return commandFromHarness(harness, task, timeoutMs);
   if (id === "pi") {