feat: add two-implementer duel eval receipts

COMMANDS.md

@@ -149,6 +149,15 @@ fable-agent plinius godmode "improve explanation quality"
 
 - `benchmark run`
   - `-b, --benchmark <id>`
+- `benchmark duel <task>`: record a two-implementer eval winner
+  - `--a <label>`
+  - `--b <label>`
+  - `--winner <a|b|tie>`
+  - `--reason <text>`
+  - `--reviewer <human|agent>`
+  - `--out <path>`
+- `benchmark duel-tally`: tally two-implementer eval receipts
+  - `--dir <path>`
 - `benchmark trend`
   - `-b, --benchmark <id>`
 - `benchmark list`

src/fable5/eval-trace.test.ts

@@ -2,7 +2,7 @@ import * as fs from "node:fs";
 import * as os from "node:os";
 import * as path from "node:path";
 import { afterEach, describe, expect, it } from "vitest";
-import { createEvalTraceReceipt, writeEvalTraceReceipt } from "./eval-trace.js";
+import { createDuelEvalReceipt, createEvalTraceReceipt, tallyDuelReceipts, writeDuelEvalReceipt, writeEvalTraceReceipt } from "./eval-trace.js";
 
 const roots: string[] = [];
 
@@ -30,4 +30,13 @@ describe("eval trace receipt", () => {
 
     expect(JSON.parse(fs.readFileSync(out, "utf-8"))).toMatchObject({ task: "task", status: "passed", gitHead: "abc123" });
   });
+
+  it("writes and tallies two-implementer duel receipts", () => {
+    const dir = tmpRoot();
+    writeDuelEvalReceipt(path.join(dir, "one.json"), createDuelEvalReceipt({ task: "fix bug", a: "pi", b: "codex", winner: "a", reason: "tests passed", now: new Date("2026-06-21T00:00:00.000Z") }));
+    writeDuelEvalReceipt(path.join(dir, "two.json"), createDuelEvalReceipt({ task: "refactor", a: "pi", b: "codex", winner: "b", reason: "smaller diff", reviewer: "agent" }));
+    writeDuelEvalReceipt(path.join(dir, "three.json"), createDuelEvalReceipt({ task: "docs", a: "pi", b: "codex", winner: "tie", reason: "equivalent" }));
+
+    expect(tallyDuelReceipts(dir)).toEqual({ total: 3, wins: { pi: 1, codex: 1 }, ties: 1 });
+  });
 });

src/fable5/eval-trace.ts

@@ -15,6 +15,22 @@ export interface EvalTraceReceipt {
   createdAt: string;
 }
 
+export interface DuelEvalReceipt {
+  schema: "fable.eval.duel.v1";
+  task: string;
+  contenders: { a: string; b: string };
+  winner: "a" | "b" | "tie";
+  reason: string;
+  reviewer: "human" | "agent";
+  createdAt: string;
+}
+
+export interface DuelTally {
+  total: number;
+  wins: Record<string, number>;
+  ties: number;
+}
+
 export function createEvalTraceReceipt(task: string, repo: string, commands: EvalTraceCommand[], gitHead?: string, now = new Date()): EvalTraceReceipt {
   return {
     task,
@@ -31,3 +47,47 @@ export function writeEvalTraceReceipt(file: string, receipt: EvalTraceReceipt):
   fs.writeFileSync(file, `${JSON.stringify(receipt, null, 2)}\n`);
   return file;
 }
+
+export function createDuelEvalReceipt(opts: {
+  task: string;
+  a: string;
+  b: string;
+  winner: "a" | "b" | "tie";
+  reason: string;
+  reviewer?: "human" | "agent";
+  now?: Date;
+}): DuelEvalReceipt {
+  return {
+    schema: "fable.eval.duel.v1",
+    task: opts.task,
+    contenders: { a: opts.a, b: opts.b },
+    winner: opts.winner,
+    reason: opts.reason,
+    reviewer: opts.reviewer ?? "human",
+    createdAt: (opts.now ?? new Date()).toISOString(),
+  };
+}
+
+export function writeDuelEvalReceipt(file: string, receipt: DuelEvalReceipt): string {
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  fs.writeFileSync(file, `${JSON.stringify(receipt, null, 2)}\n`);
+  return file;
+}
+
+export function tallyDuelReceipts(dir: string): DuelTally {
+  const tally: DuelTally = { total: 0, wins: {}, ties: 0 };
+  if (!fs.existsSync(dir)) return tally;
+  for (const file of fs.readdirSync(dir)) {
+    if (!file.endsWith(".json")) continue;
+    const receipt = JSON.parse(fs.readFileSync(path.join(dir, file), "utf-8")) as Partial<DuelEvalReceipt>;
+    if (receipt.schema !== "fable.eval.duel.v1" || !receipt.contenders || !receipt.winner) continue;
+    tally.total += 1;
+    if (receipt.winner === "tie") {
+      tally.ties += 1;
+      continue;
+    }
+    const label = receipt.contenders[receipt.winner];
+    tally.wins[label] = (tally.wins[label] ?? 0) + 1;
+  }
+  return tally;
+}

src/index.ts

@@ -833,6 +833,37 @@ benchmark
     console.log(`\n${runner.report(opts.benchmark)}\n`);
   });
 
+benchmark
+  .command("duel <task>")
+  .description("Record a two-implementer eval winner")
+  .requiredOption("--a <label>", "First contender label")
+  .requiredOption("--b <label>", "Second contender label")
+  .requiredOption("--winner <a|b|tie>", "Winning output")
+  .requiredOption("--reason <text>", "Why this output won")
+  .option("--reviewer <human|agent>", "Reviewer type", "human")
+  .option("--out <path>", "Receipt output path")
+  .action(async (task: string, opts: { a: string; b: string; winner: "a" | "b" | "tie"; reason: string; reviewer?: "human" | "agent"; out?: string }) => {
+    if (!["a", "b", "tie"].includes(opts.winner)) throw new Error("--winner must be a, b, or tie");
+    if (!["human", "agent"].includes(opts.reviewer ?? "human")) throw new Error("--reviewer must be human or agent");
+    const { createDuelEvalReceipt, writeDuelEvalReceipt } = await import("./fable5/eval-trace.js");
+    const stamp = new Date().toISOString().replace(/[:.]/g, "-");
+    const out = opts.out ?? path.join(".fable", "duel-evals", stamp + ".json");
+    const receipt = createDuelEvalReceipt({ task, a: opts.a, b: opts.b, winner: opts.winner, reason: opts.reason, reviewer: opts.reviewer });
+    writeDuelEvalReceipt(out, receipt);
+    console.log(JSON.stringify(receipt, null, 2));
+    console.log("\n  Receipt: " + out + "\n");
+  });
+
+benchmark
+  .command("duel-tally")
+  .description("Tally two-implementer eval receipts")
+  .option("--dir <path>", "Duel receipt directory", path.join(".fable", "duel-evals"))
+  .action(async (opts: { dir?: string }) => {
+    const { tallyDuelReceipts } = await import("./fable5/eval-trace.js");
+    const tally = tallyDuelReceipts(opts.dir ?? path.join(".fable", "duel-evals"));
+    console.log(JSON.stringify(tally, null, 2));
+  });
+
 benchmark
   .command("trend")
   .description("Show compounding trend over time")