feat: add verifiable reward receipts

src/fable5/eval-trace.test.ts

@@ -2,7 +2,7 @@ import * as fs from "node:fs";
 import * as os from "node:os";
 import * as path from "node:path";
 import { afterEach, describe, expect, it } from "vitest";
-import { createDuelEvalReceipt, createEvalTraceReceipt, gradeSetRecovery, tallyDuelReceipts, writeDuelEvalReceipt, writeEvalTraceReceipt } from "./eval-trace.js";
+import { createDuelEvalReceipt, createEvalTraceReceipt, gradeSetRecovery, gradeVerifiableReward, tallyDuelReceipts, writeDuelEvalReceipt, writeEvalTraceReceipt } from "./eval-trace.js";
 
 const roots: string[] = [];
 
@@ -43,4 +43,9 @@ describe("eval trace receipt", () => {
   it("grades target set recovery with Jaccard similarity", () => {
     expect(gradeSetRecovery(["A", "B", "C"], ["a", "c", "d"], 0.5)).toMatchObject({ intersection: ["a", "c"], jaccard: 0.5, passed: true });
   });
+
+  it("emits deterministic verifiable reward receipts", () => {
+    expect(gradeVerifiableReward("2+2", "4", " 4 ")).toMatchObject({ schema: "fable.eval.verifiable_reward.v1", reward: 1, passed: true });
+    expect(gradeVerifiableReward("2+2", "4", "5")).toMatchObject({ reward: 0, passed: false });
+  });
 });

src/fable5/eval-trace.ts

@@ -41,6 +41,17 @@ export interface SetGradeReceipt {
   threshold: number;
 }
 
+export interface VerifiableRewardReceipt {
+  schema: "fable.eval.verifiable_reward.v1";
+  prompt: string;
+  expected: string;
+  actual: string;
+  verifier: "exact-normalized";
+  reward: 0 | 1;
+  passed: boolean;
+  createdAt: string;
+}
+
 export function createEvalTraceReceipt(task: string, repo: string, commands: EvalTraceCommand[], gitHead?: string, now = new Date()): EvalTraceReceipt {
   return {
     task,
@@ -112,3 +123,18 @@ export function gradeSetRecovery(expected: string[], actual: string[], threshold
   const jaccard = union.size === 0 ? 1 : intersection.length / union.size;
   return { schema: "fable.benchmark.set_grade.v1", expected: e, actual: a, intersection, jaccard, passed: jaccard >= threshold, threshold };
 }
+
+export function gradeVerifiableReward(prompt: string, expected: string, actual: string, now = new Date()): VerifiableRewardReceipt {
+  const normalize = (s: string) => s.trim().replace(/\s+/g, " ").toLowerCase();
+  const passed = normalize(expected) === normalize(actual);
+  return {
+    schema: "fable.eval.verifiable_reward.v1",
+    prompt,
+    expected,
+    actual,
+    verifier: "exact-normalized",
+    reward: passed ? 1 : 0,
+    passed,
+    createdAt: now.toISOString(),
+  };
+}

src/fable5/index.ts

@@ -68,8 +68,8 @@ export type { MemoryBCellEntry, MemoryBCellResult, MemoryBCellStats } from "./me
 export { checkReceiptHealth } from "./receipt-consumer.js";
 export type { ReceiptHealth } from "./receipt-consumer.js";
 
-export { createDuelEvalReceipt, createEvalTraceReceipt, gradeSetRecovery, tallyDuelReceipts, writeDuelEvalReceipt, writeEvalTraceReceipt } from "./eval-trace.js";
-export type { DuelEvalReceipt, DuelTally, EvalTraceCommand, EvalTraceReceipt, SetGradeReceipt } from "./eval-trace.js";
+export { createDuelEvalReceipt, createEvalTraceReceipt, gradeSetRecovery, gradeVerifiableReward, tallyDuelReceipts, writeDuelEvalReceipt, writeEvalTraceReceipt } from "./eval-trace.js";
+export type { DuelEvalReceipt, DuelTally, EvalTraceCommand, EvalTraceReceipt, SetGradeReceipt, VerifiableRewardReceipt } from "./eval-trace.js";
 
 export { loadBenchHarnesses, runAgentBench, writeAgentBenchReceipt } from "./agent-bench.js";
 export type { AgentBenchOptions, AgentBenchReceipt, BenchContenderId, BenchContenderResult, BenchHarnessConfig, BenchmarkHygiene } from "./agent-bench.js";