feat: add audited goals and set graders

COMMANDS.md

@@ -158,6 +158,10 @@ fable-agent plinius godmode "improve explanation quality"
   - `--out <path>`
 - `benchmark duel-tally`: tally two-implementer eval receipts
   - `--dir <path>`
+- `benchmark set-grade`: deterministic expected/actual set grader
+  - `--expected <csv>`
+  - `--actual <csv>`
+  - `--threshold <n>`
 - `benchmark trend`
   - `-b, --benchmark <id>`
 - `benchmark list`
@@ -290,6 +294,11 @@ fable-agent plinius godmode "improve explanation quality"
 - `fable5 goal <text>`
   - `-i, --iterations <n>`
   - `-s, --min-score <n>`
+- `fable5 goal-audit <objective>`
+  - `--output <text>`
+  - `--require <label:evidence>`
+  - `--context <text>`
+  - `--out <path>`
 - `fable5 spec <task>`
   - `--repo <path>`
   - `--out <file>`

src/fable5/eval-trace.test.ts

@@ -2,7 +2,7 @@ import * as fs from "node:fs";
 import * as os from "node:os";
 import * as path from "node:path";
 import { afterEach, describe, expect, it } from "vitest";
-import { createDuelEvalReceipt, createEvalTraceReceipt, tallyDuelReceipts, writeDuelEvalReceipt, writeEvalTraceReceipt } from "./eval-trace.js";
+import { createDuelEvalReceipt, createEvalTraceReceipt, gradeSetRecovery, tallyDuelReceipts, writeDuelEvalReceipt, writeEvalTraceReceipt } from "./eval-trace.js";
 
 const roots: string[] = [];
 
@@ -39,4 +39,8 @@ describe("eval trace receipt", () => {
 
     expect(tallyDuelReceipts(dir)).toEqual({ total: 3, wins: { pi: 1, codex: 1 }, ties: 1 });
   });
+
+  it("grades target set recovery with Jaccard similarity", () => {
+    expect(gradeSetRecovery(["A", "B", "C"], ["a", "c", "d"], 0.5)).toMatchObject({ intersection: ["a", "c"], jaccard: 0.5, passed: true });
+  });
 });

src/fable5/eval-trace.ts

@@ -31,6 +31,16 @@ export interface DuelTally {
   ties: number;
 }
 
+export interface SetGradeReceipt {
+  schema: "fable.benchmark.set_grade.v1";
+  expected: string[];
+  actual: string[];
+  intersection: string[];
+  jaccard: number;
+  passed: boolean;
+  threshold: number;
+}
+
 export function createEvalTraceReceipt(task: string, repo: string, commands: EvalTraceCommand[], gitHead?: string, now = new Date()): EvalTraceReceipt {
   return {
     task,
@@ -91,3 +101,14 @@ export function tallyDuelReceipts(dir: string): DuelTally {
   }
   return tally;
 }
+
+export function gradeSetRecovery(expected: string[], actual: string[], threshold = 0.8): SetGradeReceipt {
+  const norm = (xs: string[]) => [...new Set(xs.map((x) => x.trim().toLowerCase()).filter(Boolean))].sort();
+  const e = norm(expected);
+  const a = norm(actual);
+  const aSet = new Set(a);
+  const intersection = e.filter((x) => aSet.has(x));
+  const union = new Set([...e, ...a]);
+  const jaccard = union.size === 0 ? 1 : intersection.length / union.size;
+  return { schema: "fable.benchmark.set_grade.v1", expected: e, actual: a, intersection, jaccard, passed: jaccard >= threshold, threshold };
+}

src/fable5/goal-audit.test.ts

@@ -0,0 +1,67 @@
+import * as fs from "node:fs";
+import * as os from "node:os";
+import * as path from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import { StateStore } from "../core/state-store.js";
+import { auditGoalCompletion, writeGoalAuditReceipt } from "./goal-audit.js";
+
+const roots: string[] = [];
+
+afterEach(() => {
+  for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true });
+});
+
+function tmpRoot(): string {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), "fable-goal-audit-"));
+  roots.push(root);
+  return root;
+}
+
+describe("goal completion audit", () => {
+  it("approves completion only when required evidence is present", () => {
+    const receipt = auditGoalCompletion({
+      objective: "add duel eval",
+      output: "Implemented duel receipts. Tests passed.",
+      criteria: [{ label: "tests", required: true, evidence: "tests passed" }],
+      store: new StateStore(tmpRoot()),
+      now: new Date("2026-06-21T00:00:00.000Z"),
+    });
+
+    expect(receipt.status).toBe("approved");
+    expect(receipt.schema).toBe("fable.goal.audit.v1");
+  });
+
+  it("rejects missing evidence", () => {
+    const receipt = auditGoalCompletion({
+      objective: "add duel eval",
+      output: "Implemented duel receipts.",
+      criteria: [{ label: "tests", required: true, evidence: "tests passed" }],
+      store: new StateStore(tmpRoot()),
+    });
+
+    expect(receipt.status).toBe("rejected");
+    expect(receipt.reasons).toContain("missing required evidence: tests");
+  });
+
+  it("rejects unsupported specific claims", () => {
+    const receipt = auditGoalCompletion({
+      objective: "summarize deploy route",
+      output: "Factory deploy is guaranteed on port 9999.",
+      criteria: [],
+      context: ["Factory deploy route is 8099/deploy via git-proxy."],
+      store: new StateStore(tmpRoot()),
+    });
+
+    expect(receipt.status).toBe("rejected");
+    expect(receipt.reasons.some((r) => r.includes("hallucination"))).toBe(true);
+  });
+
+  it("writes a JSON receipt", () => {
+    const out = path.join(tmpRoot(), "audit.json");
+    const receipt = auditGoalCompletion({ objective: "x", output: "done", criteria: [], store: new StateStore(tmpRoot()) });
+
+    writeGoalAuditReceipt(out, receipt);
+
+    expect(JSON.parse(fs.readFileSync(out, "utf-8"))).toMatchObject({ schema: "fable.goal.audit.v1" });
+  });
+});

src/fable5/goal-audit.ts

@@ -0,0 +1,69 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { StateStore } from "../core/state-store.js";
+import type { ContextEntry } from "../core/types.js";
+import { HallucinationDetector, type HallucinationCheck } from "../upgrades/hallucination-detector.js";
+
+export interface GoalAuditCriterion {
+  label: string;
+  required: boolean;
+  evidence: string;
+}
+
+export interface GoalAuditReceipt {
+  schema: "fable.goal.audit.v1";
+  objective: string;
+  status: "approved" | "rejected";
+  criteria: Array<GoalAuditCriterion & { passed: boolean }>;
+  hallucination: HallucinationCheck;
+  reasons: string[];
+  timestamp: string;
+}
+
+export function auditGoalCompletion(opts: {
+  objective: string;
+  output: string;
+  criteria: GoalAuditCriterion[];
+  knownFacts?: string[];
+  context?: string[];
+  now?: Date;
+  store?: StateStore;
+}): GoalAuditReceipt {
+  const detector = new HallucinationDetector(opts.store, { knownFacts: opts.knownFacts ?? [], blockThreshold: "low" });
+  const contextWindow = opts.context?.map((content, i): ContextEntry => ({
+    id: `goal-context-${i}`,
+    timestamp: (opts.now ?? new Date()).toISOString(),
+    source: "goal-audit",
+    content,
+    priority: "high",
+    tokenCount: content.split(/\s+/).length,
+  }));
+  const hallucination = detector.check(opts.output, contextWindow);
+  const criteria = opts.criteria.map((c) => ({ ...c, passed: evidenceMatches(opts.output, c.evidence) }));
+  const reasons = [
+    ...criteria.filter((c) => c.required && !c.passed).map((c) => `missing required evidence: ${c.label}`),
+    ...(hallucination.detected ? hallucination.signals.map((s) => `hallucination:${s.type}: ${s.evidence}`) : []),
+  ];
+  return {
+    schema: "fable.goal.audit.v1",
+    objective: opts.objective,
+    status: reasons.length === 0 ? "approved" : "rejected",
+    criteria,
+    hallucination,
+    reasons,
+    timestamp: (opts.now ?? new Date()).toISOString(),
+  };
+}
+
+export function writeGoalAuditReceipt(file: string, receipt: GoalAuditReceipt): string {
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  fs.writeFileSync(file, `${JSON.stringify(receipt, null, 2)}\n`);
+  return file;
+}
+
+function evidenceMatches(output: string, evidence: string): boolean {
+  const haystack = output.toLowerCase();
+  const needles = evidence.toLowerCase().split(/[\s,;]+/).filter((w) => w.length > 2);
+  // ponytail: simple lexical contract; upgrade to command-backed proof when a real goal needs it.
+  return needles.length === 0 || needles.every((w) => haystack.includes(w));
+}

src/fable5/index.ts

@@ -62,8 +62,11 @@ export type { ChannelEvent, ChannelEventSource, ChannelEventType, NewChannelEven
 export { sendMissionHeartbeat, writeMissionHeartbeatReceipt } from "./mission-heartbeat.js";
 export type { MissionHeartbeatOptions, MissionHeartbeatReceipt, MissionHeartbeatStatus } from "./mission-heartbeat.js";
 
-export { createEvalTraceReceipt, writeEvalTraceReceipt } from "./eval-trace.js";
-export type { EvalTraceCommand, EvalTraceReceipt } from "./eval-trace.js";
+export { createDuelEvalReceipt, createEvalTraceReceipt, gradeSetRecovery, tallyDuelReceipts, writeDuelEvalReceipt, writeEvalTraceReceipt } from "./eval-trace.js";
+export type { DuelEvalReceipt, DuelTally, EvalTraceCommand, EvalTraceReceipt, SetGradeReceipt } from "./eval-trace.js";
+
+export { auditGoalCompletion, writeGoalAuditReceipt } from "./goal-audit.js";
+export type { GoalAuditCriterion, GoalAuditReceipt } from "./goal-audit.js";
 
 export { appendFeedbackMemory, feedbackMemoryPath, feedbackMemoryStats } from "./feedback-memory.js";
 export type { FeedbackMemoryEntry, FeedbackMemoryType } from "./feedback-memory.js";

src/index.ts

@@ -864,6 +864,20 @@ benchmark
     console.log(JSON.stringify(tally, null, 2));
   });
 
+benchmark
+  .command("set-grade")
+  .description("Grade expected vs actual set recovery with Jaccard similarity")
+  .requiredOption("--expected <csv>", "Expected comma-separated items")
+  .requiredOption("--actual <csv>", "Actual comma-separated items")
+  .option("--threshold <n>", "Pass threshold", parseFloat, 0.8)
+  .action(async (opts: { expected: string; actual: string; threshold?: number }) => {
+    const { gradeSetRecovery } = await import("./fable5/eval-trace.js");
+    const parse = (v: string) => v.split(",").map((s) => s.trim()).filter(Boolean);
+    const receipt = gradeSetRecovery(parse(opts.expected), parse(opts.actual), opts.threshold ?? 0.8);
+    console.log(JSON.stringify(receipt, null, 2));
+    if (!receipt.passed) process.exit(1);
+  });
+
 benchmark
   .command("trend")
   .description("Show compounding trend over time")
@@ -2008,6 +2022,28 @@ fable
     console.log(`  `);
   });
 
+fable
+  .command("goal-audit <objective>")
+  .description("Audit goal completion evidence and reject hallucinated done claims")
+  .requiredOption("--output <text>", "Agent completion output to audit")
+  .option("--require <label:evidence>", "Required evidence contract; repeatable", (v, p: string[] = []) => [...p, v], [])
+  .option("--context <text>", "Verified context/facts; repeatable", (v, p: string[] = []) => [...p, v], [])
+  .option("--out <path>", "Receipt output path")
+  .action(async (objective: string, opts: { output: string; require?: string[]; context?: string[]; out?: string }) => {
+    const { auditGoalCompletion, writeGoalAuditReceipt } = await import("./fable5/goal-audit.js");
+    const criteria = (opts.require ?? []).map((raw) => {
+      const [label, ...rest] = raw.split(":");
+      return { label: label.trim(), evidence: rest.join(":").trim(), required: true };
+    });
+    const receipt = auditGoalCompletion({ objective, output: opts.output, criteria, context: opts.context });
+    const stamp = new Date().toISOString().replace(/[:.]/g, "-");
+    const out = opts.out ?? path.join(".fable", "goal-audits", stamp + ".json");
+    writeGoalAuditReceipt(out, receipt);
+    console.log(JSON.stringify(receipt, null, 2));
+    console.log("\n  Receipt: " + out + "\n");
+    if (receipt.status === "rejected") process.exit(1);
+  });
+
 fable
   .command("spec <task>")
   .description("Generate a six-section anti-hallucination spec with verifier gates")