From 161cb24da2cdd18d2a3e652240ff97aa28bed93a Mon Sep 17 00:00:00 2001 From: artale Date: Sun, 21 Jun 2026 15:36:57 +0200 Subject: [PATCH] feat: add audited goals and set graders --- COMMANDS.md | 9 +++++ src/fable5/eval-trace.test.ts | 6 ++- src/fable5/eval-trace.ts | 21 +++++++++++ src/fable5/goal-audit.test.ts | 67 ++++++++++++++++++++++++++++++++++ src/fable5/goal-audit.ts | 69 +++++++++++++++++++++++++++++++++++ src/fable5/index.ts | 7 +++- src/index.ts | 36 ++++++++++++++++++ 7 files changed, 212 insertions(+), 3 deletions(-) create mode 100644 src/fable5/goal-audit.test.ts create mode 100644 src/fable5/goal-audit.ts diff --git a/COMMANDS.md b/COMMANDS.md index 0d2fb3f..834e264 100644 --- a/COMMANDS.md +++ b/COMMANDS.md @@ -158,6 +158,10 @@ fable-agent plinius godmode "improve explanation quality" - `--out ` - `benchmark duel-tally`: tally two-implementer eval receipts - `--dir ` +- `benchmark set-grade`: deterministic expected/actual set grader + - `--expected ` + - `--actual ` + - `--threshold ` - `benchmark trend` - `-b, --benchmark ` - `benchmark list` @@ -290,6 +294,11 @@ fable-agent plinius godmode "improve explanation quality" - `fable5 goal ` - `-i, --iterations ` - `-s, --min-score ` +- `fable5 goal-audit ` + - `--output ` + - `--require ` + - `--context ` + - `--out ` - `fable5 spec ` - `--repo ` - `--out ` diff --git a/src/fable5/eval-trace.test.ts b/src/fable5/eval-trace.test.ts index f071890..8d92df3 100644 --- a/src/fable5/eval-trace.test.ts +++ b/src/fable5/eval-trace.test.ts @@ -2,7 +2,7 @@ import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; -import { createDuelEvalReceipt, createEvalTraceReceipt, tallyDuelReceipts, writeDuelEvalReceipt, writeEvalTraceReceipt } from "./eval-trace.js"; +import { createDuelEvalReceipt, createEvalTraceReceipt, gradeSetRecovery, tallyDuelReceipts, writeDuelEvalReceipt, writeEvalTraceReceipt } from "./eval-trace.js"; const roots: string[] = []; @@ -39,4 +39,8 @@ describe("eval trace receipt", () => { expect(tallyDuelReceipts(dir)).toEqual({ total: 3, wins: { pi: 1, codex: 1 }, ties: 1 }); }); + + it("grades target set recovery with Jaccard similarity", () => { + expect(gradeSetRecovery(["A", "B", "C"], ["a", "c", "d"], 0.5)).toMatchObject({ intersection: ["a", "c"], jaccard: 0.5, passed: true }); + }); }); diff --git a/src/fable5/eval-trace.ts b/src/fable5/eval-trace.ts index 69ed3ed..8f8463f 100644 --- a/src/fable5/eval-trace.ts +++ b/src/fable5/eval-trace.ts @@ -31,6 +31,16 @@ export interface DuelTally { ties: number; } +export interface SetGradeReceipt { + schema: "fable.benchmark.set_grade.v1"; + expected: string[]; + actual: string[]; + intersection: string[]; + jaccard: number; + passed: boolean; + threshold: number; +} + export function createEvalTraceReceipt(task: string, repo: string, commands: EvalTraceCommand[], gitHead?: string, now = new Date()): EvalTraceReceipt { return { task, @@ -91,3 +101,14 @@ export function tallyDuelReceipts(dir: string): DuelTally { } return tally; } + +export function gradeSetRecovery(expected: string[], actual: string[], threshold = 0.8): SetGradeReceipt { + const norm = (xs: string[]) => [...new Set(xs.map((x) => x.trim().toLowerCase()).filter(Boolean))].sort(); + const e = norm(expected); + const a = norm(actual); + const aSet = new Set(a); + const intersection = e.filter((x) => aSet.has(x)); + const union = new Set([...e, ...a]); + const jaccard = union.size === 0 ? 1 : intersection.length / union.size; + return { schema: "fable.benchmark.set_grade.v1", expected: e, actual: a, intersection, jaccard, passed: jaccard >= threshold, threshold }; +} diff --git a/src/fable5/goal-audit.test.ts b/src/fable5/goal-audit.test.ts new file mode 100644 index 0000000..8b99fd9 --- /dev/null +++ b/src/fable5/goal-audit.test.ts @@ -0,0 +1,67 @@ +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { StateStore } from "../core/state-store.js"; +import { auditGoalCompletion, writeGoalAuditReceipt } from "./goal-audit.js"; + +const roots: string[] = []; + +afterEach(() => { + for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); +}); + +function tmpRoot(): string { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "fable-goal-audit-")); + roots.push(root); + return root; +} + +describe("goal completion audit", () => { + it("approves completion only when required evidence is present", () => { + const receipt = auditGoalCompletion({ + objective: "add duel eval", + output: "Implemented duel receipts. Tests passed.", + criteria: [{ label: "tests", required: true, evidence: "tests passed" }], + store: new StateStore(tmpRoot()), + now: new Date("2026-06-21T00:00:00.000Z"), + }); + + expect(receipt.status).toBe("approved"); + expect(receipt.schema).toBe("fable.goal.audit.v1"); + }); + + it("rejects missing evidence", () => { + const receipt = auditGoalCompletion({ + objective: "add duel eval", + output: "Implemented duel receipts.", + criteria: [{ label: "tests", required: true, evidence: "tests passed" }], + store: new StateStore(tmpRoot()), + }); + + expect(receipt.status).toBe("rejected"); + expect(receipt.reasons).toContain("missing required evidence: tests"); + }); + + it("rejects unsupported specific claims", () => { + const receipt = auditGoalCompletion({ + objective: "summarize deploy route", + output: "Factory deploy is guaranteed on port 9999.", + criteria: [], + context: ["Factory deploy route is 8099/deploy via git-proxy."], + store: new StateStore(tmpRoot()), + }); + + expect(receipt.status).toBe("rejected"); + expect(receipt.reasons.some((r) => r.includes("hallucination"))).toBe(true); + }); + + it("writes a JSON receipt", () => { + const out = path.join(tmpRoot(), "audit.json"); + const receipt = auditGoalCompletion({ objective: "x", output: "done", criteria: [], store: new StateStore(tmpRoot()) }); + + writeGoalAuditReceipt(out, receipt); + + expect(JSON.parse(fs.readFileSync(out, "utf-8"))).toMatchObject({ schema: "fable.goal.audit.v1" }); + }); +}); diff --git a/src/fable5/goal-audit.ts b/src/fable5/goal-audit.ts new file mode 100644 index 0000000..c41cdb5 --- /dev/null +++ b/src/fable5/goal-audit.ts @@ -0,0 +1,69 @@ +import * as fs from "node:fs"; +import * as path from "node:path"; +import { StateStore } from "../core/state-store.js"; +import type { ContextEntry } from "../core/types.js"; +import { HallucinationDetector, type HallucinationCheck } from "../upgrades/hallucination-detector.js"; + +export interface GoalAuditCriterion { + label: string; + required: boolean; + evidence: string; +} + +export interface GoalAuditReceipt { + schema: "fable.goal.audit.v1"; + objective: string; + status: "approved" | "rejected"; + criteria: Array; + hallucination: HallucinationCheck; + reasons: string[]; + timestamp: string; +} + +export function auditGoalCompletion(opts: { + objective: string; + output: string; + criteria: GoalAuditCriterion[]; + knownFacts?: string[]; + context?: string[]; + now?: Date; + store?: StateStore; +}): GoalAuditReceipt { + const detector = new HallucinationDetector(opts.store, { knownFacts: opts.knownFacts ?? [], blockThreshold: "low" }); + const contextWindow = opts.context?.map((content, i): ContextEntry => ({ + id: `goal-context-${i}`, + timestamp: (opts.now ?? new Date()).toISOString(), + source: "goal-audit", + content, + priority: "high", + tokenCount: content.split(/\s+/).length, + })); + const hallucination = detector.check(opts.output, contextWindow); + const criteria = opts.criteria.map((c) => ({ ...c, passed: evidenceMatches(opts.output, c.evidence) })); + const reasons = [ + ...criteria.filter((c) => c.required && !c.passed).map((c) => `missing required evidence: ${c.label}`), + ...(hallucination.detected ? hallucination.signals.map((s) => `hallucination:${s.type}: ${s.evidence}`) : []), + ]; + return { + schema: "fable.goal.audit.v1", + objective: opts.objective, + status: reasons.length === 0 ? "approved" : "rejected", + criteria, + hallucination, + reasons, + timestamp: (opts.now ?? new Date()).toISOString(), + }; +} + +export function writeGoalAuditReceipt(file: string, receipt: GoalAuditReceipt): string { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, `${JSON.stringify(receipt, null, 2)}\n`); + return file; +} + +function evidenceMatches(output: string, evidence: string): boolean { + const haystack = output.toLowerCase(); + const needles = evidence.toLowerCase().split(/[\s,;]+/).filter((w) => w.length > 2); + // ponytail: simple lexical contract; upgrade to command-backed proof when a real goal needs it. + return needles.length === 0 || needles.every((w) => haystack.includes(w)); +} diff --git a/src/fable5/index.ts b/src/fable5/index.ts index 66d5561..fc8943e 100644 --- a/src/fable5/index.ts +++ b/src/fable5/index.ts @@ -62,8 +62,11 @@ export type { ChannelEvent, ChannelEventSource, ChannelEventType, NewChannelEven export { sendMissionHeartbeat, writeMissionHeartbeatReceipt } from "./mission-heartbeat.js"; export type { MissionHeartbeatOptions, MissionHeartbeatReceipt, MissionHeartbeatStatus } from "./mission-heartbeat.js"; -export { createEvalTraceReceipt, writeEvalTraceReceipt } from "./eval-trace.js"; -export type { EvalTraceCommand, EvalTraceReceipt } from "./eval-trace.js"; +export { createDuelEvalReceipt, createEvalTraceReceipt, gradeSetRecovery, tallyDuelReceipts, writeDuelEvalReceipt, writeEvalTraceReceipt } from "./eval-trace.js"; +export type { DuelEvalReceipt, DuelTally, EvalTraceCommand, EvalTraceReceipt, SetGradeReceipt } from "./eval-trace.js"; + +export { auditGoalCompletion, writeGoalAuditReceipt } from "./goal-audit.js"; +export type { GoalAuditCriterion, GoalAuditReceipt } from "./goal-audit.js"; export { appendFeedbackMemory, feedbackMemoryPath, feedbackMemoryStats } from "./feedback-memory.js"; export type { FeedbackMemoryEntry, FeedbackMemoryType } from "./feedback-memory.js"; diff --git a/src/index.ts b/src/index.ts index b1b37bc..ae296ca 100644 --- a/src/index.ts +++ b/src/index.ts @@ -864,6 +864,20 @@ benchmark console.log(JSON.stringify(tally, null, 2)); }); +benchmark + .command("set-grade") + .description("Grade expected vs actual set recovery with Jaccard similarity") + .requiredOption("--expected ", "Expected comma-separated items") + .requiredOption("--actual ", "Actual comma-separated items") + .option("--threshold ", "Pass threshold", parseFloat, 0.8) + .action(async (opts: { expected: string; actual: string; threshold?: number }) => { + const { gradeSetRecovery } = await import("./fable5/eval-trace.js"); + const parse = (v: string) => v.split(",").map((s) => s.trim()).filter(Boolean); + const receipt = gradeSetRecovery(parse(opts.expected), parse(opts.actual), opts.threshold ?? 0.8); + console.log(JSON.stringify(receipt, null, 2)); + if (!receipt.passed) process.exit(1); + }); + benchmark .command("trend") .description("Show compounding trend over time") @@ -2008,6 +2022,28 @@ fable console.log(` `); }); +fable + .command("goal-audit ") + .description("Audit goal completion evidence and reject hallucinated done claims") + .requiredOption("--output ", "Agent completion output to audit") + .option("--require ", "Required evidence contract; repeatable", (v, p: string[] = []) => [...p, v], []) + .option("--context ", "Verified context/facts; repeatable", (v, p: string[] = []) => [...p, v], []) + .option("--out ", "Receipt output path") + .action(async (objective: string, opts: { output: string; require?: string[]; context?: string[]; out?: string }) => { + const { auditGoalCompletion, writeGoalAuditReceipt } = await import("./fable5/goal-audit.js"); + const criteria = (opts.require ?? []).map((raw) => { + const [label, ...rest] = raw.split(":"); + return { label: label.trim(), evidence: rest.join(":").trim(), required: true }; + }); + const receipt = auditGoalCompletion({ objective, output: opts.output, criteria, context: opts.context }); + const stamp = new Date().toISOString().replace(/[:.]/g, "-"); + const out = opts.out ?? path.join(".fable", "goal-audits", stamp + ".json"); + writeGoalAuditReceipt(out, receipt); + console.log(JSON.stringify(receipt, null, 2)); + console.log("\n Receipt: " + out + "\n"); + if (receipt.status === "rejected") process.exit(1); + }); + fable .command("spec ") .description("Generate a six-section anti-hallucination spec with verifier gates")