From df845b63ccc5df312edcd3e8063163dab16b7bf9 Mon Sep 17 00:00:00 2001 From: artale Date: Thu, 18 Jun 2026 03:39:34 +0200 Subject: [PATCH] feat: add eval trace receipts --- COMMANDS.md | 2 ++ src/fable5/eval-trace.test.ts | 33 +++++++++++++++++++++++++++ src/fable5/eval-trace.ts | 33 +++++++++++++++++++++++++++ src/fable5/index.ts | 3 +++ src/index.ts | 42 ++++++++++++++++++++++++++--------- 5 files changed, 103 insertions(+), 10 deletions(-) create mode 100644 src/fable5/eval-trace.test.ts create mode 100644 src/fable5/eval-trace.ts diff --git a/COMMANDS.md b/COMMANDS.md index c05c1a3..4334129 100644 --- a/COMMANDS.md +++ b/COMMANDS.md @@ -246,6 +246,8 @@ fable-agent plinius godmode "improve explanation quality" - `--out ` - `fable5 verify ` - `--repo ` + - `--run ` + - `--trace-out ` - `fable5 goal ` - `-i, --iterations ` - `-s, --min-score ` diff --git a/src/fable5/eval-trace.test.ts b/src/fable5/eval-trace.test.ts new file mode 100644 index 0000000..6fc49d2 --- /dev/null +++ b/src/fable5/eval-trace.test.ts @@ -0,0 +1,33 @@ +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { createEvalTraceReceipt, writeEvalTraceReceipt } from "./eval-trace.js"; + +const roots: string[] = []; + +afterEach(() => { + for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); +}); + +function tmpRoot(): string { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "fable-eval-trace-")); + roots.push(root); + return root; +} + +describe("eval trace receipt", () => { + it("passes only when every command exits zero", () => { + expect(createEvalTraceReceipt("task", ".", [{ command: "test", status: 0 }]).status).toBe("passed"); + expect(createEvalTraceReceipt("task", ".", [{ command: "test", status: 1 }]).status).toBe("failed"); + }); + + it("writes a JSON receipt", () => { + const out = path.join(tmpRoot(), "trace.json"); + const receipt = createEvalTraceReceipt("task", ".", [{ command: "test", status: 0 }], "abc123", new Date("2026-06-18T00:00:00.000Z")); + + writeEvalTraceReceipt(out, receipt); + + expect(JSON.parse(fs.readFileSync(out, "utf-8"))).toMatchObject({ task: "task", status: "passed", gitHead: "abc123" }); + }); +}); diff --git a/src/fable5/eval-trace.ts b/src/fable5/eval-trace.ts new file mode 100644 index 0000000..29c8fa3 --- /dev/null +++ b/src/fable5/eval-trace.ts @@ -0,0 +1,33 @@ +import * as fs from "node:fs"; +import * as path from "node:path"; + +export interface EvalTraceCommand { + command: string; + status: number | null; +} + +export interface EvalTraceReceipt { + task: string; + repo: string; + status: "passed" | "failed"; + commands: EvalTraceCommand[]; + gitHead?: string; + createdAt: string; +} + +export function createEvalTraceReceipt(task: string, repo: string, commands: EvalTraceCommand[], gitHead?: string, now = new Date()): EvalTraceReceipt { + return { + task, + repo, + status: commands.every((cmd) => cmd.status === 0) ? "passed" : "failed", + commands, + gitHead, + createdAt: now.toISOString(), + }; +} + +export function writeEvalTraceReceipt(file: string, receipt: EvalTraceReceipt): string { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, `${JSON.stringify(receipt, null, 2)}\n`); + return file; +} diff --git a/src/fable5/index.ts b/src/fable5/index.ts index 7b8ed5f..8b065c9 100644 --- a/src/fable5/index.ts +++ b/src/fable5/index.ts @@ -62,6 +62,9 @@ export type { ChannelEvent, ChannelEventSource, ChannelEventType, NewChannelEven export { sendMissionHeartbeat, writeMissionHeartbeatReceipt } from "./mission-heartbeat.js"; export type { MissionHeartbeatOptions, MissionHeartbeatReceipt, MissionHeartbeatStatus } from "./mission-heartbeat.js"; +export { createEvalTraceReceipt, writeEvalTraceReceipt } from "./eval-trace.js"; +export type { EvalTraceCommand, EvalTraceReceipt } from "./eval-trace.js"; + export { runCyberPreflight, writeCyberPreflightReceipt } from "./cyber-preflight.js"; export type { CyberPreflightCheck, CyberPreflightOptions, CyberPreflightReceipt } from "./cyber-preflight.js"; diff --git a/src/index.ts b/src/index.ts index 2ae2dca..a0ddfa2 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1712,7 +1712,14 @@ fable .command("verify ") .description("Run independent verifier against a task") .option("--repo ", "Also run the repo verification gate at this path") - .action(async (task: string, opts: { repo?: string }) => { + .option("--run ", "Emit verify events to .runs//channel.jsonl") + .option("--trace-out ", "Eval trace receipt path") + .action(async (task: string, opts: { repo?: string; run?: string; traceOut?: string }) => { + if (opts.run) { + const { emitChannelEvent } = await import("./fable5/channel.js"); + emitChannelEvent(opts.run, { source: "gate", type: "started", data: { task, repo: opts.repo } }); + } + if (opts.repo) { const { spawnSync } = await import("node:child_process"); const repo = path.resolve(opts.repo); @@ -1723,28 +1730,43 @@ fable ["npx", ["tsc", "--noEmit"]], [process.execPath, [process.argv[1], "security", "scan", repo]], ]; + const commandResults: Array<{ command: string; status: number | null }> = []; console.log(` Repo Verification Gate: ${repo}`); for (const [cmd, args] of checks) { + const command = [cmd, ...args].join(" "); console.log(` - $ ${[cmd, ...args].join(" ")}`); + $ ${command}`); // ponytail: Windows npm/npx shims need cmd.exe; commands are fixed, no user shell input. const check = process.platform === "win32" && (cmd === "npm" || cmd === "npx") - ? spawnSync("cmd.exe", ["/d", "/s", "/c", [cmd, ...args].join(" ")], { cwd: repo, stdio: "inherit" }) + ? spawnSync("cmd.exe", ["/d", "/s", "/c", command], { cwd: repo, stdio: "inherit" }) : spawnSync(cmd, args, { cwd: repo, stdio: "inherit" }); - if (check.status !== 0) { - console.error(` - ✗ Repo verification failed`); - process.exit(check.status ?? 1); - } + commandResults.push({ command, status: check.status }); + if (check.status !== 0) break; } const { appendZteReceipt, createZteReceipt } = await import("./fable5/zte-protocol.js"); - const receipt = createZteReceipt(task, repo, "passed", checks.map(([cmd, args]) => [cmd, ...args].join(" "))); + const { createEvalTraceReceipt, writeEvalTraceReceipt } = await import("./fable5/eval-trace.js"); + const gitHead = spawnSync("git", ["rev-parse", "HEAD"], { cwd: repo, encoding: "utf-8" }).stdout.trim() || undefined; + const trace = createEvalTraceReceipt(task, repo, commandResults, gitHead); + const tracePath = opts.traceOut ?? path.join(repo, ".fable", "eval-traces", `${new Date().toISOString().replace(/[:.]/g, "-")}.json`); + writeEvalTraceReceipt(tracePath, trace); + if (opts.run) { + const { emitChannelEvent } = await import("./fable5/channel.js"); + emitChannelEvent(opts.run, { source: "gate", type: trace.status === "passed" ? "receipt" : "error", data: trace }); + } + const receipt = createZteReceipt(task, repo, trace.status, commandResults.map((r) => r.command), trace.status === "failed" ? "repo verification failed" : undefined); const receiptPath = appendZteReceipt(repo, receipt); + if (trace.status === "failed") { + console.error(` + ✗ Repo verification failed`); + console.error(` Eval trace: ${tracePath}`); + process.exit(commandResults.find((r) => r.status !== 0)?.status ?? 1); + } console.log(` ✓ Repo verification passed`); - console.log(` ✓ ZTE receipt written: ${receiptPath} + console.log(` ✓ ZTE receipt written: ${receiptPath}`); + console.log(` ✓ Eval trace written: ${tracePath} `); }