From e3524669b1611ef8de2e24f017d8c0230e535ea8 Mon Sep 17 00:00:00 2001 From: artale Date: Sun, 21 Jun 2026 13:14:41 +0200 Subject: [PATCH] feat: add agentic verification attestations --- COMMANDS.md | 17 ++++ src/fable5/attestation.test.ts | 48 ++++++++++ src/fable5/attestation.ts | 165 +++++++++++++++++++++++++++++++++ src/index.ts | 66 +++++++++++++ 4 files changed, 296 insertions(+) create mode 100644 src/fable5/attestation.test.ts create mode 100644 src/fable5/attestation.ts diff --git a/COMMANDS.md b/COMMANDS.md index 55e1cd2..94e7a69 100644 --- a/COMMANDS.md +++ b/COMMANDS.md @@ -23,6 +23,8 @@ fable-agent plinius godmode "improve explanation quality" | Command | Primary purpose | |---|---| | `run` | Execute full agent loop | +| `verify` | Emit agentic verification attestation | +| `attest` | Alias for `verify` | | `session` | Session lifecycle | | `skills` | Skill registry ops | | `state` | Knowledge state / stats view | @@ -50,6 +52,15 @@ fable-agent plinius godmode "improve explanation quality" ## Command matrix +### `verify` / `attest` + +- `verify `: verify a target and emit an agentic attestation + - `--out ` + - `--run ` +- `attest `: alias for `verify` + - `--out ` + - `--run ` + ### `run` - `run ` @@ -185,6 +196,12 @@ fable-agent plinius godmode "improve explanation quality" - `--out ` - `--allow-dirty` - `--run ` +- `factory verify `: verify a target and emit an agentic attestation + - `--out ` + - `--run ` +- `factory attest `: alias for `factory verify`; emits an agentic attestation + - `--out ` + - `--run ` - `factory deploy ` - `--token ` - `--gate-receipt ` diff --git a/src/fable5/attestation.test.ts b/src/fable5/attestation.test.ts new file mode 100644 index 0000000..06d1897 --- /dev/null +++ b/src/fable5/attestation.test.ts @@ -0,0 +1,48 @@ +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { describe, expect, it } from "vitest"; +import { verifyTarget } from "./attestation.js"; + +function okRunner(changed: string) { + return (command: string, args: string[]) => { + const full = [command, ...args].join(" "); + if (full === "git status --short") return { status: 0, stdout: ` M ${changed}\n`, stderr: "" }; + if (full === "npm run -s build") return { status: 0, stdout: "", stderr: "" }; + if (full === "npm run -s test") return { status: 0, stdout: "", stderr: "" }; + if (full === "npm run -s docs:check") return { status: 0, stdout: "", stderr: "" }; + return { status: 1, stdout: "", stderr: `unexpected ${full}` }; + }; +} + +describe("agentic attestation", () => { + it("emits a typed verified receipt for changed files", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "attest-")); + fs.writeFileSync(path.join(dir, "package.json"), JSON.stringify({ scripts: { build: "tsc", test: "vitest", "docs:check": "node x" } })); + const changed = path.join(dir, "src", "fable5", "safe.ts"); + fs.mkdirSync(path.dirname(changed), { recursive: true }); + fs.writeFileSync(changed, "export const add = (a: number, b: number) => a + b;\n"); + const out = path.join(dir, "receipt.json"); + + const receipt = verifyTarget({ target: dir, out, runner: okRunner(changed), now: new Date("2026-06-21T00:00:00.000Z") }); + + expect(receipt.schema).toBe("fable.verification.attestation.v1"); + expect(receipt.deploy_route).toBe("8099/deploy via git-proxy"); + expect(receipt.verification_status).toBe("verified"); + expect(receipt.human_review.required).toBe(true); + expect(receipt.human_review.reasons).toContain("sensitive path changed"); + expect(JSON.parse(fs.readFileSync(out, "utf-8")).schema).toBe("fable.verification.attestation.v1"); + }); + + it("requires review for risky capabilities without treating review as verification failure", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "attest-risk-")); + const file = path.join(dir, "index.ts"); + fs.writeFileSync(file, "import { execSync } from 'node:child_process'; execSync('echo hi');\n"); + + const receipt = verifyTarget({ target: file, runner: okRunner(file), now: new Date("2026-06-21T00:00:00.000Z") }); + + expect(receipt.verification_status).toBe("partial"); + expect(receipt.risk_findings.some((f) => f.kind === "shell_exec")).toBe(true); + expect(receipt.human_review.required).toBe(true); + }); +}); diff --git a/src/fable5/attestation.ts b/src/fable5/attestation.ts new file mode 100644 index 0000000..977ed12 --- /dev/null +++ b/src/fable5/attestation.ts @@ -0,0 +1,165 @@ +import * as fs from "node:fs"; +import * as path from "node:path"; +import { spawnSync } from "node:child_process"; +import { scanAstFile, type AstSafetyReport } from "../core/ast-safety.js"; +import { assessLegibilityRisk } from "../core/unicode-safety.js"; + +export type VerificationStatus = "verified" | "partial" | "failed" | "unverified"; +export type RiskLevel = "low" | "medium" | "high"; + +export interface VerificationCheck { + name: string; + command?: string; + status: "passed" | "failed" | "skipped"; + detail?: string; +} + +export interface RiskFinding { + kind: string; + severity: RiskLevel; + file?: string; + detail: string; +} + +export interface VerificationAttestation { + schema: "fable.verification.attestation.v1"; + run_id: string; + intent: { target: string; requested_by: "human" | "agent" | "ci"; summary: string }; + execution_boundary: { tools_allowed: string[]; network: "blocked" | "limited" | "allowed"; filesystem: "read-only" | "workspace-write" | "unrestricted" }; + observed_changes: string[]; + checks: VerificationCheck[]; + risk_findings: RiskFinding[]; + human_review: { required: boolean; reasons: string[] }; + deploy_route: "8099/deploy via git-proxy"; + verification_status: VerificationStatus; + unverified_claims: string[]; + timestamp: string; +} + +export interface VerifyTargetOptions { + target: string; + out?: string; + now?: Date; + requestedBy?: "human" | "agent" | "ci"; + runner?: (command: string, args: string[], cwd?: string) => { status: number | null; stdout: string; stderr: string }; +} + +export function verifyTarget(opts: VerifyTargetOptions): VerificationAttestation { + const target = path.resolve(opts.target); + const runner = opts.runner ?? runCommand; + const checks: VerificationCheck[] = []; + const risk_findings: RiskFinding[] = []; + + if (!fs.existsSync(target)) throw new Error(`target does not exist: ${opts.target}`); + + const changed = runGitStatus(runner); + checks.push(commandCheck("git_status", "git status --short", 0, changed.status, changed.stderr || `${changed.files.length} changed file(s)`)); + + const files = collectFiles(target, changed.files); + checks.push({ name: "target_files", status: files.length > 0 ? "passed" : "skipped", detail: `${files.length} scannable changed file(s)` }); + + for (const file of files) { + const content = fs.readFileSync(file, "utf-8"); + const legibility = assessLegibilityRisk(content); + if (legibility.risky) risk_findings.push({ kind: "human_legibility", severity: "high", file, detail: legibility.reason }); + if (/\.(ts|tsx|js|jsx)$/i.test(file)) addAstFindings(scanAstFile(file), risk_findings); + } + checks.push({ name: "glyph_legibility", status: risk_findings.some((f) => f.kind === "human_legibility") ? "failed" : "passed" }); + checks.push({ name: "ast_capabilities", status: "passed", detail: `${risk_findings.filter((f) => ["shell_exec", "network", "fs_write", "eval_like"].includes(f.kind)).length} capability finding(s)` }); + + const pkg = findPackageJson(target); + if (pkg) { + const cwd = path.dirname(pkg); + checks.push(npmCheck("build", cwd, runner)); + checks.push(npmCheck("test", cwd, runner)); + checks.push(npmCheck("docs:check", cwd, runner)); + } else { + checks.push({ name: "build", status: "skipped", detail: "no package.json found" }); + checks.push({ name: "test", status: "skipped", detail: "no package.json found" }); + checks.push({ name: "docs:check", status: "skipped", detail: "no package.json found" }); + } + + const reviewReasons = reviewReasonsFor(risk_findings, changed.files); + const failed = checks.some((c) => c.status === "failed"); + const skipped = checks.some((c) => c.status === "skipped"); + const attestation: VerificationAttestation = { + schema: "fable.verification.attestation.v1", + run_id: `attest-${Date.now().toString(36)}`, + intent: { target: opts.target, requested_by: opts.requestedBy ?? "human", summary: "Verify target and emit agentic attestation" }, + execution_boundary: { tools_allowed: ["git", "npm", "typescript-ast", "unicode-scan"], network: "limited", filesystem: "workspace-write" }, + observed_changes: changed.files, + checks, + risk_findings, + human_review: { required: reviewReasons.length > 0, reasons: reviewReasons }, + deploy_route: "8099/deploy via git-proxy", + verification_status: failed ? "failed" : skipped ? "partial" : "verified", + unverified_claims: ["8099/deploy runtime token acceptance is not proven by this local attestation"], + timestamp: (opts.now ?? new Date()).toISOString(), + }; + if (opts.out) writeAttestation(opts.out, attestation); + return attestation; +} + +export function writeAttestation(file: string, attestation: VerificationAttestation): void { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, `${JSON.stringify(attestation, null, 2)}\n`); +} + +function collectFiles(target: string, changed: string[]): string[] { + const accept = (file: string) => /\.(ts|tsx|js|jsx|md|txt|json|yaml|yml)$/i.test(file); + if (fs.statSync(target).isFile()) return accept(target) ? [target] : []; + const root = path.resolve(target); + const changedFiles = changed + .map((f) => f.trim().replace(/^[A-Z? ]+\s+/, "")) + .map((f) => path.resolve(f)) + .filter((f) => f.startsWith(root) && fs.existsSync(f) && fs.statSync(f).isFile() && accept(f)); + // ponytail: repo verify scans changed files; use a file target for deep single-file audit. + return [...new Set(changedFiles)].sort(); +} + +function addAstFindings(report: AstSafetyReport, findings: RiskFinding[]): void { + if (report.shell_exec) findings.push({ kind: "shell_exec", severity: "high", file: report.file, detail: "shell execution capability present" }); + if (report.network) findings.push({ kind: "network", severity: "medium", file: report.file, detail: "network capability present" }); + if (report.fs_write) findings.push({ kind: "fs_write", severity: "medium", file: report.file, detail: "filesystem write capability present" }); + if (report.eval_like) findings.push({ kind: "eval_like", severity: "high", file: report.file, detail: "eval-like dynamic execution present" }); + if (report.private_glyphs) findings.push({ kind: "private_glyphs", severity: "high", file: report.file, detail: report.legibility_reason }); +} + +function reviewReasonsFor(findings: RiskFinding[], changed: string[]): string[] { + const reasons = findings.filter((f) => f.severity === "high").map((f) => `${f.kind}${f.file ? ` in ${path.relative(process.cwd(), f.file)}` : ""}`); + if (changed.some((f) => /(^|\/)(deploy|security|src\/core|src\/fable5)\//.test(f.trim().replace(/^[A-Z? ]+\s+/, "").replace(/\\/g, "/")))) reasons.push("sensitive path changed"); + return [...new Set(reasons)]; +} + +function findPackageJson(target: string): string | undefined { + const start = fs.statSync(target).isDirectory() ? target : path.dirname(target); + let dir = start; + while (true) { + const pkg = path.join(dir, "package.json"); + if (fs.existsSync(pkg)) return pkg; + const parent = path.dirname(dir); + if (parent === dir) return undefined; + dir = parent; + } +} + +function npmCheck(script: string, cwd: string, runner: NonNullable): VerificationCheck { + const pkg = JSON.parse(fs.readFileSync(path.join(cwd, "package.json"), "utf-8")); + if (!pkg.scripts?.[script]) return { name: script, status: "skipped", detail: `missing npm script ${script}` }; + const r = runner("npm", ["run", "-s", script], cwd); + return commandCheck(script, `npm run -s ${script}`, 0, r.status, (r.stderr || r.stdout).slice(0, 1000)); +} + +function runGitStatus(runner: NonNullable): { status: number | null; stderr: string; files: string[] } { + const r = runner("git", ["status", "--short"]); + return { status: r.status, stderr: r.stderr, files: r.stdout.split(/\r?\n/).filter(Boolean) }; +} + +function commandCheck(name: string, command: string, want: number, got: number | null, detail?: string): VerificationCheck { + return { name, command, status: got === want ? "passed" : "failed", detail }; +} + +function runCommand(command: string, args: string[], cwd?: string) { + const result = spawnSync(command, args, { encoding: "utf-8", cwd, shell: process.platform === "win32" }); + return { status: result.status, stdout: result.stdout ?? "", stderr: result.stderr ?? result.error?.message ?? "" }; +} diff --git a/src/index.ts b/src/index.ts index 737e229..bd1a494 100644 --- a/src/index.ts +++ b/src/index.ts @@ -21,6 +21,40 @@ program .description("Harness-agnostic, model-agnostic self-improving agent system — loops, dynamic workflows, routines") .version(packageJson.version); +// ── Verify / Attest ───────────────────────────────────────── + +program + .command("verify ") + .description("Verify a target and emit an agentic attestation") + .option("--out ", "Attestation output path") + .option("--run ", "Emit attestation event to .runs//channel.jsonl") + .action(async (target: string, opts: { out?: string; run?: string }) => { + const { verifyTarget } = await import("./fable5/attestation.js"); + const stamp = new Date().toISOString().replace(/[:.]/g, "-"); + const out = opts.out ?? path.join(".fable", "attestations", stamp + ".json"); + const receipt = verifyTarget({ target, out }); + console.log(JSON.stringify(receipt, null, 2)); + console.log("\n Attestation: " + out + "\n"); + await emitRunEvent(opts.run, { source: "gate", type: receipt.verification_status === "failed" ? "error" : "receipt", data: { kind: "agentic-attestation", receipt, path: out } }); + if (receipt.verification_status === "failed") process.exit(1); + }); + +program + .command("attest ") + .description("Alias for verify: emit an agentic attestation") + .option("--out ", "Attestation output path") + .option("--run ", "Emit attestation event to .runs//channel.jsonl") + .action(async (target: string, opts: { out?: string; run?: string }) => { + const { verifyTarget } = await import("./fable5/attestation.js"); + const stamp = new Date().toISOString().replace(/[:.]/g, "-"); + const out = opts.out ?? path.join(".fable", "attestations", stamp + ".json"); + const receipt = verifyTarget({ target, out }); + console.log(JSON.stringify(receipt, null, 2)); + console.log("\n Attestation: " + out + "\n"); + await emitRunEvent(opts.run, { source: "gate", type: receipt.verification_status === "failed" ? "error" : "receipt", data: { kind: "agentic-attestation", receipt, path: out } }); + if (receipt.verification_status === "failed") process.exit(1); + }); + // ── Run ───────────────────────────────────────────────────── program @@ -1500,6 +1534,38 @@ factory if (receipt.result !== "passed") process.exit(1); }); +factory + .command("verify ") + .description("Verify a target and emit an agentic attestation") + .option("--out ", "Attestation output path") + .option("--run ", "Emit attestation event to .runs//channel.jsonl") + .action(async (target: string, opts: { out?: string; run?: string }) => { + const { verifyTarget } = await import("./fable5/attestation.js"); + const stamp = new Date().toISOString().replace(/[:.]/g, "-"); + const out = opts.out ?? path.join(".fable", "attestations", stamp + ".json"); + const receipt = verifyTarget({ target, out }); + console.log(JSON.stringify(receipt, null, 2)); + console.log("\n Attestation: " + out + "\n"); + await emitRunEvent(opts.run, { source: "gate", type: receipt.verification_status === "failed" ? "error" : "receipt", data: { kind: "agentic-attestation", receipt, path: out } }); + if (receipt.verification_status === "failed") process.exit(1); + }); + +factory + .command("attest ") + .description("Alias for factory verify: emit an agentic attestation") + .option("--out ", "Attestation output path") + .option("--run ", "Emit attestation event to .runs//channel.jsonl") + .action(async (target: string, opts: { out?: string; run?: string }) => { + const { verifyTarget } = await import("./fable5/attestation.js"); + const stamp = new Date().toISOString().replace(/[:.]/g, "-"); + const out = opts.out ?? path.join(".fable", "attestations", stamp + ".json"); + const receipt = verifyTarget({ target, out }); + console.log(JSON.stringify(receipt, null, 2)); + console.log("\n Attestation: " + out + "\n"); + await emitRunEvent(opts.run, { source: "gate", type: receipt.verification_status === "failed" ? "error" : "receipt", data: { kind: "agentic-attestation", receipt, path: out } }); + if (receipt.verification_status === "failed") process.exit(1); + }); + factory .command("deploy ") .description("Post an approved command to the factory deploy webhook")