feat: add agentic verification attestations
This commit is contained in:
artale
parent
590ce30bee
commit
e3524669b1
17
COMMANDS.md
17
COMMANDS.md
|
|
@ -23,6 +23,8 @@ fable-agent plinius godmode "improve explanation quality"
|
|||
| Command | Primary purpose |
|
||||
|---|---|
|
||||
| `run` | Execute full agent loop |
|
||||
| `verify` | Emit agentic verification attestation |
|
||||
| `attest` | Alias for `verify` |
|
||||
| `session` | Session lifecycle |
|
||||
| `skills` | Skill registry ops |
|
||||
| `state` | Knowledge state / stats view |
|
||||
|
|
@ -50,6 +52,15 @@ fable-agent plinius godmode "improve explanation quality"
|
|||
|
||||
## Command matrix
|
||||
|
||||
### `verify` / `attest`
|
||||
|
||||
- `verify <target>`: verify a target and emit an agentic attestation
|
||||
- `--out <path>`
|
||||
- `--run <id>`
|
||||
- `attest <target>`: alias for `verify`
|
||||
- `--out <path>`
|
||||
- `--run <id>`
|
||||
|
||||
### `run`
|
||||
|
||||
- `run <task>`
|
||||
|
|
@ -185,6 +196,12 @@ fable-agent plinius godmode "improve explanation quality"
|
|||
- `--out <path>`
|
||||
- `--allow-dirty`
|
||||
- `--run <id>`
|
||||
- `factory verify <target>`: verify a target and emit an agentic attestation
|
||||
- `--out <path>`
|
||||
- `--run <id>`
|
||||
- `factory attest <target>`: alias for `factory verify`; emits an agentic attestation
|
||||
- `--out <path>`
|
||||
- `--run <id>`
|
||||
- `factory deploy <command>`
|
||||
- `--token <token>`
|
||||
- `--gate-receipt <path>`
|
||||
|
|
|
|||
|
|
@ -0,0 +1,48 @@
|
|||
import * as fs from "node:fs";
|
||||
import * as os from "node:os";
|
||||
import * as path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { verifyTarget } from "./attestation.js";
|
||||
|
||||
function okRunner(changed: string) {
|
||||
return (command: string, args: string[]) => {
|
||||
const full = [command, ...args].join(" ");
|
||||
if (full === "git status --short") return { status: 0, stdout: ` M ${changed}\n`, stderr: "" };
|
||||
if (full === "npm run -s build") return { status: 0, stdout: "", stderr: "" };
|
||||
if (full === "npm run -s test") return { status: 0, stdout: "", stderr: "" };
|
||||
if (full === "npm run -s docs:check") return { status: 0, stdout: "", stderr: "" };
|
||||
return { status: 1, stdout: "", stderr: `unexpected ${full}` };
|
||||
};
|
||||
}
|
||||
|
||||
describe("agentic attestation", () => {
|
||||
it("emits a typed verified receipt for changed files", () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "attest-"));
|
||||
fs.writeFileSync(path.join(dir, "package.json"), JSON.stringify({ scripts: { build: "tsc", test: "vitest", "docs:check": "node x" } }));
|
||||
const changed = path.join(dir, "src", "fable5", "safe.ts");
|
||||
fs.mkdirSync(path.dirname(changed), { recursive: true });
|
||||
fs.writeFileSync(changed, "export const add = (a: number, b: number) => a + b;\n");
|
||||
const out = path.join(dir, "receipt.json");
|
||||
|
||||
const receipt = verifyTarget({ target: dir, out, runner: okRunner(changed), now: new Date("2026-06-21T00:00:00.000Z") });
|
||||
|
||||
expect(receipt.schema).toBe("fable.verification.attestation.v1");
|
||||
expect(receipt.deploy_route).toBe("8099/deploy via git-proxy");
|
||||
expect(receipt.verification_status).toBe("verified");
|
||||
expect(receipt.human_review.required).toBe(true);
|
||||
expect(receipt.human_review.reasons).toContain("sensitive path changed");
|
||||
expect(JSON.parse(fs.readFileSync(out, "utf-8")).schema).toBe("fable.verification.attestation.v1");
|
||||
});
|
||||
|
||||
it("requires review for risky capabilities without treating review as verification failure", () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "attest-risk-"));
|
||||
const file = path.join(dir, "index.ts");
|
||||
fs.writeFileSync(file, "import { execSync } from 'node:child_process'; execSync('echo hi');\n");
|
||||
|
||||
const receipt = verifyTarget({ target: file, runner: okRunner(file), now: new Date("2026-06-21T00:00:00.000Z") });
|
||||
|
||||
expect(receipt.verification_status).toBe("partial");
|
||||
expect(receipt.risk_findings.some((f) => f.kind === "shell_exec")).toBe(true);
|
||||
expect(receipt.human_review.required).toBe(true);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,165 @@
|
|||
import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { scanAstFile, type AstSafetyReport } from "../core/ast-safety.js";
|
||||
import { assessLegibilityRisk } from "../core/unicode-safety.js";
|
||||
|
||||
export type VerificationStatus = "verified" | "partial" | "failed" | "unverified";
|
||||
export type RiskLevel = "low" | "medium" | "high";
|
||||
|
||||
export interface VerificationCheck {
|
||||
name: string;
|
||||
command?: string;
|
||||
status: "passed" | "failed" | "skipped";
|
||||
detail?: string;
|
||||
}
|
||||
|
||||
export interface RiskFinding {
|
||||
kind: string;
|
||||
severity: RiskLevel;
|
||||
file?: string;
|
||||
detail: string;
|
||||
}
|
||||
|
||||
export interface VerificationAttestation {
|
||||
schema: "fable.verification.attestation.v1";
|
||||
run_id: string;
|
||||
intent: { target: string; requested_by: "human" | "agent" | "ci"; summary: string };
|
||||
execution_boundary: { tools_allowed: string[]; network: "blocked" | "limited" | "allowed"; filesystem: "read-only" | "workspace-write" | "unrestricted" };
|
||||
observed_changes: string[];
|
||||
checks: VerificationCheck[];
|
||||
risk_findings: RiskFinding[];
|
||||
human_review: { required: boolean; reasons: string[] };
|
||||
deploy_route: "8099/deploy via git-proxy";
|
||||
verification_status: VerificationStatus;
|
||||
unverified_claims: string[];
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
export interface VerifyTargetOptions {
|
||||
target: string;
|
||||
out?: string;
|
||||
now?: Date;
|
||||
requestedBy?: "human" | "agent" | "ci";
|
||||
runner?: (command: string, args: string[], cwd?: string) => { status: number | null; stdout: string; stderr: string };
|
||||
}
|
||||
|
||||
export function verifyTarget(opts: VerifyTargetOptions): VerificationAttestation {
|
||||
const target = path.resolve(opts.target);
|
||||
const runner = opts.runner ?? runCommand;
|
||||
const checks: VerificationCheck[] = [];
|
||||
const risk_findings: RiskFinding[] = [];
|
||||
|
||||
if (!fs.existsSync(target)) throw new Error(`target does not exist: ${opts.target}`);
|
||||
|
||||
const changed = runGitStatus(runner);
|
||||
checks.push(commandCheck("git_status", "git status --short", 0, changed.status, changed.stderr || `${changed.files.length} changed file(s)`));
|
||||
|
||||
const files = collectFiles(target, changed.files);
|
||||
checks.push({ name: "target_files", status: files.length > 0 ? "passed" : "skipped", detail: `${files.length} scannable changed file(s)` });
|
||||
|
||||
for (const file of files) {
|
||||
const content = fs.readFileSync(file, "utf-8");
|
||||
const legibility = assessLegibilityRisk(content);
|
||||
if (legibility.risky) risk_findings.push({ kind: "human_legibility", severity: "high", file, detail: legibility.reason });
|
||||
if (/\.(ts|tsx|js|jsx)$/i.test(file)) addAstFindings(scanAstFile(file), risk_findings);
|
||||
}
|
||||
checks.push({ name: "glyph_legibility", status: risk_findings.some((f) => f.kind === "human_legibility") ? "failed" : "passed" });
|
||||
checks.push({ name: "ast_capabilities", status: "passed", detail: `${risk_findings.filter((f) => ["shell_exec", "network", "fs_write", "eval_like"].includes(f.kind)).length} capability finding(s)` });
|
||||
|
||||
const pkg = findPackageJson(target);
|
||||
if (pkg) {
|
||||
const cwd = path.dirname(pkg);
|
||||
checks.push(npmCheck("build", cwd, runner));
|
||||
checks.push(npmCheck("test", cwd, runner));
|
||||
checks.push(npmCheck("docs:check", cwd, runner));
|
||||
} else {
|
||||
checks.push({ name: "build", status: "skipped", detail: "no package.json found" });
|
||||
checks.push({ name: "test", status: "skipped", detail: "no package.json found" });
|
||||
checks.push({ name: "docs:check", status: "skipped", detail: "no package.json found" });
|
||||
}
|
||||
|
||||
const reviewReasons = reviewReasonsFor(risk_findings, changed.files);
|
||||
const failed = checks.some((c) => c.status === "failed");
|
||||
const skipped = checks.some((c) => c.status === "skipped");
|
||||
const attestation: VerificationAttestation = {
|
||||
schema: "fable.verification.attestation.v1",
|
||||
run_id: `attest-${Date.now().toString(36)}`,
|
||||
intent: { target: opts.target, requested_by: opts.requestedBy ?? "human", summary: "Verify target and emit agentic attestation" },
|
||||
execution_boundary: { tools_allowed: ["git", "npm", "typescript-ast", "unicode-scan"], network: "limited", filesystem: "workspace-write" },
|
||||
observed_changes: changed.files,
|
||||
checks,
|
||||
risk_findings,
|
||||
human_review: { required: reviewReasons.length > 0, reasons: reviewReasons },
|
||||
deploy_route: "8099/deploy via git-proxy",
|
||||
verification_status: failed ? "failed" : skipped ? "partial" : "verified",
|
||||
unverified_claims: ["8099/deploy runtime token acceptance is not proven by this local attestation"],
|
||||
timestamp: (opts.now ?? new Date()).toISOString(),
|
||||
};
|
||||
if (opts.out) writeAttestation(opts.out, attestation);
|
||||
return attestation;
|
||||
}
|
||||
|
||||
export function writeAttestation(file: string, attestation: VerificationAttestation): void {
|
||||
fs.mkdirSync(path.dirname(file), { recursive: true });
|
||||
fs.writeFileSync(file, `${JSON.stringify(attestation, null, 2)}\n`);
|
||||
}
|
||||
|
||||
function collectFiles(target: string, changed: string[]): string[] {
|
||||
const accept = (file: string) => /\.(ts|tsx|js|jsx|md|txt|json|yaml|yml)$/i.test(file);
|
||||
if (fs.statSync(target).isFile()) return accept(target) ? [target] : [];
|
||||
const root = path.resolve(target);
|
||||
const changedFiles = changed
|
||||
.map((f) => f.trim().replace(/^[A-Z? ]+\s+/, ""))
|
||||
.map((f) => path.resolve(f))
|
||||
.filter((f) => f.startsWith(root) && fs.existsSync(f) && fs.statSync(f).isFile() && accept(f));
|
||||
// ponytail: repo verify scans changed files; use a file target for deep single-file audit.
|
||||
return [...new Set(changedFiles)].sort();
|
||||
}
|
||||
|
||||
function addAstFindings(report: AstSafetyReport, findings: RiskFinding[]): void {
|
||||
if (report.shell_exec) findings.push({ kind: "shell_exec", severity: "high", file: report.file, detail: "shell execution capability present" });
|
||||
if (report.network) findings.push({ kind: "network", severity: "medium", file: report.file, detail: "network capability present" });
|
||||
if (report.fs_write) findings.push({ kind: "fs_write", severity: "medium", file: report.file, detail: "filesystem write capability present" });
|
||||
if (report.eval_like) findings.push({ kind: "eval_like", severity: "high", file: report.file, detail: "eval-like dynamic execution present" });
|
||||
if (report.private_glyphs) findings.push({ kind: "private_glyphs", severity: "high", file: report.file, detail: report.legibility_reason });
|
||||
}
|
||||
|
||||
function reviewReasonsFor(findings: RiskFinding[], changed: string[]): string[] {
|
||||
const reasons = findings.filter((f) => f.severity === "high").map((f) => `${f.kind}${f.file ? ` in ${path.relative(process.cwd(), f.file)}` : ""}`);
|
||||
if (changed.some((f) => /(^|\/)(deploy|security|src\/core|src\/fable5)\//.test(f.trim().replace(/^[A-Z? ]+\s+/, "").replace(/\\/g, "/")))) reasons.push("sensitive path changed");
|
||||
return [...new Set(reasons)];
|
||||
}
|
||||
|
||||
function findPackageJson(target: string): string | undefined {
|
||||
const start = fs.statSync(target).isDirectory() ? target : path.dirname(target);
|
||||
let dir = start;
|
||||
while (true) {
|
||||
const pkg = path.join(dir, "package.json");
|
||||
if (fs.existsSync(pkg)) return pkg;
|
||||
const parent = path.dirname(dir);
|
||||
if (parent === dir) return undefined;
|
||||
dir = parent;
|
||||
}
|
||||
}
|
||||
|
||||
function npmCheck(script: string, cwd: string, runner: NonNullable<VerifyTargetOptions["runner"]>): VerificationCheck {
|
||||
const pkg = JSON.parse(fs.readFileSync(path.join(cwd, "package.json"), "utf-8"));
|
||||
if (!pkg.scripts?.[script]) return { name: script, status: "skipped", detail: `missing npm script ${script}` };
|
||||
const r = runner("npm", ["run", "-s", script], cwd);
|
||||
return commandCheck(script, `npm run -s ${script}`, 0, r.status, (r.stderr || r.stdout).slice(0, 1000));
|
||||
}
|
||||
|
||||
function runGitStatus(runner: NonNullable<VerifyTargetOptions["runner"]>): { status: number | null; stderr: string; files: string[] } {
|
||||
const r = runner("git", ["status", "--short"]);
|
||||
return { status: r.status, stderr: r.stderr, files: r.stdout.split(/\r?\n/).filter(Boolean) };
|
||||
}
|
||||
|
||||
function commandCheck(name: string, command: string, want: number, got: number | null, detail?: string): VerificationCheck {
|
||||
return { name, command, status: got === want ? "passed" : "failed", detail };
|
||||
}
|
||||
|
||||
function runCommand(command: string, args: string[], cwd?: string) {
|
||||
const result = spawnSync(command, args, { encoding: "utf-8", cwd, shell: process.platform === "win32" });
|
||||
return { status: result.status, stdout: result.stdout ?? "", stderr: result.stderr ?? result.error?.message ?? "" };
|
||||
}
|
||||
66
src/index.ts
66
src/index.ts
|
|
@ -21,6 +21,40 @@ program
|
|||
.description("Harness-agnostic, model-agnostic self-improving agent system — loops, dynamic workflows, routines")
|
||||
.version(packageJson.version);
|
||||
|
||||
// ── Verify / Attest ─────────────────────────────────────────
|
||||
|
||||
program
|
||||
.command("verify <target>")
|
||||
.description("Verify a target and emit an agentic attestation")
|
||||
.option("--out <path>", "Attestation output path")
|
||||
.option("--run <id>", "Emit attestation event to .runs/<id>/channel.jsonl")
|
||||
.action(async (target: string, opts: { out?: string; run?: string }) => {
|
||||
const { verifyTarget } = await import("./fable5/attestation.js");
|
||||
const stamp = new Date().toISOString().replace(/[:.]/g, "-");
|
||||
const out = opts.out ?? path.join(".fable", "attestations", stamp + ".json");
|
||||
const receipt = verifyTarget({ target, out });
|
||||
console.log(JSON.stringify(receipt, null, 2));
|
||||
console.log("\n Attestation: " + out + "\n");
|
||||
await emitRunEvent(opts.run, { source: "gate", type: receipt.verification_status === "failed" ? "error" : "receipt", data: { kind: "agentic-attestation", receipt, path: out } });
|
||||
if (receipt.verification_status === "failed") process.exit(1);
|
||||
});
|
||||
|
||||
program
|
||||
.command("attest <target>")
|
||||
.description("Alias for verify: emit an agentic attestation")
|
||||
.option("--out <path>", "Attestation output path")
|
||||
.option("--run <id>", "Emit attestation event to .runs/<id>/channel.jsonl")
|
||||
.action(async (target: string, opts: { out?: string; run?: string }) => {
|
||||
const { verifyTarget } = await import("./fable5/attestation.js");
|
||||
const stamp = new Date().toISOString().replace(/[:.]/g, "-");
|
||||
const out = opts.out ?? path.join(".fable", "attestations", stamp + ".json");
|
||||
const receipt = verifyTarget({ target, out });
|
||||
console.log(JSON.stringify(receipt, null, 2));
|
||||
console.log("\n Attestation: " + out + "\n");
|
||||
await emitRunEvent(opts.run, { source: "gate", type: receipt.verification_status === "failed" ? "error" : "receipt", data: { kind: "agentic-attestation", receipt, path: out } });
|
||||
if (receipt.verification_status === "failed") process.exit(1);
|
||||
});
|
||||
|
||||
// ── Run ─────────────────────────────────────────────────────
|
||||
|
||||
program
|
||||
|
|
@ -1500,6 +1534,38 @@ factory
|
|||
if (receipt.result !== "passed") process.exit(1);
|
||||
});
|
||||
|
||||
factory
|
||||
.command("verify <target>")
|
||||
.description("Verify a target and emit an agentic attestation")
|
||||
.option("--out <path>", "Attestation output path")
|
||||
.option("--run <id>", "Emit attestation event to .runs/<id>/channel.jsonl")
|
||||
.action(async (target: string, opts: { out?: string; run?: string }) => {
|
||||
const { verifyTarget } = await import("./fable5/attestation.js");
|
||||
const stamp = new Date().toISOString().replace(/[:.]/g, "-");
|
||||
const out = opts.out ?? path.join(".fable", "attestations", stamp + ".json");
|
||||
const receipt = verifyTarget({ target, out });
|
||||
console.log(JSON.stringify(receipt, null, 2));
|
||||
console.log("\n Attestation: " + out + "\n");
|
||||
await emitRunEvent(opts.run, { source: "gate", type: receipt.verification_status === "failed" ? "error" : "receipt", data: { kind: "agentic-attestation", receipt, path: out } });
|
||||
if (receipt.verification_status === "failed") process.exit(1);
|
||||
});
|
||||
|
||||
factory
|
||||
.command("attest <target>")
|
||||
.description("Alias for factory verify: emit an agentic attestation")
|
||||
.option("--out <path>", "Attestation output path")
|
||||
.option("--run <id>", "Emit attestation event to .runs/<id>/channel.jsonl")
|
||||
.action(async (target: string, opts: { out?: string; run?: string }) => {
|
||||
const { verifyTarget } = await import("./fable5/attestation.js");
|
||||
const stamp = new Date().toISOString().replace(/[:.]/g, "-");
|
||||
const out = opts.out ?? path.join(".fable", "attestations", stamp + ".json");
|
||||
const receipt = verifyTarget({ target, out });
|
||||
console.log(JSON.stringify(receipt, null, 2));
|
||||
console.log("\n Attestation: " + out + "\n");
|
||||
await emitRunEvent(opts.run, { source: "gate", type: receipt.verification_status === "failed" ? "error" : "receipt", data: { kind: "agentic-attestation", receipt, path: out } });
|
||||
if (receipt.verification_status === "failed") process.exit(1);
|
||||
});
|
||||
|
||||
factory
|
||||
.command("deploy <command>")
|
||||
.description("Post an approved command to the factory deploy webhook")
|
||||
|
|
|
|||
Loading…
Reference in New Issue