feat: harden factory receipts and agent bench
This commit is contained in:
artale
parent
b076a9525a
commit
0203b3aab1
|
|
@ -151,6 +151,11 @@ fable-agent plinius godmode "improve explanation quality"
|
|||
|
||||
- `benchmark run`
|
||||
- `-b, --benchmark <id>`
|
||||
- `benchmark agents <task>`: run the same prompt against built-in or JSON-defined harnesses
|
||||
- `--with <csv>`
|
||||
- `--timeout-ms <n>`
|
||||
- `--harnesses <path>`
|
||||
- `--out <path>`
|
||||
- `benchmark duel <task>`: record a two-implementer eval winner
|
||||
- `--a <label>`
|
||||
- `--b <label>`
|
||||
|
|
|
|||
|
|
@ -0,0 +1,52 @@
|
|||
import * as fs from "node:fs";
|
||||
import * as os from "node:os";
|
||||
import * as path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { loadBenchHarnesses, runAgentBench, writeAgentBenchReceipt, type BenchContenderResult } from "./agent-bench.js";
|
||||
|
||||
function result(id: string, stdout: string, durationMs: number): BenchContenderResult {
|
||||
return { id, mode: id === "hermes" ? "reachability_only" : "task_output", command: id, status: 0, durationMs, stdout, stderr: "" };
|
||||
}
|
||||
|
||||
describe("agent bench", () => {
|
||||
it("runs requested contenders and records the winner", async () => {
|
||||
const seen: string[] = [];
|
||||
const receipt = await runAgentBench({
|
||||
task: "summarize",
|
||||
contenders: ["pi", "hermes", "opencode"],
|
||||
now: new Date("2026-06-23T00:00:00.000Z"),
|
||||
runner: async (cmd) => {
|
||||
seen.push(cmd.id);
|
||||
if (cmd.id === "opencode") return result(cmd.id, "best detailed answer", 10);
|
||||
return result(cmd.id, "ok", 20);
|
||||
},
|
||||
});
|
||||
|
||||
expect(seen).toEqual(["pi", "hermes", "opencode"]);
|
||||
expect(receipt.schema).toBe("fable.benchmark.agent.v1");
|
||||
expect(receipt.winner).toBe("opencode");
|
||||
});
|
||||
|
||||
it("runs custom harness configs", async () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "agent-bench-config-"));
|
||||
const config = path.join(dir, "harnesses.json");
|
||||
fs.writeFileSync(config, JSON.stringify({ harnesses: [{ id: "custom", bin: "echo", args: ["model={model}", "task={task}"], model: "m1" }] }));
|
||||
const harnesses = loadBenchHarnesses(config);
|
||||
const receipt = await runAgentBench({
|
||||
task: "hello",
|
||||
contenders: ["custom"],
|
||||
harnesses,
|
||||
runner: async (cmd) => result(cmd.id, cmd.args.join(" "), 1),
|
||||
});
|
||||
|
||||
expect(receipt.winner).toBe("custom");
|
||||
expect(receipt.contenders[0].stdout).toContain("model=m1 task=hello");
|
||||
});
|
||||
|
||||
it("writes a receipt", () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "agent-bench-"));
|
||||
const file = path.join(dir, "bench.json");
|
||||
writeAgentBenchReceipt(file, { schema: "fable.benchmark.agent.v1", task: "x", contenders: [], winner: "none", createdAt: "now" });
|
||||
expect(fs.readFileSync(file, "utf-8")).toContain("fable.benchmark.agent.v1");
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,221 @@
|
|||
import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
|
||||
export type BenchContenderId = string;
|
||||
export type BenchMode = "task_output" | "reachability_only";
|
||||
|
||||
export interface BenchHarnessConfig {
|
||||
id: string;
|
||||
mode?: BenchMode;
|
||||
bin: string;
|
||||
args: string[];
|
||||
stdin?: string;
|
||||
model?: string;
|
||||
}
|
||||
|
||||
export interface BenchContenderResult {
|
||||
id: BenchContenderId;
|
||||
mode: BenchMode;
|
||||
model?: string;
|
||||
command: string;
|
||||
status: number | null;
|
||||
durationMs: number;
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
}
|
||||
|
||||
export interface AgentBenchReceipt {
|
||||
schema: "fable.benchmark.agent.v1";
|
||||
task: string;
|
||||
contenders: BenchContenderResult[];
|
||||
winner: BenchContenderId | "tie" | "none";
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
export interface AgentBenchOptions {
|
||||
task: string;
|
||||
contenders?: BenchContenderId[];
|
||||
harnesses?: BenchHarnessConfig[];
|
||||
timeoutMs?: number;
|
||||
now?: Date;
|
||||
runner?: (cmd: BenchCommand) => Promise<BenchContenderResult>;
|
||||
}
|
||||
|
||||
export interface BenchCommand {
|
||||
id: BenchContenderId;
|
||||
mode: BenchMode;
|
||||
model?: string;
|
||||
bin: string;
|
||||
args: string[];
|
||||
stdin?: string;
|
||||
timeoutMs: number;
|
||||
}
|
||||
|
||||
const MAX_CAPTURE = 20_000;
|
||||
const TIMEOUT_GRACE_MS = 5_000;
|
||||
const BUILTIN_IDS = ["pi", "hermes", "opencode"];
|
||||
|
||||
export async function runAgentBench(opts: AgentBenchOptions): Promise<AgentBenchReceipt> {
|
||||
const timeoutMs = opts.timeoutMs ?? 120_000;
|
||||
const contenders = opts.contenders ?? BUILTIN_IDS;
|
||||
const harnesses = new Map((opts.harnesses ?? []).map((h) => [h.id, h]));
|
||||
const runner = opts.runner ?? runBenchCommand;
|
||||
const results = await Promise.all(contenders.map((id) => runner(commandFor(id, opts.task, timeoutMs, harnesses.get(id)))));
|
||||
return {
|
||||
schema: "fable.benchmark.agent.v1",
|
||||
task: opts.task,
|
||||
contenders: results,
|
||||
winner: pickWinner(results),
|
||||
createdAt: (opts.now ?? new Date()).toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
export function loadBenchHarnesses(file: string): BenchHarnessConfig[] {
|
||||
const json = JSON.parse(fs.readFileSync(file, "utf-8")) as unknown;
|
||||
const raw = Array.isArray(json) ? json : asRecord(json).harnesses;
|
||||
if (!Array.isArray(raw)) throw new Error("bench harness config must be an array or { harnesses: [...] }");
|
||||
return raw.map(parseHarness);
|
||||
}
|
||||
|
||||
export function writeAgentBenchReceipt(file: string, receipt: AgentBenchReceipt): string {
|
||||
fs.mkdirSync(path.dirname(file), { recursive: true });
|
||||
fs.writeFileSync(file, `${JSON.stringify(receipt, null, 2)}\n`);
|
||||
return file;
|
||||
}
|
||||
|
||||
function commandFor(id: BenchContenderId, task: string, timeoutMs: number, harness?: BenchHarnessConfig): BenchCommand {
|
||||
if (harness) return commandFromHarness(harness, task, timeoutMs);
|
||||
if (id === "pi") {
|
||||
return { id, mode: "task_output", bin: process.platform === "win32" ? "pi.cmd" : "pi", args: ["--offline", "--no-tools", "--no-extensions", "--no-skills", "--no-prompt-templates", "--no-themes", "--no-context-files", "--print", "--no-session", task], timeoutMs };
|
||||
}
|
||||
if (id === "opencode") {
|
||||
return { id, mode: "task_output", bin: process.env.OPENCODE_BIN ?? (process.platform === "win32" ? "opencode.cmd" : "opencode"), args: ["run", "--pure", task], timeoutMs };
|
||||
}
|
||||
if (id === "hermes") {
|
||||
const model = process.env.HERMES_MODEL ?? "stepfun/step-3.7-flash:free";
|
||||
const baseUrl = process.env.HERMES_BASE_URL ?? "https://openrouter.ai/api/v1";
|
||||
const turns = process.env.HERMES_MAX_TURNS ?? "1";
|
||||
return {
|
||||
id,
|
||||
mode: "task_output",
|
||||
model,
|
||||
bin: "ssh",
|
||||
args: [
|
||||
"-p", process.env.HERMES_SSH_PORT ?? "2222",
|
||||
`root@${process.env.HERMES_HOST ?? "77.42.112.29"}`,
|
||||
`docker exec hermes /opt/hermes/.venv/bin/python /opt/hermes/run_agent.py -q ${shellQuote(task)} --model ${shellQuote(model)} --base_url ${shellQuote(baseUrl)} --max_turns ${shellQuote(turns)}`,
|
||||
],
|
||||
timeoutMs,
|
||||
};
|
||||
}
|
||||
throw new Error(`unknown benchmark harness: ${id}`);
|
||||
}
|
||||
|
||||
function commandFromHarness(harness: BenchHarnessConfig, task: string, timeoutMs: number): BenchCommand {
|
||||
return {
|
||||
id: harness.id,
|
||||
mode: harness.mode ?? "task_output",
|
||||
model: harness.model,
|
||||
bin: expandTemplate(harness.bin, task, harness.model),
|
||||
args: harness.args.map((arg) => expandTemplate(arg, task, harness.model)),
|
||||
stdin: harness.stdin === undefined ? undefined : expandTemplate(harness.stdin, task, harness.model),
|
||||
timeoutMs,
|
||||
};
|
||||
}
|
||||
|
||||
function expandTemplate(value: string, task: string, model?: string): string {
|
||||
return value.replaceAll("{task}", task).replaceAll("{model}", model ?? "");
|
||||
}
|
||||
|
||||
function parseHarness(value: unknown): BenchHarnessConfig {
|
||||
const rec = asRecord(value);
|
||||
if (typeof rec.id !== "string" || !rec.id) throw new Error("bench harness missing id");
|
||||
if (typeof rec.bin !== "string" || !rec.bin) throw new Error(`bench harness ${rec.id} missing bin`);
|
||||
if (!Array.isArray(rec.args) || rec.args.some((arg) => typeof arg !== "string")) throw new Error(`bench harness ${rec.id} args must be strings`);
|
||||
if (rec.mode !== undefined && rec.mode !== "task_output" && rec.mode !== "reachability_only") throw new Error(`bench harness ${rec.id} has invalid mode`);
|
||||
if (rec.stdin !== undefined && typeof rec.stdin !== "string") throw new Error(`bench harness ${rec.id} stdin must be a string`);
|
||||
if (rec.model !== undefined && typeof rec.model !== "string") throw new Error(`bench harness ${rec.id} model must be a string`);
|
||||
return { id: rec.id, mode: rec.mode, bin: rec.bin, args: rec.args, stdin: rec.stdin, model: rec.model } as BenchHarnessConfig;
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): Record<string, any> {
|
||||
return value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, any> : {};
|
||||
}
|
||||
|
||||
function shellQuote(value: string): string {
|
||||
return `'${value.replace(/'/g, `'\\''`)}'`;
|
||||
}
|
||||
|
||||
function pickWinner(results: BenchContenderResult[]): AgentBenchReceipt["winner"] {
|
||||
const ok = results.filter((r) => r.mode === "task_output" && r.status === 0 && outputText(r).trim());
|
||||
if (ok.length === 0) return "none";
|
||||
ok.sort((a, b) => score(b) - score(a) || a.durationMs - b.durationMs);
|
||||
if (ok.length === 1) return ok[0].id;
|
||||
return Math.abs(score(ok[0]) - score(ok[1])) < 0.001 ? "tie" : ok[0].id;
|
||||
}
|
||||
|
||||
function score(result: BenchContenderResult): number {
|
||||
const text = outputText(result).trim();
|
||||
return (result.status === 0 ? 1_000 : 0) + Math.min(text.length, 4_000) - result.durationMs / 1000;
|
||||
}
|
||||
|
||||
function outputText(result: BenchContenderResult): string {
|
||||
return result.stdout.trim() || result.stderr.replace(/\[timeout after \d+ms\]/g, "").trim();
|
||||
}
|
||||
|
||||
function runBenchCommand(cmd: BenchCommand): Promise<BenchContenderResult> {
|
||||
const started = Date.now();
|
||||
const command = [cmd.bin, ...cmd.args].join(" ");
|
||||
return new Promise((resolve) => {
|
||||
let done = false;
|
||||
let timedOut = false;
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
const finish = (status: number | null, out = stdout, err = stderr) => {
|
||||
if (done) return;
|
||||
done = true;
|
||||
clearTimeout(timeoutTimer);
|
||||
clearTimeout(graceTimer);
|
||||
const timeoutNote = timedOut ? `\n[timeout after ${cmd.timeoutMs}ms]` : "";
|
||||
const effectiveStatus = timedOut && cmd.mode === "task_output" && out.trim() ? 0 : status;
|
||||
resolve({ id: cmd.id, mode: cmd.mode, model: cmd.model, command, status: effectiveStatus, durationMs: Date.now() - started, stdout: out, stderr: `${err}${timeoutNote}`.trim() });
|
||||
};
|
||||
const isCmdShim = process.platform === "win32" && /(?:\.cmd|\.bat)$/i.test(cmd.bin);
|
||||
const bin = isCmdShim ? process.env.ComSpec ?? "cmd.exe" : cmd.bin;
|
||||
const args = isCmdShim ? ["/d", "/s", "/c", cmd.bin, ...cmd.args] : cmd.args;
|
||||
const child = spawn(bin, args, { shell: false, windowsHide: true });
|
||||
const cap = (value: string, chunk: Buffer) => (value + chunk.toString()).slice(0, MAX_CAPTURE);
|
||||
const timeoutTimer = setTimeout(() => {
|
||||
timedOut = true;
|
||||
child.kill();
|
||||
// ponytail: give chatty CLIs a moment to flush final output before killing their child tree.
|
||||
graceTimer = setTimeout(() => {
|
||||
killProcessTree(child.pid);
|
||||
child.stdout.destroy();
|
||||
child.stderr.destroy();
|
||||
child.stdin.destroy();
|
||||
finish(null);
|
||||
}, TIMEOUT_GRACE_MS);
|
||||
}, cmd.timeoutMs);
|
||||
let graceTimer: NodeJS.Timeout;
|
||||
child.stdout.on("data", (d) => { stdout = cap(stdout, d); });
|
||||
child.stderr.on("data", (d) => { stderr = cap(stderr, d); });
|
||||
child.on("error", (err) => finish(null, stdout, err.message));
|
||||
child.on("close", (status) => {
|
||||
// ponytail: timed-out agent CLIs often flush useful output after SIGTERM; let graceTimer collect it.
|
||||
if (timedOut) return;
|
||||
finish(status, stdout, stderr);
|
||||
});
|
||||
child.stdin.end(cmd.stdin ?? "");
|
||||
});
|
||||
}
|
||||
|
||||
function killProcessTree(pid: number | undefined): void {
|
||||
if (!pid) return;
|
||||
if (process.platform === "win32") {
|
||||
spawnSync("taskkill", ["/pid", String(pid), "/t", "/f"], { stdio: "ignore", windowsHide: true });
|
||||
return;
|
||||
}
|
||||
try { process.kill(-pid, "SIGTERM"); } catch { try { process.kill(pid, "SIGTERM"); } catch {} }
|
||||
}
|
||||
|
|
@ -46,7 +46,7 @@ export function runCyberPreflight(opts: CyberPreflightOptions): CyberPreflightRe
|
|||
|
||||
for (const [name, url] of REQUIRED_ENDPOINTS) {
|
||||
const probe = name === "deploy_route_health"
|
||||
? `test "$(curl -sS -m 5 -o /dev/null -w '%{http_code}' -X POST ${url})" = "401"`
|
||||
? `code="$(curl -sS -m 5 -o /dev/null -w '%{http_code}' -X POST ${url})" && (test "$code" = "401" || test "$code" = "403")`
|
||||
: `curl -fsS -m 5 ${url}`;
|
||||
checks.push(runCheck(name, "ssh", ["-p", String(port), `root@${opts.host}`, probe], runner));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import { loadFactoryCapabilities, type FactoryCapabilityService } from "./factory-capabilities.js";
|
||||
import { isGuardedDeployUrl } from "./factory-routes.js";
|
||||
import type { CyberPreflightReceipt } from "./cyber-preflight.js";
|
||||
|
||||
export interface CyberReconcileFinding {
|
||||
|
|
@ -45,11 +46,12 @@ export function writeCyberReconcileReceipt(file: string, receipt: CyberReconcile
|
|||
function findingsForService(service: FactoryCapabilityService, preflight: CyberPreflightReceipt): CyberReconcileFinding[] {
|
||||
return service.provides.map((capability) => {
|
||||
const check = matchingCheck(capability, preflight);
|
||||
const deployRouteOk = capability !== "deploy" || isGuardedDeployUrl(service.url);
|
||||
return {
|
||||
capability,
|
||||
expected: service.url,
|
||||
evidence: check ? `${check.name}: ${check.ok ? "ok" : `failed status=${check.status} ${check.stderr.trim()}`}` : "no matching preflight check",
|
||||
status: check?.ok ? "ok" : "missing",
|
||||
evidence: !deployRouteOk ? "deploy must use guarded 8099 route" : check ? `${check.name}: ${check.ok ? "ok" : `failed status=${check.status} ${check.stderr.trim()}`}` : "no matching preflight check",
|
||||
status: check?.ok && deployRouteOk ? "ok" : "missing",
|
||||
};
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,6 +19,20 @@ describe("factory capabilities", () => {
|
|||
expect(services[0]).toMatchObject({ name: "git-proxy-deploy", required: true, provides: ["deploy"] });
|
||||
});
|
||||
|
||||
it("accepts deploy endpoints with unauthorized response", async () => {
|
||||
const services = parseFactoryCapabilities(yaml);
|
||||
const report = await probeFactoryCapabilities(
|
||||
services,
|
||||
async (url) => ({ ok: url.includes("factory-status") || /\/deploy$/.test(url), status: /\/deploy$/.test(url) ? 403 : 200 }),
|
||||
);
|
||||
|
||||
expect(report.requiredOk).toBe(true);
|
||||
expect(report.available).toContain("deploy");
|
||||
expect(report.available).toContain("factory-status");
|
||||
expect(report.missing).not.toContain("deploy");
|
||||
expect(report.results.find((r) => r.name === "git-proxy-deploy")?.status).toBe(403);
|
||||
});
|
||||
|
||||
it("fails required capabilities that return 404", async () => {
|
||||
const services = parseFactoryCapabilities(yaml);
|
||||
const report = await probeFactoryCapabilities(services, async (url) => ({ ok: url.includes("/deploy"), status: url.includes("/deploy") ? 401 : 404 }));
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import * as fs from "node:fs";
|
||||
import { isGuardedDeployUrl } from "./factory-routes.js";
|
||||
|
||||
export interface FactoryCapabilityService {
|
||||
name: string;
|
||||
|
|
@ -82,13 +83,16 @@ export function formatCapabilityReport(report: CapabilityReport): string {
|
|||
|
||||
async function probeCapability(svc: FactoryCapabilityService, fetcher: FetchLike, timeoutMs: number): Promise<CapabilityProbeResult> {
|
||||
if (!svc.url) return { ...svc, ok: false, error: "missing url" };
|
||||
if (svc.provides.includes("deploy") && !isGuardedDeployUrl(svc.url)) {
|
||||
return { ...svc, ok: false, error: "deploy capability must use guarded 8099 route" };
|
||||
}
|
||||
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
||||
try {
|
||||
const res = await fetcher(svc.url, { method: svc.provides.includes("deploy") ? "POST" : "GET", signal: controller.signal });
|
||||
// ponytail: unauthenticated 401 proves the deploy route exists without requiring the deploy token.
|
||||
const ok = res.ok || (svc.provides.includes("deploy") && res.status === 401);
|
||||
// ponytail: unauthenticated 401/403 proves the deploy route exists without requiring the deploy token.
|
||||
const ok = res.ok || (svc.provides.includes("deploy") && (res.status === 401 || res.status === 403));
|
||||
return { ...svc, ok, status: res.status, error: ok ? undefined : `HTTP ${res.status}` };
|
||||
} catch (error) {
|
||||
const message = error instanceof Error && error.name === "AbortError" ? "timed out" : error instanceof Error ? error.message : String(error);
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ function passedGateReceipt(): string {
|
|||
status: "passed",
|
||||
task: "gate",
|
||||
repo: ".",
|
||||
commands: ["fable-agent fable5 verify repo gate --repo ."],
|
||||
commands: ["fable-agent factory gate gate --repo .", "fable-agent fable5 verify repo gate --repo ."],
|
||||
createdAt: "2026-06-23T00:00:00.000Z",
|
||||
})}\n`);
|
||||
return file;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import { CANONICAL_DEPLOY_ROUTE } from "./factory-routes.js";
|
||||
import { validateGateReceipt } from "./zte-protocol.js";
|
||||
|
||||
export interface FactoryDeployOptions {
|
||||
command: string;
|
||||
|
|
@ -24,7 +26,7 @@ export interface FactoryDeployReceipt {
|
|||
reason?: string;
|
||||
}
|
||||
|
||||
const DEFAULT_URL = "http://77.42.112.29:8099/deploy";
|
||||
const DEFAULT_URL = CANONICAL_DEPLOY_ROUTE;
|
||||
const SAFE_COMMANDS = [
|
||||
/^echo [A-Za-z0-9 _.,:;@/+\-=]+$/,
|
||||
/^cat \/tmp\/[A-Za-z0-9._/-]+$/,
|
||||
|
|
@ -67,24 +69,7 @@ export function isAllowedDeployUrl(url: string): boolean {
|
|||
}
|
||||
|
||||
export function hasPassedGateReceipt(file: string): boolean {
|
||||
if (!fs.existsSync(file)) return false;
|
||||
return fs.readFileSync(file, "utf-8")
|
||||
.split(/\r?\n/)
|
||||
.filter(Boolean)
|
||||
.some((line) => {
|
||||
try {
|
||||
const json = JSON.parse(line) as { status?: unknown; task?: unknown; repo?: unknown; commands?: unknown; createdAt?: unknown };
|
||||
return json.status === "passed"
|
||||
&& typeof json.task === "string"
|
||||
&& typeof json.repo === "string"
|
||||
&& Array.isArray(json.commands)
|
||||
&& json.commands.some((cmd) => typeof cmd === "string" && cmd.includes("fable5 verify"))
|
||||
&& typeof json.createdAt === "string"
|
||||
&& Number.isFinite(Date.parse(json.createdAt));
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
});
|
||||
return validateGateReceipt(file, { requiredCommands: ["factory gate", "fable5 verify"] }).ok;
|
||||
}
|
||||
|
||||
export function writeFactoryDeployReceipt(file: string, value: FactoryDeployReceipt): FactoryDeployReceipt {
|
||||
|
|
@ -96,10 +81,10 @@ export function writeFactoryDeployReceipt(file: string, value: FactoryDeployRece
|
|||
function blockReason(opts: FactoryDeployOptions, url: string): string | undefined {
|
||||
if (!opts.yes) return "missing --yes human approval";
|
||||
if (!isAllowedDeployUrl(url)) return "deploy url must be the guarded 8099 git-proxy route";
|
||||
if (!isAllowedFactoryCommand(opts.command)) return "command is outside the guarded deploy allowlist";
|
||||
if (!opts.gateReceipt) return "missing --gate-receipt verifier evidence";
|
||||
if (!hasPassedGateReceipt(opts.gateReceipt)) return "gate receipt missing passed verifier status";
|
||||
if (!opts.token) return "missing deploy token";
|
||||
if (!isAllowedFactoryCommand(opts.command)) return "command is outside the guarded deploy allowlist";
|
||||
return undefined;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,34 @@
|
|||
export const CANONICAL_DEPLOY_HOST = "77.42.112.29";
|
||||
export const CANONICAL_DEPLOY_PORT = "8099";
|
||||
export const LEGACY_DEPLOY_PORT = "8098";
|
||||
|
||||
export const CANONICAL_DEPLOY_ROUTE = `http://${CANONICAL_DEPLOY_HOST}:${CANONICAL_DEPLOY_PORT}/deploy`;
|
||||
export const LOCAL_DEPLOY_ROUTE = `http://127.0.0.1:${CANONICAL_DEPLOY_PORT}/deploy`;
|
||||
export const CANONICAL_FACTORY_STATUS_ROUTE = `http://${CANONICAL_DEPLOY_HOST}:${CANONICAL_DEPLOY_PORT}/factory-status`;
|
||||
export const LOCAL_FACTORY_STATUS_ROUTE = `http://127.0.0.1:${CANONICAL_DEPLOY_PORT}/factory-status`;
|
||||
export const FACTORY_STATUS_PATH = "/factory-status";
|
||||
export const DEPLOY_PATH = "/deploy";
|
||||
export const DEPLOY_WEBHOOK_LEGACY_PATH = "/deploy-webhook";
|
||||
export const FACTORY_RECEIPT_SCHEMA = "fable.zte.receipt.v1";
|
||||
|
||||
const LEGACY_ROUTE_PATTERNS = [
|
||||
new RegExp(`https?:\\/\\/[^/]+:${LEGACY_DEPLOY_PORT}/`, "i"),
|
||||
/https?:\/\/[^/]+\/deploy-webhook/i,
|
||||
];
|
||||
|
||||
export function isAllowedDeployUrl(url: string): boolean {
|
||||
return url === CANONICAL_DEPLOY_ROUTE;
|
||||
}
|
||||
|
||||
export function isGuardedDeployUrl(url: string): boolean {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
return parsed.protocol === "http:" && parsed.port === CANONICAL_DEPLOY_PORT && parsed.pathname === DEPLOY_PATH && !isForbiddenDeployRoute(url);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function isForbiddenDeployRoute(url: string): boolean {
|
||||
return LEGACY_ROUTE_PATTERNS.some((pat) => pat.test(url));
|
||||
}
|
||||
|
|
@ -1,3 +1,5 @@
|
|||
import { CANONICAL_DEPLOY_ROUTE, CANONICAL_FACTORY_STATUS_ROUTE } from "./factory-routes.js";
|
||||
|
||||
export interface FactoryStatus {
|
||||
status?: string;
|
||||
phase?: number;
|
||||
|
|
@ -22,8 +24,8 @@ export interface FactoryCheckResult {
|
|||
errors: string[];
|
||||
}
|
||||
|
||||
const FACTORY_STATUS_URL = "http://77.42.112.29:8099/factory-status";
|
||||
const DEPLOY_HEALTH_URL = "http://77.42.112.29:8099/deploy";
|
||||
const FACTORY_STATUS_URL = CANONICAL_FACTORY_STATUS_ROUTE;
|
||||
const DEPLOY_HEALTH_URL = CANONICAL_DEPLOY_ROUTE;
|
||||
|
||||
type FetchLike = (url: string, init?: { method?: string; signal?: AbortSignal }) => Promise<{ ok: boolean; status: number; json(): Promise<unknown> }>;
|
||||
|
||||
|
|
@ -74,7 +76,7 @@ async function checkDeployRoute(fetcher: FetchLike, timeoutMs: number): Promise<
|
|||
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
||||
try {
|
||||
const res = await fetcher(DEPLOY_HEALTH_URL, { method: "POST", signal: controller.signal });
|
||||
if (res.status !== 401 && !res.ok) throw new Error(`${DEPLOY_HEALTH_URL} returned ${res.status}`);
|
||||
if (res.status !== 401 && res.status !== 403) throw new Error(`${DEPLOY_HEALTH_URL} returned ${res.status}`);
|
||||
return { status: "ok", service: "git-proxy-deploy" };
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.name === "AbortError") throw new Error(`${DEPLOY_HEALTH_URL} timed out`);
|
||||
|
|
|
|||
|
|
@ -102,13 +102,23 @@ export function createForgejoIntakeReceipt(item: ForgejoItem, now = new Date()):
|
|||
export async function intakeForgejoItem(opts: ForgejoIntakeOptions): Promise<{ receipt: ForgejoIntakeReceipt; path?: string }> {
|
||||
const parsed = parseForgejoRef(opts.ref, opts.repo);
|
||||
if (opts.runId) emitChannelEvent(opts.runId, { source: "intake", type: "started", data: { ref: opts.ref, repo: parsed.repo, kind: parsed.kind } }, opts.channelRoot, opts.now);
|
||||
const baseUrl = opts.baseUrl?.replace(/\/$/, "") ?? parsed.baseUrl;
|
||||
|
||||
const configuredBase = opts.baseUrl?.replace(/\/$/, "")?.replace(/\/$/, "");
|
||||
const baseUrl = configuredBase ?? parsed.baseUrl;
|
||||
if (!baseUrl) throw new Error("FORGEJO_URL is required unless ISSUE is a full Forgejo URL");
|
||||
const item = await fetchForgejoItem(baseUrl, parsed, opts.token, opts.fetcher ?? fetch);
|
||||
|
||||
if (parsed.baseUrl && configuredBase && normalizedOrigin(parsed.baseUrl) !== normalizedOrigin(configuredBase)) {
|
||||
throw new Error("Forgejo reference host does not match configured FORGEJO_URL");
|
||||
}
|
||||
|
||||
// ponytail: only send FORGEJO_TOKEN to configured base URL; ref-selected origins without a configured base get no token.
|
||||
const token = configuredBase ? opts.token : undefined;
|
||||
const item = await fetchForgejoItem(baseUrl, parsed, token, opts.fetcher ?? fetch);
|
||||
const receipt = createForgejoIntakeReceipt(item, opts.now);
|
||||
|
||||
if (opts.runId) emitChannelEvent(opts.runId, { source: "intake", type: "receipt", data: receipt }, opts.channelRoot, opts.now);
|
||||
if (opts.dryRun) return { receipt };
|
||||
|
||||
const outDir = opts.outDir ?? ".fable/tasks";
|
||||
fs.mkdirSync(outDir, { recursive: true });
|
||||
const file = path.join(outDir, `forgejo-${parsed.kind}-${safeFilePart(parsed.id)}.json`);
|
||||
|
|
@ -160,5 +170,11 @@ function parseUrl(value: string): URL | undefined {
|
|||
}
|
||||
|
||||
function safeFilePart(value: string): string {
|
||||
return value.replace(/[^A-Za-z0-9_.-]+/g, "-").replace(/^-|-$/g, "") || "item";
|
||||
return value
|
||||
.replace(/[^A-Za-z0-9_.-]+/g, "-")
|
||||
.replace(/(^-+)|(-+$)/g, "") || "item";
|
||||
}
|
||||
|
||||
function normalizedOrigin(value: string): string {
|
||||
return new URL(value).origin.replace(/\/$/, "").toLowerCase();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -41,8 +41,8 @@ export type { TeamRole, TeamAgent, TeamConfig, TeamResult } from "./agent-teams.
|
|||
export { isActionDenied, mergeToolPolicies } from "./orchestrator-policy.js";
|
||||
export type { ToolPolicy, EffectiveToolPolicy } from "./orchestrator-policy.js";
|
||||
|
||||
export { appendZteReceipt, createZteReceipt, createZteSpec } from "./zte-protocol.js";
|
||||
export type { ZteReceipt, ZteSpec, ZteSpecOptions } from "./zte-protocol.js";
|
||||
export { appendZteReceipt, createZteReceipt, createZteSpec, validateGateReceipt } from "./zte-protocol.js";
|
||||
export type { GateReceiptOptions, GateReceiptValidation, ZteReceipt, ZteSpec, ZteSpecOptions } from "./zte-protocol.js";
|
||||
|
||||
export { formatCapabilityReport, loadFactoryCapabilities, parseFactoryCapabilities, probeFactoryCapabilities } from "./factory-capabilities.js";
|
||||
export type { CapabilityProbeResult, CapabilityReport, FactoryCapabilityService } from "./factory-capabilities.js";
|
||||
|
|
@ -71,6 +71,9 @@ export type { ReceiptHealth } from "./receipt-consumer.js";
|
|||
export { createDuelEvalReceipt, createEvalTraceReceipt, gradeSetRecovery, tallyDuelReceipts, writeDuelEvalReceipt, writeEvalTraceReceipt } from "./eval-trace.js";
|
||||
export type { DuelEvalReceipt, DuelTally, EvalTraceCommand, EvalTraceReceipt, SetGradeReceipt } from "./eval-trace.js";
|
||||
|
||||
export { loadBenchHarnesses, runAgentBench, writeAgentBenchReceipt } from "./agent-bench.js";
|
||||
export type { AgentBenchOptions, AgentBenchReceipt, BenchContenderId, BenchContenderResult, BenchHarnessConfig } from "./agent-bench.js";
|
||||
|
||||
export { auditGoalCompletion, writeGoalAuditReceipt } from "./goal-audit.js";
|
||||
export type { GoalAuditCriterion, GoalAuditReceipt } from "./goal-audit.js";
|
||||
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ export function reconcileRsi(opts: RsiReconcileOptions): RsiReconcileReceipt {
|
|||
check("skill_health", port, opts.host, "test -f /tmp/skill_health.py && echo present", runner),
|
||||
check("latest_rsi", port, opts.host, "tail -160 /tmp/rsi-diagnosis.log 2>/dev/null | grep -E 'RSI:|All healthy|auto-patching|Rolled back|All patched'", runner),
|
||||
check("factory_watcher", port, opts.host, "systemctl is-active factory-watcher.service 2>/dev/null", runner),
|
||||
check("deploy_route", port, opts.host, "test \"$(curl -sS -m 5 -o /dev/null -w '%{http_code}' -X POST http://127.0.0.1:8099/deploy)\" = \"401\" && echo present", runner),
|
||||
check("deploy_route", port, opts.host, "code=\"$(curl -sS -m 5 -o /dev/null -w '%{http_code}' -X POST http://127.0.0.1:8099/deploy)\" && (test \"$code\" = \"401\" || test \"$code\" = \"403\") && echo present", runner),
|
||||
];
|
||||
const latest = checks.find((c) => c.name === "latest_rsi")?.stdout ?? "";
|
||||
const facts = {
|
||||
|
|
@ -43,7 +43,7 @@ export function reconcileRsi(opts: RsiReconcileOptions): RsiReconcileReceipt {
|
|||
latest_score: latest.match(/RSI:\s*([^\n]+)/)?.[1]?.trim(),
|
||||
factory_watcher_active: /active/.test(checks.find((c) => c.name === "factory_watcher")?.stdout ?? ""),
|
||||
deploy_route_present: isOk("deploy_route", checks),
|
||||
auto_patch_proven: /All patched|auto-patching/.test(latest),
|
||||
auto_patch_proven: /auto-patching/.test(latest) && /All patched/.test(latest),
|
||||
};
|
||||
// ponytail: cron name drift is a warning; live health + watcher + deploy route are the gate.
|
||||
const coreHealthy = facts.skill_health_present && /100%|65\/65/.test(facts.latest_score ?? "") && facts.factory_watcher_active && facts.deploy_route_present;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,104 @@
|
|||
import * as fs from "node:fs";
|
||||
import * as os from "node:os";
|
||||
import * as path from "node:path";
|
||||
import { execSync } from "node:child_process";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { WorktreeManager } from "./worktree-isolation.js";
|
||||
|
||||
const dirs: string[] = [];
|
||||
|
||||
afterEach(() => {
|
||||
for (const dir of dirs.splice(0)) {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
function mkRepo(): { root: string; worktrees: string } {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "fable-worktree-"));
|
||||
const worktrees = fs.mkdtempSync(path.join(os.tmpdir(), "fable-worktrees-"));
|
||||
dirs.push(root, worktrees);
|
||||
|
||||
execSync("git init", { cwd: root, stdio: "ignore" });
|
||||
execSync("git config user.email test@example.com", { cwd: root, stdio: "ignore" });
|
||||
execSync("git config user.name test", { cwd: root, stdio: "ignore" });
|
||||
fs.writeFileSync(path.join(root, "README.md"), "seed\n");
|
||||
execSync("git add README.md", { cwd: root, stdio: "ignore" });
|
||||
execSync("git commit -m seed", { cwd: root, stdio: "ignore" });
|
||||
|
||||
return { root, worktrees };
|
||||
}
|
||||
|
||||
function uniqueName(prefix: string): string {
|
||||
return `${prefix}-${Date.now()}-${Math.floor(Math.random() * 1_000_000)}`;
|
||||
}
|
||||
|
||||
function worktreePathFor(worktrees: string, name: string): string {
|
||||
return path.join(worktrees, name);
|
||||
}
|
||||
|
||||
describe("worktree isolation", () => {
|
||||
it("creates and reads a managed worktree", () => {
|
||||
const { root: repo, worktrees } = mkRepo();
|
||||
const manager = new WorktreeManager(repo, worktrees);
|
||||
|
||||
const base = manager.createForAgent("Agent/Test-One");
|
||||
const spec = { ...base, name: uniqueName(base.name) };
|
||||
const wt = worktreePathFor(worktrees, spec.name);
|
||||
if (fs.existsSync(wt)) fs.rmSync(wt, { recursive: true, force: true });
|
||||
|
||||
expect(spec.name).not.toBe(base.name);
|
||||
expect(spec.branch).toMatch(/agent-agent-test-one-\d+/);
|
||||
|
||||
const created = manager.create(spec);
|
||||
expect(created).toBe(wt);
|
||||
expect(fs.existsSync(wt)).toBe(true);
|
||||
expect(manager.readFileFromWorktree(wt, "README.md")?.replace(/\r/g, "")).toBe("seed\n");
|
||||
expect(manager.readFileFromWorktree(wt, "does-not-exist.txt")).toBe(null);
|
||||
expect(manager.hasChanges(wt)).toBe(false);
|
||||
|
||||
fs.writeFileSync(path.join(wt, "notes.txt"), "todo\n");
|
||||
expect(manager.hasChanges(wt)).toBe(true);
|
||||
|
||||
const listed = manager.list().map((item) => item.path.replace(/\\/g, "/"));
|
||||
expect(listed).toContain(wt.replace(/\\/g, "/"));
|
||||
|
||||
manager.remove(spec.name);
|
||||
expect(fs.existsSync(wt)).toBe(false);
|
||||
expect(manager.list().map((item) => item.name)).not.toContain(spec.name);
|
||||
});
|
||||
|
||||
it("creates and checks out a branch in a single step", () => {
|
||||
const { root: repo, worktrees } = mkRepo();
|
||||
const manager = new WorktreeManager(repo, worktrees);
|
||||
|
||||
const base = manager.createForAgent("agent-quick");
|
||||
const spec = { ...base, name: uniqueName(base.name) };
|
||||
const expected = worktreePathFor(worktrees, spec.name);
|
||||
if (fs.existsSync(expected)) fs.rmSync(expected, { recursive: true, force: true });
|
||||
|
||||
const checked = manager.createAndCheckout(spec);
|
||||
expect(checked).toBe(expected);
|
||||
expect(fs.existsSync(path.join(checked, "README.md"))).toBe(true);
|
||||
|
||||
manager.remove(spec.name);
|
||||
expect(fs.existsSync(expected)).toBe(false);
|
||||
});
|
||||
|
||||
it("parses worktree list safely", () => {
|
||||
const { root: repo, worktrees } = mkRepo();
|
||||
const manager = new WorktreeManager(repo, worktrees);
|
||||
|
||||
const base = manager.createForAgent("prune-tester");
|
||||
const spec = { ...base, name: uniqueName(base.name) };
|
||||
const wt = manager.create(spec);
|
||||
|
||||
const listed = manager.list().map((item) => item.path.replace(/\\/g, "/"));
|
||||
const normalizedWt = wt.replace(/\\/g, "/");
|
||||
expect(listed).toContain(normalizedWt);
|
||||
|
||||
manager.prune();
|
||||
expect(manager.list().map((item) => item.path.replace(/\\/g, "/"))).toContain(normalizedWt);
|
||||
|
||||
manager.remove(spec.name);
|
||||
});
|
||||
});
|
||||
|
|
@ -1,6 +1,14 @@
|
|||
import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import { execSync } from "node:child_process";
|
||||
import { execFileSync } from "node:child_process";
|
||||
|
||||
const CLEAN_GIT_ENV = {
|
||||
...process.env,
|
||||
GIT_DIR: undefined,
|
||||
GIT_WORK_TREE: undefined,
|
||||
GIT_INDEX_FILE: undefined,
|
||||
GIT_PREFIX: undefined,
|
||||
};
|
||||
|
||||
export interface WorktreeSpec {
|
||||
name: string;
|
||||
|
|
@ -20,9 +28,9 @@ export class WorktreeManager {
|
|||
private repoPath: string;
|
||||
private baseDir: string;
|
||||
|
||||
constructor(repoPath?: string) {
|
||||
constructor(repoPath?: string, baseDir?: string) {
|
||||
this.repoPath = repoPath ?? process.cwd();
|
||||
this.baseDir = path.join(this.repoPath, "..", ".worktrees");
|
||||
this.baseDir = baseDir ?? path.join(this.repoPath, "..", ".worktrees");
|
||||
}
|
||||
|
||||
isGitRepo(): boolean {
|
||||
|
|
@ -37,15 +45,18 @@ export class WorktreeManager {
|
|||
|
||||
create(spec: WorktreeSpec): string {
|
||||
this.initDir();
|
||||
const worktreePath = path.join(this.baseDir, spec.name);
|
||||
const safeName = this.safeName(spec.name);
|
||||
const safeBranch = this.safeBranch(spec.branch);
|
||||
const worktreePath = path.join(this.baseDir, safeName);
|
||||
|
||||
this.git(["worktree", "add", "-b", spec.branch, worktreePath]);
|
||||
this.git(["worktree", "add", "-b", safeBranch, worktreePath]);
|
||||
|
||||
return worktreePath;
|
||||
}
|
||||
|
||||
remove(name: string): void {
|
||||
const worktreePath = path.join(this.baseDir, name);
|
||||
const safeName = this.safeName(name);
|
||||
const worktreePath = path.join(this.baseDir, safeName);
|
||||
if (fs.existsSync(worktreePath)) {
|
||||
this.git(["worktree", "remove", worktreePath, "--force"]);
|
||||
}
|
||||
|
|
@ -66,10 +77,11 @@ export class WorktreeManager {
|
|||
|
||||
createForAgent(agentName: string): WorktreeSpec {
|
||||
const branch = `agent/${agentName.toLowerCase().replace(/[^a-z0-9-]/g, "-")}/${Date.now()}`;
|
||||
const safeName = this.safeName(agentName.toLowerCase().replace(/[^a-z0-9-]/g, "-"));
|
||||
const spec: WorktreeSpec = {
|
||||
name: agentName.toLowerCase().replace(/[^a-z0-9-]/g, "-"),
|
||||
branch,
|
||||
targetDir: path.join(this.baseDir, agentName.toLowerCase().replace(/[^a-z0-9-]/g, "-")),
|
||||
name: safeName,
|
||||
branch: this.safeBranch(branch),
|
||||
targetDir: path.join(this.baseDir, safeName),
|
||||
};
|
||||
return spec;
|
||||
}
|
||||
|
|
@ -97,11 +109,12 @@ export class WorktreeManager {
|
|||
|
||||
private git(args: string[]): string {
|
||||
try {
|
||||
return execSync(`git ${args.join(" ")}`, {
|
||||
return execFileSync("git", args, {
|
||||
cwd: this.repoPath,
|
||||
encoding: "utf-8",
|
||||
env: CLEAN_GIT_ENV,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
}) as string;
|
||||
} catch (err) {
|
||||
if (err instanceof Error) throw new Error(`git ${args[0]} failed: ${err.message}`);
|
||||
throw err;
|
||||
|
|
@ -110,11 +123,12 @@ export class WorktreeManager {
|
|||
|
||||
private gitInDir(dir: string, args: string[]): string {
|
||||
try {
|
||||
return execSync(`git ${args.join(" ")}`, {
|
||||
return execFileSync("git", args, {
|
||||
cwd: dir,
|
||||
encoding: "utf-8",
|
||||
env: CLEAN_GIT_ENV,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
}) as string;
|
||||
} catch (err) {
|
||||
if (err instanceof Error) throw new Error(`git in ${dir} failed: ${err.message}`);
|
||||
throw err;
|
||||
|
|
@ -123,23 +137,34 @@ export class WorktreeManager {
|
|||
|
||||
private parseWorktreeList(output: string): WorktreeStatus[] {
|
||||
const worktrees: WorktreeStatus[] = [];
|
||||
const blocks = output.split("\n\n").filter(Boolean);
|
||||
let current: Partial<WorktreeStatus> = { isDirty: false, lastCommit: "" };
|
||||
|
||||
for (const block of blocks) {
|
||||
const lines = block.split("\n");
|
||||
let w: Partial<WorktreeStatus> = { isDirty: false, lastCommit: "" };
|
||||
for (const line of lines) {
|
||||
if (line.startsWith("worktree ")) w.path = line.slice(9);
|
||||
if (line.startsWith("HEAD ")) w.lastCommit = line.slice(5);
|
||||
if (line.startsWith("branch ")) w.branch = line.slice(7).replace("refs/heads/", "");
|
||||
}
|
||||
if (w.path) {
|
||||
w.name = path.basename(w.path);
|
||||
w.isDirty = this.hasChanges(w.path);
|
||||
worktrees.push(w as WorktreeStatus);
|
||||
const pushCurrent = () => {
|
||||
if (!current.path) return;
|
||||
current.name = path.basename(current.path);
|
||||
current.isDirty = this.hasChanges(current.path);
|
||||
worktrees.push(current as WorktreeStatus);
|
||||
};
|
||||
|
||||
for (const line of output.split("\n")) {
|
||||
if (line.startsWith("worktree ")) {
|
||||
pushCurrent();
|
||||
current = { path: line.slice(9), isDirty: false, lastCommit: "" };
|
||||
continue;
|
||||
}
|
||||
|
||||
if (line.startsWith("HEAD ")) current.lastCommit = line.slice(5);
|
||||
if (line.startsWith("branch ")) current.branch = line.slice(7).replace("refs/heads/", "");
|
||||
}
|
||||
|
||||
pushCurrent();
|
||||
return worktrees;
|
||||
}
|
||||
private safeName(value: string): string {
|
||||
return value.trim().replace(/[^a-zA-Z0-9._-]/g, "-").replace(/(^-+)|(-+$)/g, "");
|
||||
}
|
||||
|
||||
private safeBranch(value: string): string {
|
||||
return this.safeName(value).replace(/\/+/, "-");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { describe, expect, it } from "vitest";
|
|||
import * as fs from "node:fs";
|
||||
import * as os from "node:os";
|
||||
import * as path from "node:path";
|
||||
import { appendZteReceipt, createZteReceipt, createZteSpec } from "./zte-protocol.js";
|
||||
import { appendZteReceipt, createZteReceipt, createZteSpec, validateGateReceipt } from "./zte-protocol.js";
|
||||
|
||||
describe("ZTE protocol", () => {
|
||||
it("creates a verifier-gated spec with inherited policy", () => {
|
||||
|
|
@ -30,4 +30,22 @@ describe("ZTE protocol", () => {
|
|||
expect(lines).toHaveLength(1);
|
||||
expect(JSON.parse(lines[0]).status).toBe("passed");
|
||||
});
|
||||
|
||||
it("validates fresh context-matched gate receipts", () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "zte-valid-"));
|
||||
const file = path.join(dir, "receipts.jsonl");
|
||||
fs.writeFileSync(file, `${JSON.stringify({
|
||||
schema: "fable.zte.receipt.v1",
|
||||
task: "deploy",
|
||||
repo: dir,
|
||||
status: "passed",
|
||||
commands: ["factory gate", "fable5 verify"],
|
||||
createdAt: "2026-06-23T00:00:00.000Z",
|
||||
})}\n`);
|
||||
|
||||
expect(validateGateReceipt(file, { repo: dir, task: "deploy", requiredCommands: ["factory gate", "fable5 verify"], now: new Date("2026-06-23T00:01:00.000Z") }).ok).toBe(true);
|
||||
expect(validateGateReceipt(file, { repo: "other", now: new Date("2026-06-23T00:01:00.000Z") }).ok).toBe(false);
|
||||
expect(validateGateReceipt(file, { maxAgeMs: 1, now: new Date("2026-06-23T00:01:00.000Z") }).ok).toBe(false);
|
||||
expect(validateGateReceipt(file, { requiredCommands: ["missing"], now: new Date("2026-06-23T00:01:00.000Z") }).ok).toBe(false);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import { FACTORY_RECEIPT_SCHEMA } from "./factory-routes.js";
|
||||
import { isActionDenied, mergeToolPolicies, type ToolPolicy } from "./orchestrator-policy.js";
|
||||
|
||||
export interface ZteSpecOptions {
|
||||
|
|
@ -16,6 +17,7 @@ export interface ZteSpec {
|
|||
}
|
||||
|
||||
export interface ZteReceipt {
|
||||
schema?: string;
|
||||
task: string;
|
||||
repo: string;
|
||||
status: "passed" | "failed" | "blocked";
|
||||
|
|
@ -24,6 +26,20 @@ export interface ZteReceipt {
|
|||
createdAt: string;
|
||||
}
|
||||
|
||||
export interface GateReceiptOptions {
|
||||
repo?: string;
|
||||
task?: string;
|
||||
requiredCommands?: string[];
|
||||
maxAgeMs?: number;
|
||||
now?: Date;
|
||||
}
|
||||
|
||||
export interface GateReceiptValidation {
|
||||
ok: boolean;
|
||||
receipt?: ZteReceipt;
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
const DEFAULT_VALIDATION = [
|
||||
"npm run -s test",
|
||||
"npm run -s build",
|
||||
|
|
@ -94,7 +110,7 @@ export function createZteReceipt(
|
|||
commands: string[],
|
||||
failureReason?: string,
|
||||
): ZteReceipt {
|
||||
return { task, repo, status, commands, failureReason, createdAt: new Date().toISOString() };
|
||||
return { schema: FACTORY_RECEIPT_SCHEMA, task, repo, status, commands, failureReason, createdAt: new Date().toISOString() };
|
||||
}
|
||||
|
||||
export function appendZteReceipt(repo: string, receipt: ZteReceipt): string {
|
||||
|
|
@ -103,3 +119,42 @@ export function appendZteReceipt(repo: string, receipt: ZteReceipt): string {
|
|||
fs.appendFileSync(file, JSON.stringify(receipt) + "\n");
|
||||
return file;
|
||||
}
|
||||
|
||||
export function validateGateReceipt(file: string, options: GateReceiptOptions = {}): GateReceiptValidation {
|
||||
if (!fs.existsSync(file)) return { ok: false, reason: "gate receipt file missing" };
|
||||
const lines = fs.readFileSync(file, "utf-8").split(/\r?\n/).filter(Boolean).reverse();
|
||||
for (const line of lines) {
|
||||
const receipt = parseReceipt(line);
|
||||
if (!receipt) continue;
|
||||
const reason = invalidReceiptReason(receipt, options);
|
||||
if (!reason) return { ok: true, receipt };
|
||||
}
|
||||
return { ok: false, reason: "no fresh passed verifier receipt found" };
|
||||
}
|
||||
|
||||
function parseReceipt(line: string): ZteReceipt | undefined {
|
||||
try {
|
||||
const value = JSON.parse(line) as Partial<ZteReceipt>;
|
||||
if (typeof value.task !== "string" || typeof value.repo !== "string" || typeof value.createdAt !== "string") return undefined;
|
||||
if (value.status !== "passed" && value.status !== "failed" && value.status !== "blocked") return undefined;
|
||||
if (!Array.isArray(value.commands) || value.commands.some((cmd) => typeof cmd !== "string")) return undefined;
|
||||
if (value.schema !== undefined && value.schema !== FACTORY_RECEIPT_SCHEMA) return undefined;
|
||||
return value as ZteReceipt;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function invalidReceiptReason(receipt: ZteReceipt, options: GateReceiptOptions): string | undefined {
|
||||
if (receipt.status !== "passed") return "receipt did not pass";
|
||||
if (options.repo && receipt.repo !== options.repo) return "receipt repo mismatch";
|
||||
if (options.task && receipt.task !== options.task) return "receipt task mismatch";
|
||||
const created = Date.parse(receipt.createdAt);
|
||||
if (!Number.isFinite(created)) return "receipt timestamp invalid";
|
||||
const maxAgeMs = options.maxAgeMs ?? 7 * 24 * 60 * 60 * 1000;
|
||||
if ((options.now ?? new Date()).getTime() - created > maxAgeMs) return "receipt is stale";
|
||||
for (const required of options.requiredCommands ?? ["fable5 verify"]) {
|
||||
if (!receipt.commands.some((cmd) => cmd.includes(required))) return `receipt missing command evidence: ${required}`;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
|
|
|||
25
src/index.ts
25
src/index.ts
|
|
@ -835,6 +835,26 @@ benchmark
|
|||
console.log(`\n${runner.report(opts.benchmark)}\n`);
|
||||
});
|
||||
|
||||
benchmark
|
||||
.command("agents <task>")
|
||||
.description("Run the same prompt against Pi, Hermes, and OpenCode")
|
||||
.option("--with <csv>", "Contenders: pi,hermes,opencode", "pi,hermes,opencode")
|
||||
.option("--timeout-ms <n>", "Timeout per contender", (v) => Number(v), 120000)
|
||||
.option("--harnesses <path>", "JSON harness config: array or { harnesses: [...] }")
|
||||
.option("--out <path>", "Receipt output path")
|
||||
.action(async (task: string, opts: { with?: string; timeoutMs?: number; harnesses?: string; out?: string }) => {
|
||||
const { loadBenchHarnesses, runAgentBench, writeAgentBenchReceipt } = await import("./fable5/agent-bench.js");
|
||||
const contenders = (opts.with ?? "pi,hermes,opencode").split(",").map((s) => s.trim()).filter(Boolean);
|
||||
const harnesses = opts.harnesses ? loadBenchHarnesses(path.resolve(opts.harnesses)) : undefined;
|
||||
const receipt = await runAgentBench({ task, contenders, harnesses, timeoutMs: opts.timeoutMs });
|
||||
const stamp = new Date().toISOString().replace(/[:.]/g, "-");
|
||||
const out = opts.out ?? path.join(".fable", "agent-bench", stamp + ".json");
|
||||
writeAgentBenchReceipt(out, receipt);
|
||||
console.log(JSON.stringify(receipt, null, 2));
|
||||
console.log("\n Receipt: " + out + "\n");
|
||||
if (receipt.winner === "none") process.exit(1);
|
||||
});
|
||||
|
||||
benchmark
|
||||
.command("duel <task>")
|
||||
.description("Record a two-implementer eval winner")
|
||||
|
|
@ -1708,8 +1728,11 @@ factory
|
|||
await emitRunEvent(opts.run, { source: "gate", type: "error", data: { kind: "factory-gate", reason: "repo verification failed", status: check.status } });
|
||||
process.exit(check.status ?? 1);
|
||||
}
|
||||
await emitRunEvent(opts.run, { source: "gate", type: "done", data: { kind: "factory-gate", task, repo } });
|
||||
const { appendZteReceipt, createZteReceipt } = await import("./fable5/zte-protocol.js");
|
||||
const receiptPath = appendZteReceipt(repo, createZteReceipt(task, repo, "passed", ["factory gate", "factory check", "factory capabilities", "cyber preflight", "cyber reconcile", "fable5 verify"]));
|
||||
await emitRunEvent(opts.run, { source: "gate", type: "done", data: { kind: "factory-gate", task, repo, receiptPath } });
|
||||
console.log(` ✓ Factory gate passed; deploy still requires explicit token/contract.`);
|
||||
console.log(` ✓ Gate receipt: ${receiptPath}`);
|
||||
console.log(``);
|
||||
});
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue