fix: fail closed on stale factory status

This commit is contained in:
artale 2026-06-16 11:52:24 +02:00
parent c7b50d867f
commit 9b2e6a91dc
8 changed files with 188 additions and 24 deletions

1
.gitignore vendored
View File

@ -8,3 +8,4 @@ TEMP/
.familiar-test .familiar-test
.DS_Store .DS_Store
*.tsbuildinfo *.tsbuildinfo
.fable/

View File

@ -156,6 +156,7 @@ fable-agent plinius godmode "improve explanation quality"
- `factory check` - `factory check`
- `factory gate <task>` - `factory gate <task>`
- `--repo <path>` - `--repo <path>`
- `--diagnostic-only`
### `familiar` ### `familiar`

View File

@ -1,7 +1,10 @@
import { describe, expect, it } from "vitest"; import { describe, expect, it } from "vitest";
import { formatFactoryCheck } from "./factory-status.js"; import { checkFactory, formatFactoryCheck } from "./factory-status.js";
describe("factory check formatting", () => { const ok = (json: unknown, status = 200) => ({ ok: true, status, json: async () => json });
const fail = (status: number) => ({ ok: false, status, json: async () => ({}) });
describe("factory check", () => {
it("prints live counts from response data", () => { it("prints live counts from response data", () => {
const out = formatFactoryCheck( const out = formatFactoryCheck(
{ status: "ok", phase: 4, containers: 24, agents: 6, infra: 14, monitoring: 5, skills: 63 }, { status: "ok", phase: 4, containers: 24, agents: 6, infra: 14, monitoring: 5, skills: 63 },
@ -11,4 +14,52 @@ describe("factory check formatting", () => {
expect(out).toContain("Containers: 24"); expect(out).toContain("Containers: 24");
expect(out).toContain("Deploy webhook: ok (deploy-webhook)"); expect(out).toContain("Deploy webhook: ok (deploy-webhook)");
}); });
it("accepts factory-status 200", async () => {
const result = await checkFactory(async (url) =>
url.includes(":8099/")
? ok({ status: "ok", containers: 25 })
: ok({ status: "ok", service: "deploy-webhook" }),
);
expect(result.factoryAvailable).toBe(true);
expect(result.deployAvailable).toBe(true);
expect(result.factory.containers).toBe(25);
});
it("degrades factory-status 404 to unknown while preserving deploy health", async () => {
const result = await checkFactory(async (url) =>
url.includes(":8099/") ? fail(404) : ok({ status: "ok", service: "deploy-webhook" }),
);
expect(result.factoryAvailable).toBe(false);
expect(result.deploy.status).toBe("ok");
const out = formatFactoryCheck(result.factory, result.deploy, result.errors);
expect(out).toContain("Factory API: unknown (returned 404)");
expect(out).toContain("Factory: unknown");
expect(result.errors[0]).toContain("returned 404");
});
it("degrades malformed factory-status JSON to unknown", async () => {
const result = await checkFactory(async (url) =>
url.includes(":8099/") ? ok(null) : ok({ status: "ok", service: "deploy-webhook" }),
);
expect(result.factoryAvailable).toBe(false);
expect(result.factory.status).toBeUndefined();
expect(result.errors[0]).toContain("malformed JSON");
});
it("degrades factory-status timeout without using stale memory", async () => {
const result = await checkFactory(async (url, init) => {
if (!url.includes(":8099/")) return ok({ status: "ok", service: "deploy-webhook" });
await new Promise((_resolve, reject) => init?.signal?.addEventListener("abort", () => reject(Object.assign(new Error("aborted"), { name: "AbortError" }))));
return ok({ status: "stale-memory-should-not-appear" });
}, 1);
expect(result.factoryAvailable).toBe(false);
expect(result.factory.status).toBeUndefined();
expect(result.deploy.status).toBe("ok");
expect(result.errors[0]).toContain("timed out");
});
}); });

View File

@ -14,11 +14,77 @@ export interface DeployHealth {
phase?: number; phase?: number;
} }
export function formatFactoryCheck(factory: FactoryStatus, deploy: DeployHealth): string { export interface FactoryCheckResult {
factory: FactoryStatus;
deploy: DeployHealth;
factoryAvailable: boolean;
deployAvailable: boolean;
errors: string[];
}
const FACTORY_STATUS_URL = "http://77.42.112.29:8099/factory-status";
const DEPLOY_HEALTH_URL = "http://77.42.112.29:8098/health";
type FetchLike = (url: string, init?: { signal?: AbortSignal }) => Promise<{ ok: boolean; status: number; json(): Promise<unknown> }>;
function asRecord(value: unknown): Record<string, unknown> {
return value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
}
async function getJson(fetcher: FetchLike, url: string, timeoutMs: number): Promise<Record<string, unknown>> {
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), timeoutMs);
try {
const res = await fetcher(url, { signal: controller.signal });
if (!res.ok) throw new Error(`${url} returned ${res.status}`);
const json = await res.json();
const record = asRecord(json);
if (Object.keys(record).length === 0 && json !== record) throw new Error(`${url} returned malformed JSON`);
return record;
} catch (error) {
if (error instanceof Error && error.name === "AbortError") throw new Error(`${url} timed out`);
throw error;
} finally {
clearTimeout(timeout);
}
}
export async function checkFactory(fetcher: FetchLike = fetch, timeoutMs = 5_000): Promise<FactoryCheckResult> {
// ponytail: live HTTP only; no memory fallback, because stale factory state must not open the deploy gate.
const [factoryResult, deployResult] = await Promise.allSettled([
getJson(fetcher, FACTORY_STATUS_URL, timeoutMs),
getJson(fetcher, DEPLOY_HEALTH_URL, timeoutMs),
]);
const errors: string[] = [];
if (factoryResult.status === "rejected") errors.push(factoryResult.reason instanceof Error ? factoryResult.reason.message : String(factoryResult.reason));
if (deployResult.status === "rejected") errors.push(deployResult.reason instanceof Error ? deployResult.reason.message : String(deployResult.reason));
return {
factory: factoryResult.status === "fulfilled" ? factoryResult.value as FactoryStatus : {},
deploy: deployResult.status === "fulfilled" ? deployResult.value as DeployHealth : {},
factoryAvailable: factoryResult.status === "fulfilled",
deployAvailable: deployResult.status === "fulfilled",
errors,
};
}
export function factoryGateReady(result: FactoryCheckResult): boolean {
return result.factoryAvailable && result.deployAvailable && result.factory.status === "ok" && result.deploy.status === "ok";
}
function factoryApiStatus(factory: FactoryStatus, errors: string[]): string {
if (factory.status) return "ok";
const factoryError = errors.find((e) => e.includes("factory-status"));
const detail = factoryError?.match(/returned \d+|timed out|malformed JSON/)?.[0] ?? "unavailable";
return `unknown (${detail})`;
}
export function formatFactoryCheck(factory: FactoryStatus, deploy: DeployHealth, errors: string[] = []): string {
return [ return [
"", "",
" Factory Check", " Factory Check",
" ─────────────────────────────", " ─────────────────────────────",
` Factory API: ${factoryApiStatus(factory, errors)}`,
` Factory: ${factory.status ?? "unknown"}`, ` Factory: ${factory.status ?? "unknown"}`,
` Phase: ${factory.phase ?? "unknown"}`, ` Phase: ${factory.phase ?? "unknown"}`,
` Containers: ${factory.containers ?? "unknown"}`, ` Containers: ${factory.containers ?? "unknown"}`,
@ -27,6 +93,7 @@ export function formatFactoryCheck(factory: FactoryStatus, deploy: DeployHealth)
` Monitoring: ${factory.monitoring ?? "unknown"}`, ` Monitoring: ${factory.monitoring ?? "unknown"}`,
` Skills: ${factory.skills ?? "unknown"}`, ` Skills: ${factory.skills ?? "unknown"}`,
` Deploy webhook: ${deploy.status ?? "unknown"}${deploy.service ? ` (${deploy.service})` : ""}`, ` Deploy webhook: ${deploy.status ?? "unknown"}${deploy.service ? ` (${deploy.service})` : ""}`,
...errors.map((e) => ` Warning: ${e}`),
"", "",
].join("\n"); ].join("\n");
} }

View File

@ -41,5 +41,5 @@ export type { TeamRole, TeamAgent, TeamConfig, TeamResult } from "./agent-teams.
export { isActionDenied, mergeToolPolicies } from "./orchestrator-policy.js"; export { isActionDenied, mergeToolPolicies } from "./orchestrator-policy.js";
export type { ToolPolicy, EffectiveToolPolicy } from "./orchestrator-policy.js"; export type { ToolPolicy, EffectiveToolPolicy } from "./orchestrator-policy.js";
export { createZteSpec } from "./zte-protocol.js"; export { appendZteReceipt, createZteReceipt, createZteSpec } from "./zte-protocol.js";
export type { ZteSpec, ZteSpecOptions } from "./zte-protocol.js"; export type { ZteReceipt, ZteSpec, ZteSpecOptions } from "./zte-protocol.js";

View File

@ -1,5 +1,8 @@
import { describe, expect, it } from "vitest"; import { describe, expect, it } from "vitest";
import { createZteSpec } from "./zte-protocol.js"; import * as fs from "node:fs";
import * as os from "node:os";
import * as path from "node:path";
import { appendZteReceipt, createZteReceipt, createZteSpec } from "./zte-protocol.js";
describe("ZTE protocol", () => { describe("ZTE protocol", () => {
it("creates a verifier-gated spec with inherited policy", () => { it("creates a verifier-gated spec with inherited policy", () => {
@ -17,4 +20,14 @@ describe("ZTE protocol", () => {
expect(spec.policy.deny).toContain("git push --force"); expect(spec.policy.deny).toContain("git push --force");
expect(spec.policy.deny).toContain("curl /deploy"); expect(spec.policy.deny).toContain("curl /deploy");
}); });
it("appends a durable JSONL receipt", () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "zte-receipt-"));
const receipt = createZteReceipt("verify repo", dir, "passed", ["npm run -s test"]);
const file = appendZteReceipt(dir, receipt);
const lines = fs.readFileSync(file, "utf-8").trim().split("\n");
expect(lines).toHaveLength(1);
expect(JSON.parse(lines[0]).status).toBe("passed");
});
}); });

View File

@ -1,3 +1,5 @@
import * as fs from "node:fs";
import * as path from "node:path";
import { isActionDenied, mergeToolPolicies, type ToolPolicy } from "./orchestrator-policy.js"; import { isActionDenied, mergeToolPolicies, type ToolPolicy } from "./orchestrator-policy.js";
export interface ZteSpecOptions { export interface ZteSpecOptions {
@ -13,6 +15,15 @@ export interface ZteSpec {
policy: ReturnType<typeof mergeToolPolicies>; policy: ReturnType<typeof mergeToolPolicies>;
} }
export interface ZteReceipt {
task: string;
repo: string;
status: "passed" | "failed" | "blocked";
commands: string[];
failureReason?: string;
createdAt: string;
}
const DEFAULT_VALIDATION = [ const DEFAULT_VALIDATION = [
"npm run -s test", "npm run -s test",
"npm run -s build", "npm run -s build",
@ -75,3 +86,20 @@ ${validationCommands.map((cmd) => `- \`${cmd}\``).join("\n")}
`, `,
}; };
} }
export function createZteReceipt(
task: string,
repo: string,
status: ZteReceipt["status"],
commands: string[],
failureReason?: string,
): ZteReceipt {
return { task, repo, status, commands, failureReason, createdAt: new Date().toISOString() };
}
export function appendZteReceipt(repo: string, receipt: ZteReceipt): string {
const file = path.join(repo, ".fable", "zte-receipts.jsonl");
fs.mkdirSync(path.dirname(file), { recursive: true });
fs.appendFileSync(file, JSON.stringify(receipt) + "\n");
return file;
}

View File

@ -1426,20 +1426,11 @@ const factory = program
.command("factory") .command("factory")
.description("Factory/VPS status and deployment guards"); .description("Factory/VPS status and deployment guards");
async function printFactoryCheck(): Promise<void> { async function printFactoryCheck(): Promise<{ ready: boolean }> {
const { formatFactoryCheck } = await import("./fable5/factory-status.js"); const { checkFactory, factoryGateReady, formatFactoryCheck } = await import("./fable5/factory-status.js");
const getJson = async (url: string) => { const result = await checkFactory();
const res = await fetch(url); console.log(formatFactoryCheck(result.factory, result.deploy, result.errors));
if (!res.ok) throw new Error(url + " returned " + res.status); return { ready: factoryGateReady(result) };
return res.json();
};
const [factoryStatus, deployHealth] = await Promise.all([
getJson("http://77.42.112.29:8099/factory-status"),
getJson("http://77.42.112.29:8098/health"),
]);
console.log(formatFactoryCheck(factoryStatus, deployHealth));
} }
factory factory
@ -1453,11 +1444,19 @@ factory
.command("gate <task>") .command("gate <task>")
.description("Run factory check plus local ZTE/repo verification gate") .description("Run factory check plus local ZTE/repo verification gate")
.option("--repo <path>", "Repo path to verify", ".") .option("--repo <path>", "Repo path to verify", ".")
.action(async (task: string, opts: { repo?: string }) => { .option("--diagnostic-only", "Bypass live factory/deploy readiness failures for non-deploy diagnostics")
.action(async (task: string, opts: { repo?: string; diagnosticOnly?: boolean }) => {
const { spawnSync } = await import("node:child_process"); const { spawnSync } = await import("node:child_process");
await printFactoryCheck();
const repo = path.resolve(opts.repo ?? "."); const repo = path.resolve(opts.repo ?? ".");
const status = await printFactoryCheck();
if (!status.ready && !opts.diagnosticOnly) {
const { appendZteReceipt, createZteReceipt } = await import("./fable5/zte-protocol.js");
const reason = "live factory/deploy status is not ok";
appendZteReceipt(repo, createZteReceipt(task, repo, "blocked", ["factory check"], reason));
console.error(` ✗ Factory gate failed: ${reason}; use --diagnostic-only for non-deploy diagnostics.`);
process.exit(1);
}
console.log(` ZTE Gate: ${task}`); console.log(` ZTE Gate: ${task}`);
console.log(` Repo: ${repo}`); console.log(` Repo: ${repo}`);
console.log(``); console.log(``);
@ -1564,8 +1563,12 @@ fable
process.exit(check.status ?? 1); process.exit(check.status ?? 1);
} }
} }
const { appendZteReceipt, createZteReceipt } = await import("./fable5/zte-protocol.js");
const receipt = createZteReceipt(task, repo, "passed", checks.map(([cmd, args]) => [cmd, ...args].join(" ")));
const receiptPath = appendZteReceipt(repo, receipt);
console.log(` console.log(`
Repo verification passed Repo verification passed`);
console.log(` ✓ ZTE receipt written: ${receiptPath}
`); `);
} }