From 9b2e6a91dc96de69a88aef0627736e7f5ec69c63 Mon Sep 17 00:00:00 2001 From: artale Date: Tue, 16 Jun 2026 11:52:24 +0200 Subject: [PATCH] fix: fail closed on stale factory status --- .gitignore | 1 + COMMANDS.md | 1 + src/fable5/factory-status.test.ts | 55 +++++++++++++++++++++++- src/fable5/factory-status.ts | 69 ++++++++++++++++++++++++++++++- src/fable5/index.ts | 4 +- src/fable5/zte-protocol.test.ts | 15 ++++++- src/fable5/zte-protocol.ts | 28 +++++++++++++ src/index.ts | 39 +++++++++-------- 8 files changed, 188 insertions(+), 24 deletions(-) diff --git a/.gitignore b/.gitignore index 82a610b..3889649 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ TEMP/ .familiar-test .DS_Store *.tsbuildinfo +.fable/ diff --git a/COMMANDS.md b/COMMANDS.md index b3c6505..68585a9 100644 --- a/COMMANDS.md +++ b/COMMANDS.md @@ -156,6 +156,7 @@ fable-agent plinius godmode "improve explanation quality" - `factory check` - `factory gate ` - `--repo ` + - `--diagnostic-only` ### `familiar` diff --git a/src/fable5/factory-status.test.ts b/src/fable5/factory-status.test.ts index 7582f84..e462d63 100644 --- a/src/fable5/factory-status.test.ts +++ b/src/fable5/factory-status.test.ts @@ -1,7 +1,10 @@ import { describe, expect, it } from "vitest"; -import { formatFactoryCheck } from "./factory-status.js"; +import { checkFactory, formatFactoryCheck } from "./factory-status.js"; -describe("factory check formatting", () => { +const ok = (json: unknown, status = 200) => ({ ok: true, status, json: async () => json }); +const fail = (status: number) => ({ ok: false, status, json: async () => ({}) }); + +describe("factory check", () => { it("prints live counts from response data", () => { const out = formatFactoryCheck( { status: "ok", phase: 4, containers: 24, agents: 6, infra: 14, monitoring: 5, skills: 63 }, @@ -11,4 +14,52 @@ describe("factory check formatting", () => { expect(out).toContain("Containers: 24"); expect(out).toContain("Deploy webhook: ok (deploy-webhook)"); }); + + it("accepts factory-status 200", async () => { + const result = await checkFactory(async (url) => + url.includes(":8099/") + ? ok({ status: "ok", containers: 25 }) + : ok({ status: "ok", service: "deploy-webhook" }), + ); + + expect(result.factoryAvailable).toBe(true); + expect(result.deployAvailable).toBe(true); + expect(result.factory.containers).toBe(25); + }); + + it("degrades factory-status 404 to unknown while preserving deploy health", async () => { + const result = await checkFactory(async (url) => + url.includes(":8099/") ? fail(404) : ok({ status: "ok", service: "deploy-webhook" }), + ); + + expect(result.factoryAvailable).toBe(false); + expect(result.deploy.status).toBe("ok"); + const out = formatFactoryCheck(result.factory, result.deploy, result.errors); + expect(out).toContain("Factory API: unknown (returned 404)"); + expect(out).toContain("Factory: unknown"); + expect(result.errors[0]).toContain("returned 404"); + }); + + it("degrades malformed factory-status JSON to unknown", async () => { + const result = await checkFactory(async (url) => + url.includes(":8099/") ? ok(null) : ok({ status: "ok", service: "deploy-webhook" }), + ); + + expect(result.factoryAvailable).toBe(false); + expect(result.factory.status).toBeUndefined(); + expect(result.errors[0]).toContain("malformed JSON"); + }); + + it("degrades factory-status timeout without using stale memory", async () => { + const result = await checkFactory(async (url, init) => { + if (!url.includes(":8099/")) return ok({ status: "ok", service: "deploy-webhook" }); + await new Promise((_resolve, reject) => init?.signal?.addEventListener("abort", () => reject(Object.assign(new Error("aborted"), { name: "AbortError" })))); + return ok({ status: "stale-memory-should-not-appear" }); + }, 1); + + expect(result.factoryAvailable).toBe(false); + expect(result.factory.status).toBeUndefined(); + expect(result.deploy.status).toBe("ok"); + expect(result.errors[0]).toContain("timed out"); + }); }); diff --git a/src/fable5/factory-status.ts b/src/fable5/factory-status.ts index 30a1f69..78e55be 100644 --- a/src/fable5/factory-status.ts +++ b/src/fable5/factory-status.ts @@ -14,11 +14,77 @@ export interface DeployHealth { phase?: number; } -export function formatFactoryCheck(factory: FactoryStatus, deploy: DeployHealth): string { +export interface FactoryCheckResult { + factory: FactoryStatus; + deploy: DeployHealth; + factoryAvailable: boolean; + deployAvailable: boolean; + errors: string[]; +} + +const FACTORY_STATUS_URL = "http://77.42.112.29:8099/factory-status"; +const DEPLOY_HEALTH_URL = "http://77.42.112.29:8098/health"; + +type FetchLike = (url: string, init?: { signal?: AbortSignal }) => Promise<{ ok: boolean; status: number; json(): Promise }>; + +function asRecord(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; +} + +async function getJson(fetcher: FetchLike, url: string, timeoutMs: number): Promise> { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + try { + const res = await fetcher(url, { signal: controller.signal }); + if (!res.ok) throw new Error(`${url} returned ${res.status}`); + const json = await res.json(); + const record = asRecord(json); + if (Object.keys(record).length === 0 && json !== record) throw new Error(`${url} returned malformed JSON`); + return record; + } catch (error) { + if (error instanceof Error && error.name === "AbortError") throw new Error(`${url} timed out`); + throw error; + } finally { + clearTimeout(timeout); + } +} + +export async function checkFactory(fetcher: FetchLike = fetch, timeoutMs = 5_000): Promise { + // ponytail: live HTTP only; no memory fallback, because stale factory state must not open the deploy gate. + const [factoryResult, deployResult] = await Promise.allSettled([ + getJson(fetcher, FACTORY_STATUS_URL, timeoutMs), + getJson(fetcher, DEPLOY_HEALTH_URL, timeoutMs), + ]); + const errors: string[] = []; + if (factoryResult.status === "rejected") errors.push(factoryResult.reason instanceof Error ? factoryResult.reason.message : String(factoryResult.reason)); + if (deployResult.status === "rejected") errors.push(deployResult.reason instanceof Error ? deployResult.reason.message : String(deployResult.reason)); + + return { + factory: factoryResult.status === "fulfilled" ? factoryResult.value as FactoryStatus : {}, + deploy: deployResult.status === "fulfilled" ? deployResult.value as DeployHealth : {}, + factoryAvailable: factoryResult.status === "fulfilled", + deployAvailable: deployResult.status === "fulfilled", + errors, + }; +} + +export function factoryGateReady(result: FactoryCheckResult): boolean { + return result.factoryAvailable && result.deployAvailable && result.factory.status === "ok" && result.deploy.status === "ok"; +} + +function factoryApiStatus(factory: FactoryStatus, errors: string[]): string { + if (factory.status) return "ok"; + const factoryError = errors.find((e) => e.includes("factory-status")); + const detail = factoryError?.match(/returned \d+|timed out|malformed JSON/)?.[0] ?? "unavailable"; + return `unknown (${detail})`; +} + +export function formatFactoryCheck(factory: FactoryStatus, deploy: DeployHealth, errors: string[] = []): string { return [ "", " Factory Check", " ─────────────────────────────", + ` Factory API: ${factoryApiStatus(factory, errors)}`, ` Factory: ${factory.status ?? "unknown"}`, ` Phase: ${factory.phase ?? "unknown"}`, ` Containers: ${factory.containers ?? "unknown"}`, @@ -27,6 +93,7 @@ export function formatFactoryCheck(factory: FactoryStatus, deploy: DeployHealth) ` Monitoring: ${factory.monitoring ?? "unknown"}`, ` Skills: ${factory.skills ?? "unknown"}`, ` Deploy webhook: ${deploy.status ?? "unknown"}${deploy.service ? ` (${deploy.service})` : ""}`, + ...errors.map((e) => ` Warning: ${e}`), "", ].join("\n"); } diff --git a/src/fable5/index.ts b/src/fable5/index.ts index eba8df9..1ae9077 100644 --- a/src/fable5/index.ts +++ b/src/fable5/index.ts @@ -41,5 +41,5 @@ export type { TeamRole, TeamAgent, TeamConfig, TeamResult } from "./agent-teams. export { isActionDenied, mergeToolPolicies } from "./orchestrator-policy.js"; export type { ToolPolicy, EffectiveToolPolicy } from "./orchestrator-policy.js"; -export { createZteSpec } from "./zte-protocol.js"; -export type { ZteSpec, ZteSpecOptions } from "./zte-protocol.js"; +export { appendZteReceipt, createZteReceipt, createZteSpec } from "./zte-protocol.js"; +export type { ZteReceipt, ZteSpec, ZteSpecOptions } from "./zte-protocol.js"; diff --git a/src/fable5/zte-protocol.test.ts b/src/fable5/zte-protocol.test.ts index 265f3e7..aa3e8c2 100644 --- a/src/fable5/zte-protocol.test.ts +++ b/src/fable5/zte-protocol.test.ts @@ -1,5 +1,8 @@ import { describe, expect, it } from "vitest"; -import { createZteSpec } from "./zte-protocol.js"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { appendZteReceipt, createZteReceipt, createZteSpec } from "./zte-protocol.js"; describe("ZTE protocol", () => { it("creates a verifier-gated spec with inherited policy", () => { @@ -17,4 +20,14 @@ describe("ZTE protocol", () => { expect(spec.policy.deny).toContain("git push --force"); expect(spec.policy.deny).toContain("curl /deploy"); }); + + it("appends a durable JSONL receipt", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "zte-receipt-")); + const receipt = createZteReceipt("verify repo", dir, "passed", ["npm run -s test"]); + const file = appendZteReceipt(dir, receipt); + + const lines = fs.readFileSync(file, "utf-8").trim().split("\n"); + expect(lines).toHaveLength(1); + expect(JSON.parse(lines[0]).status).toBe("passed"); + }); }); diff --git a/src/fable5/zte-protocol.ts b/src/fable5/zte-protocol.ts index 50e9c7d..4b5ebdb 100644 --- a/src/fable5/zte-protocol.ts +++ b/src/fable5/zte-protocol.ts @@ -1,3 +1,5 @@ +import * as fs from "node:fs"; +import * as path from "node:path"; import { isActionDenied, mergeToolPolicies, type ToolPolicy } from "./orchestrator-policy.js"; export interface ZteSpecOptions { @@ -13,6 +15,15 @@ export interface ZteSpec { policy: ReturnType; } +export interface ZteReceipt { + task: string; + repo: string; + status: "passed" | "failed" | "blocked"; + commands: string[]; + failureReason?: string; + createdAt: string; +} + const DEFAULT_VALIDATION = [ "npm run -s test", "npm run -s build", @@ -75,3 +86,20 @@ ${validationCommands.map((cmd) => `- \`${cmd}\``).join("\n")} `, }; } + +export function createZteReceipt( + task: string, + repo: string, + status: ZteReceipt["status"], + commands: string[], + failureReason?: string, +): ZteReceipt { + return { task, repo, status, commands, failureReason, createdAt: new Date().toISOString() }; +} + +export function appendZteReceipt(repo: string, receipt: ZteReceipt): string { + const file = path.join(repo, ".fable", "zte-receipts.jsonl"); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.appendFileSync(file, JSON.stringify(receipt) + "\n"); + return file; +} diff --git a/src/index.ts b/src/index.ts index a3cfb4a..ae02f7b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1426,20 +1426,11 @@ const factory = program .command("factory") .description("Factory/VPS status and deployment guards"); -async function printFactoryCheck(): Promise { - const { formatFactoryCheck } = await import("./fable5/factory-status.js"); - const getJson = async (url: string) => { - const res = await fetch(url); - if (!res.ok) throw new Error(url + " returned " + res.status); - return res.json(); - }; - - const [factoryStatus, deployHealth] = await Promise.all([ - getJson("http://77.42.112.29:8099/factory-status"), - getJson("http://77.42.112.29:8098/health"), - ]); - - console.log(formatFactoryCheck(factoryStatus, deployHealth)); +async function printFactoryCheck(): Promise<{ ready: boolean }> { + const { checkFactory, factoryGateReady, formatFactoryCheck } = await import("./fable5/factory-status.js"); + const result = await checkFactory(); + console.log(formatFactoryCheck(result.factory, result.deploy, result.errors)); + return { ready: factoryGateReady(result) }; } factory @@ -1453,11 +1444,19 @@ factory .command("gate ") .description("Run factory check plus local ZTE/repo verification gate") .option("--repo ", "Repo path to verify", ".") - .action(async (task: string, opts: { repo?: string }) => { + .option("--diagnostic-only", "Bypass live factory/deploy readiness failures for non-deploy diagnostics") + .action(async (task: string, opts: { repo?: string; diagnosticOnly?: boolean }) => { const { spawnSync } = await import("node:child_process"); - await printFactoryCheck(); - const repo = path.resolve(opts.repo ?? "."); + const status = await printFactoryCheck(); + if (!status.ready && !opts.diagnosticOnly) { + const { appendZteReceipt, createZteReceipt } = await import("./fable5/zte-protocol.js"); + const reason = "live factory/deploy status is not ok"; + appendZteReceipt(repo, createZteReceipt(task, repo, "blocked", ["factory check"], reason)); + console.error(` ✗ Factory gate failed: ${reason}; use --diagnostic-only for non-deploy diagnostics.`); + process.exit(1); + } + console.log(` ZTE Gate: ${task}`); console.log(` Repo: ${repo}`); console.log(``); @@ -1564,8 +1563,12 @@ fable process.exit(check.status ?? 1); } } + const { appendZteReceipt, createZteReceipt } = await import("./fable5/zte-protocol.js"); + const receipt = createZteReceipt(task, repo, "passed", checks.map(([cmd, args]) => [cmd, ...args].join(" "))); + const receiptPath = appendZteReceipt(repo, receipt); console.log(` - ✓ Repo verification passed + ✓ Repo verification passed`); + console.log(` ✓ ZTE receipt written: ${receiptPath} `); }