fix: fail closed on stale factory status
This commit is contained in:
parent
c7b50d867f
commit
9b2e6a91dc
|
|
@ -8,3 +8,4 @@ TEMP/
|
|||
.familiar-test
|
||||
.DS_Store
|
||||
*.tsbuildinfo
|
||||
.fable/
|
||||
|
|
|
|||
|
|
@ -156,6 +156,7 @@ fable-agent plinius godmode "improve explanation quality"
|
|||
- `factory check`
|
||||
- `factory gate <task>`
|
||||
- `--repo <path>`
|
||||
- `--diagnostic-only`
|
||||
|
||||
### `familiar`
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,10 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { formatFactoryCheck } from "./factory-status.js";
|
||||
import { checkFactory, formatFactoryCheck } from "./factory-status.js";
|
||||
|
||||
describe("factory check formatting", () => {
|
||||
const ok = (json: unknown, status = 200) => ({ ok: true, status, json: async () => json });
|
||||
const fail = (status: number) => ({ ok: false, status, json: async () => ({}) });
|
||||
|
||||
describe("factory check", () => {
|
||||
it("prints live counts from response data", () => {
|
||||
const out = formatFactoryCheck(
|
||||
{ status: "ok", phase: 4, containers: 24, agents: 6, infra: 14, monitoring: 5, skills: 63 },
|
||||
|
|
@ -11,4 +14,52 @@ describe("factory check formatting", () => {
|
|||
expect(out).toContain("Containers: 24");
|
||||
expect(out).toContain("Deploy webhook: ok (deploy-webhook)");
|
||||
});
|
||||
|
||||
it("accepts factory-status 200", async () => {
|
||||
const result = await checkFactory(async (url) =>
|
||||
url.includes(":8099/")
|
||||
? ok({ status: "ok", containers: 25 })
|
||||
: ok({ status: "ok", service: "deploy-webhook" }),
|
||||
);
|
||||
|
||||
expect(result.factoryAvailable).toBe(true);
|
||||
expect(result.deployAvailable).toBe(true);
|
||||
expect(result.factory.containers).toBe(25);
|
||||
});
|
||||
|
||||
it("degrades factory-status 404 to unknown while preserving deploy health", async () => {
|
||||
const result = await checkFactory(async (url) =>
|
||||
url.includes(":8099/") ? fail(404) : ok({ status: "ok", service: "deploy-webhook" }),
|
||||
);
|
||||
|
||||
expect(result.factoryAvailable).toBe(false);
|
||||
expect(result.deploy.status).toBe("ok");
|
||||
const out = formatFactoryCheck(result.factory, result.deploy, result.errors);
|
||||
expect(out).toContain("Factory API: unknown (returned 404)");
|
||||
expect(out).toContain("Factory: unknown");
|
||||
expect(result.errors[0]).toContain("returned 404");
|
||||
});
|
||||
|
||||
it("degrades malformed factory-status JSON to unknown", async () => {
|
||||
const result = await checkFactory(async (url) =>
|
||||
url.includes(":8099/") ? ok(null) : ok({ status: "ok", service: "deploy-webhook" }),
|
||||
);
|
||||
|
||||
expect(result.factoryAvailable).toBe(false);
|
||||
expect(result.factory.status).toBeUndefined();
|
||||
expect(result.errors[0]).toContain("malformed JSON");
|
||||
});
|
||||
|
||||
it("degrades factory-status timeout without using stale memory", async () => {
|
||||
const result = await checkFactory(async (url, init) => {
|
||||
if (!url.includes(":8099/")) return ok({ status: "ok", service: "deploy-webhook" });
|
||||
await new Promise((_resolve, reject) => init?.signal?.addEventListener("abort", () => reject(Object.assign(new Error("aborted"), { name: "AbortError" }))));
|
||||
return ok({ status: "stale-memory-should-not-appear" });
|
||||
}, 1);
|
||||
|
||||
expect(result.factoryAvailable).toBe(false);
|
||||
expect(result.factory.status).toBeUndefined();
|
||||
expect(result.deploy.status).toBe("ok");
|
||||
expect(result.errors[0]).toContain("timed out");
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -14,11 +14,77 @@ export interface DeployHealth {
|
|||
phase?: number;
|
||||
}
|
||||
|
||||
export function formatFactoryCheck(factory: FactoryStatus, deploy: DeployHealth): string {
|
||||
export interface FactoryCheckResult {
|
||||
factory: FactoryStatus;
|
||||
deploy: DeployHealth;
|
||||
factoryAvailable: boolean;
|
||||
deployAvailable: boolean;
|
||||
errors: string[];
|
||||
}
|
||||
|
||||
const FACTORY_STATUS_URL = "http://77.42.112.29:8099/factory-status";
|
||||
const DEPLOY_HEALTH_URL = "http://77.42.112.29:8098/health";
|
||||
|
||||
type FetchLike = (url: string, init?: { signal?: AbortSignal }) => Promise<{ ok: boolean; status: number; json(): Promise<unknown> }>;
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> {
|
||||
return value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||||
}
|
||||
|
||||
async function getJson(fetcher: FetchLike, url: string, timeoutMs: number): Promise<Record<string, unknown>> {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
||||
try {
|
||||
const res = await fetcher(url, { signal: controller.signal });
|
||||
if (!res.ok) throw new Error(`${url} returned ${res.status}`);
|
||||
const json = await res.json();
|
||||
const record = asRecord(json);
|
||||
if (Object.keys(record).length === 0 && json !== record) throw new Error(`${url} returned malformed JSON`);
|
||||
return record;
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.name === "AbortError") throw new Error(`${url} timed out`);
|
||||
throw error;
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
export async function checkFactory(fetcher: FetchLike = fetch, timeoutMs = 5_000): Promise<FactoryCheckResult> {
|
||||
// ponytail: live HTTP only; no memory fallback, because stale factory state must not open the deploy gate.
|
||||
const [factoryResult, deployResult] = await Promise.allSettled([
|
||||
getJson(fetcher, FACTORY_STATUS_URL, timeoutMs),
|
||||
getJson(fetcher, DEPLOY_HEALTH_URL, timeoutMs),
|
||||
]);
|
||||
const errors: string[] = [];
|
||||
if (factoryResult.status === "rejected") errors.push(factoryResult.reason instanceof Error ? factoryResult.reason.message : String(factoryResult.reason));
|
||||
if (deployResult.status === "rejected") errors.push(deployResult.reason instanceof Error ? deployResult.reason.message : String(deployResult.reason));
|
||||
|
||||
return {
|
||||
factory: factoryResult.status === "fulfilled" ? factoryResult.value as FactoryStatus : {},
|
||||
deploy: deployResult.status === "fulfilled" ? deployResult.value as DeployHealth : {},
|
||||
factoryAvailable: factoryResult.status === "fulfilled",
|
||||
deployAvailable: deployResult.status === "fulfilled",
|
||||
errors,
|
||||
};
|
||||
}
|
||||
|
||||
export function factoryGateReady(result: FactoryCheckResult): boolean {
|
||||
return result.factoryAvailable && result.deployAvailable && result.factory.status === "ok" && result.deploy.status === "ok";
|
||||
}
|
||||
|
||||
function factoryApiStatus(factory: FactoryStatus, errors: string[]): string {
|
||||
if (factory.status) return "ok";
|
||||
const factoryError = errors.find((e) => e.includes("factory-status"));
|
||||
const detail = factoryError?.match(/returned \d+|timed out|malformed JSON/)?.[0] ?? "unavailable";
|
||||
return `unknown (${detail})`;
|
||||
}
|
||||
|
||||
export function formatFactoryCheck(factory: FactoryStatus, deploy: DeployHealth, errors: string[] = []): string {
|
||||
return [
|
||||
"",
|
||||
" Factory Check",
|
||||
" ─────────────────────────────",
|
||||
` Factory API: ${factoryApiStatus(factory, errors)}`,
|
||||
` Factory: ${factory.status ?? "unknown"}`,
|
||||
` Phase: ${factory.phase ?? "unknown"}`,
|
||||
` Containers: ${factory.containers ?? "unknown"}`,
|
||||
|
|
@ -27,6 +93,7 @@ export function formatFactoryCheck(factory: FactoryStatus, deploy: DeployHealth)
|
|||
` Monitoring: ${factory.monitoring ?? "unknown"}`,
|
||||
` Skills: ${factory.skills ?? "unknown"}`,
|
||||
` Deploy webhook: ${deploy.status ?? "unknown"}${deploy.service ? ` (${deploy.service})` : ""}`,
|
||||
...errors.map((e) => ` Warning: ${e}`),
|
||||
"",
|
||||
].join("\n");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -41,5 +41,5 @@ export type { TeamRole, TeamAgent, TeamConfig, TeamResult } from "./agent-teams.
|
|||
export { isActionDenied, mergeToolPolicies } from "./orchestrator-policy.js";
|
||||
export type { ToolPolicy, EffectiveToolPolicy } from "./orchestrator-policy.js";
|
||||
|
||||
export { createZteSpec } from "./zte-protocol.js";
|
||||
export type { ZteSpec, ZteSpecOptions } from "./zte-protocol.js";
|
||||
export { appendZteReceipt, createZteReceipt, createZteSpec } from "./zte-protocol.js";
|
||||
export type { ZteReceipt, ZteSpec, ZteSpecOptions } from "./zte-protocol.js";
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { createZteSpec } from "./zte-protocol.js";
|
||||
import * as fs from "node:fs";
|
||||
import * as os from "node:os";
|
||||
import * as path from "node:path";
|
||||
import { appendZteReceipt, createZteReceipt, createZteSpec } from "./zte-protocol.js";
|
||||
|
||||
describe("ZTE protocol", () => {
|
||||
it("creates a verifier-gated spec with inherited policy", () => {
|
||||
|
|
@ -17,4 +20,14 @@ describe("ZTE protocol", () => {
|
|||
expect(spec.policy.deny).toContain("git push --force");
|
||||
expect(spec.policy.deny).toContain("curl /deploy");
|
||||
});
|
||||
|
||||
it("appends a durable JSONL receipt", () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "zte-receipt-"));
|
||||
const receipt = createZteReceipt("verify repo", dir, "passed", ["npm run -s test"]);
|
||||
const file = appendZteReceipt(dir, receipt);
|
||||
|
||||
const lines = fs.readFileSync(file, "utf-8").trim().split("\n");
|
||||
expect(lines).toHaveLength(1);
|
||||
expect(JSON.parse(lines[0]).status).toBe("passed");
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import { isActionDenied, mergeToolPolicies, type ToolPolicy } from "./orchestrator-policy.js";
|
||||
|
||||
export interface ZteSpecOptions {
|
||||
|
|
@ -13,6 +15,15 @@ export interface ZteSpec {
|
|||
policy: ReturnType<typeof mergeToolPolicies>;
|
||||
}
|
||||
|
||||
export interface ZteReceipt {
|
||||
task: string;
|
||||
repo: string;
|
||||
status: "passed" | "failed" | "blocked";
|
||||
commands: string[];
|
||||
failureReason?: string;
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
const DEFAULT_VALIDATION = [
|
||||
"npm run -s test",
|
||||
"npm run -s build",
|
||||
|
|
@ -75,3 +86,20 @@ ${validationCommands.map((cmd) => `- \`${cmd}\``).join("\n")}
|
|||
`,
|
||||
};
|
||||
}
|
||||
|
||||
export function createZteReceipt(
|
||||
task: string,
|
||||
repo: string,
|
||||
status: ZteReceipt["status"],
|
||||
commands: string[],
|
||||
failureReason?: string,
|
||||
): ZteReceipt {
|
||||
return { task, repo, status, commands, failureReason, createdAt: new Date().toISOString() };
|
||||
}
|
||||
|
||||
export function appendZteReceipt(repo: string, receipt: ZteReceipt): string {
|
||||
const file = path.join(repo, ".fable", "zte-receipts.jsonl");
|
||||
fs.mkdirSync(path.dirname(file), { recursive: true });
|
||||
fs.appendFileSync(file, JSON.stringify(receipt) + "\n");
|
||||
return file;
|
||||
}
|
||||
|
|
|
|||
39
src/index.ts
39
src/index.ts
|
|
@ -1426,20 +1426,11 @@ const factory = program
|
|||
.command("factory")
|
||||
.description("Factory/VPS status and deployment guards");
|
||||
|
||||
async function printFactoryCheck(): Promise<void> {
|
||||
const { formatFactoryCheck } = await import("./fable5/factory-status.js");
|
||||
const getJson = async (url: string) => {
|
||||
const res = await fetch(url);
|
||||
if (!res.ok) throw new Error(url + " returned " + res.status);
|
||||
return res.json();
|
||||
};
|
||||
|
||||
const [factoryStatus, deployHealth] = await Promise.all([
|
||||
getJson("http://77.42.112.29:8099/factory-status"),
|
||||
getJson("http://77.42.112.29:8098/health"),
|
||||
]);
|
||||
|
||||
console.log(formatFactoryCheck(factoryStatus, deployHealth));
|
||||
async function printFactoryCheck(): Promise<{ ready: boolean }> {
|
||||
const { checkFactory, factoryGateReady, formatFactoryCheck } = await import("./fable5/factory-status.js");
|
||||
const result = await checkFactory();
|
||||
console.log(formatFactoryCheck(result.factory, result.deploy, result.errors));
|
||||
return { ready: factoryGateReady(result) };
|
||||
}
|
||||
|
||||
factory
|
||||
|
|
@ -1453,11 +1444,19 @@ factory
|
|||
.command("gate <task>")
|
||||
.description("Run factory check plus local ZTE/repo verification gate")
|
||||
.option("--repo <path>", "Repo path to verify", ".")
|
||||
.action(async (task: string, opts: { repo?: string }) => {
|
||||
.option("--diagnostic-only", "Bypass live factory/deploy readiness failures for non-deploy diagnostics")
|
||||
.action(async (task: string, opts: { repo?: string; diagnosticOnly?: boolean }) => {
|
||||
const { spawnSync } = await import("node:child_process");
|
||||
await printFactoryCheck();
|
||||
|
||||
const repo = path.resolve(opts.repo ?? ".");
|
||||
const status = await printFactoryCheck();
|
||||
if (!status.ready && !opts.diagnosticOnly) {
|
||||
const { appendZteReceipt, createZteReceipt } = await import("./fable5/zte-protocol.js");
|
||||
const reason = "live factory/deploy status is not ok";
|
||||
appendZteReceipt(repo, createZteReceipt(task, repo, "blocked", ["factory check"], reason));
|
||||
console.error(` ✗ Factory gate failed: ${reason}; use --diagnostic-only for non-deploy diagnostics.`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(` ZTE Gate: ${task}`);
|
||||
console.log(` Repo: ${repo}`);
|
||||
console.log(``);
|
||||
|
|
@ -1564,8 +1563,12 @@ fable
|
|||
process.exit(check.status ?? 1);
|
||||
}
|
||||
}
|
||||
const { appendZteReceipt, createZteReceipt } = await import("./fable5/zte-protocol.js");
|
||||
const receipt = createZteReceipt(task, repo, "passed", checks.map(([cmd, args]) => [cmd, ...args].join(" ")));
|
||||
const receiptPath = appendZteReceipt(repo, receipt);
|
||||
console.log(`
|
||||
✓ Repo verification passed
|
||||
✓ Repo verification passed`);
|
||||
console.log(` ✓ ZTE receipt written: ${receiptPath}
|
||||
`);
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue