fix: fail rsi on stale container canary
This commit is contained in:
parent
451267d8bc
commit
e59a0f0ec4
|
|
@ -43,6 +43,8 @@ function runner(overrides: Record<string, { status: number; stdout: string; stde
|
|||
if (script.includes("tail -160")) return { status: 0, stdout: "RSI: 65/65 (100%)\nAll healthy — ZTE idle\n", stderr: "" };
|
||||
if (script.includes("systemctl is-active")) return { status: 0, stdout: "active\n", stderr: "" };
|
||||
if (script.includes("/deploy")) return { status: 0, stdout: "present\n", stderr: "" };
|
||||
if (script.includes("docker exec hermes test -f /tmp/skill_health.py")) return { status: 0, stdout: "present\n", stderr: "" };
|
||||
if (script.includes("/opt/data/skills/rsi_canary/test.sh")) return { status: 0, stdout: "rsi_canary: recovered\n", stderr: "" };
|
||||
return { status: 1, stdout: "", stderr: "unexpected" };
|
||||
};
|
||||
}
|
||||
|
|
@ -103,6 +105,14 @@ describe("rsi reconcile", () => {
|
|||
expect(receipt.decision).toBe("degraded");
|
||||
});
|
||||
|
||||
it("marks degraded when host RSI logs are stale but container canary fails", () => {
|
||||
const receipt = reconcileRsi({ host: "example", runner: runner({ "/opt/data/skills/rsi_canary/test.sh": { status: 1, stdout: "rsi_canary: degraded\n" } }) });
|
||||
|
||||
expect(receipt.facts.latest_score).toBe("65/65 (100%)");
|
||||
expect(receipt.facts.canary_healthy).toBe(false);
|
||||
expect(receipt.decision).toBe("degraded");
|
||||
});
|
||||
|
||||
it("writes a receipt", () => {
|
||||
const out = tmpFile();
|
||||
reconcileRsi({ host: "example", out, runner: runner() });
|
||||
|
|
|
|||
|
|
@ -51,6 +51,8 @@ export interface RsiReconcileReceipt {
|
|||
latest_score?: string;
|
||||
factory_watcher_active: boolean;
|
||||
deploy_route_present: boolean;
|
||||
hermes_skill_health_present: boolean;
|
||||
canary_healthy: boolean;
|
||||
auto_patch_proven: boolean;
|
||||
canary_proof?: string;
|
||||
};
|
||||
|
|
@ -66,6 +68,8 @@ export function reconcileRsi(opts: RsiReconcileOptions): RsiReconcileReceipt {
|
|||
check("latest_rsi", port, opts.host, "tail -160 /tmp/rsi-diagnosis.log 2>/dev/null | grep -E 'RSI:|All healthy|auto-patching|Rolled back|All patched'", runner),
|
||||
check("factory_watcher", port, opts.host, "systemctl is-active factory-watcher.service 2>/dev/null", runner),
|
||||
check("deploy_route", port, opts.host, "code=\"$(curl -sS -m 5 -o /dev/null -w '%{http_code}' -X POST http://127.0.0.1:8099/deploy)\" && (test \"$code\" = \"401\" || test \"$code\" = \"403\") && echo present", runner),
|
||||
check("hermes_skill_health", port, opts.host, "docker exec hermes test -f /tmp/skill_health.py && echo present", runner),
|
||||
check("rsi_canary", port, opts.host, "docker exec hermes bash -lc 'test -x /opt/data/skills/rsi_canary/test.sh && /opt/data/skills/rsi_canary/test.sh'", runner),
|
||||
];
|
||||
const latest = checks.find((c) => c.name === "latest_rsi")?.stdout ?? "";
|
||||
const canaryProofOk = opts.canaryProof ? verifyCanaryProof(opts.canaryProof, opts.now).ok : false;
|
||||
|
|
@ -75,11 +79,13 @@ export function reconcileRsi(opts: RsiReconcileOptions): RsiReconcileReceipt {
|
|||
latest_score: latest.match(/RSI:\s*([^\n]+)/)?.[1]?.trim(),
|
||||
factory_watcher_active: /active/.test(checks.find((c) => c.name === "factory_watcher")?.stdout ?? ""),
|
||||
deploy_route_present: isOk("deploy_route", checks),
|
||||
hermes_skill_health_present: isOk("hermes_skill_health", checks),
|
||||
canary_healthy: isOk("rsi_canary", checks),
|
||||
auto_patch_proven: canaryProofOk,
|
||||
...(canaryProofOk ? { canary_proof: opts.canaryProof } : {}),
|
||||
};
|
||||
// ponytail: cron/log text is advisory; fresh structured canary proof is required for auto_patch_proven.
|
||||
const coreHealthy = facts.skill_health_present && /100%|65\/65|66\/66/.test(facts.latest_score ?? "") && facts.factory_watcher_active && facts.deploy_route_present;
|
||||
// ponytail: host cron/log text is advisory; container canary evidence wins over stale logs.
|
||||
const coreHealthy = facts.skill_health_present && facts.hermes_skill_health_present && facts.canary_healthy && /100%|65\/65|66\/66/.test(facts.latest_score ?? "") && facts.factory_watcher_active && facts.deploy_route_present;
|
||||
const createdAt = (opts.now ?? new Date()).toISOString();
|
||||
const decision = coreHealthy ? (facts.auto_patch_proven ? "healthy" : "healthy_but_autopatch_unproven") : "degraded";
|
||||
const receipt: RsiReconcileReceipt = {
|
||||
|
|
|
|||
Loading…
Reference in New Issue