fix: require fresh structured RSI canary proof
This commit is contained in:
parent
5f2a3bb1a1
commit
98486d3a55
|
|
@ -16,6 +16,23 @@ function tmpFile(): string {
|
||||||
return path.join(root, "receipt.json");
|
return path.join(root, "receipt.json");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function canaryProof(createdAt: string) {
|
||||||
|
return {
|
||||||
|
schema: "fable.rsi.canary-proof.v1",
|
||||||
|
created_at: createdAt,
|
||||||
|
skill: "rsi_canary",
|
||||||
|
degraded_before: true,
|
||||||
|
status_before: 1,
|
||||||
|
auto_patch: {
|
||||||
|
command: "python3 /tmp/skill_health.py --auto-patch",
|
||||||
|
status: 0,
|
||||||
|
evidence: "auto-patching 1 degraded skills\nAll patched successfully",
|
||||||
|
},
|
||||||
|
recovered_after: true,
|
||||||
|
status_after: 0,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
function runner(overrides: Record<string, { status: number; stdout: string; stderr?: string }> = {}) {
|
function runner(overrides: Record<string, { status: number; stdout: string; stderr?: string }> = {}) {
|
||||||
return (_command: string, args: string[]) => {
|
return (_command: string, args: string[]) => {
|
||||||
const script = args.at(-1) ?? "";
|
const script = args.at(-1) ?? "";
|
||||||
|
|
@ -46,25 +63,27 @@ describe("rsi reconcile", () => {
|
||||||
expect(receipt.decision).toBe("healthy");
|
expect(receipt.decision).toBe("healthy");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("marks healthy when a fresh canary proof receipt exists", () => {
|
it("marks healthy when a fresh structured canary proof receipt exists", () => {
|
||||||
const proof = tmpFile();
|
const proof = tmpFile();
|
||||||
fs.writeFileSync(proof, [
|
fs.writeFileSync(proof, JSON.stringify(canaryProof("2026-06-18T00:00:00.000Z")));
|
||||||
"rsi_canary: degraded",
|
|
||||||
"STATUS_BEFORE:1",
|
|
||||||
"auto-patching 1 degraded skills",
|
|
||||||
"All patched successfully",
|
|
||||||
"rsi_canary: recovered",
|
|
||||||
"STATUS_PATCH:0",
|
|
||||||
"STATUS_AFTER:0",
|
|
||||||
].join("\n"));
|
|
||||||
|
|
||||||
expect(verifyCanaryProof(proof)).toBe(true);
|
expect(verifyCanaryProof(proof, new Date("2026-06-18T01:00:00.000Z"))).toEqual({ ok: true });
|
||||||
const receipt = reconcileRsi({ host: "example", canaryProof: proof, runner: runner() });
|
const receipt = reconcileRsi({ host: "example", canaryProof: proof, runner: runner(), now: new Date("2026-06-18T01:00:00.000Z") });
|
||||||
expect(receipt.facts.auto_patch_proven).toBe(true);
|
expect(receipt.facts.auto_patch_proven).toBe(true);
|
||||||
expect(receipt.facts.canary_proof).toBe(proof);
|
expect(receipt.facts.canary_proof).toBe(proof);
|
||||||
expect(receipt.decision).toBe("healthy");
|
expect(receipt.decision).toBe("healthy");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("rejects stale or unstructured canary proof", () => {
|
||||||
|
const old = tmpFile();
|
||||||
|
fs.writeFileSync(old, JSON.stringify(canaryProof("2026-06-16T00:00:00.000Z")));
|
||||||
|
expect(verifyCanaryProof(old, new Date("2026-06-18T01:00:00.000Z"))).toEqual({ ok: false, reason: "stale" });
|
||||||
|
|
||||||
|
const text = tmpFile();
|
||||||
|
fs.writeFileSync(text, "rsi_canary: degraded\nSTATUS_BEFORE:1\n");
|
||||||
|
expect(verifyCanaryProof(text, new Date("2026-06-18T01:00:00.000Z"))).toEqual({ ok: false, reason: "invalid_json" });
|
||||||
|
});
|
||||||
|
|
||||||
it("keeps healthy when only cron naming drifts", () => {
|
it("keeps healthy when only cron naming drifts", () => {
|
||||||
const receipt = reconcileRsi({ host: "example", runner: runner({ "skill_health.py|rsi-diagnosis": { status: 1, stdout: "" } }) });
|
const receipt = reconcileRsi({ host: "example", runner: runner({ "skill_health.py|rsi-diagnosis": { status: 1, stdout: "" } }) });
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -11,6 +11,17 @@ export interface RsiReconcileOptions {
|
||||||
runner?: (command: string, args: string[]) => { status: number | null; stdout: string; stderr: string };
|
runner?: (command: string, args: string[]) => { status: number | null; stdout: string; stderr: string };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface RsiCanaryProofReceipt {
|
||||||
|
schema: "fable.rsi.canary-proof.v1";
|
||||||
|
created_at: string;
|
||||||
|
skill: string;
|
||||||
|
degraded_before: true;
|
||||||
|
status_before: 1;
|
||||||
|
auto_patch: { command: string; status: 0; evidence: string };
|
||||||
|
recovered_after: true;
|
||||||
|
status_after: 0;
|
||||||
|
}
|
||||||
|
|
||||||
export interface RsiReconcileReceipt {
|
export interface RsiReconcileReceipt {
|
||||||
created_at: string;
|
created_at: string;
|
||||||
host: string;
|
host: string;
|
||||||
|
|
@ -39,7 +50,7 @@ export function reconcileRsi(opts: RsiReconcileOptions): RsiReconcileReceipt {
|
||||||
check("deploy_route", port, opts.host, "code=\"$(curl -sS -m 5 -o /dev/null -w '%{http_code}' -X POST http://127.0.0.1:8099/deploy)\" && (test \"$code\" = \"401\" || test \"$code\" = \"403\") && echo present", runner),
|
check("deploy_route", port, opts.host, "code=\"$(curl -sS -m 5 -o /dev/null -w '%{http_code}' -X POST http://127.0.0.1:8099/deploy)\" && (test \"$code\" = \"401\" || test \"$code\" = \"403\") && echo present", runner),
|
||||||
];
|
];
|
||||||
const latest = checks.find((c) => c.name === "latest_rsi")?.stdout ?? "";
|
const latest = checks.find((c) => c.name === "latest_rsi")?.stdout ?? "";
|
||||||
const canaryProofOk = opts.canaryProof ? verifyCanaryProof(opts.canaryProof) : false;
|
const canaryProofOk = opts.canaryProof ? verifyCanaryProof(opts.canaryProof, opts.now).ok : false;
|
||||||
const facts = {
|
const facts = {
|
||||||
cron_present: isOk("cron", checks),
|
cron_present: isOk("cron", checks),
|
||||||
skill_health_present: isOk("skill_health", checks),
|
skill_health_present: isOk("skill_health", checks),
|
||||||
|
|
@ -63,16 +74,29 @@ export function reconcileRsi(opts: RsiReconcileOptions): RsiReconcileReceipt {
|
||||||
return receipt;
|
return receipt;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function verifyCanaryProof(file: string): boolean {
|
export function verifyCanaryProof(file: string, now = new Date()): { ok: boolean; reason?: string } {
|
||||||
if (!fs.existsSync(file)) return false;
|
if (!fs.existsSync(file)) return { ok: false, reason: "missing" };
|
||||||
const text = fs.readFileSync(file, "utf-8");
|
let receipt: RsiCanaryProofReceipt;
|
||||||
return /rsi_canary: degraded/.test(text)
|
try {
|
||||||
&& /STATUS_BEFORE:1/.test(text)
|
receipt = JSON.parse(fs.readFileSync(file, "utf-8")) as RsiCanaryProofReceipt;
|
||||||
&& /auto-patching 1 degraded skills/.test(text)
|
} catch {
|
||||||
&& /All patched successfully/.test(text)
|
return { ok: false, reason: "invalid_json" };
|
||||||
&& /rsi_canary: recovered/.test(text)
|
}
|
||||||
&& /STATUS_PATCH:0/.test(text)
|
const created = Date.parse(receipt.created_at);
|
||||||
&& /STATUS_AFTER:0/.test(text);
|
if (!Number.isFinite(created)) return { ok: false, reason: "invalid_created_at" };
|
||||||
|
const ageMs = Math.abs(now.getTime() - created);
|
||||||
|
if (ageMs > 24 * 60 * 60 * 1000) return { ok: false, reason: "stale" };
|
||||||
|
const ok = receipt.schema === "fable.rsi.canary-proof.v1"
|
||||||
|
&& receipt.skill === "rsi_canary"
|
||||||
|
&& receipt.degraded_before === true
|
||||||
|
&& receipt.status_before === 1
|
||||||
|
&& receipt.auto_patch?.status === 0
|
||||||
|
&& /skill_health.py --auto-patch/.test(receipt.auto_patch.command)
|
||||||
|
&& /auto-patching 1 degraded skills/.test(receipt.auto_patch.evidence)
|
||||||
|
&& /All patched successfully/.test(receipt.auto_patch.evidence)
|
||||||
|
&& receipt.recovered_after === true
|
||||||
|
&& receipt.status_after === 0;
|
||||||
|
return ok ? { ok: true } : { ok: false, reason: "missing_required_evidence" };
|
||||||
}
|
}
|
||||||
|
|
||||||
export function writeRsiReconcileReceipt(file: string, receipt: RsiReconcileReceipt): void {
|
export function writeRsiReconcileReceipt(file: string, receipt: RsiReconcileReceipt): void {
|
||||||
|
|
|
||||||
|
|
@ -26,6 +26,23 @@ function runner(command: string, args: string[]) {
|
||||||
return { status: 1, stdout: "", stderr: `unexpected: ${full}` };
|
return { status: 1, stdout: "", stderr: `unexpected: ${full}` };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function canaryProof(createdAt: string) {
|
||||||
|
return {
|
||||||
|
schema: "fable.rsi.canary-proof.v1",
|
||||||
|
created_at: createdAt,
|
||||||
|
skill: "rsi_canary",
|
||||||
|
degraded_before: true,
|
||||||
|
status_before: 1,
|
||||||
|
auto_patch: {
|
||||||
|
command: "python3 /tmp/skill_health.py --auto-patch",
|
||||||
|
status: 0,
|
||||||
|
evidence: "auto-patching 1 degraded skills\nAll patched successfully",
|
||||||
|
},
|
||||||
|
recovered_after: true,
|
||||||
|
status_after: 0,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
describe("verifyCycle", () => {
|
describe("verifyCycle", () => {
|
||||||
it("writes a passing receipt with 8099 deploy evidence", async () => {
|
it("writes a passing receipt with 8099 deploy evidence", async () => {
|
||||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "verify-cycle-"));
|
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "verify-cycle-"));
|
||||||
|
|
@ -51,16 +68,8 @@ describe("verifyCycle", () => {
|
||||||
|
|
||||||
it("passes reconciled stale factory only with explicit opt-in and canary proof", async () => {
|
it("passes reconciled stale factory only with explicit opt-in and canary proof", async () => {
|
||||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "verify-cycle-proof-"));
|
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "verify-cycle-proof-"));
|
||||||
const proof = path.join(dir, "canary.txt");
|
const proof = path.join(dir, "canary.json");
|
||||||
fs.writeFileSync(proof, [
|
fs.writeFileSync(proof, JSON.stringify(canaryProof("2026-06-19T00:00:00.000Z")));
|
||||||
"rsi_canary: degraded",
|
|
||||||
"STATUS_BEFORE:1",
|
|
||||||
"auto-patching 1 degraded skills",
|
|
||||||
"All patched successfully",
|
|
||||||
"rsi_canary: recovered",
|
|
||||||
"STATUS_PATCH:0",
|
|
||||||
"STATUS_AFTER:0",
|
|
||||||
].join("\n"));
|
|
||||||
const staleFactory = {
|
const staleFactory = {
|
||||||
...factoryOk,
|
...factoryOk,
|
||||||
factory: {
|
factory: {
|
||||||
|
|
@ -72,16 +81,27 @@ describe("verifyCycle", () => {
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
const blocked = await verifyCycle({ host: "127.0.0.1", skill: "rsi_canary", allowDirty: true, canaryProof: proof, factoryCheck: async () => staleFactory, runner, rsiRunner: runner });
|
const blocked = await verifyCycle({ host: "127.0.0.1", skill: "rsi_canary", allowDirty: true, canaryProof: proof, now: new Date("2026-06-19T01:00:00.000Z"), factoryCheck: async () => staleFactory, runner, rsiRunner: runner });
|
||||||
expect(blocked.result).toBe("failed");
|
expect(blocked.result).toBe("failed");
|
||||||
expect(blocked.factory_ready).toBe(false);
|
expect(blocked.factory_ready).toBe(false);
|
||||||
|
|
||||||
const allowed = await verifyCycle({ host: "127.0.0.1", skill: "rsi_canary", allowDirty: true, allowReconciledStaleFactory: true, canaryProof: proof, factoryCheck: async () => staleFactory, runner, rsiRunner: runner });
|
const allowed = await verifyCycle({ host: "127.0.0.1", skill: "rsi_canary", allowDirty: true, allowReconciledStaleFactory: true, canaryProof: proof, now: new Date("2026-06-19T01:00:00.000Z"), factoryCheck: async () => staleFactory, runner, rsiRunner: runner });
|
||||||
expect(allowed.result).toBe("passed");
|
expect(allowed.result).toBe("passed");
|
||||||
expect(allowed.factory_ready).toBe(true);
|
expect(allowed.factory_ready).toBe(true);
|
||||||
expect(allowed.rsi.facts.canary_proof).toBe(proof);
|
expect(allowed.rsi.facts.canary_proof).toBe(proof);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("fails when RSI health is unproven", async () => {
|
||||||
|
const unprovenRunner = (command: string, args: string[]) => {
|
||||||
|
const full = [command, ...args].join(" ");
|
||||||
|
if (full.includes("tail -160 /tmp/rsi-diagnosis.log")) return { status: 0, stdout: "RSI: 65/65 (100%)\nAll healthy — ZTE idle\n", stderr: "" };
|
||||||
|
return runner(command, args);
|
||||||
|
};
|
||||||
|
const receipt = await verifyCycle({ host: "127.0.0.1", skill: "rsi_canary", allowDirty: true, factoryCheck: async () => factoryOk, runner, rsiRunner: unprovenRunner });
|
||||||
|
expect(receipt.rsi.decision).toBe("healthy_but_autopatch_unproven");
|
||||||
|
expect(receipt.result).toBe("failed");
|
||||||
|
});
|
||||||
|
|
||||||
it("fails dirty worktrees unless explicitly allowed", async () => {
|
it("fails dirty worktrees unless explicitly allowed", async () => {
|
||||||
const receipt = await verifyCycle({ host: "127.0.0.1", skill: "rsi_canary", factoryCheck: async () => factoryOk, runner, rsiRunner: runner });
|
const receipt = await verifyCycle({ host: "127.0.0.1", skill: "rsi_canary", factoryCheck: async () => factoryOk, runner, rsiRunner: runner });
|
||||||
expect(receipt.result).toBe("failed");
|
expect(receipt.result).toBe("failed");
|
||||||
|
|
|
||||||
|
|
@ -50,7 +50,7 @@ export async function verifyCycle(opts: VerifyCycleOptions): Promise<VerifyCycle
|
||||||
const changed = (commands.find((c) => c.name === "changed_files")?.stdout ?? "").split(/\r?\n/).filter(Boolean);
|
const changed = (commands.find((c) => c.name === "changed_files")?.stdout ?? "").split(/\r?\n/).filter(Boolean);
|
||||||
const cleanEnough = opts.allowDirty || changed.length === 0;
|
const cleanEnough = opts.allowDirty || changed.length === 0;
|
||||||
const factoryReady = factoryGateReady(factory) || (opts.allowReconciledStaleFactory === true && factoryStaleButReconciled(factory));
|
const factoryReady = factoryGateReady(factory) || (opts.allowReconciledStaleFactory === true && factoryStaleButReconciled(factory));
|
||||||
const passed = cleanEnough && factoryReady && rsi.decision !== "degraded" && commands.every((c) => c.status === 0);
|
const passed = cleanEnough && factoryReady && rsi.decision === "healthy" && commands.every((c) => c.status === 0);
|
||||||
const receipt: VerifyCycleReceipt = {
|
const receipt: VerifyCycleReceipt = {
|
||||||
created_at: created,
|
created_at: created,
|
||||||
commit_sha: commit,
|
commit_sha: commit,
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue