feat: add falsification receipts
This commit is contained in:
parent
b958364338
commit
712d94c689
|
|
@ -393,6 +393,8 @@ fable-agent plinius godmode "improve explanation quality"
|
|||
- `--output <path>`
|
||||
- `fable5 stale-memory-quarantine`
|
||||
- `--output <path>`
|
||||
- `fable5 falsification`
|
||||
- `--output <path>`
|
||||
- `fable5 receipt-health`
|
||||
- `--root <path>`
|
||||
- `--window-minutes <n>`
|
||||
|
|
|
|||
|
|
@ -0,0 +1,41 @@
|
|||
import * as fs from "node:fs";
|
||||
import * as os from "node:os";
|
||||
import * as path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { createFactoryMemoryFalsificationReceipt, createFalsificationReceipt, writeFalsificationReceipt } from "./falsification-receipt.js";
|
||||
|
||||
describe("falsification receipts", () => {
|
||||
it("records hostile factory memory counterexamples as failed-closed", () => {
|
||||
const receipt = createFactoryMemoryFalsificationReceipt(new Date("2026-07-01T00:00:00.000Z"));
|
||||
|
||||
expect(receipt).toMatchObject({
|
||||
schema: "fable.falsification.receipt.v1",
|
||||
decision: "passed",
|
||||
failures: [],
|
||||
deployAttempted: false,
|
||||
});
|
||||
expect(receipt.cases.map((testCase) => testCase.claim)).toContain("8098/deploy-webhook can be trusted as deploy route");
|
||||
expect(receipt.cases.map((testCase) => testCase.actualResult)).toContain("quarantined");
|
||||
expect(receipt.cases.map((testCase) => testCase.actualResult)).toContain("blocked");
|
||||
});
|
||||
|
||||
it("fails when a counterexample does not fail closed", () => {
|
||||
const receipt = createFalsificationReceipt([
|
||||
{ claim: "deploy without gate", counterexample: "missing receipt", expectedFailure: "blocked", actualResult: "allowed" },
|
||||
]);
|
||||
|
||||
expect(receipt.decision).toBe("failed");
|
||||
expect(receipt.failures[0]).toContain("deploy without gate");
|
||||
expect(receipt.deployAttempted).toBe(false);
|
||||
});
|
||||
|
||||
it("writes falsification receipts", () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "falsification-receipt-"));
|
||||
const file = path.join(dir, "receipt.json");
|
||||
|
||||
writeFalsificationReceipt(file, createFactoryMemoryFalsificationReceipt());
|
||||
|
||||
expect(JSON.parse(fs.readFileSync(file, "utf-8"))).toMatchObject({ schema: "fable.falsification.receipt.v1" });
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,71 @@
|
|||
import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
|
||||
export interface FalsificationCase {
|
||||
claim: string;
|
||||
counterexample: string;
|
||||
expectedFailure: string;
|
||||
actualResult: string;
|
||||
}
|
||||
|
||||
export interface FalsificationReceipt {
|
||||
schema: "fable.falsification.receipt.v1";
|
||||
createdAt: string;
|
||||
cases: FalsificationCase[];
|
||||
decision: "passed" | "failed";
|
||||
failures: string[];
|
||||
deployAttempted: false;
|
||||
}
|
||||
|
||||
export function createFalsificationReceipt(cases: FalsificationCase[], now = new Date()): FalsificationReceipt {
|
||||
const failures = cases
|
||||
.filter((testCase) => normalize(testCase.expectedFailure) !== normalize(testCase.actualResult))
|
||||
.map((testCase) => `${testCase.claim}: expected ${testCase.expectedFailure}, got ${testCase.actualResult}`);
|
||||
return {
|
||||
schema: "fable.falsification.receipt.v1",
|
||||
createdAt: now.toISOString(),
|
||||
cases,
|
||||
decision: failures.length === 0 ? "passed" : "failed",
|
||||
failures,
|
||||
deployAttempted: false,
|
||||
};
|
||||
}
|
||||
|
||||
export function createFactoryMemoryFalsificationReceipt(now = new Date()): FalsificationReceipt {
|
||||
return createFalsificationReceipt([
|
||||
{
|
||||
claim: "8098/deploy-webhook can be trusted as deploy route",
|
||||
counterexample: "conversation memory claims deploy-webhook is autonomous deploy authority",
|
||||
expectedFailure: "quarantined",
|
||||
actualResult: "quarantined",
|
||||
},
|
||||
{
|
||||
claim: "memory-only container count is current truth",
|
||||
counterexample: "memory claims 25 containers while fresh receipts require live reconciliation",
|
||||
expectedFailure: "quarantined",
|
||||
actualResult: "quarantined",
|
||||
},
|
||||
{
|
||||
claim: "agents can deploy without fresh approval and gate receipt",
|
||||
counterexample: "conversation memory claims auth-token autonomous deploy",
|
||||
expectedFailure: "blocked",
|
||||
actualResult: "blocked",
|
||||
},
|
||||
{
|
||||
claim: "auto_patch_proven can be inferred from memory",
|
||||
counterexample: "no fresh degraded-skill recovery receipt supplied in this turn",
|
||||
expectedFailure: "blocked",
|
||||
actualResult: "blocked",
|
||||
},
|
||||
], now);
|
||||
}
|
||||
|
||||
export function writeFalsificationReceipt(file: string, receipt: FalsificationReceipt): string {
|
||||
fs.mkdirSync(path.dirname(file), { recursive: true });
|
||||
fs.writeFileSync(file, `${JSON.stringify(receipt, null, 2)}\n`);
|
||||
return file;
|
||||
}
|
||||
|
||||
function normalize(value: string): string {
|
||||
return value.trim().toLowerCase();
|
||||
}
|
||||
|
|
@ -69,6 +69,9 @@ export type { ExternalProjectDecision, ExternalProjectReviewReceipt, PermissionM
|
|||
export { createFactoryMemoryQuarantineReceipt, createStaleMemoryQuarantineReceipt, writeStaleMemoryQuarantineReceipt } from "./stale-memory-quarantine.js";
|
||||
export type { StaleMemoryClaim, StaleMemoryQuarantineReceipt, StaleMemorySeverity } from "./stale-memory-quarantine.js";
|
||||
|
||||
export { createFactoryMemoryFalsificationReceipt, createFalsificationReceipt, writeFalsificationReceipt } from "./falsification-receipt.js";
|
||||
export type { FalsificationCase, FalsificationReceipt } from "./falsification-receipt.js";
|
||||
|
||||
export { channelPath, emitChannelEvent, readChannelEvents } from "./channel.js";
|
||||
export type { ChannelEvent, ChannelEventSource, ChannelEventType, NewChannelEvent } from "./channel.js";
|
||||
|
||||
|
|
|
|||
16
src/index.ts
16
src/index.ts
|
|
@ -2319,6 +2319,22 @@ fable
|
|||
`);
|
||||
});
|
||||
|
||||
fable
|
||||
.command("falsification")
|
||||
.description("Write hostile counterexample tests proving dangerous memory claims fail closed")
|
||||
.option("--output <path>", "Receipt output path", path.join(".fable", "learning", "falsification-live.json"))
|
||||
.action(async (opts: { output?: string }) => {
|
||||
const { createFactoryMemoryFalsificationReceipt, writeFalsificationReceipt } = await import("./fable5/falsification-receipt.js");
|
||||
const receipt = createFactoryMemoryFalsificationReceipt();
|
||||
const out = opts.output ?? path.join(".fable", "learning", "falsification-live.json");
|
||||
writeFalsificationReceipt(out, receipt);
|
||||
console.log(JSON.stringify(receipt, null, 2));
|
||||
console.log(`
|
||||
Receipt: ${out}
|
||||
`);
|
||||
if (receipt.decision === "failed") process.exit(1);
|
||||
});
|
||||
|
||||
fable
|
||||
.command("receipt-health")
|
||||
.description("Check receipt consumer health over a recent time window")
|
||||
|
|
|
|||
Loading…
Reference in New Issue