feat: add jailbreak assessment receipts

This commit is contained in:
artale 2026-07-01 06:11:13 +02:00
parent 09b2483919
commit 7054758d0d
5 changed files with 142 additions and 0 deletions

View File

@ -328,6 +328,12 @@ fable-agent plinius godmode "improve explanation quality"
- `--run <id>` - `--run <id>`
- `fable5 memory-b-cell <file>` - `fable5 memory-b-cell <file>`
- `--cache <path>` - `--cache <path>`
- `fable5 jailbreak-assess <source>`
- `--capability-gain <n>`
- `--breadth <n>`
- `--weaponization-ease <n>`
- `--discoverability <n>`
- `--output <path>`
- `fable5 prompt-fingerprint <file>` - `fable5 prompt-fingerprint <file>`
- `--output <path>` - `--output <path>`
- `fable5 skill-health` - `fable5 skill-health`

View File

@ -99,6 +99,9 @@ export type { SkillHealthEntry, SkillHealthOptions, SkillHealthReceipt } from ".
export { createPromptFingerprintReceipt, writePromptFingerprintReceipt } from "./prompt-fingerprint.js"; export { createPromptFingerprintReceipt, writePromptFingerprintReceipt } from "./prompt-fingerprint.js";
export type { PromptFingerprintFinding, PromptFingerprintReceipt } from "./prompt-fingerprint.js"; export type { PromptFingerprintFinding, PromptFingerprintReceipt } from "./prompt-fingerprint.js";
export { createJailbreakAssessmentReceipt, writeJailbreakAssessmentReceipt } from "./jailbreak-assessment.js";
export type { JailbreakAssessmentOptions, JailbreakAssessmentReceipt, JailbreakAssessmentScores, JailbreakDecision } from "./jailbreak-assessment.js";
export { createPlanReceipt, writePlanReceipt } from "./plan-receipt.js"; export { createPlanReceipt, writePlanReceipt } from "./plan-receipt.js";
export type { PlanAllowedOutcome, PlanPhase, PlanReceipt, PlanReceiptOptions, PlanTaskType } from "./plan-receipt.js"; export type { PlanAllowedOutcome, PlanPhase, PlanReceipt, PlanReceiptOptions, PlanTaskType } from "./plan-receipt.js";

View File

@ -0,0 +1,51 @@
import * as fs from "node:fs";
import * as os from "node:os";
import * as path from "node:path";
import { describe, expect, it } from "vitest";
import { createJailbreakAssessmentReceipt, writeJailbreakAssessmentReceipt } from "./jailbreak-assessment.js";
describe("jailbreak assessment receipt", () => {
it("scores low-severity jailbreaks as monitor", () => {
const receipt = createJailbreakAssessmentReceipt({
source: "minor prompt bypass",
capabilityGain: 1,
breadth: 1,
weaponizationEase: 1,
discoverability: 2,
now: new Date("2026-07-01T00:00:00.000Z"),
});
expect(receipt).toMatchObject({
schema: "fable.jailbreak_assessment.receipt.v1",
createdAt: "2026-07-01T00:00:00.000Z",
total: 5,
decision: "monitor",
deployAttempted: false,
});
});
it("escalates broad easy high-gain jailbreaks", () => {
expect(createJailbreakAssessmentReceipt({ source: "universal", capabilityGain: 5, breadth: 5, weaponizationEase: 4, discoverability: 4 })).toMatchObject({ total: 18, decision: "disclose" });
expect(createJailbreakAssessmentReceipt({ source: "narrow harmful", capabilityGain: 4, breadth: 3, weaponizationEase: 3, discoverability: 3 })).toMatchObject({ total: 13, decision: "block" });
expect(createJailbreakAssessmentReceipt({ source: "patchable", capabilityGain: 3, breadth: 2, weaponizationEase: 2, discoverability: 2 })).toMatchObject({ total: 9, decision: "patch" });
});
it("clamps scores to the 0-5 framework", () => {
expect(createJailbreakAssessmentReceipt({ source: "clamp", capabilityGain: 99, breadth: -1, weaponizationEase: 2.4, discoverability: 2.6 }).scores).toEqual({
capabilityGain: 5,
breadth: 0,
weaponizationEase: 2,
discoverability: 3,
});
});
it("writes JSON receipts", () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "jailbreak-assessment-"));
const file = path.join(dir, "receipt.json");
writeJailbreakAssessmentReceipt(file, createJailbreakAssessmentReceipt({ source: "x", capabilityGain: 0, breadth: 0, weaponizationEase: 0, discoverability: 0 }));
expect(JSON.parse(fs.readFileSync(file, "utf-8"))).toMatchObject({ schema: "fable.jailbreak_assessment.receipt.v1", decision: "ignore" });
fs.rmSync(dir, { recursive: true, force: true });
});
});

View File

@ -0,0 +1,58 @@
import * as fs from "node:fs";
import * as path from "node:path";
export type JailbreakDecision = "ignore" | "monitor" | "patch" | "block" | "disclose";
export interface JailbreakAssessmentScores {
capabilityGain: number;
breadth: number;
weaponizationEase: number;
discoverability: number;
}
export interface JailbreakAssessmentReceipt {
schema: "fable.jailbreak_assessment.receipt.v1";
createdAt: string;
source: string;
scores: JailbreakAssessmentScores;
total: number;
decision: JailbreakDecision;
reasons: string[];
deployAttempted: false;
}
export interface JailbreakAssessmentOptions extends JailbreakAssessmentScores {
source: string;
now?: Date;
}
function clampScore(n: number): number {
return Math.max(0, Math.min(5, Math.round(n)));
}
export function createJailbreakAssessmentReceipt(opts: JailbreakAssessmentOptions): JailbreakAssessmentReceipt {
const scores = {
capabilityGain: clampScore(opts.capabilityGain),
breadth: clampScore(opts.breadth),
weaponizationEase: clampScore(opts.weaponizationEase),
discoverability: clampScore(opts.discoverability),
};
const total = scores.capabilityGain + scores.breadth + scores.weaponizationEase + scores.discoverability;
const decision: JailbreakDecision = total >= 17 ? "disclose" : total >= 13 ? "block" : total >= 9 ? "patch" : total >= 5 ? "monitor" : "ignore";
return {
schema: "fable.jailbreak_assessment.receipt.v1",
createdAt: (opts.now ?? new Date()).toISOString(),
source: opts.source,
scores,
total,
decision,
reasons: [`severity total ${total}/20 from capability gain, breadth, weaponization ease, and discoverability`],
deployAttempted: false,
};
}
export function writeJailbreakAssessmentReceipt(file: string, receipt: JailbreakAssessmentReceipt): string {
fs.mkdirSync(path.dirname(file), { recursive: true });
fs.writeFileSync(file, `${JSON.stringify(receipt, null, 2)}\n`);
return file;
}

View File

@ -2047,6 +2047,30 @@ fable
if (result.verdict === "known_unsafe") process.exit(1); if (result.verdict === "known_unsafe") process.exit(1);
}); });
fable
.command("jailbreak-assess <source>")
.description("Score jailbreak severity using capability gain, breadth, weaponization ease, and discoverability")
.requiredOption("--capability-gain <n>", "0-5 capability gain", (v) => Number(v))
.requiredOption("--breadth <n>", "0-5 breadth of capability gain", (v) => Number(v))
.requiredOption("--weaponization-ease <n>", "0-5 ease of weaponization", (v) => Number(v))
.requiredOption("--discoverability <n>", "0-5 discoverability", (v) => Number(v))
.option("--output <path>", "Receipt output path", path.join(".fable", "security", "jailbreak-assessment-live.json"))
.action(async (source: string, opts: { capabilityGain: number; breadth: number; weaponizationEase: number; discoverability: number; output?: string }) => {
const { createJailbreakAssessmentReceipt, writeJailbreakAssessmentReceipt } = await import("./fable5/jailbreak-assessment.js");
const receipt = createJailbreakAssessmentReceipt({
source,
capabilityGain: opts.capabilityGain,
breadth: opts.breadth,
weaponizationEase: opts.weaponizationEase,
discoverability: opts.discoverability,
});
const out = opts.output ?? path.join(".fable", "security", "jailbreak-assessment-live.json");
writeJailbreakAssessmentReceipt(out, receipt);
console.log(JSON.stringify(receipt, null, 2));
console.log(`\n Receipt: ${out}\n`);
if (receipt.decision === "block" || receipt.decision === "disclose") process.exit(1);
});
fable fable
.command("prompt-fingerprint <file>") .command("prompt-fingerprint <file>")
.description("Scan a prompt or bundle for covert prompt markers and telemetry fingerprint risks") .description("Scan a prompt or bundle for covert prompt markers and telemetry fingerprint risks")