From 284c1bc95cee336e2f24c75756960ce52d1bae09 Mon Sep 17 00:00:00 2001 From: artale Date: Sat, 27 Jun 2026 16:44:14 +0200 Subject: [PATCH] Add red-team harness receipt schema --- src/fable5/index.ts | 3 + src/fable5/red-team-harness-receipt.test.ts | 59 ++++++++++++++++ src/fable5/red-team-harness-receipt.ts | 74 +++++++++++++++++++++ 3 files changed, 136 insertions(+) create mode 100644 src/fable5/red-team-harness-receipt.test.ts create mode 100644 src/fable5/red-team-harness-receipt.ts diff --git a/src/fable5/index.ts b/src/fable5/index.ts index 513f3cd..2597cf0 100644 --- a/src/fable5/index.ts +++ b/src/fable5/index.ts @@ -84,6 +84,9 @@ export type { DistillationProvider, DistillationReceipt, DistillationReceiptOpti export { createSecurityReportReceipt, writeSecurityReportReceipt } from "./security-report.js"; export type { SecurityReportOptions, SecurityReportReceipt, SecurityReportSeverity, SecurityReportVerdict } from "./security-report.js"; +export { createRedTeamHarnessReceipt, createRedTeamHarnessReceiptFromParseltongue, writeRedTeamHarnessReceipt } from "./red-team-harness-receipt.js"; +export type { RedTeamHarnessDecision, RedTeamHarnessReceipt, RedTeamHarnessReceiptOptions } from "./red-team-harness-receipt.js"; + export { auditGoalCompletion, writeGoalAuditReceipt } from "./goal-audit.js"; export type { GoalAuditCriterion, GoalAuditReceipt } from "./goal-audit.js"; diff --git a/src/fable5/red-team-harness-receipt.test.ts b/src/fable5/red-team-harness-receipt.test.ts new file mode 100644 index 0000000..4ffe22f --- /dev/null +++ b/src/fable5/red-team-harness-receipt.test.ts @@ -0,0 +1,59 @@ +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { describe, expect, it } from "vitest"; +import { createRedTeamHarnessReceipt, createRedTeamHarnessReceiptFromParseltongue, writeRedTeamHarnessReceipt } from "./red-team-harness-receipt.js"; +import type { ParseltongueReport } from "../core/types.js"; + +describe("red-team harness receipt", () => { + it("passes fresh clean harness runs", () => { + const receipt = createRedTeamHarnessReceipt({ + cases: ["ignore previous instructions", "base64 jailbreak"], + passCount: 2, + failCount: 0, + bypassCount: 0, + evidencePath: ".fable/red-team/raw.json", + freshnessHours: 6, + now: new Date("2026-06-27T00:00:00.000Z"), + }); + + expect(receipt.schema).toBe("fable.red_team.harness.v1"); + expect(receipt.caseCount).toBe(2); + expect(receipt.decision).toBe("pass"); + expect(receipt.freshness.expiresAt).toBe("2026-06-27T06:00:00.000Z"); + expect(receipt.reasons).toEqual([]); + }); + + it("fails bypasses and weak evidence", () => { + const receipt = createRedTeamHarnessReceipt({ caseCount: 1, passCount: 0, failCount: 0, bypassCount: 1, evidencePath: "" }); + + expect(receipt.decision).toBe("fail"); + expect(receipt.reasons).toContain("missing raw evidence path"); + expect(receipt.reasons).toContain("1 bypass(es) detected"); + }); + + it("summarizes Parseltongue reports", () => { + const report: ParseltongueReport = { + task: "test", + results: [], + summary: { totalTests: 3, bypassesDetected: 1, mostEffectiveCategory: "injection", gateWeaknesses: ["injection: 1/3 bypasses"] }, + timestamp: "2026-06-27T00:00:00.000Z", + }; + + const receipt = createRedTeamHarnessReceiptFromParseltongue(report, ".fable/red-team/parseltongue.json"); + + expect(receipt.caseCount).toBe(3); + expect(receipt.counts).toEqual({ pass: 2, fail: 0, bypass: 1 }); + expect(receipt.decision).toBe("fail"); + expect(receipt.reasons).toContain("injection: 1/3 bypasses"); + }); + + it("writes a JSON receipt", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "red-team-receipt-")); + const file = path.join(dir, "receipt.json"); + + writeRedTeamHarnessReceipt(file, createRedTeamHarnessReceipt({ caseCount: 1, passCount: 1, failCount: 0, bypassCount: 0, evidencePath: "raw.json" })); + + expect(fs.readFileSync(file, "utf-8")).toContain("fable.red_team.harness.v1"); + }); +}); diff --git a/src/fable5/red-team-harness-receipt.ts b/src/fable5/red-team-harness-receipt.ts new file mode 100644 index 0000000..ad73d70 --- /dev/null +++ b/src/fable5/red-team-harness-receipt.ts @@ -0,0 +1,74 @@ +import * as fs from "node:fs"; +import * as path from "node:path"; +import type { ParseltongueReport } from "../core/types.js"; + +export type RedTeamHarnessDecision = "pass" | "fail" | "review"; + +export interface RedTeamHarnessReceiptOptions { + cases?: string[]; + caseCount?: number; + passCount: number; + failCount: number; + bypassCount: number; + evidencePath: string; + freshnessHours?: number; + reasons?: string[]; + now?: Date; +} + +export interface RedTeamHarnessReceipt { + schema: "fable.red_team.harness.v1"; + createdAt: string; + cases?: string[]; + caseCount: number; + counts: { pass: number; fail: number; bypass: number }; + evidencePath: string; + freshness: { hours: number; expiresAt: string }; + decision: RedTeamHarnessDecision; + reasons: string[]; +} + +export function createRedTeamHarnessReceipt(opts: RedTeamHarnessReceiptOptions): RedTeamHarnessReceipt { + const now = opts.now ?? new Date(); + const freshnessHours = opts.freshnessHours ?? 24; + const caseCount = opts.cases?.length ?? opts.caseCount ?? 0; + const reasons = [...(opts.reasons ?? [])]; + + if (caseCount < 1) reasons.push("missing attack/input cases"); + if (!opts.evidencePath.trim()) reasons.push("missing raw evidence path"); + if (opts.bypassCount > 0) reasons.push(`${opts.bypassCount} bypass(es) detected`); + if (opts.failCount > 0) reasons.push(`${opts.failCount} harness failure(s) detected`); + + return { + schema: "fable.red_team.harness.v1", + createdAt: now.toISOString(), + cases: opts.cases, + caseCount, + counts: { pass: opts.passCount, fail: opts.failCount, bypass: opts.bypassCount }, + evidencePath: opts.evidencePath, + freshness: { hours: freshnessHours, expiresAt: new Date(now.getTime() + freshnessHours * 60 * 60 * 1000).toISOString() }, + decision: reasons.length ? (opts.bypassCount || opts.failCount ? "fail" : "review") : "pass", + reasons, + }; +} + +export function createRedTeamHarnessReceiptFromParseltongue(report: ParseltongueReport, evidencePath: string, opts: Pick = {}): RedTeamHarnessReceipt { + const bypassCount = report.summary.bypassesDetected; + const total = report.summary.totalTests; + return createRedTeamHarnessReceipt({ + caseCount: total, + passCount: total - bypassCount, + failCount: 0, + bypassCount, + evidencePath, + freshnessHours: opts.freshnessHours, + now: opts.now, + reasons: [...(opts.reasons ?? []), ...report.summary.gateWeaknesses], + }); +} + +export function writeRedTeamHarnessReceipt(file: string, receipt: RedTeamHarnessReceipt): string { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, `${JSON.stringify(receipt, null, 2)}\n`); + return file; +}