feat: add model access receipts

This commit is contained in:
artale 2026-06-28 01:04:34 +02:00
parent 265c42edbb
commit 3c4684f4c5
5 changed files with 136 additions and 0 deletions

View File

@ -291,6 +291,9 @@ fable-agent plinius godmode "improve explanation quality"
- `fable5 route <task>` - `fable5 route <task>`
- `-d, --domain <domain>` - `-d, --domain <domain>`
- `fable5 models` - `fable5 models`
- `fable5 model-access`
- `--output <path>`
- `--run <id>`
- `fable5 flue` - `fable5 flue`
- `fable5 channels` - `fable5 channels`
- `--run <id>` - `--run <id>`

View File

@ -90,6 +90,9 @@ export type { SecurityReportOptions, SecurityReportReceipt, SecurityReportSeveri
export { createRedTeamHarnessReceipt, createRedTeamHarnessReceiptFromParseltongue, writeRedTeamHarnessReceipt } from "./red-team-harness-receipt.js"; export { createRedTeamHarnessReceipt, createRedTeamHarnessReceiptFromParseltongue, writeRedTeamHarnessReceipt } from "./red-team-harness-receipt.js";
export type { RedTeamHarnessDecision, RedTeamHarnessReceipt, RedTeamHarnessReceiptOptions } from "./red-team-harness-receipt.js"; export type { RedTeamHarnessDecision, RedTeamHarnessReceipt, RedTeamHarnessReceiptOptions } from "./red-team-harness-receipt.js";
export { createModelAccessReceipt, writeModelAccessReceipt } from "./model-access-receipt.js";
export type { ModelAccessProbe, ModelAccessReceipt, ModelAccessReceiptOptions, ModelAccessStatus, ModelAccessSurface } from "./model-access-receipt.js";
export { auditGoalCompletion, writeGoalAuditReceipt } from "./goal-audit.js"; export { auditGoalCompletion, writeGoalAuditReceipt } from "./goal-audit.js";
export type { GoalAuditCriterion, GoalAuditReceipt } from "./goal-audit.js"; export type { GoalAuditCriterion, GoalAuditReceipt } from "./goal-audit.js";

View File

@ -0,0 +1,44 @@
import * as fs from "node:fs";
import * as os from "node:os";
import * as path from "node:path";
import { describe, expect, it } from "vitest";
import { createModelAccessReceipt, writeModelAccessReceipt } from "./model-access-receipt.js";
describe("model access receipt", () => {
it("verifies available model surfaces with freshness", () => {
const receipt = createModelAccessReceipt({
now: new Date("2026-06-27T00:00:00.000Z"),
freshnessHours: 6,
probes: [
{ surface: "openrouter", command: "fixture", status: "available", model: "openai/codex", evidence: "API_OK" },
],
});
expect(receipt.schema).toBe("fable.model_access.receipt.v1");
expect(receipt.decision).toBe("verified");
expect(receipt.freshness.expiresAt).toBe("2026-06-27T06:00:00.000Z");
expect(receipt.claims).toEqual(["openrouter:openai/codex available"]);
});
it("does not claim ChatGPT Cyber when blocked", () => {
const receipt = createModelAccessReceipt({
probes: [
{ surface: "chatgpt-cyber", command: "curl https://chatgpt.com/cyber", status: "blocked", evidence: "403 Forbidden" },
{ surface: "pi", command: "pi --list-models cyber", status: "missing", evidence: "No models matching cyber" },
],
});
expect(receipt.decision).toBe("blocked");
expect(receipt.claims).toEqual([]);
expect(receipt.reasons).toContain("ChatGPT Cyber access not verified");
});
it("writes a JSON receipt", () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "model-access-"));
const file = path.join(dir, "receipt.json");
writeModelAccessReceipt(file, createModelAccessReceipt({ probes: [{ surface: "other", command: "echo ok", status: "available", evidence: "ok" }] }));
expect(fs.readFileSync(file, "utf-8")).toContain("fable.model_access.receipt.v1");
});
});

View File

@ -0,0 +1,61 @@
import * as fs from "node:fs";
import * as path from "node:path";
export type ModelAccessSurface = "pi" | "openrouter" | "openai" | "chatgpt-cyber" | "other";
export type ModelAccessStatus = "available" | "blocked" | "missing" | "unknown";
export interface ModelAccessProbe {
surface: ModelAccessSurface;
command: string;
status: ModelAccessStatus;
evidence: string;
model?: string;
}
export interface ModelAccessReceiptOptions {
probes: ModelAccessProbe[];
output?: string;
freshnessHours?: number;
now?: Date;
}
export interface ModelAccessReceipt {
schema: "fable.model_access.receipt.v1";
createdAt: string;
probes: ModelAccessProbe[];
summary: Record<ModelAccessStatus, number>;
freshness: { hours: number; expiresAt: string };
decision: "verified" | "partial" | "blocked";
claims: string[];
reasons: string[];
}
export function createModelAccessReceipt(opts: ModelAccessReceiptOptions): ModelAccessReceipt {
const now = opts.now ?? new Date();
const freshnessHours = opts.freshnessHours ?? 24;
const summary = opts.probes.reduce(
(acc, probe) => ({ ...acc, [probe.status]: acc[probe.status] + 1 }),
{ available: 0, blocked: 0, missing: 0, unknown: 0 },
);
const reasons: string[] = [];
if (opts.probes.length === 0) reasons.push("no model access probes supplied");
if (opts.probes.some((probe) => probe.surface === "chatgpt-cyber" && probe.status !== "available")) reasons.push("ChatGPT Cyber access not verified");
if (summary.unknown > 0) reasons.push("one or more model surfaces are unknown");
return {
schema: "fable.model_access.receipt.v1",
createdAt: now.toISOString(),
probes: opts.probes,
summary,
freshness: { hours: freshnessHours, expiresAt: new Date(now.getTime() + freshnessHours * 60 * 60 * 1000).toISOString() },
decision: opts.probes.length === 0 || summary.available === 0 ? "blocked" : reasons.length ? "partial" : "verified",
claims: opts.probes.filter((probe) => probe.status === "available").map((probe) => `${probe.surface}${probe.model ? `:${probe.model}` : ""} available`),
reasons,
};
}
export function writeModelAccessReceipt(file: string, receipt: ModelAccessReceipt): string {
fs.mkdirSync(path.dirname(file), { recursive: true });
fs.writeFileSync(file, `${JSON.stringify(receipt, null, 2)}\n`);
return file;
}

View File

@ -1855,6 +1855,31 @@ fable
console.log(` `); console.log(` `);
}); });
fable
.command("model-access")
.description("Write a fail-closed model access receipt")
.option("--output <path>", "Receipt output path", path.join(".fable", "api", "model-access-live.json"))
.option("--run <id>", "Emit receipt event to .runs/<id>/channel.jsonl")
.action(async (opts: { output?: string; run?: string }) => {
const { createModelAccessReceipt, writeModelAccessReceipt } = await import("./fable5/model-access-receipt.js");
// ponytail: env presence is not access proof; external probes can overwrite this receipt with live evidence.
const probes = [
{ surface: "openrouter" as const, command: "env OPENROUTER_API_KEY", status: process.env.OPENROUTER_API_KEY ? "unknown" as const : "missing" as const, evidence: process.env.OPENROUTER_API_KEY ? "key present; API not probed" : "OPENROUTER_API_KEY missing" },
{ surface: "openai" as const, command: "env OPENAI_API_KEY", status: process.env.OPENAI_API_KEY ? "unknown" as const : "missing" as const, evidence: process.env.OPENAI_API_KEY ? "key present; API not probed" : "OPENAI_API_KEY missing" },
{ surface: "pi" as const, command: "pi --list-models cyber", status: "unknown" as const, evidence: "not probed by this fail-closed receipt command" },
{ surface: "chatgpt-cyber" as const, command: "https://chatgpt.com/cyber", status: "unknown" as const, evidence: "browser/login-gated surface not probed by this command" },
];
const out = opts.output ?? path.join(".fable", "api", "model-access-live.json");
const receipt = createModelAccessReceipt({ probes });
writeModelAccessReceipt(out, receipt);
console.log(JSON.stringify(receipt, null, 2));
console.log(`
Receipt: ${out}
`);
await emitRunEvent(opts.run, { source: "gate", type: receipt.decision === "blocked" ? "error" : "receipt", data: { kind: "model-access", receipt, path: out } });
if (receipt.decision === "blocked") process.exit(1);
});
fable fable
.command("flue") .command("flue")
.description("Show Flue-inspired interop gaps without vendoring Flue") .description("Show Flue-inspired interop gaps without vendoring Flue")