feat: add model access receipts
This commit is contained in:
parent
265c42edbb
commit
3c4684f4c5
|
|
@ -291,6 +291,9 @@ fable-agent plinius godmode "improve explanation quality"
|
||||||
- `fable5 route <task>`
|
- `fable5 route <task>`
|
||||||
- `-d, --domain <domain>`
|
- `-d, --domain <domain>`
|
||||||
- `fable5 models`
|
- `fable5 models`
|
||||||
|
- `fable5 model-access`
|
||||||
|
- `--output <path>`
|
||||||
|
- `--run <id>`
|
||||||
- `fable5 flue`
|
- `fable5 flue`
|
||||||
- `fable5 channels`
|
- `fable5 channels`
|
||||||
- `--run <id>`
|
- `--run <id>`
|
||||||
|
|
|
||||||
|
|
@ -90,6 +90,9 @@ export type { SecurityReportOptions, SecurityReportReceipt, SecurityReportSeveri
|
||||||
export { createRedTeamHarnessReceipt, createRedTeamHarnessReceiptFromParseltongue, writeRedTeamHarnessReceipt } from "./red-team-harness-receipt.js";
|
export { createRedTeamHarnessReceipt, createRedTeamHarnessReceiptFromParseltongue, writeRedTeamHarnessReceipt } from "./red-team-harness-receipt.js";
|
||||||
export type { RedTeamHarnessDecision, RedTeamHarnessReceipt, RedTeamHarnessReceiptOptions } from "./red-team-harness-receipt.js";
|
export type { RedTeamHarnessDecision, RedTeamHarnessReceipt, RedTeamHarnessReceiptOptions } from "./red-team-harness-receipt.js";
|
||||||
|
|
||||||
|
export { createModelAccessReceipt, writeModelAccessReceipt } from "./model-access-receipt.js";
|
||||||
|
export type { ModelAccessProbe, ModelAccessReceipt, ModelAccessReceiptOptions, ModelAccessStatus, ModelAccessSurface } from "./model-access-receipt.js";
|
||||||
|
|
||||||
export { auditGoalCompletion, writeGoalAuditReceipt } from "./goal-audit.js";
|
export { auditGoalCompletion, writeGoalAuditReceipt } from "./goal-audit.js";
|
||||||
export type { GoalAuditCriterion, GoalAuditReceipt } from "./goal-audit.js";
|
export type { GoalAuditCriterion, GoalAuditReceipt } from "./goal-audit.js";
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,44 @@
|
||||||
|
import * as fs from "node:fs";
|
||||||
|
import * as os from "node:os";
|
||||||
|
import * as path from "node:path";
|
||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { createModelAccessReceipt, writeModelAccessReceipt } from "./model-access-receipt.js";
|
||||||
|
|
||||||
|
describe("model access receipt", () => {
|
||||||
|
it("verifies available model surfaces with freshness", () => {
|
||||||
|
const receipt = createModelAccessReceipt({
|
||||||
|
now: new Date("2026-06-27T00:00:00.000Z"),
|
||||||
|
freshnessHours: 6,
|
||||||
|
probes: [
|
||||||
|
{ surface: "openrouter", command: "fixture", status: "available", model: "openai/codex", evidence: "API_OK" },
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(receipt.schema).toBe("fable.model_access.receipt.v1");
|
||||||
|
expect(receipt.decision).toBe("verified");
|
||||||
|
expect(receipt.freshness.expiresAt).toBe("2026-06-27T06:00:00.000Z");
|
||||||
|
expect(receipt.claims).toEqual(["openrouter:openai/codex available"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not claim ChatGPT Cyber when blocked", () => {
|
||||||
|
const receipt = createModelAccessReceipt({
|
||||||
|
probes: [
|
||||||
|
{ surface: "chatgpt-cyber", command: "curl https://chatgpt.com/cyber", status: "blocked", evidence: "403 Forbidden" },
|
||||||
|
{ surface: "pi", command: "pi --list-models cyber", status: "missing", evidence: "No models matching cyber" },
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(receipt.decision).toBe("blocked");
|
||||||
|
expect(receipt.claims).toEqual([]);
|
||||||
|
expect(receipt.reasons).toContain("ChatGPT Cyber access not verified");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("writes a JSON receipt", () => {
|
||||||
|
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "model-access-"));
|
||||||
|
const file = path.join(dir, "receipt.json");
|
||||||
|
|
||||||
|
writeModelAccessReceipt(file, createModelAccessReceipt({ probes: [{ surface: "other", command: "echo ok", status: "available", evidence: "ok" }] }));
|
||||||
|
|
||||||
|
expect(fs.readFileSync(file, "utf-8")).toContain("fable.model_access.receipt.v1");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
@ -0,0 +1,61 @@
|
||||||
|
import * as fs from "node:fs";
|
||||||
|
import * as path from "node:path";
|
||||||
|
|
||||||
|
export type ModelAccessSurface = "pi" | "openrouter" | "openai" | "chatgpt-cyber" | "other";
|
||||||
|
export type ModelAccessStatus = "available" | "blocked" | "missing" | "unknown";
|
||||||
|
|
||||||
|
export interface ModelAccessProbe {
|
||||||
|
surface: ModelAccessSurface;
|
||||||
|
command: string;
|
||||||
|
status: ModelAccessStatus;
|
||||||
|
evidence: string;
|
||||||
|
model?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ModelAccessReceiptOptions {
|
||||||
|
probes: ModelAccessProbe[];
|
||||||
|
output?: string;
|
||||||
|
freshnessHours?: number;
|
||||||
|
now?: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ModelAccessReceipt {
|
||||||
|
schema: "fable.model_access.receipt.v1";
|
||||||
|
createdAt: string;
|
||||||
|
probes: ModelAccessProbe[];
|
||||||
|
summary: Record<ModelAccessStatus, number>;
|
||||||
|
freshness: { hours: number; expiresAt: string };
|
||||||
|
decision: "verified" | "partial" | "blocked";
|
||||||
|
claims: string[];
|
||||||
|
reasons: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createModelAccessReceipt(opts: ModelAccessReceiptOptions): ModelAccessReceipt {
|
||||||
|
const now = opts.now ?? new Date();
|
||||||
|
const freshnessHours = opts.freshnessHours ?? 24;
|
||||||
|
const summary = opts.probes.reduce(
|
||||||
|
(acc, probe) => ({ ...acc, [probe.status]: acc[probe.status] + 1 }),
|
||||||
|
{ available: 0, blocked: 0, missing: 0, unknown: 0 },
|
||||||
|
);
|
||||||
|
const reasons: string[] = [];
|
||||||
|
if (opts.probes.length === 0) reasons.push("no model access probes supplied");
|
||||||
|
if (opts.probes.some((probe) => probe.surface === "chatgpt-cyber" && probe.status !== "available")) reasons.push("ChatGPT Cyber access not verified");
|
||||||
|
if (summary.unknown > 0) reasons.push("one or more model surfaces are unknown");
|
||||||
|
|
||||||
|
return {
|
||||||
|
schema: "fable.model_access.receipt.v1",
|
||||||
|
createdAt: now.toISOString(),
|
||||||
|
probes: opts.probes,
|
||||||
|
summary,
|
||||||
|
freshness: { hours: freshnessHours, expiresAt: new Date(now.getTime() + freshnessHours * 60 * 60 * 1000).toISOString() },
|
||||||
|
decision: opts.probes.length === 0 || summary.available === 0 ? "blocked" : reasons.length ? "partial" : "verified",
|
||||||
|
claims: opts.probes.filter((probe) => probe.status === "available").map((probe) => `${probe.surface}${probe.model ? `:${probe.model}` : ""} available`),
|
||||||
|
reasons,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function writeModelAccessReceipt(file: string, receipt: ModelAccessReceipt): string {
|
||||||
|
fs.mkdirSync(path.dirname(file), { recursive: true });
|
||||||
|
fs.writeFileSync(file, `${JSON.stringify(receipt, null, 2)}\n`);
|
||||||
|
return file;
|
||||||
|
}
|
||||||
25
src/index.ts
25
src/index.ts
|
|
@ -1855,6 +1855,31 @@ fable
|
||||||
console.log(` `);
|
console.log(` `);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
fable
|
||||||
|
.command("model-access")
|
||||||
|
.description("Write a fail-closed model access receipt")
|
||||||
|
.option("--output <path>", "Receipt output path", path.join(".fable", "api", "model-access-live.json"))
|
||||||
|
.option("--run <id>", "Emit receipt event to .runs/<id>/channel.jsonl")
|
||||||
|
.action(async (opts: { output?: string; run?: string }) => {
|
||||||
|
const { createModelAccessReceipt, writeModelAccessReceipt } = await import("./fable5/model-access-receipt.js");
|
||||||
|
// ponytail: env presence is not access proof; external probes can overwrite this receipt with live evidence.
|
||||||
|
const probes = [
|
||||||
|
{ surface: "openrouter" as const, command: "env OPENROUTER_API_KEY", status: process.env.OPENROUTER_API_KEY ? "unknown" as const : "missing" as const, evidence: process.env.OPENROUTER_API_KEY ? "key present; API not probed" : "OPENROUTER_API_KEY missing" },
|
||||||
|
{ surface: "openai" as const, command: "env OPENAI_API_KEY", status: process.env.OPENAI_API_KEY ? "unknown" as const : "missing" as const, evidence: process.env.OPENAI_API_KEY ? "key present; API not probed" : "OPENAI_API_KEY missing" },
|
||||||
|
{ surface: "pi" as const, command: "pi --list-models cyber", status: "unknown" as const, evidence: "not probed by this fail-closed receipt command" },
|
||||||
|
{ surface: "chatgpt-cyber" as const, command: "https://chatgpt.com/cyber", status: "unknown" as const, evidence: "browser/login-gated surface not probed by this command" },
|
||||||
|
];
|
||||||
|
const out = opts.output ?? path.join(".fable", "api", "model-access-live.json");
|
||||||
|
const receipt = createModelAccessReceipt({ probes });
|
||||||
|
writeModelAccessReceipt(out, receipt);
|
||||||
|
console.log(JSON.stringify(receipt, null, 2));
|
||||||
|
console.log(`
|
||||||
|
Receipt: ${out}
|
||||||
|
`);
|
||||||
|
await emitRunEvent(opts.run, { source: "gate", type: receipt.decision === "blocked" ? "error" : "receipt", data: { kind: "model-access", receipt, path: out } });
|
||||||
|
if (receipt.decision === "blocked") process.exit(1);
|
||||||
|
});
|
||||||
|
|
||||||
fable
|
fable
|
||||||
.command("flue")
|
.command("flue")
|
||||||
.description("Show Flue-inspired interop gaps without vendoring Flue")
|
.description("Show Flue-inspired interop gaps without vendoring Flue")
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue