diff --git a/COMMANDS.md b/COMMANDS.md index 41bd71a..6bd891f 100644 --- a/COMMANDS.md +++ b/COMMANDS.md @@ -328,6 +328,17 @@ fable-agent plinius godmode "improve explanation quality" - `--run ` - `fable5 memory-b-cell ` - `--cache ` +- `fable5 skill-health` + - `--input ` + - `--skill ` + - `--agent ` + - `--test-command ` + - `--passed ` + - `--failed ` + - `--evidence ` + - `--source ` + - `--threshold ` + - `--output ` - `fable5 receipt-health` - `--root ` - `--window-minutes ` diff --git a/src/fable5/index.ts b/src/fable5/index.ts index ce1df0a..48857c3 100644 --- a/src/fable5/index.ts +++ b/src/fable5/index.ts @@ -93,6 +93,9 @@ export type { RedTeamHarnessDecision, RedTeamHarnessReceipt, RedTeamHarnessRecei export { createModelAccessReceipt, writeModelAccessReceipt } from "./model-access-receipt.js"; export type { ModelAccessProbe, ModelAccessReceipt, ModelAccessReceiptOptions, ModelAccessStatus, ModelAccessSurface } from "./model-access-receipt.js"; +export { createSkillHealthReceipt, writeSkillHealthReceipt } from "./skill-health.js"; +export type { SkillHealthEntry, SkillHealthOptions, SkillHealthReceipt } from "./skill-health.js"; + export { createPlanReceipt, writePlanReceipt } from "./plan-receipt.js"; export type { PlanAllowedOutcome, PlanPhase, PlanReceipt, PlanReceiptOptions, PlanTaskType } from "./plan-receipt.js"; diff --git a/src/fable5/skill-health.test.ts b/src/fable5/skill-health.test.ts new file mode 100644 index 0000000..54cfe3b --- /dev/null +++ b/src/fable5/skill-health.test.ts @@ -0,0 +1,52 @@ +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { describe, expect, it } from "vitest"; +import { createSkillHealthReceipt, writeSkillHealthReceipt } from "./skill-health.js"; + +describe("skill health receipt", () => { + it("marks all passing skills ready without auto-patch or deploy authority", () => { + const receipt = createSkillHealthReceipt({ + now: new Date("2026-07-01T00:00:00.000Z"), + source: "factory", + skills: [{ skill: "rsi_canary", agent: "hermes", testCommand: "test.sh", passed: 1, failed: 0, passRate: 0, evidence: "local test passed" }], + }); + + expect(receipt).toMatchObject({ + schema: "fable.skill_health.receipt.v1", + createdAt: "2026-07-01T00:00:00.000Z", + totalSkills: 1, + healthy: 1, + degraded: 0, + decision: "ready", + autoPatchAttempted: false, + deployAttempted: false, + }); + expect(receipt.skills[0]).toMatchObject({ passRate: 1, status: "healthy" }); + }); + + it("degrades below threshold instead of claiming repair", () => { + const receipt = createSkillHealthReceipt({ + threshold: 0.8, + skills: [{ skill: "bad", passed: 3, failed: 2, passRate: 0 }], + }); + + expect(receipt).toMatchObject({ degraded: 1, decision: "degraded", autoPatchAttempted: false, deployAttempted: false }); + expect(receipt.reasons[0]).toContain("below 80% pass threshold"); + }); + + it("blocks empty evidence", () => { + expect(createSkillHealthReceipt({ skills: [] })).toMatchObject({ decision: "blocked", reasons: ["no skill health evidence supplied"] }); + }); + + it("writes JSON receipts", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "skill-health-")); + const file = path.join(dir, "receipt.json"); + const receipt = createSkillHealthReceipt({ skills: [{ skill: "ok", passed: 1, failed: 0, passRate: 0 }] }); + + writeSkillHealthReceipt(file, receipt); + + expect(JSON.parse(fs.readFileSync(file, "utf-8"))).toMatchObject({ schema: "fable.skill_health.receipt.v1", decision: "ready" }); + fs.rmSync(dir, { recursive: true, force: true }); + }); +}); diff --git a/src/fable5/skill-health.ts b/src/fable5/skill-health.ts new file mode 100644 index 0000000..c2f35eb --- /dev/null +++ b/src/fable5/skill-health.ts @@ -0,0 +1,73 @@ +import * as fs from "node:fs"; +import * as path from "node:path"; + +export interface SkillHealthInput { + skill: string; + agent?: string; + testCommand?: string; + passed: number; + failed: number; + passRate?: number; + evidence?: string; +} + +export interface SkillHealthEntry extends SkillHealthInput { + passRate: number; + status: "healthy" | "degraded"; +} + +export interface SkillHealthReceipt { + schema: "fable.skill_health.receipt.v1"; + createdAt: string; + source: "manual" | "factory" | "cron"; + totalSkills: number; + healthy: number; + degraded: number; + threshold: number; + skills: SkillHealthEntry[]; + decision: "ready" | "degraded" | "blocked"; + reasons: string[]; + autoPatchAttempted: false; + deployAttempted: false; +} + +export interface SkillHealthOptions { + skills: SkillHealthInput[]; + source?: SkillHealthReceipt["source"]; + threshold?: number; + now?: Date; +} + +export function createSkillHealthReceipt(opts: SkillHealthOptions): SkillHealthReceipt { + const threshold = opts.threshold ?? 0.8; + const skills = opts.skills.map((skill) => { + const total = skill.passed + skill.failed; + const passRate = total === 0 ? skill.passRate ?? 0 : skill.passed / total; + return { ...skill, passRate, status: passRate >= threshold ? "healthy" as const : "degraded" as const }; + }); + const degraded = skills.filter((skill) => skill.status === "degraded").length; + const reasons: string[] = []; + if (skills.length === 0) reasons.push("no skill health evidence supplied"); + if (degraded > 0) reasons.push(`${degraded} skill(s) below ${(threshold * 100).toFixed(0)}% pass threshold`); + + return { + schema: "fable.skill_health.receipt.v1", + createdAt: (opts.now ?? new Date()).toISOString(), + source: opts.source ?? "manual", + totalSkills: skills.length, + healthy: skills.length - degraded, + degraded, + threshold, + skills, + decision: skills.length === 0 ? "blocked" : degraded > 0 ? "degraded" : "ready", + reasons, + autoPatchAttempted: false, + deployAttempted: false, + }; +} + +export function writeSkillHealthReceipt(file: string, receipt: SkillHealthReceipt): string { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, `${JSON.stringify(receipt, null, 2)}\n`); + return file; +} diff --git a/src/index.ts b/src/index.ts index 1c92a74..31c3766 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2047,6 +2047,34 @@ fable if (result.verdict === "known_unsafe") process.exit(1); }); +fable + .command("skill-health") + .description("Write a read-only skill health receipt without auto-patch or deploy authority") + .option("--input ", "JSON array of skill health entries") + .option("--skill ", "Single skill name") + .option("--agent ", "Agent responsible for the skill") + .option("--test-command ", "Command used as evidence") + .option("--passed ", "Passed checks", (v) => Number(v), 0) + .option("--failed ", "Failed checks", (v) => Number(v), 0) + .option("--evidence ", "Evidence summary") + .option("--source ", "manual|factory|cron", "manual") + .option("--threshold ", "Pass-rate threshold", (v) => Number(v), 0.8) + .option("--output ", "Receipt output path", path.join(".fable", "skills", "skill-health-live.json")) + .action(async (opts: { input?: string; skill?: string; agent?: string; testCommand?: string; passed?: number; failed?: number; evidence?: string; source?: string; threshold?: number; output?: string }) => { + const { createSkillHealthReceipt, writeSkillHealthReceipt } = await import("./fable5/skill-health.js"); + const skills = opts.input + ? JSON.parse(fs.readFileSync(opts.input, "utf-8")) + : opts.skill + ? [{ skill: opts.skill, agent: opts.agent, testCommand: opts.testCommand, passed: opts.passed ?? 0, failed: opts.failed ?? 0, passRate: 0, evidence: opts.evidence }] + : []; + const receipt = createSkillHealthReceipt({ skills, source: opts.source as never, threshold: opts.threshold }); + const out = opts.output ?? path.join(".fable", "skills", "skill-health-live.json"); + writeSkillHealthReceipt(out, receipt); + console.log(JSON.stringify(receipt, null, 2)); + console.log(`\n Receipt: ${out}\n`); + if (receipt.decision === "blocked") process.exit(1); + }); + fable .command("receipt-health") .description("Check receipt consumer health over a recent time window")