feat: add skill health receipts

This commit is contained in:
artale 2026-07-01 05:17:20 +02:00
parent ac800d106f
commit 0030455324
5 changed files with 167 additions and 0 deletions

View File

@ -328,6 +328,17 @@ fable-agent plinius godmode "improve explanation quality"
- `--run <id>` - `--run <id>`
- `fable5 memory-b-cell <file>` - `fable5 memory-b-cell <file>`
- `--cache <path>` - `--cache <path>`
- `fable5 skill-health`
- `--input <path>`
- `--skill <name>`
- `--agent <name>`
- `--test-command <command>`
- `--passed <n>`
- `--failed <n>`
- `--evidence <text>`
- `--source <source>`
- `--threshold <n>`
- `--output <path>`
- `fable5 receipt-health` - `fable5 receipt-health`
- `--root <path>` - `--root <path>`
- `--window-minutes <n>` - `--window-minutes <n>`

View File

@ -93,6 +93,9 @@ export type { RedTeamHarnessDecision, RedTeamHarnessReceipt, RedTeamHarnessRecei
export { createModelAccessReceipt, writeModelAccessReceipt } from "./model-access-receipt.js"; export { createModelAccessReceipt, writeModelAccessReceipt } from "./model-access-receipt.js";
export type { ModelAccessProbe, ModelAccessReceipt, ModelAccessReceiptOptions, ModelAccessStatus, ModelAccessSurface } from "./model-access-receipt.js"; export type { ModelAccessProbe, ModelAccessReceipt, ModelAccessReceiptOptions, ModelAccessStatus, ModelAccessSurface } from "./model-access-receipt.js";
export { createSkillHealthReceipt, writeSkillHealthReceipt } from "./skill-health.js";
export type { SkillHealthEntry, SkillHealthOptions, SkillHealthReceipt } from "./skill-health.js";
export { createPlanReceipt, writePlanReceipt } from "./plan-receipt.js"; export { createPlanReceipt, writePlanReceipt } from "./plan-receipt.js";
export type { PlanAllowedOutcome, PlanPhase, PlanReceipt, PlanReceiptOptions, PlanTaskType } from "./plan-receipt.js"; export type { PlanAllowedOutcome, PlanPhase, PlanReceipt, PlanReceiptOptions, PlanTaskType } from "./plan-receipt.js";

View File

@ -0,0 +1,52 @@
import * as fs from "node:fs";
import * as os from "node:os";
import * as path from "node:path";
import { describe, expect, it } from "vitest";
import { createSkillHealthReceipt, writeSkillHealthReceipt } from "./skill-health.js";
describe("skill health receipt", () => {
it("marks all passing skills ready without auto-patch or deploy authority", () => {
const receipt = createSkillHealthReceipt({
now: new Date("2026-07-01T00:00:00.000Z"),
source: "factory",
skills: [{ skill: "rsi_canary", agent: "hermes", testCommand: "test.sh", passed: 1, failed: 0, passRate: 0, evidence: "local test passed" }],
});
expect(receipt).toMatchObject({
schema: "fable.skill_health.receipt.v1",
createdAt: "2026-07-01T00:00:00.000Z",
totalSkills: 1,
healthy: 1,
degraded: 0,
decision: "ready",
autoPatchAttempted: false,
deployAttempted: false,
});
expect(receipt.skills[0]).toMatchObject({ passRate: 1, status: "healthy" });
});
it("degrades below threshold instead of claiming repair", () => {
const receipt = createSkillHealthReceipt({
threshold: 0.8,
skills: [{ skill: "bad", passed: 3, failed: 2, passRate: 0 }],
});
expect(receipt).toMatchObject({ degraded: 1, decision: "degraded", autoPatchAttempted: false, deployAttempted: false });
expect(receipt.reasons[0]).toContain("below 80% pass threshold");
});
it("blocks empty evidence", () => {
expect(createSkillHealthReceipt({ skills: [] })).toMatchObject({ decision: "blocked", reasons: ["no skill health evidence supplied"] });
});
it("writes JSON receipts", () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "skill-health-"));
const file = path.join(dir, "receipt.json");
const receipt = createSkillHealthReceipt({ skills: [{ skill: "ok", passed: 1, failed: 0, passRate: 0 }] });
writeSkillHealthReceipt(file, receipt);
expect(JSON.parse(fs.readFileSync(file, "utf-8"))).toMatchObject({ schema: "fable.skill_health.receipt.v1", decision: "ready" });
fs.rmSync(dir, { recursive: true, force: true });
});
});

View File

@ -0,0 +1,73 @@
import * as fs from "node:fs";
import * as path from "node:path";
export interface SkillHealthInput {
skill: string;
agent?: string;
testCommand?: string;
passed: number;
failed: number;
passRate?: number;
evidence?: string;
}
export interface SkillHealthEntry extends SkillHealthInput {
passRate: number;
status: "healthy" | "degraded";
}
export interface SkillHealthReceipt {
schema: "fable.skill_health.receipt.v1";
createdAt: string;
source: "manual" | "factory" | "cron";
totalSkills: number;
healthy: number;
degraded: number;
threshold: number;
skills: SkillHealthEntry[];
decision: "ready" | "degraded" | "blocked";
reasons: string[];
autoPatchAttempted: false;
deployAttempted: false;
}
export interface SkillHealthOptions {
skills: SkillHealthInput[];
source?: SkillHealthReceipt["source"];
threshold?: number;
now?: Date;
}
export function createSkillHealthReceipt(opts: SkillHealthOptions): SkillHealthReceipt {
const threshold = opts.threshold ?? 0.8;
const skills = opts.skills.map((skill) => {
const total = skill.passed + skill.failed;
const passRate = total === 0 ? skill.passRate ?? 0 : skill.passed / total;
return { ...skill, passRate, status: passRate >= threshold ? "healthy" as const : "degraded" as const };
});
const degraded = skills.filter((skill) => skill.status === "degraded").length;
const reasons: string[] = [];
if (skills.length === 0) reasons.push("no skill health evidence supplied");
if (degraded > 0) reasons.push(`${degraded} skill(s) below ${(threshold * 100).toFixed(0)}% pass threshold`);
return {
schema: "fable.skill_health.receipt.v1",
createdAt: (opts.now ?? new Date()).toISOString(),
source: opts.source ?? "manual",
totalSkills: skills.length,
healthy: skills.length - degraded,
degraded,
threshold,
skills,
decision: skills.length === 0 ? "blocked" : degraded > 0 ? "degraded" : "ready",
reasons,
autoPatchAttempted: false,
deployAttempted: false,
};
}
export function writeSkillHealthReceipt(file: string, receipt: SkillHealthReceipt): string {
fs.mkdirSync(path.dirname(file), { recursive: true });
fs.writeFileSync(file, `${JSON.stringify(receipt, null, 2)}\n`);
return file;
}

View File

@ -2047,6 +2047,34 @@ fable
if (result.verdict === "known_unsafe") process.exit(1); if (result.verdict === "known_unsafe") process.exit(1);
}); });
fable
.command("skill-health")
.description("Write a read-only skill health receipt without auto-patch or deploy authority")
.option("--input <path>", "JSON array of skill health entries")
.option("--skill <name>", "Single skill name")
.option("--agent <name>", "Agent responsible for the skill")
.option("--test-command <command>", "Command used as evidence")
.option("--passed <n>", "Passed checks", (v) => Number(v), 0)
.option("--failed <n>", "Failed checks", (v) => Number(v), 0)
.option("--evidence <text>", "Evidence summary")
.option("--source <source>", "manual|factory|cron", "manual")
.option("--threshold <n>", "Pass-rate threshold", (v) => Number(v), 0.8)
.option("--output <path>", "Receipt output path", path.join(".fable", "skills", "skill-health-live.json"))
.action(async (opts: { input?: string; skill?: string; agent?: string; testCommand?: string; passed?: number; failed?: number; evidence?: string; source?: string; threshold?: number; output?: string }) => {
const { createSkillHealthReceipt, writeSkillHealthReceipt } = await import("./fable5/skill-health.js");
const skills = opts.input
? JSON.parse(fs.readFileSync(opts.input, "utf-8"))
: opts.skill
? [{ skill: opts.skill, agent: opts.agent, testCommand: opts.testCommand, passed: opts.passed ?? 0, failed: opts.failed ?? 0, passRate: 0, evidence: opts.evidence }]
: [];
const receipt = createSkillHealthReceipt({ skills, source: opts.source as never, threshold: opts.threshold });
const out = opts.output ?? path.join(".fable", "skills", "skill-health-live.json");
writeSkillHealthReceipt(out, receipt);
console.log(JSON.stringify(receipt, null, 2));
console.log(`\n Receipt: ${out}\n`);
if (receipt.decision === "blocked") process.exit(1);
});
fable fable
.command("receipt-health") .command("receipt-health")
.description("Check receipt consumer health over a recent time window") .description("Check receipt consumer health over a recent time window")