feat: add skill health receipts
This commit is contained in:
parent
ac800d106f
commit
0030455324
11
COMMANDS.md
11
COMMANDS.md
|
|
@ -328,6 +328,17 @@ fable-agent plinius godmode "improve explanation quality"
|
|||
- `--run <id>`
|
||||
- `fable5 memory-b-cell <file>`
|
||||
- `--cache <path>`
|
||||
- `fable5 skill-health`
|
||||
- `--input <path>`
|
||||
- `--skill <name>`
|
||||
- `--agent <name>`
|
||||
- `--test-command <command>`
|
||||
- `--passed <n>`
|
||||
- `--failed <n>`
|
||||
- `--evidence <text>`
|
||||
- `--source <source>`
|
||||
- `--threshold <n>`
|
||||
- `--output <path>`
|
||||
- `fable5 receipt-health`
|
||||
- `--root <path>`
|
||||
- `--window-minutes <n>`
|
||||
|
|
|
|||
|
|
@ -93,6 +93,9 @@ export type { RedTeamHarnessDecision, RedTeamHarnessReceipt, RedTeamHarnessRecei
|
|||
export { createModelAccessReceipt, writeModelAccessReceipt } from "./model-access-receipt.js";
|
||||
export type { ModelAccessProbe, ModelAccessReceipt, ModelAccessReceiptOptions, ModelAccessStatus, ModelAccessSurface } from "./model-access-receipt.js";
|
||||
|
||||
export { createSkillHealthReceipt, writeSkillHealthReceipt } from "./skill-health.js";
|
||||
export type { SkillHealthEntry, SkillHealthOptions, SkillHealthReceipt } from "./skill-health.js";
|
||||
|
||||
export { createPlanReceipt, writePlanReceipt } from "./plan-receipt.js";
|
||||
export type { PlanAllowedOutcome, PlanPhase, PlanReceipt, PlanReceiptOptions, PlanTaskType } from "./plan-receipt.js";
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,52 @@
|
|||
import * as fs from "node:fs";
|
||||
import * as os from "node:os";
|
||||
import * as path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { createSkillHealthReceipt, writeSkillHealthReceipt } from "./skill-health.js";
|
||||
|
||||
describe("skill health receipt", () => {
|
||||
it("marks all passing skills ready without auto-patch or deploy authority", () => {
|
||||
const receipt = createSkillHealthReceipt({
|
||||
now: new Date("2026-07-01T00:00:00.000Z"),
|
||||
source: "factory",
|
||||
skills: [{ skill: "rsi_canary", agent: "hermes", testCommand: "test.sh", passed: 1, failed: 0, passRate: 0, evidence: "local test passed" }],
|
||||
});
|
||||
|
||||
expect(receipt).toMatchObject({
|
||||
schema: "fable.skill_health.receipt.v1",
|
||||
createdAt: "2026-07-01T00:00:00.000Z",
|
||||
totalSkills: 1,
|
||||
healthy: 1,
|
||||
degraded: 0,
|
||||
decision: "ready",
|
||||
autoPatchAttempted: false,
|
||||
deployAttempted: false,
|
||||
});
|
||||
expect(receipt.skills[0]).toMatchObject({ passRate: 1, status: "healthy" });
|
||||
});
|
||||
|
||||
it("degrades below threshold instead of claiming repair", () => {
|
||||
const receipt = createSkillHealthReceipt({
|
||||
threshold: 0.8,
|
||||
skills: [{ skill: "bad", passed: 3, failed: 2, passRate: 0 }],
|
||||
});
|
||||
|
||||
expect(receipt).toMatchObject({ degraded: 1, decision: "degraded", autoPatchAttempted: false, deployAttempted: false });
|
||||
expect(receipt.reasons[0]).toContain("below 80% pass threshold");
|
||||
});
|
||||
|
||||
it("blocks empty evidence", () => {
|
||||
expect(createSkillHealthReceipt({ skills: [] })).toMatchObject({ decision: "blocked", reasons: ["no skill health evidence supplied"] });
|
||||
});
|
||||
|
||||
it("writes JSON receipts", () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "skill-health-"));
|
||||
const file = path.join(dir, "receipt.json");
|
||||
const receipt = createSkillHealthReceipt({ skills: [{ skill: "ok", passed: 1, failed: 0, passRate: 0 }] });
|
||||
|
||||
writeSkillHealthReceipt(file, receipt);
|
||||
|
||||
expect(JSON.parse(fs.readFileSync(file, "utf-8"))).toMatchObject({ schema: "fable.skill_health.receipt.v1", decision: "ready" });
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
|
||||
export interface SkillHealthInput {
|
||||
skill: string;
|
||||
agent?: string;
|
||||
testCommand?: string;
|
||||
passed: number;
|
||||
failed: number;
|
||||
passRate?: number;
|
||||
evidence?: string;
|
||||
}
|
||||
|
||||
export interface SkillHealthEntry extends SkillHealthInput {
|
||||
passRate: number;
|
||||
status: "healthy" | "degraded";
|
||||
}
|
||||
|
||||
export interface SkillHealthReceipt {
|
||||
schema: "fable.skill_health.receipt.v1";
|
||||
createdAt: string;
|
||||
source: "manual" | "factory" | "cron";
|
||||
totalSkills: number;
|
||||
healthy: number;
|
||||
degraded: number;
|
||||
threshold: number;
|
||||
skills: SkillHealthEntry[];
|
||||
decision: "ready" | "degraded" | "blocked";
|
||||
reasons: string[];
|
||||
autoPatchAttempted: false;
|
||||
deployAttempted: false;
|
||||
}
|
||||
|
||||
export interface SkillHealthOptions {
|
||||
skills: SkillHealthInput[];
|
||||
source?: SkillHealthReceipt["source"];
|
||||
threshold?: number;
|
||||
now?: Date;
|
||||
}
|
||||
|
||||
export function createSkillHealthReceipt(opts: SkillHealthOptions): SkillHealthReceipt {
|
||||
const threshold = opts.threshold ?? 0.8;
|
||||
const skills = opts.skills.map((skill) => {
|
||||
const total = skill.passed + skill.failed;
|
||||
const passRate = total === 0 ? skill.passRate ?? 0 : skill.passed / total;
|
||||
return { ...skill, passRate, status: passRate >= threshold ? "healthy" as const : "degraded" as const };
|
||||
});
|
||||
const degraded = skills.filter((skill) => skill.status === "degraded").length;
|
||||
const reasons: string[] = [];
|
||||
if (skills.length === 0) reasons.push("no skill health evidence supplied");
|
||||
if (degraded > 0) reasons.push(`${degraded} skill(s) below ${(threshold * 100).toFixed(0)}% pass threshold`);
|
||||
|
||||
return {
|
||||
schema: "fable.skill_health.receipt.v1",
|
||||
createdAt: (opts.now ?? new Date()).toISOString(),
|
||||
source: opts.source ?? "manual",
|
||||
totalSkills: skills.length,
|
||||
healthy: skills.length - degraded,
|
||||
degraded,
|
||||
threshold,
|
||||
skills,
|
||||
decision: skills.length === 0 ? "blocked" : degraded > 0 ? "degraded" : "ready",
|
||||
reasons,
|
||||
autoPatchAttempted: false,
|
||||
deployAttempted: false,
|
||||
};
|
||||
}
|
||||
|
||||
export function writeSkillHealthReceipt(file: string, receipt: SkillHealthReceipt): string {
|
||||
fs.mkdirSync(path.dirname(file), { recursive: true });
|
||||
fs.writeFileSync(file, `${JSON.stringify(receipt, null, 2)}\n`);
|
||||
return file;
|
||||
}
|
||||
28
src/index.ts
28
src/index.ts
|
|
@ -2047,6 +2047,34 @@ fable
|
|||
if (result.verdict === "known_unsafe") process.exit(1);
|
||||
});
|
||||
|
||||
fable
|
||||
.command("skill-health")
|
||||
.description("Write a read-only skill health receipt without auto-patch or deploy authority")
|
||||
.option("--input <path>", "JSON array of skill health entries")
|
||||
.option("--skill <name>", "Single skill name")
|
||||
.option("--agent <name>", "Agent responsible for the skill")
|
||||
.option("--test-command <command>", "Command used as evidence")
|
||||
.option("--passed <n>", "Passed checks", (v) => Number(v), 0)
|
||||
.option("--failed <n>", "Failed checks", (v) => Number(v), 0)
|
||||
.option("--evidence <text>", "Evidence summary")
|
||||
.option("--source <source>", "manual|factory|cron", "manual")
|
||||
.option("--threshold <n>", "Pass-rate threshold", (v) => Number(v), 0.8)
|
||||
.option("--output <path>", "Receipt output path", path.join(".fable", "skills", "skill-health-live.json"))
|
||||
.action(async (opts: { input?: string; skill?: string; agent?: string; testCommand?: string; passed?: number; failed?: number; evidence?: string; source?: string; threshold?: number; output?: string }) => {
|
||||
const { createSkillHealthReceipt, writeSkillHealthReceipt } = await import("./fable5/skill-health.js");
|
||||
const skills = opts.input
|
||||
? JSON.parse(fs.readFileSync(opts.input, "utf-8"))
|
||||
: opts.skill
|
||||
? [{ skill: opts.skill, agent: opts.agent, testCommand: opts.testCommand, passed: opts.passed ?? 0, failed: opts.failed ?? 0, passRate: 0, evidence: opts.evidence }]
|
||||
: [];
|
||||
const receipt = createSkillHealthReceipt({ skills, source: opts.source as never, threshold: opts.threshold });
|
||||
const out = opts.output ?? path.join(".fable", "skills", "skill-health-live.json");
|
||||
writeSkillHealthReceipt(out, receipt);
|
||||
console.log(JSON.stringify(receipt, null, 2));
|
||||
console.log(`\n Receipt: ${out}\n`);
|
||||
if (receipt.decision === "blocked") process.exit(1);
|
||||
});
|
||||
|
||||
fable
|
||||
.command("receipt-health")
|
||||
.description("Check receipt consumer health over a recent time window")
|
||||
|
|
|
|||
Loading…
Reference in New Issue