diff --git a/src/fable5/distillation-receipt.test.ts b/src/fable5/distillation-receipt.test.ts new file mode 100644 index 0000000..d8e3113 --- /dev/null +++ b/src/fable5/distillation-receipt.test.ts @@ -0,0 +1,41 @@ +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { describe, expect, it } from "vitest"; +import { createDistillationReceipt, writeDistillationReceipt } from "./distillation-receipt.js"; + +describe("distillation receipt", () => { + it("allows open-weight teacher provenance", () => { + const receipt = createDistillationReceipt({ + dataset: "math_train_sample.json", + output: "distill.json", + teacherModel: "qwen3:4b", + provider: "ollama", + teacherKind: "open-weight", + samples: 10, + now: new Date("2026-06-27T00:00:00.000Z"), + }); + + expect(receipt.schema).toBe("fable.distillation.receipt.v1"); + expect(receipt.decision).toBe("ok"); + expect(receipt.reasons).toEqual([]); + }); + + it("flags closed or unknown teachers for review", () => { + const closed = createDistillationReceipt({ dataset: "x", output: "y", teacherModel: "claude", provider: "openrouter", teacherKind: "closed-frontier" }); + const unknown = createDistillationReceipt({ dataset: "x", output: "y", teacherModel: "mystery", provider: "other" }); + + expect(closed.decision).toBe("needs-review"); + expect(closed.reasons.join("\n")).toContain("closed-frontier teacher output"); + expect(unknown.decision).toBe("needs-review"); + expect(unknown.reasons.join("\n")).toContain("teacher kind unknown"); + }); + + it("writes a JSON receipt", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "distill-receipt-")); + const file = path.join(dir, "receipt.json"); + writeDistillationReceipt(file, createDistillationReceipt({ dataset: "x", output: "y", teacherModel: "qwen", provider: "ollama", teacherKind: "open-weight" })); + + expect(fs.readFileSync(file, "utf-8")).toContain("fable.distillation.receipt.v1"); + }); +}); diff --git a/src/fable5/distillation-receipt.ts b/src/fable5/distillation-receipt.ts new file mode 100644 index 0000000..4a439b8 --- /dev/null +++ b/src/fable5/distillation-receipt.ts @@ -0,0 +1,55 @@ +import * as fs from "node:fs"; +import * as path from "node:path"; + +export type DistillationTeacherKind = "open-weight" | "closed-frontier" | "unknown"; +export type DistillationProvider = "ollama" | "openrouter" | "other"; + +export interface DistillationReceiptOptions { + dataset: string; + output: string; + teacherModel: string; + provider: DistillationProvider; + teacherKind?: DistillationTeacherKind; + samples?: number; + now?: Date; +} + +export interface DistillationReceipt { + schema: "fable.distillation.receipt.v1"; + createdAt: string; + dataset: string; + output: string; + teacherModel: string; + provider: DistillationProvider; + teacherKind: DistillationTeacherKind; + samples?: number; + decision: "ok" | "needs-review"; + reasons: string[]; +} + +export function createDistillationReceipt(opts: DistillationReceiptOptions): DistillationReceipt { + const teacherKind = opts.teacherKind ?? "unknown"; + const reasons: string[] = []; + if (teacherKind === "closed-frontier") reasons.push("closed-frontier teacher output requires explicit terms/provenance approval before training"); + if (teacherKind === "unknown") reasons.push("teacher kind unknown; do not claim compliant/open distillation"); + if (opts.provider === "openrouter" && teacherKind !== "open-weight") reasons.push("OpenRouter can route many model types; record the exact teacher model and policy posture"); + + return { + schema: "fable.distillation.receipt.v1", + createdAt: (opts.now ?? new Date()).toISOString(), + dataset: opts.dataset, + output: opts.output, + teacherModel: opts.teacherModel, + provider: opts.provider, + teacherKind, + samples: opts.samples, + decision: reasons.length ? "needs-review" : "ok", + reasons, + }; +} + +export function writeDistillationReceipt(file: string, receipt: DistillationReceipt): string { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, `${JSON.stringify(receipt, null, 2)}\n`); + return file; +} diff --git a/src/fable5/index.ts b/src/fable5/index.ts index ba83ad4..71500b5 100644 --- a/src/fable5/index.ts +++ b/src/fable5/index.ts @@ -72,7 +72,10 @@ export { createDuelEvalReceipt, createEvalTraceReceipt, gradeSetRecovery, tallyD export type { DuelEvalReceipt, DuelTally, EvalTraceCommand, EvalTraceReceipt, SetGradeReceipt } from "./eval-trace.js"; export { loadBenchHarnesses, runAgentBench, writeAgentBenchReceipt } from "./agent-bench.js"; -export type { AgentBenchOptions, AgentBenchReceipt, BenchContenderId, BenchContenderResult, BenchHarnessConfig } from "./agent-bench.js"; +export type { AgentBenchOptions, AgentBenchReceipt, BenchContenderId, BenchContenderResult, BenchHarnessConfig, BenchmarkHygiene } from "./agent-bench.js"; + +export { createDistillationReceipt, writeDistillationReceipt } from "./distillation-receipt.js"; +export type { DistillationProvider, DistillationReceipt, DistillationReceiptOptions, DistillationTeacherKind } from "./distillation-receipt.js"; export { auditGoalCompletion, writeGoalAuditReceipt } from "./goal-audit.js"; export type { GoalAuditCriterion, GoalAuditReceipt } from "./goal-audit.js";