fable-agent/src/fable5/verify-cycle.ts

import * as fs from "node:fs";
import * as path from "node:path";
import { spawnSync } from "node:child_process";
import { checkFactory, factoryGateReady, type FactoryCheckResult } from "./factory-status.js";
import { reconcileRsi, type ReceiptRubric, type RsiReconcileOptions, type RsiReconcileReceipt } from "./rsi-reconcile.js";

export interface VerifyCycleOptions {
  host: string;
  port?: number;
  skill?: string;
  out?: string;
  allowDirty?: boolean;
  canaryProof?: string;
  allowReconciledStaleFactory?: boolean;
  now?: Date;
  factoryCheck?: () => Promise<FactoryCheckResult>;
  rsiRunner?: RsiReconcileOptions["runner"];
  runner?: (command: string, args: string[]) => { status: number | null; stdout: string; stderr: string };
}

export interface VerifyCycleEvidence {
  container_count?: number;
  factory_ok: boolean;
  cron_present: boolean;
  rsi_healthy: boolean;
  deploy_target: "8099/deploy";
  verify_cycle_passed: boolean;
  receipt_id: string;
}

export interface VerifyCycleReceipt {
  created_at: string;
  commit_sha: string;
  skill: string;
  deploy: "8099/deploy gated";
  evidence: VerifyCycleEvidence;
  rubric: ReceiptRubric;
  factory_ready: boolean;
  factory: FactoryCheckResult;
  rsi: RsiReconcileReceipt;
  commands: Array<{ name: string; command: string; status: number | null; stdout: string; stderr: string }>;
  changed_files: string[];
  result: "passed" | "failed";
}

export async function verifyCycle(opts: VerifyCycleOptions): Promise<VerifyCycleReceipt> {
  const skill = opts.skill ?? "rsi_canary";
  if (!/^[a-zA-Z0-9_-]+$/.test(skill)) throw new Error("skill must be a simple name");

  const runner = opts.runner ?? runCommand;
  const port = opts.port ?? 2222;
  const created = (opts.now ?? new Date()).toISOString();
  const factory = await (opts.factoryCheck ?? checkFactory)();
  const rsi = reconcileRsi({ host: opts.host, port, runner: opts.rsiRunner, canaryProof: opts.canaryProof, now: opts.now });
  const commands = [
    local("commit", "git", ["rev-parse", "HEAD"], runner),
    local("changed_files", "git", ["status", "--short"], runner),
    remote("auto_patch", port, opts.host, "docker exec hermes python3 /tmp/skill_health.py --auto-patch", runner),
    remote("verify_skill", port, opts.host, `docker exec hermes bash -lc 'test -x /opt/data/skills/${skill}/test.sh && /opt/data/skills/${skill}/test.sh'`, runner),
  ];
  const commit = commands.find((c) => c.name === "commit")?.stdout.trim() || "unknown";
  const changed = (commands.find((c) => c.name === "changed_files")?.stdout ?? "").split(/\r?\n/).filter(Boolean);
  const cleanEnough = opts.allowDirty || changed.length === 0;
  const factoryReady = factoryGateReady(factory) || (opts.allowReconciledStaleFactory === true && factoryStaleButReconciled(factory));
  const passed = cleanEnough && factoryReady && rsi.decision === "healthy" && commands.every((c) => c.status === 0);
  const receipt: VerifyCycleReceipt = {
    created_at: created,
    commit_sha: commit,
    skill,
    deploy: "8099/deploy gated",
    evidence: {
      container_count: factory.factory.containers,
      factory_ok: factoryReady,
      cron_present: rsi.facts.cron_present,
      rsi_healthy: rsi.decision === "healthy",
      deploy_target: "8099/deploy",
      verify_cycle_passed: passed,
      receipt_id: `verify-cycle:${commit}:${created}`,
    },
    rubric: receiptRubric(passed),
    factory_ready: factoryReady,
    factory,
    rsi,
    commands,
    changed_files: changed,
    result: passed ? "passed" : "failed",
  };
  if (opts.out) writeVerifyCycleReceipt(opts.out, receipt);
  return receipt;
}

export function writeVerifyCycleReceipt(file: string, receipt: VerifyCycleReceipt): void {
  fs.mkdirSync(path.dirname(file), { recursive: true });
  fs.writeFileSync(file, `${JSON.stringify(receipt, null, 2)}\n`);
}

function factoryStaleButReconciled(result: FactoryCheckResult): boolean {
  const factory = result.factory as Record<string, unknown>;
  return result.factoryAvailable
    && result.deployAvailable
    && result.deploy.status === "ok"
    && factory.status === "stale"
    && factory.state === "STALE_FACTORY_STATE"
    && typeof factory.live_containers === "number"
    && typeof factory.catalog_containers === "number"
    && factory.truth === "live docker ps wins over catalog/memory when counts disagree";
}

function local(name: string, command: string, args: string[], runner: NonNullable<VerifyCycleOptions["runner"]>) {
  const r = runner(command, args);
  return { name, command: [command, ...args].join(" "), status: r.status, stdout: r.stdout.slice(0, 4000), stderr: r.stderr.slice(0, 1000) };
}

function remote(name: string, port: number, host: string, script: string, runner: NonNullable<VerifyCycleOptions["runner"]>) {
  const args = ["-p", String(port), `root@${host}`, script];
  const r = runner("ssh", args);
  return { name, command: ["ssh", ...args].join(" "), status: r.status, stdout: r.stdout.slice(0, 4000), stderr: r.stderr.slice(0, 1000) };
}

function runCommand(command: string, args: string[]) {
  const result = spawnSync(command, args, { encoding: "utf-8" });
  return { status: result.status, stdout: result.stdout ?? "", stderr: result.stderr ?? "" };
}

function receiptRubric(ok: boolean): ReceiptRubric {
  return {
    format: 5,
    factuality: ok ? 5 : 3,
    consistency: ok ? 5 : 3,
    realism: 4,
    quality: ok ? 5 : 3,
  };
}