feat: add system card fields to agent traces
This commit is contained in:
parent
98729709d7
commit
ac800d106f
|
|
@ -4,6 +4,8 @@ This document mirrors `src/index.ts` command registration and is the canonical C
|
||||||
|
|
||||||
Trust boundary: live receipt > committed receipt > memory summary. Memory is advisory only; `8099/deploy` via git-proxy is the guarded deploy route, and `8098/deploy-webhook` is legacy/stale for fable-agent deploy trust.
|
Trust boundary: live receipt > committed receipt > memory summary. Memory is advisory only; `8099/deploy` via git-proxy is the guarded deploy route, and `8098/deploy-webhook` is legacy/stale for fable-agent deploy trust.
|
||||||
|
|
||||||
|
Agent-run traces follow system-card discipline: `fable.agent_run.trace.v1` records budget, environment, safety posture, receipts, latency, and `deployAttempted: false`; capability evidence is not deploy authority.
|
||||||
|
|
||||||
Last sync: 2026-06-14.
|
Last sync: 2026-06-14.
|
||||||
|
|
||||||
## Quick examples
|
## Quick examples
|
||||||
|
|
|
||||||
|
|
@ -43,6 +43,9 @@ describe("eval trace receipt", () => {
|
||||||
receiptsWritten: ["fable.agent_run.trace.v1"],
|
receiptsWritten: ["fable.agent_run.trace.v1"],
|
||||||
tokensIn: 100,
|
tokensIn: 100,
|
||||||
tokensOut: 20,
|
tokensOut: 20,
|
||||||
|
budget: { tokens: 500, wallClockMs: 10_000, toolCalls: 8 },
|
||||||
|
environment: { internet: false, repoAccess: true, deployAuthority: "gated" },
|
||||||
|
safetyPosture: { cyber: "read-only", securityScan: "passed", approval: "required" },
|
||||||
decision: "ok",
|
decision: "ok",
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
@ -52,6 +55,22 @@ describe("eval trace receipt", () => {
|
||||||
schema: "fable.agent_run.trace.v1",
|
schema: "fable.agent_run.trace.v1",
|
||||||
runId: "run-1",
|
runId: "run-1",
|
||||||
latencyMs: 2500,
|
latencyMs: 2500,
|
||||||
|
budget: { tokens: 500, wallClockMs: 10000, toolCalls: 8 },
|
||||||
|
environment: { internet: false, repoAccess: true, secretsAccess: false, deployAuthority: "gated" },
|
||||||
|
safetyPosture: { cyber: "read-only", securityScan: "passed", approval: "required" },
|
||||||
|
deployAttempted: false,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("defaults agent run traces to no internet, no secrets, and no deploy authority", () => {
|
||||||
|
expect(createAgentRunTraceReceipt({
|
||||||
|
runId: "run-2",
|
||||||
|
task: "inspect",
|
||||||
|
startedAt: new Date("2026-06-29T00:00:00.000Z"),
|
||||||
|
endedAt: new Date("2026-06-29T00:00:00.001Z"),
|
||||||
|
})).toMatchObject({
|
||||||
|
environment: { internet: false, repoAccess: true, secretsAccess: false, deployAuthority: "none" },
|
||||||
|
safetyPosture: { cyber: "not-run", securityScan: "not-run", approval: "not-required" },
|
||||||
deployAttempted: false,
|
deployAttempted: false,
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -15,6 +15,25 @@ export interface EvalTraceReceipt {
|
||||||
createdAt: string;
|
createdAt: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface AgentRunTraceBudget {
|
||||||
|
tokens?: number;
|
||||||
|
wallClockMs?: number;
|
||||||
|
toolCalls?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface AgentRunTraceEnvironment {
|
||||||
|
internet: boolean;
|
||||||
|
repoAccess: boolean;
|
||||||
|
secretsAccess: false;
|
||||||
|
deployAuthority: "none" | "gated";
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface AgentRunTraceSafetyPosture {
|
||||||
|
cyber: "not-run" | "read-only" | "approved-active";
|
||||||
|
securityScan?: "not-run" | "passed" | "failed";
|
||||||
|
approval?: "not-required" | "required" | "granted";
|
||||||
|
}
|
||||||
|
|
||||||
export interface AgentRunTraceReceipt {
|
export interface AgentRunTraceReceipt {
|
||||||
schema: "fable.agent_run.trace.v1";
|
schema: "fable.agent_run.trace.v1";
|
||||||
runId: string;
|
runId: string;
|
||||||
|
|
@ -27,6 +46,9 @@ export interface AgentRunTraceReceipt {
|
||||||
tokensIn?: number;
|
tokensIn?: number;
|
||||||
tokensOut?: number;
|
tokensOut?: number;
|
||||||
latencyMs: number;
|
latencyMs: number;
|
||||||
|
budget: AgentRunTraceBudget;
|
||||||
|
environment: AgentRunTraceEnvironment;
|
||||||
|
safetyPosture: AgentRunTraceSafetyPosture;
|
||||||
decision: "ok" | "needs-human" | "blocked" | "failed";
|
decision: "ok" | "needs-human" | "blocked" | "failed";
|
||||||
reasons: string[];
|
reasons: string[];
|
||||||
deployAttempted: false;
|
deployAttempted: false;
|
||||||
|
|
@ -42,6 +64,9 @@ export interface AgentRunTraceOptions {
|
||||||
receiptsWritten?: string[];
|
receiptsWritten?: string[];
|
||||||
tokensIn?: number;
|
tokensIn?: number;
|
||||||
tokensOut?: number;
|
tokensOut?: number;
|
||||||
|
budget?: Partial<AgentRunTraceBudget>;
|
||||||
|
environment?: Partial<AgentRunTraceEnvironment>;
|
||||||
|
safetyPosture?: Partial<AgentRunTraceSafetyPosture>;
|
||||||
decision?: AgentRunTraceReceipt["decision"];
|
decision?: AgentRunTraceReceipt["decision"];
|
||||||
reasons?: string[];
|
reasons?: string[];
|
||||||
}
|
}
|
||||||
|
|
@ -109,6 +134,22 @@ export function createAgentRunTraceReceipt(opts: AgentRunTraceOptions): AgentRun
|
||||||
tokensIn: opts.tokensIn,
|
tokensIn: opts.tokensIn,
|
||||||
tokensOut: opts.tokensOut,
|
tokensOut: opts.tokensOut,
|
||||||
latencyMs,
|
latencyMs,
|
||||||
|
budget: {
|
||||||
|
tokens: opts.budget?.tokens,
|
||||||
|
wallClockMs: opts.budget?.wallClockMs,
|
||||||
|
toolCalls: opts.budget?.toolCalls,
|
||||||
|
},
|
||||||
|
environment: {
|
||||||
|
internet: opts.environment?.internet ?? false,
|
||||||
|
repoAccess: opts.environment?.repoAccess ?? true,
|
||||||
|
secretsAccess: false,
|
||||||
|
deployAuthority: opts.environment?.deployAuthority ?? "none",
|
||||||
|
},
|
||||||
|
safetyPosture: {
|
||||||
|
cyber: opts.safetyPosture?.cyber ?? "not-run",
|
||||||
|
securityScan: opts.safetyPosture?.securityScan ?? "not-run",
|
||||||
|
approval: opts.safetyPosture?.approval ?? "not-required",
|
||||||
|
},
|
||||||
decision: opts.decision ?? "ok",
|
decision: opts.decision ?? "ok",
|
||||||
reasons: opts.reasons ?? [],
|
reasons: opts.reasons ?? [],
|
||||||
deployAttempted: false,
|
deployAttempted: false,
|
||||||
|
|
|
||||||
|
|
@ -76,7 +76,7 @@ export { checkReceiptHealth } from "./receipt-consumer.js";
|
||||||
export type { ReceiptHealth } from "./receipt-consumer.js";
|
export type { ReceiptHealth } from "./receipt-consumer.js";
|
||||||
|
|
||||||
export { createAgentRunTraceReceipt, createDuelEvalReceipt, createEvalTraceReceipt, gradeSetRecovery, gradeVerifiableReward, tallyDuelReceipts, writeAgentRunTraceReceipt, writeDuelEvalReceipt, writeEvalTraceReceipt } from "./eval-trace.js";
|
export { createAgentRunTraceReceipt, createDuelEvalReceipt, createEvalTraceReceipt, gradeSetRecovery, gradeVerifiableReward, tallyDuelReceipts, writeAgentRunTraceReceipt, writeDuelEvalReceipt, writeEvalTraceReceipt } from "./eval-trace.js";
|
||||||
export type { AgentRunTraceOptions, AgentRunTraceReceipt, DuelEvalReceipt, DuelTally, EvalTraceCommand, EvalTraceReceipt, SetGradeReceipt, VerifiableRewardReceipt } from "./eval-trace.js";
|
export type { AgentRunTraceBudget, AgentRunTraceEnvironment, AgentRunTraceOptions, AgentRunTraceReceipt, AgentRunTraceSafetyPosture, DuelEvalReceipt, DuelTally, EvalTraceCommand, EvalTraceReceipt, SetGradeReceipt, VerifiableRewardReceipt } from "./eval-trace.js";
|
||||||
|
|
||||||
export { loadBenchHarnesses, runAgentBench, writeAgentBenchReceipt } from "./agent-bench.js";
|
export { loadBenchHarnesses, runAgentBench, writeAgentBenchReceipt } from "./agent-bench.js";
|
||||||
export type { AgentBenchOptions, AgentBenchReceipt, BenchContenderId, BenchContenderResult, BenchHarnessConfig, BenchmarkHygiene } from "./agent-bench.js";
|
export type { AgentBenchOptions, AgentBenchReceipt, BenchContenderId, BenchContenderResult, BenchHarnessConfig, BenchmarkHygiene } from "./agent-bench.js";
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue