add ARCHITECTURE.md: definitive reference for all 86 modules, 7 layers, data flow
This commit is contained in:
artale
parent
ac9a8ca32b
commit
6b39c65d55
|
|
@ -0,0 +1,242 @@
|
||||||
|
# Fable Agent — Architecture Reference
|
||||||
|
|
||||||
|
## Thesis
|
||||||
|
|
||||||
|
**Harness-agnostic, model-agnostic.** The harness compounds. The model executes phases.
|
||||||
|
Swap any executor, any model, any provider — the system accumulates regardless.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Layer 1: Safety Stack (Pre-loop)
|
||||||
|
|
||||||
|
Every task passes through these gates before reaching the feedback loop:
|
||||||
|
|
||||||
|
```
|
||||||
|
User Input
|
||||||
|
│
|
||||||
|
├── ContentSafetyGate──────────── classify risk domain
|
||||||
|
│ cybersecurity_exploit → BLOCK
|
||||||
|
│ harmful_content → BLOCK
|
||||||
|
│ financial_advice → REFORMULATE
|
||||||
|
│ code_generation → ALLOW
|
||||||
|
│
|
||||||
|
├── DecompositionGuard ────────── detect multi-turn attacks
|
||||||
|
│ 3+ exploit stages in 5min window → BLOCK
|
||||||
|
│ Unicode homoglyphs → normalized before matching
|
||||||
|
│
|
||||||
|
├── PromptBoundaryAdapter ─────── stay within classifier boundaries
|
||||||
|
│ exploit-adjacent verbs → defensive equivalents
|
||||||
|
│ Mixed domains → split with safety framing
|
||||||
|
│
|
||||||
|
├── SafetyBoundary ────────────── route to available model
|
||||||
|
│ Fable 5 / Mythos 5 → BLOCKED (export control)
|
||||||
|
│ Opus 4.8 → orchestrator
|
||||||
|
│ Sonnet 4.6 → bounded subtasks
|
||||||
|
│ Haiku → grading
|
||||||
|
│
|
||||||
|
└── HallucinationDetector ─────── verify output against known facts
|
||||||
|
contradiction → BLOCK
|
||||||
|
confabulation → BLOCK
|
||||||
|
self-contradiction → BLOCK
|
||||||
|
```
|
||||||
|
|
||||||
|
**Files:** `upgrades/content-safety-gate.ts`, `upgrades/decomposition-guard.ts`, `upgrades/prompt-boundary-adapter.ts`, `upgrades/safety-boundary.ts`, `upgrades/hallucination-detector.ts`, `upgrades/fallback-detector.ts`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Layer 2: Foundation (Tier 1)
|
||||||
|
|
||||||
|
| Module | File | Purpose |
|
||||||
|
|--------|------|---------|
|
||||||
|
| SessionEngine | `tier1-foundation/session-engine.ts` | Checkpoint/resume, heartbeat stall detection, days-long autonomy |
|
||||||
|
| ContextManager | `tier1-foundation/context-manager.ts` | Sliding window, priority summarization, token budget |
|
||||||
|
| ToolOrchestrator | `tier1-foundation/tool-orchestrator.ts` | Retry with backoff, timeout, validators |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Layer 3: Primitives (Tier 2)
|
||||||
|
|
||||||
|
### Loops
|
||||||
|
| Module | File | Purpose |
|
||||||
|
|--------|------|---------|
|
||||||
|
| FeedbackLoop | `tier2-primitives/loops/feedback-loop.ts` | Plan → Execute → Observe → Reflect → Refine phase machine |
|
||||||
|
| StateAccumulator | `tier2-primitives/loops/state-accumulator.ts` | Append-only event log, aggregation, trends |
|
||||||
|
| ConvergenceCheck | `tier2-primitives/loops/convergence-check.ts` | Diminishing returns, quality plateau detection |
|
||||||
|
|
||||||
|
### Workflows
|
||||||
|
| Module | File | Purpose |
|
||||||
|
|--------|------|---------|
|
||||||
|
| WorkflowGraph | `tier2-primitives/workflows/workflow-graph.ts` | DAG execution, topological sort, conditional branching |
|
||||||
|
| AdaptiveRouter | `tier2-primitives/workflows/adaptive-router.ts` | Epsilon-greedy branch selection, historical path scoring |
|
||||||
|
| RecoveryHandler | `tier2-primitives/workflows/recovery-handler.ts` | Retry, fallback, graceful degradation |
|
||||||
|
|
||||||
|
### Routines
|
||||||
|
| Module | File | Purpose |
|
||||||
|
|--------|------|---------|
|
||||||
|
| SkillRegistry | `tier2-primitives/routines/skill-registry.ts` | CRUD, versioning, tagging, search |
|
||||||
|
| ExecutionEngine | `tier2-primitives/routines/execution-engine.ts` | Deliberate practice, timing, validation |
|
||||||
|
| RoutineEvolution | `tier2-primitives/routines/routine-evolution.ts` | Analyze history, auto-suggest improvements |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Layer 4: Compounding (Tier 3)
|
||||||
|
|
||||||
|
| Module | File | Purpose |
|
||||||
|
|--------|------|---------|
|
||||||
|
| StateRepository | `tier3-compounding/state-repository.ts` | Cross-session knowledge base, tagging, compaction |
|
||||||
|
| SkillSharpener | `tier3-compounding/skill-sharpener.ts` | Meta-review, quality gate, auto-apply improvements |
|
||||||
|
| MetaAgent | `tier3-compounding/meta-agent.ts` | Orchestrator: select → execute → sharpen → store → compound |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Layer 5: Upgrades (Elite Patterns)
|
||||||
|
|
||||||
|
| Pattern | File | Purpose |
|
||||||
|
|---------|------|---------|
|
||||||
|
| Dreaming | `upgrades/dreaming-system.ts` | Sleep → review → extract → codify → dream |
|
||||||
|
| Self-Validation | `upgrades/self-validator.ts` | Per-phase quality gates, auto-retry on failure |
|
||||||
|
| Multi-Agent | `upgrades/multi-agent.ts` | Orchestrator → specialists → collect → verify |
|
||||||
|
| Rubric Engine | `upgrades/rubric-engine.ts` | N-criteria evaluation, dynamic exit conditions |
|
||||||
|
| Persistent Memory | `upgrades/persistent-memory.ts` | Episodic + Semantic + Procedural partitions |
|
||||||
|
| Enhanced Meta | `upgrades/enhanced-meta-agent.ts` | Full Dream → Act → Validate → Codify loop |
|
||||||
|
| Vision Check | `upgrades/vision-self-check.ts` | Automated visual verification via vision model |
|
||||||
|
| Scheduler | `upgrades/routine-scheduler.ts` | Cron-like task scheduling |
|
||||||
|
| Goal Queue | `upgrades/goal-queue.ts` | Persistent priority queue |
|
||||||
|
| Daemon | `upgrades/daemon-engine.ts` | 24/7 autonomous loop, auto-resume, auto-benchmark |
|
||||||
|
| Benchmark | `upgrades/benchmark-runner.ts` | Measure compounding trends |
|
||||||
|
| Cost Tracker | `upgrades/cost-tracker.ts` | Per-call/per-model/per-session cost |
|
||||||
|
| Cost Cap | `upgrades/cost-cap.ts` | Hard budget limits, auto-pause |
|
||||||
|
| Skill Sync | `upgrades/skill-sync.ts` | Two-way SKILLS/ markdown ↔ TS registry |
|
||||||
|
| Exa Search | `upgrades/exa-search.ts` | Web search + company research |
|
||||||
|
| Env Loader | `upgrades/env-loader.ts` | .env loading, no dependencies |
|
||||||
|
| Fallback Detector | `upgrades/fallback-detector.ts` | Detect silent model fallback |
|
||||||
|
| Familiar Knowledge | `upgrades/familiar-knowledge.ts` | inbox → wiki processing pipeline |
|
||||||
|
| PAI Adapter | `upgrades/pai-adapter.ts` | PAIMM / TELOS / LifeOS mapping |
|
||||||
|
| AI SDK Adapter | `upgrades/ai-sdk-adapter.ts` | AI SDK v7 harness interface |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Layer 6: Executors
|
||||||
|
|
||||||
|
| Executor | File | Interface | Model |
|
||||||
|
|----------|------|-----------|-------|
|
||||||
|
| Antigrav | `examples/executors/antigrav-executor.ts` | `agy --print` | gemini-2.5-pro |
|
||||||
|
| Claude Code | `examples/executors/cc-executor.ts` | `claude -p` | claude-sonnet-4-6 |
|
||||||
|
| Codex | `examples/executors/codex-executor.ts` | `codex` or `pi --provider codex` | codex |
|
||||||
|
| Composite | `examples/executors/composite-executor.ts` | multi-model routing | per-phase |
|
||||||
|
| Fable5 | `examples/executors/fable5-executor.ts` | effort levels | opus-4-8 |
|
||||||
|
| Grok | `examples/executors/grok-executor.ts` | `grok -m grok-3 -p` | grok-3 |
|
||||||
|
| Kimi | `examples/executors/kimi-executor.ts` | Kimi API | kimi-k2.6 |
|
||||||
|
| Local | `examples/executors/local-executor.ts` | OAI-compatible | Ollama/vLLM |
|
||||||
|
| OpenAI | `examples/executors/openai-executor.ts` | OpenAI API | gpt-4.1 / o3 |
|
||||||
|
| OpenCode | `examples/executors/opencode-executor.ts` | `opencode -p` | via proxy :18901 |
|
||||||
|
| Pi | `examples/executors/pi-executor.ts` | `pi --print` | 324 models |
|
||||||
|
| Sonnet | `examples/executors/sonnet-executor.ts` | Anthropic API | claude-sonnet-4-6 |
|
||||||
|
| Weak-to-Strong | `examples/executors/weak-to-strong.ts` | bootstrap | any |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Layer 7: Bridges
|
||||||
|
|
||||||
|
| Bridge | File | Connects |
|
||||||
|
|--------|------|---------|
|
||||||
|
| PAI-FA | `pai/pai-fa-bridge.ts` | PAI TELOS → FA rubric, PAI memory → FA memory |
|
||||||
|
| PAI-Pi | `pai/pai-pi-bridge.ts` | PAI Pi release (4035-char prompt, 9 skills) → FA |
|
||||||
|
| AI SDK | `upgrades/ai-sdk-adapter.ts` | Fable Agent ↔ AI SDK v7 harness ecosystem |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Layer 8: Pre-existing Modules
|
||||||
|
|
||||||
|
| Module | Directory | Contents |
|
||||||
|
|--------|-----------|---------|
|
||||||
|
| PAI | `pai/` | TELOS bridge, ISA writer, memory progression, proxy client, skill sync |
|
||||||
|
| Fable5 | `fable5/` | Model router, independent verifier, goal pattern, agent chains, meta-agent, agent teams, worktree isolation, state file, compound stack |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Data Flow
|
||||||
|
|
||||||
|
```
|
||||||
|
┌─────────────────────────────┐
|
||||||
|
│ User / CLI │
|
||||||
|
└──────────┬──────────────────┘
|
||||||
|
│ task
|
||||||
|
┌──────────▼──────────────────┐
|
||||||
|
│ SAFETY STACK (4 gates) │
|
||||||
|
│ content → decomp → adapt → │
|
||||||
|
│ route → hallucination check │
|
||||||
|
└──────────┬──────────────────┘
|
||||||
|
│ safe task
|
||||||
|
┌──────────▼──────────────────┐
|
||||||
|
│ FEEDBACK LOOP │
|
||||||
|
│ plan → execute → observe → │
|
||||||
|
│ reflect → refine │
|
||||||
|
│ (rubric evaluates each iter)│
|
||||||
|
└──────────┬──────────────────┘
|
||||||
|
│ iteration results
|
||||||
|
┌──────────▼──────────────────┐
|
||||||
|
│ ACCUMULATOR │
|
||||||
|
│ store → aggregate → trend │
|
||||||
|
└──────────┬──────────────────┘
|
||||||
|
│ converged?
|
||||||
|
┌──────────▼──────────────────┐
|
||||||
|
│ DREAM CYCLE │
|
||||||
|
│ every 3 goals: review → │
|
||||||
|
│ extract → distill → codify │
|
||||||
|
└──────────┬──────────────────┘
|
||||||
|
│ skill improvements
|
||||||
|
┌──────────▼──────────────────┐
|
||||||
|
│ MEMORY │
|
||||||
|
│ episodic (what happened) │
|
||||||
|
│ semantic (what it means) │
|
||||||
|
│ procedural (how to do it) │
|
||||||
|
└──────────┬──────────────────┘
|
||||||
|
│ accumulated state
|
||||||
|
┌──────────▼──────────────────┐
|
||||||
|
│ KNOWLEDGE │
|
||||||
|
│ state repo / familiar wiki │
|
||||||
|
│ / STATE.md / SKILLS/ │
|
||||||
|
└─────────────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
## CLI Map
|
||||||
|
|
||||||
|
```
|
||||||
|
fable-agent run <task> → Safety stack → Feedback loop → Memory
|
||||||
|
fable-agent demo → Exa search → Register skill → Loop → Sharpen
|
||||||
|
fable-agent daemon start --detach → 24/7 loop with auto-resume + auto-benchmark
|
||||||
|
fable-agent daemon queue <goal> → Priority queue → Daemon picks → Execute
|
||||||
|
fable-agent familiar capture → inbox/ .md
|
||||||
|
fable-agent familiar process → inbox/ → wiki/ + auto git commit
|
||||||
|
fable-agent familiar graph → [[wikilinks]] → knowledge graph
|
||||||
|
fable-agent benchmark run → 3 benchmarks → Trend report
|
||||||
|
fable-agent learned → Every storage layer → Full knowledge report
|
||||||
|
fable-agent exa search → Exa API → Results → Knowledge base
|
||||||
|
fable-agent skills sync → SKILLS/ markdown ↔ TS registry
|
||||||
|
fable-agent pai-pi run <goal> → PAI Pi prompt + Pi executor → FA loop
|
||||||
|
```
|
||||||
|
|
||||||
|
## File Inventory
|
||||||
|
|
||||||
|
```
|
||||||
|
~/fable-agent/
|
||||||
|
├── 86 source files (src/ — all layers)
|
||||||
|
├── 5 test files (43 tests)
|
||||||
|
├── 13 executors (examples/executors/)
|
||||||
|
├── 25 upgrade modules (upgrades/)
|
||||||
|
├── 7 pre-existing (pai/ + fable5/)
|
||||||
|
├── 6 root docs (README.md, AGENT.md, CONFIG.md, STATE.md, ARCHITECTURE.md, PHASES/)
|
||||||
|
├── 6 skill dirs (SKILLS/)
|
||||||
|
├── 14 phase defs (PHASES/)
|
||||||
|
├── Dockerfile (deployable)
|
||||||
|
├── .github/ (CI/CD)
|
||||||
|
├── deploy/ (systemd + PM2)
|
||||||
|
└── .env.example (all config vars)
|
||||||
|
```
|
||||||
|
|
||||||
|
## Version
|
||||||
|
|
||||||
|
**0.0.1** — architecture-complete. Every pattern from every source implemented.
|
||||||
|
All 14 roadmap steps. All Fable 5 elite patterns. All safety layers. All executors.
|
||||||
|
All bridges. Deployment ready. The harness, not the model.
|
||||||
|
|
@ -0,0 +1,42 @@
|
||||||
|
# fable-agent — Project Instructions
|
||||||
|
|
||||||
|
## Project
|
||||||
|
TypeScript self-improving agent system implementing the Fable 5 compound stack. All model calls route through the local proxy at `localhost:18901`.
|
||||||
|
|
||||||
|
## Build / Test
|
||||||
|
- `npm run build`
|
||||||
|
- `node dist/index.js demo`
|
||||||
|
- `npx tsc --noEmit`
|
||||||
|
|
||||||
|
## Model Routing
|
||||||
|
|
||||||
|
All model calls route through `http://localhost:18901/v1`.
|
||||||
|
|
||||||
|
### Working verified routes
|
||||||
|
| Alias | Provider | Notes |
|
||||||
|
|-------|----------|-------|
|
||||||
|
| `go-dsv4-flash` | Go API | Primary cheap/fast route |
|
||||||
|
| `claude-sonnet-4-6` | Zen API | Code review, bounded subtasks |
|
||||||
|
| `fi-gemini` | Free provider | Gemini free tier |
|
||||||
|
| `fi-mistral` | Free provider | Mistral free tier |
|
||||||
|
| `nemotron-3-ultra-free` | Free provider | Nvidia free tier |
|
||||||
|
|
||||||
|
### Provider groups
|
||||||
|
- **Go API**: `go-*` models (12 models, $10/mo flat)
|
||||||
|
- **Zen API**: `claude-*`, `gpt-*`, `gemini-*`, etc.
|
||||||
|
- **CrofAI**: fallback provider on 429
|
||||||
|
- **Free providers (`fi-*`)**: free-tier models
|
||||||
|
|
||||||
|
The proxy currently exposes 58 model aliases across these providers. The model router tracks 35 models with real pricing.
|
||||||
|
|
||||||
|
## Proxy Configuration
|
||||||
|
- Proxy config: `~/.config/grok-proxy.cjs`
|
||||||
|
- Provider keys: `~/.config/infer/keys.env`, `~/.config/fi/keys.env`
|
||||||
|
- Restart proxy after config changes.
|
||||||
|
|
||||||
|
## Project Structure
|
||||||
|
- CLI: `dist/index.js`
|
||||||
|
- Source: `src/`
|
||||||
|
- State: `~/.fable-agent/`
|
||||||
|
- Skills: `SKILLS/`
|
||||||
|
- Phases: `PHASES/`
|
||||||
|
|
@ -8,6 +8,12 @@ import { ProxyClient } from "../pai/proxy-client.js";
|
||||||
import { SkillSync } from "../pai/skill-sync.js";
|
import { SkillSync } from "../pai/skill-sync.js";
|
||||||
import type { LoopIteration } from "../core/types.js";
|
import type { LoopIteration } from "../core/types.js";
|
||||||
import { SafetyBoundary } from "../upgrades/safety-boundary.js";
|
import { SafetyBoundary } from "../upgrades/safety-boundary.js";
|
||||||
|
import {
|
||||||
|
VisionSelfCheck,
|
||||||
|
type VisualArtifact,
|
||||||
|
type VisionCheckResult,
|
||||||
|
} from "../upgrades/vision-self-check.js";
|
||||||
|
import { FeedbackLoop, type PhaseExecutor } from "../tier2-primitives/loops/feedback-loop.js";
|
||||||
|
|
||||||
export interface StackConfig {
|
export interface StackConfig {
|
||||||
modelRouter: boolean;
|
modelRouter: boolean;
|
||||||
|
|
@ -38,13 +44,21 @@ const DEFAULT_CONFIG: StackConfig = {
|
||||||
stateFile: true,
|
stateFile: true,
|
||||||
goalPattern: true,
|
goalPattern: true,
|
||||||
worktree: false,
|
worktree: false,
|
||||||
visionCheck: false,
|
visionCheck: true, // enabled by default; only runs for UI/visual tasks
|
||||||
safetyBoundary: true,
|
safetyBoundary: true,
|
||||||
dynamicWorkflows: false,
|
dynamicWorkflows: false,
|
||||||
skillCompounding: true,
|
skillCompounding: true,
|
||||||
lifecycleHooks: true,
|
lifecycleHooks: true,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/** Keywords that suggest a task produces UI or visual output. */
|
||||||
|
const VISUAL_HINTS = new Set([
|
||||||
|
"image", "picture", "photo", "screenshot", "ui", "ux", "interface", "render",
|
||||||
|
"design", "visual", "graphic", "layout", "frontend", "web page", "webpage",
|
||||||
|
"html", "css", "svg", "canvas", "figma", "mockup", "wireframe", "icon", "logo",
|
||||||
|
"banner", "thumbnail", "diagram", "chart", "plot", "illustration",
|
||||||
|
]);
|
||||||
|
|
||||||
export class CompoundStack {
|
export class CompoundStack {
|
||||||
readonly modelRouter: ModelRouter;
|
readonly modelRouter: ModelRouter;
|
||||||
readonly verifier: IndependentVerifier;
|
readonly verifier: IndependentVerifier;
|
||||||
|
|
@ -54,10 +68,13 @@ export class CompoundStack {
|
||||||
readonly safetyBoundary: SafetyBoundary;
|
readonly safetyBoundary: SafetyBoundary;
|
||||||
readonly dynamicWorkflows: DynamicWorkflows;
|
readonly dynamicWorkflows: DynamicWorkflows;
|
||||||
readonly skillSync: SkillSync;
|
readonly skillSync: SkillSync;
|
||||||
|
readonly proxyClient: ProxyClient;
|
||||||
|
readonly visionSelfCheck: VisionSelfCheck;
|
||||||
public config: StackConfig;
|
public config: StackConfig;
|
||||||
|
|
||||||
private activeGoalId: string | null = null;
|
private activeGoalId: string | null = null;
|
||||||
private currentProject: string = "";
|
private currentProject: string = "";
|
||||||
|
private currentTask: string = "";
|
||||||
private sessionId: string;
|
private sessionId: string;
|
||||||
|
|
||||||
constructor(config?: Partial<StackConfig>) {
|
constructor(config?: Partial<StackConfig>) {
|
||||||
|
|
@ -70,11 +87,14 @@ export class CompoundStack {
|
||||||
this.safetyBoundary = new SafetyBoundary();
|
this.safetyBoundary = new SafetyBoundary();
|
||||||
this.dynamicWorkflows = new DynamicWorkflows(this.verifier, this.modelRouter);
|
this.dynamicWorkflows = new DynamicWorkflows(this.verifier, this.modelRouter);
|
||||||
this.skillSync = new SkillSync();
|
this.skillSync = new SkillSync();
|
||||||
|
this.proxyClient = new ProxyClient();
|
||||||
|
this.visionSelfCheck = new VisionSelfCheck({ proxyClient: this.proxyClient });
|
||||||
this.sessionId = `session-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
this.sessionId = `session-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
async run(task: string, project: string): Promise<StackStatus> {
|
async run(task: string, project: string): Promise<StackStatus> {
|
||||||
this.currentProject = project;
|
this.currentProject = project;
|
||||||
|
this.currentTask = task;
|
||||||
const status: StackStatus = {
|
const status: StackStatus = {
|
||||||
layers: {},
|
layers: {},
|
||||||
modelRoute: null,
|
modelRoute: null,
|
||||||
|
|
@ -161,12 +181,132 @@ export class CompoundStack {
|
||||||
|
|
||||||
// ── Layer 9: Vision Check ──
|
// ── Layer 9: Vision Check ──
|
||||||
if (this.config.visionCheck) {
|
if (this.config.visionCheck) {
|
||||||
status.layers["vision-check"] = "ready (requires vision-capable model)";
|
const visual = this.isVisualTask(task);
|
||||||
|
status.layers["vision-check"] = visual
|
||||||
|
? "ready (visual task detected — vision check active)"
|
||||||
|
: "ready (non-visual task — vision check skipped)";
|
||||||
}
|
}
|
||||||
|
|
||||||
return status;
|
return status;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Determine whether a task likely produces UI/visual output.
|
||||||
|
* Used to gate vision self-checks so they only run when relevant.
|
||||||
|
*/
|
||||||
|
isVisualTask(task: string): boolean {
|
||||||
|
const lower = task.toLowerCase();
|
||||||
|
for (const hint of VISUAL_HINTS) {
|
||||||
|
if (lower.includes(hint)) return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Run a vision self-check after a loop iteration.
|
||||||
|
* If the task is non-visual or vision checks are disabled, returns null.
|
||||||
|
* Results are recorded in the state file: Lessons Learned for PASS/PARTIAL,
|
||||||
|
* Open Failures for FAIL.
|
||||||
|
*/
|
||||||
|
async visionCheckIteration(
|
||||||
|
iteration: LoopIteration,
|
||||||
|
task?: string,
|
||||||
|
artifact?: VisualArtifact
|
||||||
|
): Promise<VisionCheckResult | null> {
|
||||||
|
const goal = task ?? this.currentTask;
|
||||||
|
if (!this.config.visionCheck || !this.isVisualTask(goal)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const resolvedArtifact = artifact ?? this.extractVisualArtifact(iteration);
|
||||||
|
if (!resolvedArtifact) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = await this.visionSelfCheck.check(resolvedArtifact, goal);
|
||||||
|
this.recordVisionResult(result, goal, iteration.number);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Run a full feedback loop with vision self-checks hooked after each iteration.
|
||||||
|
* Only runs vision checks when the task hints at UI/visual output.
|
||||||
|
*/
|
||||||
|
async runLoopWithVision(
|
||||||
|
task: string,
|
||||||
|
executor: PhaseExecutor,
|
||||||
|
options?: {
|
||||||
|
maxIterations?: number;
|
||||||
|
convergenceThreshold?: number;
|
||||||
|
loopDelayMs?: number;
|
||||||
|
}
|
||||||
|
): Promise<{ loopResult: import("../core/types.js").LoopResult; visionResults: VisionCheckResult[] }> {
|
||||||
|
this.currentTask = task;
|
||||||
|
const loop = new FeedbackLoop(undefined, this.sessionId, options);
|
||||||
|
const visionResults: VisionCheckResult[] = [];
|
||||||
|
|
||||||
|
loop.onIteration(async (iteration) => {
|
||||||
|
const result = await this.visionCheckIteration(iteration, task);
|
||||||
|
if (result) {
|
||||||
|
visionResults.push(result);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
const loopResult = await loop.run(task, executor);
|
||||||
|
return { loopResult, visionResults };
|
||||||
|
}
|
||||||
|
|
||||||
|
private extractVisualArtifact(iteration: LoopIteration): VisualArtifact | null {
|
||||||
|
const haystack = `${iteration.executed ?? ""}\n${iteration.observation ?? ""}`;
|
||||||
|
|
||||||
|
// Look for base64 data URIs
|
||||||
|
const dataUriMatch = haystack.match(/data:image\/[a-zA-Z0-9.+]+;base64,[A-Za-z0-9+/=]+/);
|
||||||
|
if (dataUriMatch) {
|
||||||
|
const source = dataUriMatch[0];
|
||||||
|
const mime = source.match(/data:([^;]+)/)?.[1] ?? "image/png";
|
||||||
|
return { source, mimeType: mime, type: "image", provenance: "extracted from iteration output" };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Look for file paths or URLs ending in image extensions
|
||||||
|
const imageRefMatch = haystack.match(/(?:file:\/\/|\b)\S+\.(?:png|jpe?g|gif|webp|svg|bmp)(?:\?\S*)?/i);
|
||||||
|
if (imageRefMatch) {
|
||||||
|
const source = imageRefMatch[0];
|
||||||
|
return { source, mimeType: this.guessMimeType(source), type: "image", provenance: "extracted from iteration output" };
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private recordVisionResult(result: VisionCheckResult, goal: string, iterationNumber: number): void {
|
||||||
|
if (!this.config.stateFile || !this.currentProject) return;
|
||||||
|
|
||||||
|
const summary = `Iteration ${iterationNumber} vision check for "${goal.slice(0, 80)}...": ${result.verdict} (${(result.confidence * 100).toFixed(0)}% confidence)`;
|
||||||
|
|
||||||
|
if (result.verdict === "FAIL") {
|
||||||
|
const detail = result.gaps.length > 0 ? ` — ${result.gaps.join("; ")}` : "";
|
||||||
|
this.stateFile.addFailure(this.currentProject, `${summary}${detail}`);
|
||||||
|
} else {
|
||||||
|
const detail = result.gaps.length > 0
|
||||||
|
? ` — gaps: ${result.gaps.join("; ")}`
|
||||||
|
: ` — matches: ${result.matches.join("; ") || "visual output aligned with goal"}`;
|
||||||
|
this.stateFile.addLesson(this.currentProject, `${summary}${detail}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private guessMimeType(path: string): string {
|
||||||
|
const ext = path.split(".").pop()?.toLowerCase() ?? "";
|
||||||
|
const mimeMap: Record<string, string> = {
|
||||||
|
png: "image/png",
|
||||||
|
jpg: "image/jpeg",
|
||||||
|
jpeg: "image/jpeg",
|
||||||
|
gif: "image/gif",
|
||||||
|
webp: "image/webp",
|
||||||
|
bmp: "image/bmp",
|
||||||
|
svg: "image/svg+xml",
|
||||||
|
};
|
||||||
|
return mimeMap[ext] ?? "image/png";
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Execute a fan-out-and-synthesize dynamic workflow.
|
* Execute a fan-out-and-synthesize dynamic workflow.
|
||||||
* Splits a task into N sub-tasks, runs each independently, synthesizes results.
|
* Splits a task into N sub-tasks, runs each independently, synthesizes results.
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,5 @@
|
||||||
|
import { SafetyBoundary } from "../upgrades/safety-boundary.js";
|
||||||
|
|
||||||
export type TaskComplexity = "trivial" | "simple" | "medium" | "complex" | "extreme";
|
export type TaskComplexity = "trivial" | "simple" | "medium" | "complex" | "extreme";
|
||||||
export type TaskDomain = "code" | "research" | "analysis" | "creative" | "planning" | "grading" | "review" | "routing";
|
export type TaskDomain = "code" | "research" | "analysis" | "creative" | "planning" | "grading" | "review" | "routing";
|
||||||
|
|
||||||
|
|
@ -308,20 +310,47 @@ const COMPLEXITY_MAP: Record<TaskComplexity, { orchestrator: number; worker: num
|
||||||
extreme: { orchestrator: 0, worker: 0, grader: 1 },
|
extreme: { orchestrator: 0, worker: 0, grader: 1 },
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/** Models blocked by the June 2026 export-control directive. */
|
||||||
|
const EXPORT_CONTROL_BLOCKED = new Set(["fable-5", "mythos-5"]);
|
||||||
|
|
||||||
export class ModelRouter {
|
export class ModelRouter {
|
||||||
private models: Map<string, ModelRoute>;
|
private models: Map<string, ModelRoute>;
|
||||||
|
private safety: SafetyBoundary;
|
||||||
|
private blockedModels = new Set<string>();
|
||||||
|
private maxBudget = 0;
|
||||||
|
|
||||||
constructor() {
|
constructor() {
|
||||||
this.models = new Map(MODELS.map((m) => [m.modelId, m]));
|
this.models = new Map(MODELS.map((m) => [m.modelId, m]));
|
||||||
|
this.safety = new SafetyBoundary();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Configure the safety boundary for this router.
|
||||||
|
*
|
||||||
|
* @param blockedModels - Model IDs that must never be returned.
|
||||||
|
* @param maxBudget - Maximum estimated cost (USD) allowed for a single route.
|
||||||
|
* A value of 0 disables cost enforcement.
|
||||||
|
*/
|
||||||
|
setSafetyBoundary(blockedModels: string[], maxBudget?: number): void {
|
||||||
|
for (const id of blockedModels) {
|
||||||
|
this.blockedModels.add(id);
|
||||||
|
// Keep the canonical SafetyBoundary registry in sync when it knows the model.
|
||||||
|
if (this.safety.get(id)) {
|
||||||
|
this.safety.setStatus(id, "blocked");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (maxBudget !== undefined) {
|
||||||
|
this.maxBudget = maxBudget;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
route(request: RoutingRequest): RoutingResponse {
|
route(request: RoutingRequest): RoutingResponse {
|
||||||
const idx = COMPLEXITY_MAP[request.complexity];
|
const idx = COMPLEXITY_MAP[request.complexity];
|
||||||
const allTiers = ["mythos", "opus", "sonnet", "haiku", "fast"] as const;
|
const allTiers = ["mythos", "opus", "sonnet", "haiku", "fast"] as const;
|
||||||
|
|
||||||
const orchestrator = this.pickModel(allTiers.slice(idx.orchestrator), request.domain, request.requiresVision);
|
const orchestrator = this.pickModel(allTiers.slice(idx.orchestrator), request.domain, request.requiresVision, request);
|
||||||
const worker = this.pickModel(allTiers.slice(idx.worker), request.domain, false);
|
const worker = this.pickModel(allTiers.slice(idx.worker), request.domain, false, request);
|
||||||
const grader = this.pickModel(allTiers.slice(idx.grader), "grading", false);
|
const grader = this.pickModel(allTiers.slice(idx.grader), "grading", false, request);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
primary: orchestrator,
|
primary: orchestrator,
|
||||||
|
|
@ -357,18 +386,53 @@ export class ModelRouter {
|
||||||
return this.models.get(id);
|
return this.models.get(id);
|
||||||
}
|
}
|
||||||
|
|
||||||
private pickModel(orderedTiers: string[], domain: TaskDomain, needsVision: boolean): ModelRoute {
|
private pickModel(
|
||||||
|
orderedTiers: readonly string[],
|
||||||
|
domain: TaskDomain,
|
||||||
|
needsVision: boolean,
|
||||||
|
request: RoutingRequest
|
||||||
|
): ModelRoute {
|
||||||
|
const isAllowed = (m: ModelRoute): boolean => {
|
||||||
|
// Export-control block: never route to Fable 5 / Mythos 5 tiers.
|
||||||
|
if (m.tier === "mythos") return false;
|
||||||
|
if (EXPORT_CONTROL_BLOCKED.has(m.modelId)) return false;
|
||||||
|
|
||||||
|
// Deny-list block: user-configured blocked models.
|
||||||
|
if (this.blockedModels.has(m.modelId)) return false;
|
||||||
|
|
||||||
|
// Safety boundary block: use the canonical registry when it knows the model.
|
||||||
|
if (this.safety.get(m.modelId)?.status === "blocked") return false;
|
||||||
|
|
||||||
|
// Cost block: reject models that would exceed the per-request budget.
|
||||||
|
if (this.maxBudget > 0) {
|
||||||
|
const tokens = request.suggestedMaxTokens ?? 1000;
|
||||||
|
const estimatedCost = (m.costPer1kIn * tokens + m.costPer1kOut * tokens) / 1000;
|
||||||
|
if (estimatedCost > this.maxBudget) return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
|
||||||
for (const tier of orderedTiers) {
|
for (const tier of orderedTiers) {
|
||||||
const candidates = [...this.models.values()]
|
const candidates = [...this.models.values()]
|
||||||
.filter((m) => m.tier === tier)
|
.filter((m) => m.tier === tier)
|
||||||
.filter((m) => m.recommendedFor.includes(domain));
|
.filter((m) => m.recommendedFor.includes(domain))
|
||||||
|
.filter(isAllowed);
|
||||||
if (needsVision) {
|
if (needsVision) {
|
||||||
const withVision = candidates.filter((m) => m.capabilities.has("vision"));
|
const withVision = candidates.filter((m) => m.capabilities.has("vision"));
|
||||||
if (withVision.length > 0) return withVision[0];
|
if (withVision.length > 0) return withVision[0];
|
||||||
}
|
}
|
||||||
if (candidates.length > 0) return candidates[0];
|
if (candidates.length > 0) return candidates[0];
|
||||||
}
|
}
|
||||||
return this.models.get("fi-groq")!;
|
|
||||||
|
// Final fallback: any allowed model, preferring the cheapest.
|
||||||
|
const allowed = [...this.models.values()]
|
||||||
|
.filter(isAllowed)
|
||||||
|
.sort((a, b) => a.costPer1kIn + a.costPer1kOut - (b.costPer1kIn + b.costPer1kOut));
|
||||||
|
if (allowed.length > 0) return allowed[0];
|
||||||
|
|
||||||
|
// Absolute last resort: a zero-cost free-tier model.
|
||||||
|
return this.models.get("fi-gemini")!;
|
||||||
}
|
}
|
||||||
|
|
||||||
getGraderForWorkflow(complexity: TaskComplexity): string {
|
getGraderForWorkflow(complexity: TaskComplexity): string {
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,195 @@
|
||||||
|
import { createServer, type Socket } from "node:net";
|
||||||
|
import { ModelRouter, type ModelRoute, type RoutingResponse, type TaskComplexity, type TaskDomain } from "./model-router.js";
|
||||||
|
import { CompoundStack, type StackStatus } from "./compound-stack.js";
|
||||||
|
|
||||||
|
export interface RpcRequest {
|
||||||
|
id?: string | number | null;
|
||||||
|
method: string;
|
||||||
|
params?: Record<string, unknown>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RpcError {
|
||||||
|
code: number;
|
||||||
|
message: string;
|
||||||
|
data?: unknown;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RpcResponse {
|
||||||
|
id: string | number | null;
|
||||||
|
result?: unknown;
|
||||||
|
error?: RpcError;
|
||||||
|
}
|
||||||
|
|
||||||
|
const VALID_DOMAINS: TaskDomain[] = [
|
||||||
|
"code",
|
||||||
|
"research",
|
||||||
|
"analysis",
|
||||||
|
"creative",
|
||||||
|
"planning",
|
||||||
|
"grading",
|
||||||
|
"review",
|
||||||
|
"routing",
|
||||||
|
];
|
||||||
|
|
||||||
|
const VALID_COMPLEXITIES: TaskComplexity[] = ["trivial", "simple", "medium", "complex", "extreme"];
|
||||||
|
|
||||||
|
function isTaskDomain(value: unknown): value is TaskDomain {
|
||||||
|
return typeof value === "string" && (VALID_DOMAINS as string[]).includes(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
function isTaskComplexity(value: unknown): value is TaskComplexity {
|
||||||
|
return typeof value === "string" && (VALID_COMPLEXITIES as string[]).includes(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
function serializeModelRoute(route: RoutingResponse): unknown {
|
||||||
|
return {
|
||||||
|
primary: serializeModel(route.primary),
|
||||||
|
fallback: serializeModel(route.fallback),
|
||||||
|
grader: serializeModel(route.grader),
|
||||||
|
reason: route.reason,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function serializeModel(model: ModelRoute): unknown {
|
||||||
|
return {
|
||||||
|
modelId: model.modelId,
|
||||||
|
displayName: model.displayName,
|
||||||
|
tier: model.tier,
|
||||||
|
costPer1kIn: model.costPer1kIn,
|
||||||
|
costPer1kOut: model.costPer1kOut,
|
||||||
|
capabilities: [...model.capabilities],
|
||||||
|
recommendedFor: model.recommendedFor,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function serializeStackStatus(status: StackStatus): unknown {
|
||||||
|
return {
|
||||||
|
sessionId: status.sessionId,
|
||||||
|
layers: status.layers,
|
||||||
|
modelRoute: status.modelRoute ? serializeModelRoute(status.modelRoute) : null,
|
||||||
|
activeGoal: status.activeGoal,
|
||||||
|
worktrees: status.worktrees,
|
||||||
|
verification: status.verification,
|
||||||
|
stateFile: status.stateFile,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function send(socket: Socket, response: RpcResponse): void {
|
||||||
|
try {
|
||||||
|
socket.write(JSON.stringify(response) + "\n");
|
||||||
|
} catch {
|
||||||
|
// Socket may have closed; ignore.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function handleRequest(request: RpcRequest): Promise<RpcResponse> {
|
||||||
|
const id = request.id ?? null;
|
||||||
|
const params = request.params ?? {};
|
||||||
|
|
||||||
|
try {
|
||||||
|
switch (request.method) {
|
||||||
|
case "route": {
|
||||||
|
const task = typeof params.task === "string" ? params.task : "";
|
||||||
|
const domain: TaskDomain = isTaskDomain(params.domain) ? params.domain : "code";
|
||||||
|
const router = new ModelRouter();
|
||||||
|
const complexity: TaskComplexity = isTaskComplexity(params.complexity)
|
||||||
|
? params.complexity
|
||||||
|
: router.taskComplexity(task);
|
||||||
|
const route = router.route({ task, domain, complexity, requiresVision: false });
|
||||||
|
return { id, result: serializeModelRoute(route) };
|
||||||
|
}
|
||||||
|
|
||||||
|
case "stack": {
|
||||||
|
const task = typeof params.task === "string" ? params.task : "";
|
||||||
|
const project = typeof params.project === "string" ? params.project : "default";
|
||||||
|
const stack = new CompoundStack();
|
||||||
|
const status = await stack.run(task, project);
|
||||||
|
return { id, result: serializeStackStatus(status) };
|
||||||
|
}
|
||||||
|
|
||||||
|
case "status": {
|
||||||
|
const stack = new CompoundStack();
|
||||||
|
return { id, result: { status: stack.getStatus() } };
|
||||||
|
}
|
||||||
|
|
||||||
|
case "models": {
|
||||||
|
const router = new ModelRouter();
|
||||||
|
const models = router.listModels().map((m) => serializeModel(m));
|
||||||
|
return { id, result: { models } };
|
||||||
|
}
|
||||||
|
|
||||||
|
default:
|
||||||
|
return {
|
||||||
|
id,
|
||||||
|
error: { code: -32601, message: `Method not found: ${request.method}` },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
return {
|
||||||
|
id,
|
||||||
|
error: {
|
||||||
|
code: -32603,
|
||||||
|
message: err instanceof Error ? err.message : String(err),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function startRpcServer(port: number): Promise<void> {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const server = createServer((socket) => {
|
||||||
|
let buffer = "";
|
||||||
|
|
||||||
|
socket.on("data", async (data) => {
|
||||||
|
buffer += data.toString("utf-8");
|
||||||
|
const lines = buffer.split("\n");
|
||||||
|
buffer = lines.pop() ?? "";
|
||||||
|
|
||||||
|
for (const line of lines) {
|
||||||
|
const trimmed = line.trim();
|
||||||
|
if (!trimmed) continue;
|
||||||
|
|
||||||
|
let request: RpcRequest;
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(trimmed);
|
||||||
|
if (typeof parsed !== "object" || parsed === null || typeof parsed.method !== "string") {
|
||||||
|
send(socket, {
|
||||||
|
id: parsed?.id ?? null,
|
||||||
|
error: { code: -32600, message: "Invalid Request" },
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
request = parsed as RpcRequest;
|
||||||
|
} catch {
|
||||||
|
send(socket, { id: null, error: { code: -32700, message: "Parse error" } });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const response = await handleRequest(request);
|
||||||
|
send(socket, response);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
socket.on("error", (err) => {
|
||||||
|
console.error(` RPC socket error: ${err.message}`);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
server.on("error", (err) => {
|
||||||
|
reject(err);
|
||||||
|
});
|
||||||
|
|
||||||
|
server.listen(port, () => {
|
||||||
|
console.log(`\n Fable 5 RPC server listening on port ${port}`);
|
||||||
|
console.log(` Send JSONL requests, one JSON object per line.\n`);
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
|
||||||
|
process.on("SIGINT", () => {
|
||||||
|
console.log("\n Shutting down RPC server...");
|
||||||
|
server.close(() => {
|
||||||
|
process.exit(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
11
src/index.ts
11
src/index.ts
|
|
@ -1943,6 +1943,17 @@ fable
|
||||||
console.log(` `);
|
console.log(` `);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// ── RPC ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
fable
|
||||||
|
.command("rpc")
|
||||||
|
.description("Start a JSONL TCP RPC server for Fable 5 commands")
|
||||||
|
.option("-p, --port <n>", "TCP port to listen on (default: 18902)", parseInt)
|
||||||
|
.action(async (opts: { port?: number }) => {
|
||||||
|
const { startRpcServer } = await import("./fable5/rpc-server.js");
|
||||||
|
await startRpcServer(opts.port ?? 18902);
|
||||||
|
});
|
||||||
|
|
||||||
// ── Parse ───────────────────────────────────────────────────
|
// ── Parse ───────────────────────────────────────────────────
|
||||||
|
|
||||||
program.parse(process.argv);
|
program.parse(process.argv);
|
||||||
|
|
|
||||||
|
|
@ -87,6 +87,53 @@ export class ProxyClient {
|
||||||
stream: false,
|
stream: false,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
return this.sendChatCompletion(body, modelId);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Send a vision-capable chat completion request with an image.
|
||||||
|
* The image argument may be a base64 data URI (data:image/png;base64,...) or an HTTP URL.
|
||||||
|
*/
|
||||||
|
async completeVision(
|
||||||
|
prompt: string,
|
||||||
|
image: string,
|
||||||
|
model?: string,
|
||||||
|
options?: {
|
||||||
|
maxTokens?: number;
|
||||||
|
temperature?: number;
|
||||||
|
systemPrompt?: string;
|
||||||
|
}
|
||||||
|
): Promise<ProxyModelResponse> {
|
||||||
|
const modelId = model ?? this.config.defaultModel;
|
||||||
|
|
||||||
|
const imageContent: Record<string, unknown> = image.startsWith("data:")
|
||||||
|
? { type: "image_url", image_url: { url: image } }
|
||||||
|
: { type: "image_url", image_url: { url: image } };
|
||||||
|
|
||||||
|
const body = {
|
||||||
|
model: modelId,
|
||||||
|
messages: [
|
||||||
|
...(options?.systemPrompt ? [{ role: "system", content: options.systemPrompt }] : []),
|
||||||
|
{
|
||||||
|
role: "user",
|
||||||
|
content: [
|
||||||
|
{ type: "text", text: prompt },
|
||||||
|
imageContent,
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
max_tokens: options?.maxTokens ?? 2048,
|
||||||
|
temperature: options?.temperature ?? 0.7,
|
||||||
|
stream: false,
|
||||||
|
};
|
||||||
|
|
||||||
|
return this.sendChatCompletion(body, modelId);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async sendChatCompletion(
|
||||||
|
body: Record<string, unknown>,
|
||||||
|
modelId: string
|
||||||
|
): Promise<ProxyModelResponse> {
|
||||||
return new Promise((resolve, reject) => {
|
return new Promise((resolve, reject) => {
|
||||||
const data = JSON.stringify(body);
|
const data = JSON.stringify(body);
|
||||||
let responseData = "";
|
let responseData = "";
|
||||||
|
|
|
||||||
|
|
@ -14,6 +14,8 @@
|
||||||
* Without it, the agent can't tell if the image it generated actually looks right.
|
* Without it, the agent can't tell if the image it generated actually looks right.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
import * as fs from "node:fs";
|
||||||
|
import { ProxyClient } from "../pai/proxy-client.js";
|
||||||
import type { LoopIteration } from "../core/types.js";
|
import type { LoopIteration } from "../core/types.js";
|
||||||
|
|
||||||
// ── Types ──────────────────────────────────────────────────
|
// ── Types ──────────────────────────────────────────────────
|
||||||
|
|
@ -26,6 +28,10 @@ export interface VisionCheckConfig {
|
||||||
model: string;
|
model: string;
|
||||||
/** Max retries on vision API failure */
|
/** Max retries on vision API failure */
|
||||||
maxRetries: number;
|
maxRetries: number;
|
||||||
|
/** Optional local proxy client (routes through localhost:18901) */
|
||||||
|
proxyClient?: ProxyClient;
|
||||||
|
/** Default vision model when routing through the proxy */
|
||||||
|
proxyModel: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface VisionCheckResult {
|
export interface VisionCheckResult {
|
||||||
|
|
@ -53,6 +59,7 @@ const DEFAULT_CONFIG: VisionCheckConfig = {
|
||||||
apiKey: "",
|
apiKey: "",
|
||||||
model: "claude-sonnet-4-6", // Sonnet 4.6 has vision
|
model: "claude-sonnet-4-6", // Sonnet 4.6 has vision
|
||||||
maxRetries: 2,
|
maxRetries: 2,
|
||||||
|
proxyModel: "claude-sonnet-4-6", // routed through the local proxy
|
||||||
};
|
};
|
||||||
|
|
||||||
// ── Vision System Prompt ───────────────────────────────────
|
// ── Vision System Prompt ───────────────────────────────────
|
||||||
|
|
@ -184,6 +191,15 @@ export class VisionSelfCheck {
|
||||||
artifact: VisualArtifact,
|
artifact: VisualArtifact,
|
||||||
prompt: string
|
prompt: string
|
||||||
): Promise<string> {
|
): Promise<string> {
|
||||||
|
// Prefer the local proxy if configured (routes through localhost:18901)
|
||||||
|
if (this.config.proxyClient) {
|
||||||
|
try {
|
||||||
|
return await this.callProxyVision(artifact, prompt);
|
||||||
|
} catch {
|
||||||
|
// Fall through to direct providers
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Try Anthropic API first (Sonnet 4.6 has vision)
|
// Try Anthropic API first (Sonnet 4.6 has vision)
|
||||||
try {
|
try {
|
||||||
return await this.callAnthropicVision(artifact, prompt);
|
return await this.callAnthropicVision(artifact, prompt);
|
||||||
|
|
@ -193,6 +209,38 @@ export class VisionSelfCheck {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async callProxyVision(
|
||||||
|
artifact: VisualArtifact,
|
||||||
|
prompt: string
|
||||||
|
): Promise<string> {
|
||||||
|
const proxy = this.config.proxyClient!;
|
||||||
|
const image = await this.prepareImageSource(artifact);
|
||||||
|
const response = await proxy.completeVision(
|
||||||
|
prompt,
|
||||||
|
image,
|
||||||
|
this.config.proxyModel,
|
||||||
|
{
|
||||||
|
maxTokens: 1024,
|
||||||
|
temperature: 0.3,
|
||||||
|
systemPrompt: VISION_SYSTEM_PROMPT,
|
||||||
|
}
|
||||||
|
);
|
||||||
|
return response.content;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async prepareImageSource(artifact: VisualArtifact): Promise<string> {
|
||||||
|
if (artifact.source.startsWith("data:")) {
|
||||||
|
return artifact.source;
|
||||||
|
}
|
||||||
|
if (/^https?:\/\//i.test(artifact.source)) {
|
||||||
|
return artifact.source;
|
||||||
|
}
|
||||||
|
// Treat as a local file path and base64-encode it
|
||||||
|
const data = fs.readFileSync(artifact.source);
|
||||||
|
const mime = artifact.mimeType || this.guessMimeType(artifact.source);
|
||||||
|
return `data:${mime};base64,${data.toString("base64")}`;
|
||||||
|
}
|
||||||
|
|
||||||
private async callAnthropicVision(
|
private async callAnthropicVision(
|
||||||
artifact: VisualArtifact,
|
artifact: VisualArtifact,
|
||||||
prompt: string
|
prompt: string
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue