From fcfa2277ba1464768320e8490915da68aafe4a96 Mon Sep 17 00:00:00 2001 From: Yuriy Mayatnikov Date: Wed, 27 May 2026 17:39:43 +0300 Subject: [PATCH] v0.3.0-rc.1: live skill registry + fast advisor scaffold MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Roots out the v0.2.x failure mode: Pi-extracted lessons routinely named hallucinated skill ids (relocate.surface, choose.safe.surface, survive.shelter, gather.visible_log, …). All 47 Pi-lessons in the live DB had applied_count=0 because normalisePreferSkill couldn't find them. Fix: 1. runtime/skill-registry.js — single source of truth derived from skills/index.js. Exports listSkillIds, isRegistered, and a prompt-ready block (skillRegistryPrompt) grouped by namespace. 2. Pi prompts (coach/postmortem, coach/reflect) embed the live registry with a "USE ONLY THESE, never invent" instruction. Lessons are filtered at write-time too — anything not in the registry and not a known mode name gets dropped. 3. coach/advice.js — normalisePreferSkill now returns null for unknown ids, hardening consult() against any hallucinations that slip through. Warn-logged for visibility. Also lays the LLM substrate for the rest of v0.3.0: - runtime/llm/provider.js — OpenAI-compatible chat client. Configured via PEPA_FAST_LLM_{BASE_URL,API_KEY,MODEL,TIMEOUT_MS}. Safe no-op unless API_KEY is set. Supports JSON-mode. - runtime/coach/fast-advisor.js — tactical advisor tier (scaffold). Exposes advise() that asks the fast LLM what to do RIGHT NOW when the reflex is wedged/stuck. Rejects hallucinated skill ids using the registry. Rate-limited 6/h, 30s cooldown. Not auto-triggered yet — wired into reflex in rc.3 (awareness layer). Tests: 279 green (+24 vs rc.3): 5 registry, 9 provider, 10 advisor. See dev/v0.3.0/PLAN.md for the full iteration design (manifesto needs ladder, event-driven awareness, skill pre-emption) and STATUS.md for shipped/pending tracking. Co-Authored-By: Claude Opus 4.7 --- dev/v0.3.0/PLAN.md | 155 ++++++++++++++++++++++++++ dev/v0.3.0/STATUS.md | 86 +++++++++++++++ package.json | 4 +- runtime/coach/advice.js | 20 ++-- runtime/coach/advice.test.js | 12 ++- runtime/coach/fast-advisor.js | 167 +++++++++++++++++++++++++++++ runtime/coach/fast-advisor.test.js | 163 ++++++++++++++++++++++++++++ runtime/coach/postmortem.js | 41 ++++++- runtime/coach/reflect.js | 36 ++++++- runtime/llm/provider.js | 152 ++++++++++++++++++++++++++ runtime/llm/provider.test.js | 164 ++++++++++++++++++++++++++++ runtime/skill-registry.js | 68 ++++++++++++ runtime/skill-registry.test.js | 65 +++++++++++ 13 files changed, 1113 insertions(+), 20 deletions(-) create mode 100644 dev/v0.3.0/PLAN.md create mode 100644 dev/v0.3.0/STATUS.md create mode 100644 runtime/coach/fast-advisor.js create mode 100644 runtime/coach/fast-advisor.test.js create mode 100644 runtime/llm/provider.js create mode 100644 runtime/llm/provider.test.js create mode 100644 runtime/skill-registry.js create mode 100644 runtime/skill-registry.test.js diff --git a/dev/v0.3.0/PLAN.md b/dev/v0.3.0/PLAN.md new file mode 100644 index 0000000..62a191c --- /dev/null +++ b/dev/v0.3.0/PLAN.md @@ -0,0 +1,155 @@ +# pepa v0.3.0 — Maslow + Awareness + +Concept: needs-based hierarchical agent with real-time event awareness and a +fast LLM tactical advisor. Pi (CLI) stays for slow deep analytics +(postmortem, reflection). A second, fast LLM tier (TimeWeb / OpenAI- +compatible) handles "what to do right now" decisions when the reflex +detects wedged/stuck/changed environment. + +This is a major behavioural rewrite over v0.2.x: + +- v0.2.x had a flat reflex chain (modes → defend → eat → sleep → + curriculum → idle) — curriculum was just "next mode". The bot had no + internal concept of "do I have a pickaxe?" let alone "do I have a + shelter?" Lessons from Pi were hallucinated skill names (47/47 + Pi-extracted lessons applied_count=0 as of v0.2.0-rc.3). +- v0.3.0 introduces a **needs ladder** (Maslow-like) that drives the + bot's intent end-to-end, an **awareness layer** that reacts to env + changes within ~100ms instead of waiting for the next tick boundary, + and a **fast LLM** tier that closes the loop when the bot is stuck. + +## Concept terms + +This pattern is called variously in the AI literature: +- **Hierarchical Task Network (HTN)** planning — Voyager uses this +- **Needs-based / utility AI** — game-AI mainstream +- **BDI agent** (Beliefs-Desires-Intentions) — academic AI +- **Subsumption architecture** (Brooks) — reactive layers preempt + deliberative layers when conditions trigger + +Pepa v0.3.0 is essentially **Maslow-stack curriculum + Brooks-style +preemption + dual-tier LLM (fast tactical + slow analytical)**. + +## Manifesto / Needs ladder + +``` +L0 alive HP>5, не тонет, не горит, не падает с фатальной высоты +L1 food ≥6 насыщения (готов кушать на месте) +L2 tools_wood wooden_pickaxe + wooden_axe + wooden_sword +L3 shelter_basic 4 стены + крыша + кровать в радиусе 8 от спавн-base +L4 tools_stone stone_pickaxe + stone_axe + stone_sword +L5 armor_basic хотя бы один кусок (predпочтительно нагрудник) +L6 food_security ≥16 еды + источник (ферма / стая коров рядом) +L7 tools_iron iron_pickaxe + iron_axe + iron_sword +L8 armor_iron полный iron set +L9 village_seed 2+ постройки, забор/оградка, базовая ферма +L10 village_full глобальная цель (ферма + дом + сосед-NPC мечта) +``` + +На каждом тике reflex выбирает **самую нижнюю неудовлетворённую** нужду. +Эта нужда становится **активной**. Curriculum.next() и Pi-coach подсказки +дальше выбираются **внутри** активной нужды. Если нужда сменилась +(например, HP упало → L0 проснулся), текущий skill прерывается. + +## v0.3.0 release plan (3 rc) + +### rc.1 — Skill registry hardening + Fast advisor scaffold +**Цель**: убрать главную проблему v0.2.x — Pi инвентит skill names. +Не вводим манифест ещё, но строим инфраструктуру для него. + +- `runtime/skill-registry.js` — exported `listSkillIds()`, `isRegistered(id)`, + `skillRegistryPrompt()` (готовый блок текста для LLM-промпта со списком + валидных id, по группам) +- Все Pi-промпты (`coach/postmortem.js`, `coach/reflect.js`) передают + реестр в system prompt +- `runtime/coach/advice.js`: `normalisePreferSkill` строго отбрасывает + всё, что не в реестре (raise log, не дрейфит на fuzzy) +- `runtime/llm/provider.js` — OpenAI-совместимый клиент, конфигурируется + через env (`PEPA_FAST_LLM_BASE_URL`, `_API_KEY`, `_MODEL`); graceful + fallback "no-op" если env не задан (бот не падает) +- `runtime/coach/fast-advisor.js` — функция `advise({snapshot, reason})` + с rate-limit (макс. 6 вызовов/час), таймаут 8с, JSON-парсинг ответа + через тот же `extractJson()` что у Pi. **Пока не подключаем к reflex** — + scaffold + тесты +- Тесты для каждого нового модуля + регрессионный тест: + `advice.test.js` проверяет что hallucinated `relocate.surface` falls + through (никакой override) +- Минимум 270+ зелёных тестов + +### rc.2 — Manifesto / Needs ladder + curriculum integration +**Цель**: bot acts toward concrete needs, not toward "explore further". + +- `runtime/manifesto/needs.js` — каталог 11 нужд, каждая со схемой: + ``` + { id, level, detect(snapshot) → boolean satisfied, prefer_skill_for_pursuit, ... } + ``` +- `runtime/manifesto/state.js` — `pickActiveNeed(snapshot)` возвращает + самую нижнюю неудовлетворённую. Кеширует на 5с. +- `runtime/reflex.js`: + - В `curriculumReflex` сначала `activeNeed = pickActiveNeed(...)` + - Skill подбирается в первую очередь по `activeNeed.prefer_skill_for_pursuit` + - Fallback на curriculum.next() только если нужда не дала однозначного skill +- `runtime/coach/advice.js`: `consult()` теперь принимает `activeNeed` и + отбрасывает lessons чьи trigger_situation противоречит текущей нужде + (например, "избегай ночью гулять" не применяется когда L0=alive в опасности) +- Pi-промпты (postmortem, reflect) получают `currentNeed: "L2 tools_wood"` + и просят Pi дать совет именно для этого уровня +- Новый персонаж reflex hook: при смене activeNeed бот произносит в чате + "пора заняться X" (Russian narration tying into chatter.js) +- Тесты: каждая нужда имеет 2-3 теста (detect satisfied/unsatisfied, + правильный prefer_skill) + +### rc.3 — Event-driven awareness + skill pre-emption +**Цель**: bot reacts within ~100ms to env changes (fall, teleport, +damage, hostile spawn near). + +- `runtime/awareness/events.js` — установка listeners: + - `bot.on('move')` — детект position-jump >5 блоков за тик → событие + `forced-move` → инвалидация current dispatch context + - `bot.on('health')` — снижение HP > 2 за тик → reflex.preempt() + - `bot.on('entitySpawn')` — враждебный <12 блоков → reflex.preempt() + - `bot.on('blockUpdate')` около бота (manhattan <4) → пометка + `environment_changed=true` +- `runtime/awareness/state.js` — храним flags `(forcedMove, lastDamage, + hostileAdded, envChanged)`, expose `consumeFlags()` для reflex +- Skill protocol extended: `execute(ctx, args)` теперь получает + `ctx.abortSignal` (AbortSignal). Длинные операции (pathfinder.goto, + collectBlock loops) проверяют `signal.aborted` между шагами и сразу + возвращают `{ok: false, code: 'preempted'}` +- `runtime/reflex.js`: при срабатывании preempt-флагов вызывается + `currentDispatch?.abort()`, и reflex запускает следующий тик + немедленно (не ждёт `DISPATCH_INTERVAL_MS`) +- `recovery.tunnel-out`, `survive.pillar-up`, `gather.logs`, `explore.far` + адаптируются под AbortSignal (минимальное — `if (signal.aborted) + return { ok:false, code:'preempted' }` после каждого `await`) +- **Связка с fast advisor**: когда preempt сработал из-за `forcedMove` + или environment_changed, и reflex не находит очевидный skill, вызывает + `fastAdvisor.advise(...)` чтобы получить тактический совет (rc.1 + scaffolding активируется здесь) + +### Acceptance signals (после rc.3) + +- В живой БД: `lessons WHERE applied_count > 0` растёт (сейчас 3, + должно стать 20+ за сутки) +- Bot движется к конкретным целям: видимый прогресс инвентаря (wood → + pickaxe → stone → axe), а не "блуждание в одном квадранте" +- При forcedMove бот меняет план в течение секунды, не продолжает + старый skill +- Fast advisor пакетно срабатывает <10 раз/час, каждый раз приводит к + смене skill (логируется) + +## Что отложено в v0.3.1 + +- **Vision** (multimodal LLM на скриншотах) — требует prismarine-viewer + pipeline + multimodal model в провайдере; не в первом релизе +- **Vector memory of scenarios** — embeddings от похожих ситуаций +- **Auto-curriculum from wiki** — фоновый паук minecraft.wiki + +## Workflow notes + +- Каждый rc — отдельный PR, мержим после approve +- main защищён, auto-patch открывает PR с тегом `auto-patch` +- Если что-то ломается в проде (живой бот в петле >30 мин), откатываем + на v0.2.0-rc.3 commit `865aae1` через `git checkout ` на + ветке `revert/v0.3.0-stability` +- Тестовые данные строго в `/tmp/pepa-test-state-*` (исправлено в v0.2.0-rc.2) diff --git a/dev/v0.3.0/STATUS.md b/dev/v0.3.0/STATUS.md new file mode 100644 index 0000000..2e41401 --- /dev/null +++ b/dev/v0.3.0/STATUS.md @@ -0,0 +1,86 @@ +# pepa v0.3.0 — status + +Live tracking document for the v0.3.0 iteration ("Maslow + Awareness"). +See [`PLAN.md`](./PLAN.md) for the full design. + +## Shipped + +### rc.1 — Live skill registry + Fast advisor scaffold +**Root problem solved**: 47/47 Pi-extracted lessons in v0.2.x had +`applied_count = 0` because Pi was hallucinating skill ids +(`relocate.surface`, `choose.safe.surface`, `survive.shelter`, +`gather.visible_log`, …) that don't exist in the registry. Both halves +fixed: (a) Pi now sees the real registry in its system prompt, +(b) anything that still slips through gets rejected at consult time. + +- [`runtime/skill-registry.js`](../../runtime/skill-registry.js) — + single source of truth wrapping `skills/index.js`. Exports: + - `listSkillIds()` — live id list + - `isRegistered(id)` — bool check + - `describeSkill(id)` — id/title/timeoutMs + - `skillRegistryPrompt({ limit })` — prompt-ready block grouped by + namespace, with "USE ONLY THESE, never invent" instruction +- [`runtime/llm/provider.js`](../../runtime/llm/provider.js) — + OpenAI-compatible chat client, env-driven: + - `PEPA_FAST_LLM_BASE_URL` (default `https://api.openai.com/v1`) + - `PEPA_FAST_LLM_API_KEY` (required to enable; safe no-op otherwise) + - `PEPA_FAST_LLM_MODEL` (required) + - `PEPA_FAST_LLM_TIMEOUT_MS` (default 8000) + - Supports JSON-mode via `response_format: { type: "json_object" }` + - Surfaces `not_configured`, `no_model`, `http_`, + `network_error`, `timeout`, `bad_json` codes +- [`runtime/coach/fast-advisor.js`](../../runtime/coach/fast-advisor.js) + — tactical "what now?" tier. Scaffold only in rc.1; auto-trigger + comes in rc.3. + - `advise({snapshot, reason, recentSkillIds, lessonsTail})` → + `{action: 'switch_skill'|'continue'|'wait', skillId?, rationale}` + - Rejects any returned `skill_id` not in the live registry + - Rate-limit: 6 calls/hour, 30s cooldown between calls + - System prompt embeds registry; user prompt carries snapshot + trigger +- [`runtime/coach/advice.js`](../../runtime/coach/advice.js): + - `normalisePreferSkill()` now returns `null` for anything not in + registry/mode-map (was: passed through unchanged → dispatcher + crashed at `runSkill()`) + - Logs `warn` line when a hallucinated prefer_skill is dropped +- [`runtime/coach/postmortem.js`](../../runtime/coach/postmortem.js): + - Pi prompt includes the live registry block (`skillRegistryPrompt`) + with a "CRITICAL: USE ONLY THESE" instruction + - On insert, drops `prefer_skill`/`avoid_skill` that's neither a + registered id nor a known mode name; warn-logs the count +- [`runtime/coach/reflect.js`](../../runtime/coach/reflect.js) — same + treatment as postmortem (registry in prompt + write-time filter) + +Tests: 279 green (was 257 on rc.3). Added: +- `runtime/skill-registry.test.js` — 5 tests +- `runtime/llm/provider.test.js` — 9 tests +- `runtime/coach/fast-advisor.test.js` — 10 tests + +### rc.2 — (pending) Manifesto / Needs ladder +### rc.3 — (pending) Event-driven awareness + skill pre-emption + +## Next session quick start + +1. **Read PLAN.md** for the full design and per-rc breakdown. +2. **Check live DB** to see if Pi-lesson application is improving: + ```bash + sqlite3 state/play.xmatic.team_25565/knowledge.db \ + "SELECT source, COUNT(*) AS n, SUM(applied_count > 0) AS applied + FROM lessons GROUP BY source ORDER BY n DESC;" + ``` + After rc.1 deploys, expect Pi-coach/Pi-reflect `applied` count to + start growing as the registry feedback closes the loop. +3. **Set fast-advisor env when ready to test**: + ```bash + export PEPA_FAST_LLM_BASE_URL="https:///v1" + export PEPA_FAST_LLM_API_KEY="" + export PEPA_FAST_LLM_MODEL="gpt-5-mini" + ``` + The advisor still isn't auto-triggered in rc.1 — it's wired in rc.3. +4. **Pick the next rc** from PLAN.md. + +## Workflow notes + +- main is protected — only operator merges PRs +- Tests: `npm test` (279 green at last check), isolated under `/tmp/` +- The bot supervisor hot-restarts on file changes in `runtime/**/*.js` +- If something regresses badly, revert to v0.2.0-rc.3 commit `865aae1` diff --git a/package.json b/package.json index d07bf00..a1653f5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "pepa-pi-bot", - "version": "0.2.0-rc.3", + "version": "0.3.0-rc.1", "private": true, "description": "An autonomous, self-extending Minecraft player powered by Pi and Mineflayer.", "license": "MIT", @@ -16,7 +16,7 @@ "tui": "tsx tui/tui.tsx", "propose:apply": "node scripts/propose-apply.js", "stop": "bash scripts/stop.sh", - "test": "node --test runtime/skills/contract.test.js runtime/skills/groups.test.js runtime/skills/compat.test.js runtime/skills/recovery-tunnel-out.test.js runtime/skills/pillar-up.test.js runtime/curriculum.test.js runtime/social/social.test.js runtime/social/conversation.test.js runtime/social/chat-history.test.js runtime/social/reply-pi.test.js runtime/stuck-incident.test.js runtime/compat.test.js runtime/reflex.test.js runtime/base-site.test.js runtime/locations.test.js runtime/watch-filter.test.js runtime/world-journal.test.js runtime/scenario-memory.test.js runtime/critic.test.js runtime/skill-library.test.js runtime/modes.test.js runtime/pathfinder-watchdog.test.js runtime/knowledge/knowledge.test.js runtime/coach/postmortem.test.js runtime/coach/advice.test.js runtime/coach/reflect.test.js runtime/persona/chatter.test.js scripts/edit-scope.test.js scripts/lint-patch.test.js" + "test": "node --test runtime/skills/contract.test.js runtime/skills/groups.test.js runtime/skills/compat.test.js runtime/skills/recovery-tunnel-out.test.js runtime/skills/pillar-up.test.js runtime/curriculum.test.js runtime/social/social.test.js runtime/social/conversation.test.js runtime/social/chat-history.test.js runtime/social/reply-pi.test.js runtime/stuck-incident.test.js runtime/compat.test.js runtime/reflex.test.js runtime/base-site.test.js runtime/locations.test.js runtime/watch-filter.test.js runtime/world-journal.test.js runtime/scenario-memory.test.js runtime/critic.test.js runtime/skill-library.test.js runtime/skill-registry.test.js runtime/modes.test.js runtime/pathfinder-watchdog.test.js runtime/knowledge/knowledge.test.js runtime/llm/provider.test.js runtime/coach/postmortem.test.js runtime/coach/advice.test.js runtime/coach/reflect.test.js runtime/coach/fast-advisor.test.js runtime/persona/chatter.test.js scripts/edit-scope.test.js scripts/lint-patch.test.js" }, "dependencies": { "better-sqlite3": "^11.10.0", diff --git a/runtime/coach/advice.js b/runtime/coach/advice.js index 47c19ec..e05cf79 100644 --- a/runtime/coach/advice.js +++ b/runtime/coach/advice.js @@ -10,7 +10,8 @@ // recall → behavioural change. Without this, the DB is just a log. import { isAvailable as knowledgeAvailable, topAdvice, markApplied } from "../knowledge/index.js"; -import { info } from "../log.js"; +import { isRegistered } from "../skill-registry.js"; +import { info, warn } from "../log.js"; // Skills we will not blindly swap into — they require their own // preconditions (e.g. survive.flee needs a known threat direction). @@ -47,15 +48,19 @@ const MODE_TO_SKILL = Object.freeze({ }); function normalisePreferSkill(raw) { - if (!raw || typeof raw !== "string") return raw; - if (SAFE_OVERRIDES.has(raw)) return raw; + if (!raw || typeof raw !== "string") return null; + if (SAFE_OVERRIDES.has(raw) && isRegistered(raw)) return raw; const lower = raw.toLowerCase().trim(); if (MODE_TO_SKILL[lower]) return MODE_TO_SKILL[lower]; // Pi sometimes writes "survive_flee" or "survive flee"; normalise. const dot = lower.replace(/[_\s]+/g, "."); - if (SAFE_OVERRIDES.has(dot)) return dot; + if (SAFE_OVERRIDES.has(dot) && isRegistered(dot)) return dot; if (MODE_TO_SKILL[dot]) return MODE_TO_SKILL[dot]; - return raw; + // Anything else (Pi hallucinated names like "relocate.surface", + // "choose.safe.surface", "survive.shelter", "gather.visible_log") — + // hard reject. We'd rather fall through to 'avoid' / 'proceed' than + // dispatch a nonexistent skill. + return null; } /** @@ -81,7 +86,7 @@ export function consult({ plannedSkillId, snapshot } = {}) { // avoid_skill matches? if (advice.avoid && advice.avoid === plannedSkillId) { const normalisedPrefer = normalisePreferSkill(advice.prefer); - if (normalisedPrefer && SAFE_OVERRIDES.has(normalisedPrefer)) { + if (normalisedPrefer && SAFE_OVERRIDES.has(normalisedPrefer) && isRegistered(normalisedPrefer)) { if (normalisedPrefer !== advice.prefer) { info("coach", `advice: normalised prefer "${advice.prefer}" → "${normalisedPrefer}"`); } @@ -93,6 +98,9 @@ export function consult({ plannedSkillId, snapshot } = {}) { lesson: advice.lesson, }; } + if (advice.prefer && !normalisedPrefer) { + warn("coach", `advice: rejected hallucinated prefer_skill "${advice.prefer}" (lesson #${advice.lessonId})`); + } info("coach", `advice: avoid ${plannedSkillId} (lesson #${advice.lessonId})`); return { action: "avoid", lessonId: advice.lessonId, lesson: advice.lesson }; } diff --git a/runtime/coach/advice.test.js b/runtime/coach/advice.test.js index 04d6612..ae71c42 100644 --- a/runtime/coach/advice.test.js +++ b/runtime/coach/advice.test.js @@ -117,10 +117,16 @@ test("normalisePreferSkill: passes through known dot-form skills unchanged", () assert.equal(normalisePreferSkill("explore.far"), "explore.far"); }); -test("normalisePreferSkill: unknown values returned as-is", () => { - assert.equal(normalisePreferSkill("some.unknown.skill"), "some.unknown.skill"); +test("normalisePreferSkill: unknown values rejected (returns null)", () => { + // v0.3.0-rc.1: anything not in the live registry and not a known mode + // name is rejected outright. We'd rather fall through to 'avoid' than + // dispatch a hallucinated skill id. + assert.equal(normalisePreferSkill("some.unknown.skill"), null); + assert.equal(normalisePreferSkill("relocate.surface"), null); + assert.equal(normalisePreferSkill("choose.safe.surface"), null); + assert.equal(normalisePreferSkill("survive.shelter"), null); assert.equal(normalisePreferSkill(null), null); - assert.equal(normalisePreferSkill(""), ""); + assert.equal(normalisePreferSkill(""), null); }); test("consult: Pi-style mode-name prefer is normalised to override target", async () => { diff --git a/runtime/coach/fast-advisor.js b/runtime/coach/fast-advisor.js new file mode 100644 index 0000000..2d4f818 --- /dev/null +++ b/runtime/coach/fast-advisor.js @@ -0,0 +1,167 @@ +// Fast tactical advisor — second LLM tier, parallel to Pi. +// +// Pi (the CLI coach) is great for deep post-mortems and 30-min reflection, +// but it's slow (5-15s) and rate-limited. When the reflex detects the bot +// is wedged, stuck, or just took an environment shock (forced teleport, +// HP plunge, hostile spawn), we want a sub-2-second "what do I do?" +// answer from a cheap, hosted model. That's this module. +// +// In rc.1 this is a scaffold: complete() + advise() + rate-limiting + +// integration tests, but no auto-trigger from the reflex yet. rc.3 wires +// the trigger paths (awareness layer) into here. +// +// The advisor MUST return a JSON shape whose `prefer_skill` field is a +// real, registered skill id — anything else is rejected. The system +// prompt embeds the live registry so the model has the source of truth. + +import { complete, isAvailable as llmAvailable } from "../llm/provider.js"; +import { isRegistered, skillRegistryPrompt } from "../skill-registry.js"; +import { info, warn } from "../log.js"; + +const HOURLY_BUDGET = 6; +const COOLDOWN_MS = 30_000; + +let _callTimes = []; +let _lastCallAt = 0; + +export function isAvailable() { + return llmAvailable(); +} + +export function _resetForTest() { + _callTimes = []; + _lastCallAt = 0; +} + +/** + * advise({ snapshot, reason, recentSkillIds, lessonsTail }) → + * { ok: true, action: 'switch_skill'|'continue'|'wait', skillId?, rationale, raw, latencyMs } + * | { ok: false, code, detail, latencyMs } + * + * `reason` is a free-text trigger ("wedged_60s", "forced_move", + * "hp_plunge", "stuck_3_dispatches"). It goes verbatim into the prompt + * so the model can tailor its advice. + */ +export async function advise({ + snapshot, + reason = "unknown", + recentSkillIds = [], + lessonsTail = [], + force = false, +} = {}) { + if (!isAvailable()) { + return { ok: false, code: "not_configured", detail: "set PEPA_FAST_LLM_API_KEY", latencyMs: 0 }; + } + + const now = Date.now(); + _callTimes = _callTimes.filter((t) => t > now - 3600_000); + if (!force && _callTimes.length >= HOURLY_BUDGET) { + return { ok: false, code: "budget_exhausted", detail: `${_callTimes.length}/${HOURLY_BUDGET} per hour`, latencyMs: 0 }; + } + if (!force && now - _lastCallAt < COOLDOWN_MS) { + return { ok: false, code: "cooldown", detail: `${Math.round((COOLDOWN_MS - (now - _lastCallAt)) / 1000)}s`, latencyMs: 0 }; + } + + const system = buildSystemPrompt(); + const user = buildUserPrompt({ snapshot, reason, recentSkillIds, lessonsTail }); + + _callTimes.push(now); + _lastCallAt = now; + + const res = await complete({ system, user, json: true }); + if (!res.ok) { + warn("advisor", `complete failed: ${res.code} (${res.detail})`); + return { ok: false, code: res.code, detail: res.detail, latencyMs: res.latencyMs }; + } + + const parsed = res.text; + if (!parsed || typeof parsed !== "object") { + return { ok: false, code: "bad_shape", detail: "no object in reply", latencyMs: res.latencyMs }; + } + + const action = String(parsed.action ?? "").toLowerCase(); + const skillId = parsed.skill_id ?? parsed.prefer_skill ?? null; + const rationale = parsed.rationale ?? parsed.reason ?? ""; + + if (action === "switch_skill") { + if (!skillId || !isRegistered(skillId)) { + warn("advisor", `rejected hallucinated skill "${skillId}"`); + return { + ok: false, + code: "hallucinated_skill", + detail: skillId ?? "(null)", + rationale, + raw: parsed, + latencyMs: res.latencyMs, + }; + } + info("advisor", `switch_skill → ${skillId} (${rationale.slice(0, 80)})`); + return { + ok: true, + action: "switch_skill", + skillId, + rationale, + raw: parsed, + latencyMs: res.latencyMs, + }; + } + + if (action === "continue" || action === "wait") { + info("advisor", `${action} (${rationale.slice(0, 80)})`); + return { ok: true, action, rationale, raw: parsed, latencyMs: res.latencyMs }; + } + + return { ok: false, code: "bad_action", detail: action || "missing", raw: parsed, latencyMs: res.latencyMs }; +} + +function buildSystemPrompt() { + return [ + "You are the tactical advisor for pepa, an autonomous Minecraft survival bot.", + "You are called when the bot's reflex layer detects something wrong (wedged, stuck,", + "forced move, HP plunge). Your job: produce a single fast decision.", + "", + "Reply STRICTLY with a JSON object:", + '{', + ' "action": "switch_skill" | "continue" | "wait",', + ' "skill_id": "",', + ' "rationale": "<≤25 words explaining why>"', + '}', + "", + "Rules:", + '- "switch_skill" REQUIRES skill_id to be one of the registered ids below.', + '- "continue" means current skill is fine, just give it more time.', + '- "wait" means stop dispatching for ~10s (e.g. waiting for night to pass).', + '- If unsure, return "continue".', + "", + skillRegistryPrompt({ limit: 1800 }), + ].join("\n"); +} + +function buildUserPrompt({ snapshot, reason, recentSkillIds, lessonsTail }) { + const pos = snapshot?.position; + const inv = snapshot?.inventory ? Object.keys(snapshot.inventory).slice(0, 10).join(", ") : "(empty)"; + const recent = (recentSkillIds ?? []).slice(-8).join(" → ") || "(none)"; + const lessons = (lessonsTail ?? []).slice(0, 4).map((l) => ` - ${l.text ?? l}`).join("\n"); + + return [ + `Trigger: ${reason}`, + `Position: ${pos ? `(${Math.round(pos.x)}, ${Math.round(pos.y)}, ${Math.round(pos.z)})` : "?"}`, + `HP: ${snapshot?.health ?? "?"} food: ${snapshot?.food ?? "?"} day: ${snapshot?.isDay ? "yes" : "no"}`, + `Active skill: ${snapshot?.activeSkill ?? "(idle)"}`, + `Recent dispatches: ${recent}`, + `Inventory keys: ${inv}`, + `Nearby threats: ${formatThreats(snapshot?.threats)}`, + `No-progress reason: ${snapshot?.noProgressReason ?? "(none)"}`, + "", + lessons ? `Relevant lessons:\n${lessons}\n` : "", + "What should the bot do RIGHT NOW? Return the JSON decision.", + ].filter(Boolean).join("\n"); +} + +function formatThreats(threats) { + if (!Array.isArray(threats) || threats.length === 0) return "(none)"; + return threats.slice(0, 3).map((t) => `${t.name ?? "?"}@${Math.round(t.distance ?? 0)}m`).join(", "); +} + +// Test exports +export const __testing = { buildSystemPrompt, buildUserPrompt, formatThreats, HOURLY_BUDGET, COOLDOWN_MS }; diff --git a/runtime/coach/fast-advisor.test.js b/runtime/coach/fast-advisor.test.js new file mode 100644 index 0000000..909d09b --- /dev/null +++ b/runtime/coach/fast-advisor.test.js @@ -0,0 +1,163 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { advise, isAvailable, _resetForTest, __testing } from "./fast-advisor.js"; + +const API_KEY = "PEPA_FAST_LLM_API_KEY"; +const MODEL = "PEPA_FAST_LLM_MODEL"; +const BASE = "PEPA_FAST_LLM_BASE_URL"; + +function withEnv(env, fn) { + const prev = {}; + for (const k of Object.keys(env)) { + prev[k] = process.env[k]; + if (env[k] === undefined) delete process.env[k]; + else process.env[k] = env[k]; + } + return Promise.resolve(fn()).finally(() => { + for (const [k, v] of Object.entries(prev)) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + }); +} + +function stubFetch(reply) { + const calls = []; + const orig = globalThis.fetch; + globalThis.fetch = async (url, opts) => { + calls.push({ url, opts }); + return { + ok: true, + json: async () => ({ + choices: [{ message: { content: typeof reply === "string" ? reply : JSON.stringify(reply) } }], + }), + }; + }; + return { calls, restore() { globalThis.fetch = orig; } }; +} + +test("advise: not_configured without API key", async () => { + await withEnv({ [API_KEY]: undefined }, async () => { + _resetForTest(); + const res = await advise({ reason: "stuck" }); + assert.equal(res.ok, false); + assert.equal(res.code, "not_configured"); + }); +}); + +test("advise: accepts a registered skill", async () => { + const f = stubFetch({ action: "switch_skill", skill_id: "survive.flee", rationale: "creeper close" }); + try { + await withEnv({ [API_KEY]: "k", [MODEL]: "m", [BASE]: "https://x/v1" }, async () => { + _resetForTest(); + const res = await advise({ + snapshot: { health: 10, food: 18, isDay: true, position: { x: 1, y: 64, z: 1 } }, + reason: "wedged_60s", + recentSkillIds: ["explore.far", "explore.far", "explore.far"], + force: true, + }); + assert.equal(res.ok, true); + assert.equal(res.action, "switch_skill"); + assert.equal(res.skillId, "survive.flee"); + assert.match(res.rationale, /creeper/); + // system prompt should mention the live registry + const sent = JSON.parse(f.calls[0].opts.body); + assert.match(sent.messages[0].content, /Valid skill ids/); + assert.match(sent.messages[0].content, /survive\.flee/); + assert.match(sent.messages[1].content, /wedged_60s/); + }); + } finally { f.restore(); } +}); + +test("advise: rejects hallucinated skill id with code=hallucinated_skill", async () => { + const f = stubFetch({ action: "switch_skill", skill_id: "relocate.surface", rationale: "fresh spot" }); + try { + await withEnv({ [API_KEY]: "k", [MODEL]: "m" }, async () => { + _resetForTest(); + const res = await advise({ reason: "loop", force: true }); + assert.equal(res.ok, false); + assert.equal(res.code, "hallucinated_skill"); + assert.equal(res.detail, "relocate.surface"); + }); + } finally { f.restore(); } +}); + +test("advise: accepts 'continue' and 'wait' without skill_id", async () => { + const f = stubFetch({ action: "continue", rationale: "skill is making slow progress" }); + try { + await withEnv({ [API_KEY]: "k", [MODEL]: "m" }, async () => { + _resetForTest(); + const res = await advise({ reason: "tick", force: true }); + assert.equal(res.ok, true); + assert.equal(res.action, "continue"); + }); + } finally { f.restore(); } +}); + +test("advise: rate-limit cooldown blocks rapid calls", async () => { + const f = stubFetch({ action: "continue", rationale: "ok" }); + try { + await withEnv({ [API_KEY]: "k", [MODEL]: "m" }, async () => { + _resetForTest(); + const r1 = await advise({ reason: "x" }); + assert.equal(r1.ok, true); + const r2 = await advise({ reason: "y" }); + assert.equal(r2.ok, false); + assert.equal(r2.code, "cooldown"); + }); + } finally { f.restore(); } +}); + +test("advise: hourly budget enforced with force=true override", async () => { + const f = stubFetch({ action: "continue", rationale: "ok" }); + try { + await withEnv({ [API_KEY]: "k", [MODEL]: "m" }, async () => { + _resetForTest(); + for (let i = 0; i < __testing.HOURLY_BUDGET; i++) { + await advise({ reason: `t${i}`, force: true }); + } + const over = await advise({ reason: "over" }); + assert.equal(over.ok, false); + assert.equal(over.code, "budget_exhausted"); + }); + } finally { f.restore(); } +}); + +test("buildSystemPrompt: contains registry block and JSON schema", () => { + const sys = __testing.buildSystemPrompt(); + assert.match(sys, /switch_skill/); + assert.match(sys, /Valid skill ids/); + assert.match(sys, /survive\.flee/); +}); + +test("buildUserPrompt: includes trigger and recent skills", () => { + const u = __testing.buildUserPrompt({ + snapshot: { health: 4, food: 3, isDay: false, position: { x: 10, y: 65, z: 10 }, activeSkill: "explore.far" }, + reason: "hp_plunge", + recentSkillIds: ["explore.far", "explore.far"], + lessonsTail: [{ text: "do not fight at night" }], + }); + assert.match(u, /hp_plunge/); + assert.match(u, /HP: 4/); + assert.match(u, /Recent dispatches: explore\.far → explore\.far/); + assert.match(u, /do not fight at night/); +}); + +test("formatThreats: empty / formatted", () => { + assert.equal(__testing.formatThreats(undefined), "(none)"); + assert.equal(__testing.formatThreats([]), "(none)"); + assert.equal( + __testing.formatThreats([{ name: "zombie", distance: 4.3 }, { name: "creeper", distance: 7 }]), + "zombie@4m, creeper@7m", + ); +}); + +test("isAvailable mirrors provider availability", async () => { + await withEnv({ [API_KEY]: undefined }, async () => { + assert.equal(isAvailable(), false); + }); + await withEnv({ [API_KEY]: "k" }, async () => { + assert.equal(isAvailable(), true); + }); +}); diff --git a/runtime/coach/postmortem.js b/runtime/coach/postmortem.js index f26495e..4d9ea4a 100644 --- a/runtime/coach/postmortem.js +++ b/runtime/coach/postmortem.js @@ -26,6 +26,7 @@ import { poiNearby, recordPOI, } from "../knowledge/index.js"; +import { isRegistered, skillRegistryPrompt } from "../skill-registry.js"; import { info, warn } from "../log.js"; const COACH_INTERVAL_MS = 5 * 60 * 1000; // 5 min between coach passes @@ -278,22 +279,38 @@ export async function drainOnce({ askPi, stateDir, force = false } = {}) { } let lessonsCount = 0; + let rejectedPreferCount = 0; for (const item of asArray(parsed.lessons ?? parsed)) { if (!item || !item.lesson) continue; + // Skill ids referenced by Pi must be in the live registry. + // Mode names (e.g. "night_shelter") are tolerated at write time and + // translated at consult time by advice.js#normalisePreferSkill. + let preferSkill = item.prefer_skill ?? null; + if (preferSkill && !isRegistered(preferSkill) && !isLikelyModeName(preferSkill)) { + rejectedPreferCount += 1; + preferSkill = null; + } + let avoidSkill = item.avoid_skill ?? null; + if (avoidSkill && !isRegistered(avoidSkill) && !isLikelyModeName(avoidSkill)) { + avoidSkill = null; + } recordLesson({ text: item.lesson, category: item.category ?? "survival", triggerSkill: item.trigger_skill ?? null, triggerHostile: item.trigger_hostile ?? null, triggerSituation: item.trigger_situation ?? null, - avoidSkill: item.avoid_skill ?? null, - preferSkill: item.prefer_skill ?? null, + avoidSkill, + preferSkill, confidence: clamp(Number(item.confidence) || 0.6, 0.1, 0.95), source: "pi-coach", sourceRef: item.source_ref ?? null, }); lessonsCount += 1; } + if (rejectedPreferCount > 0) { + warn("coach", `dropped prefer_skill from ${rejectedPreferCount} lessons (not in registry)`); + } // Write one postmortem per death; if Pi grouped them, share the same lesson. const groupLesson = parsed.lessons?.[0]?.lesson ?? parsed.lesson ?? null; @@ -313,6 +330,17 @@ export async function drainOnce({ askPi, stateDir, force = false } = {}) { return { ok: true, analysed: pending.length, lessons: lessonsCount }; } +// Mode names from runtime/modes.js (advice.js#MODE_TO_SKILL) — we accept +// these at write time because advice.js maps them to real skills at consult. +const KNOWN_MODE_NAMES = new Set([ + "self_preservation", "night_shelter", "hunger", "shelter", + "flee", "sleep", "eat", "tunnel_out", "tunnel-out", "explore", "wander", +]); +function isLikelyModeName(s) { + if (!s || typeof s !== "string") return false; + return KNOWN_MODE_NAMES.has(s.toLowerCase().trim()); +} + function buildPrompt(deaths) { const summary = deaths.map((d) => { const ctx = safeParse(d.context_blob); @@ -334,6 +362,8 @@ function buildPrompt(deaths) { "The bot is trying to gather wood, craft tools, build a small village, and survive nights.", "It's currently dying repeatedly. Your job: extract 1-3 short, generalised lessons it can apply on respawn.", "", + skillRegistryPrompt({ limit: 1800 }), + "", "DEATHS:", summary, "", @@ -343,12 +373,13 @@ function buildPrompt(deaths) { ' { "lesson": "...", "category": "combat|pathing|crafting|survival|social",', ' "trigger_skill": "",', ' "trigger_hostile": "",', - ' "avoid_skill": "",', - ' "prefer_skill": "",', + ' "avoid_skill": "",', + ' "prefer_skill": "",', ' "confidence": 0.7 }', ' ] }', "", "Keep each lesson under 30 words. Be specific (e.g., \"attack creeper with fists\" rather than \"don't fight\").", + "CRITICAL: avoid_skill and prefer_skill MUST be one of the registered ids above, or null.", ].join("\n"); } @@ -390,4 +421,4 @@ function clamp(v, lo, hi) { return Math.max(lo, Math.min(hi, v)); } function safeParse(s) { try { return JSON.parse(s); } catch { return null; } } // Test-only exports -export const __testing = { captureDeath, buildPrompt, extractJson, inferCause }; +export const __testing = { captureDeath, buildPrompt, extractJson, inferCause, isLikelyModeName, KNOWN_MODE_NAMES }; diff --git a/runtime/coach/reflect.js b/runtime/coach/reflect.js index 3820b99..e1f3376 100644 --- a/runtime/coach/reflect.js +++ b/runtime/coach/reflect.js @@ -16,8 +16,20 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { resolve } from "node:path"; import { isAvailable as knowledgeAvailable, record as recordLesson } from "../knowledge/index.js"; +import { isRegistered, skillRegistryPrompt } from "../skill-registry.js"; import { info, warn } from "../log.js"; +// Mode-name allow-list, mirrors postmortem.js (advice.js maps them to +// real skills at consult-time). Anything else is hallucination → dropped. +const KNOWN_MODE_NAMES = new Set([ + "self_preservation", "night_shelter", "hunger", "shelter", + "flee", "sleep", "eat", "tunnel_out", "tunnel-out", "explore", "wander", +]); +function isLikelyModeName(s) { + if (!s || typeof s !== "string") return false; + return KNOWN_MODE_NAMES.has(s.toLowerCase().trim()); +} + const DEFAULT_INTERVAL_MS = 30 * 60 * 1000; const HOURLY_BUDGET = 2; const HISTORY_TAIL_LINES = 80; @@ -78,20 +90,33 @@ export async function runOnce({ stateDir, askPi, getSnapshot, force = false } = } const path = writeReflection(stateDir, parsed, reply); + let rejectedPrefer = 0; for (const l of asArray(parsed.lessons)) { if (!l?.lesson) continue; + let preferSkill = l.prefer_skill ?? null; + if (preferSkill && !isRegistered(preferSkill) && !isLikelyModeName(preferSkill)) { + rejectedPrefer += 1; + preferSkill = null; + } + let avoidSkill = l.avoid_skill ?? null; + if (avoidSkill && !isRegistered(avoidSkill) && !isLikelyModeName(avoidSkill)) { + avoidSkill = null; + } recordLesson({ text: l.lesson, category: l.category ?? "self-improve", triggerSkill: l.trigger_skill ?? null, triggerHostile: l.trigger_hostile ?? null, - avoidSkill: l.avoid_skill ?? null, - preferSkill: l.prefer_skill ?? null, + avoidSkill, + preferSkill, confidence: clamp(Number(l.confidence) || 0.5, 0.1, 0.9), source: "pi-reflect", sourceRef: path, }); } + if (rejectedPrefer > 0) { + warn("reflect", `dropped prefer_skill from ${rejectedPrefer} reflection lessons (not in registry)`); + } info("reflect", `verdict=${parsed.verdict ?? "?"} ${parsed.summary?.slice(0, 80) ?? ""} (${path ?? "no file"})`); return { ok: true, verdict: parsed.verdict, summary: parsed.summary, lessons: parsed.lessons ?? [] }; } @@ -136,6 +161,8 @@ function buildPrompt({ snap, journal, scenarios, diary, plan }) { "You are pepa, an autonomous Minecraft survival bot, reflecting on your own progress.", "Look at the last ~30 minutes of activity below. Answer honestly: are you actually making progress, or stuck in a loop?", "", + skillRegistryPrompt({ limit: 1800 }), + "", "## Current state", `- position: ${pos ? `(${Math.round(pos.x)}, ${Math.round(pos.y)}, ${Math.round(pos.z)})` : "?"}`, `- hp: ${snap?.health ?? "?"} food: ${snap?.food ?? "?"} day: ${snap?.isDay ? "yes" : "no"}`, @@ -176,8 +203,8 @@ function buildPrompt({ snap, journal, scenarios, diary, plan }) { ' "category": "combat|pathing|crafting|survival|self-improve",', ' "trigger_skill": "",', ' "trigger_hostile": "",', - ' "avoid_skill": "",', - ' "prefer_skill": "",', + ' "avoid_skill": "",', + ' "prefer_skill": "",', ' "confidence": 0.6 }', ' ]', '}', @@ -185,6 +212,7 @@ function buildPrompt({ snap, journal, scenarios, diary, plan }) { "If you're clearly in a loop (same activity, no inventory growth, same position), say so honestly.", "If you're stuck in a bad terrain (deep pit, hostile-rich area), recommend choosing a new base.", "Lessons should be SHORT and ACTIONABLE. Don't repeat lessons the dispatcher already learned.", + "CRITICAL: avoid_skill and prefer_skill MUST be one of the registered ids listed at the top of this prompt, or null. Do NOT invent new ids.", ].join("\n"); } diff --git a/runtime/llm/provider.js b/runtime/llm/provider.js new file mode 100644 index 0000000..c78f271 --- /dev/null +++ b/runtime/llm/provider.js @@ -0,0 +1,152 @@ +// OpenAI-compatible chat client for the "fast advisor" tier. +// +// The original "coach" loop uses Pi via the CLI subprocess (5-15s latency, +// rate-limited to a few calls per hour). That's appropriate for deep +// post-mortem analytics but useless when the bot needs tactical advice +// right now ("I'm wedged in a pit, what should I do?"). +// +// This provider opens a parallel path: any OpenAI-compatible HTTP endpoint +// (TimeWeb, OpenAI direct, Groq, OpenRouter, local Ollama with the OpenAI +// shim, …) producing a structured JSON answer in ≤8 seconds. +// +// Configuration is strictly env-driven. The provider is a NO-OP unless +// PEPA_FAST_LLM_API_KEY is set, so it's safe to ship the code disabled. + +import { info, warn } from "../log.js"; + +const ENV = { + BASE_URL: "PEPA_FAST_LLM_BASE_URL", + API_KEY: "PEPA_FAST_LLM_API_KEY", + MODEL: "PEPA_FAST_LLM_MODEL", + TIMEOUT_MS: "PEPA_FAST_LLM_TIMEOUT_MS", +}; + +const DEFAULT_BASE_URL = "https://api.openai.com/v1"; +const DEFAULT_TIMEOUT_MS = 8000; + +export function isAvailable() { + return !!process.env[ENV.API_KEY]; +} + +export function getConfig() { + return { + baseUrl: (process.env[ENV.BASE_URL] || DEFAULT_BASE_URL).replace(/\/+$/, ""), + apiKey: process.env[ENV.API_KEY] || null, + model: process.env[ENV.MODEL] || null, + timeoutMs: Number(process.env[ENV.TIMEOUT_MS]) || DEFAULT_TIMEOUT_MS, + }; +} + +/** + * complete({ system, user, json, model?, timeoutMs? }) + * → { ok: true, text, raw, latencyMs } | { ok: false, code, detail, latencyMs } + * + * `json: true` requests JSON-mode (response_format) and returns the + * parsed object as `text`. If the provider doesn't honour JSON-mode the + * call still works but caller is responsible for parsing. + */ +export async function complete({ + system, + user, + json = false, + model, + timeoutMs, +} = {}) { + const cfg = getConfig(); + if (!cfg.apiKey) { + return { ok: false, code: "not_configured", detail: `set ${ENV.API_KEY}`, latencyMs: 0 }; + } + const useModel = model || cfg.model; + if (!useModel) { + return { ok: false, code: "no_model", detail: `set ${ENV.MODEL} or pass model arg`, latencyMs: 0 }; + } + + const body = { + model: useModel, + messages: [ + system ? { role: "system", content: system } : null, + { role: "user", content: user ?? "" }, + ].filter(Boolean), + temperature: 0.3, + }; + if (json) { + body.response_format = { type: "json_object" }; + } + + const url = `${cfg.baseUrl}/chat/completions`; + const startedAt = Date.now(); + const controller = new AbortController(); + const t = setTimeout(() => controller.abort(), timeoutMs ?? cfg.timeoutMs); + + let resp; + try { + resp = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${cfg.apiKey}`, + }, + body: JSON.stringify(body), + signal: controller.signal, + }); + } catch (e) { + clearTimeout(t); + const latency = Date.now() - startedAt; + const aborted = e?.name === "AbortError"; + return { + ok: false, + code: aborted ? "timeout" : "network_error", + detail: e?.message ?? String(e), + latencyMs: latency, + }; + } + clearTimeout(t); + + const latencyMs = Date.now() - startedAt; + if (!resp.ok) { + let body; + try { body = await resp.text(); } catch { body = ""; } + warn("llm", `${useModel} ${resp.status}: ${body.slice(0, 200)}`); + return { + ok: false, + code: `http_${resp.status}`, + detail: body.slice(0, 500), + latencyMs, + }; + } + + let payload; + try { + payload = await resp.json(); + } catch (e) { + return { ok: false, code: "bad_json", detail: e?.message ?? "parse error", latencyMs }; + } + + const text = payload?.choices?.[0]?.message?.content; + if (typeof text !== "string") { + return { ok: false, code: "no_content", detail: "no choices[0].message.content", latencyMs }; + } + + let parsed = text; + if (json) { + parsed = tryParseJson(text); + if (parsed === null) { + return { ok: false, code: "bad_json", detail: text.slice(0, 200), latencyMs }; + } + } + + info("llm", `${useModel} ok (${latencyMs}ms, ${text.length}ch)`); + return { ok: true, text: parsed, raw: text, latencyMs }; +} + +function tryParseJson(text) { + if (!text) return null; + const trimmed = text.trim().replace(/^```(?:json)?/i, "").replace(/```$/, "").trim(); + try { return JSON.parse(trimmed); } catch {} + const m = trimmed.match(/\{[\s\S]*\}/); + if (!m) return null; + try { return JSON.parse(m[0]); } catch { return null; } +} + +// Test exports +export const __testing = { ENV, DEFAULT_BASE_URL, DEFAULT_TIMEOUT_MS, tryParseJson }; diff --git a/runtime/llm/provider.test.js b/runtime/llm/provider.test.js new file mode 100644 index 0000000..7a11796 --- /dev/null +++ b/runtime/llm/provider.test.js @@ -0,0 +1,164 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { isAvailable, getConfig, complete, __testing } from "./provider.js"; + +const { tryParseJson, ENV } = __testing; + +test("isAvailable: false when no API key in env", () => { + const prev = process.env[ENV.API_KEY]; + delete process.env[ENV.API_KEY]; + try { + assert.equal(isAvailable(), false); + } finally { + if (prev !== undefined) process.env[ENV.API_KEY] = prev; + } +}); + +test("isAvailable: true when API key set", () => { + const prev = process.env[ENV.API_KEY]; + process.env[ENV.API_KEY] = "test-key"; + try { + assert.equal(isAvailable(), true); + } finally { + if (prev === undefined) delete process.env[ENV.API_KEY]; + else process.env[ENV.API_KEY] = prev; + } +}); + +test("getConfig: reflects env overrides and strips trailing slash", () => { + const prev = { + base: process.env[ENV.BASE_URL], + key: process.env[ENV.API_KEY], + model: process.env[ENV.MODEL], + }; + process.env[ENV.BASE_URL] = "https://api.example.com/v1/"; + process.env[ENV.API_KEY] = "abc"; + process.env[ENV.MODEL] = "gpt-fast"; + try { + const cfg = getConfig(); + assert.equal(cfg.baseUrl, "https://api.example.com/v1"); + assert.equal(cfg.apiKey, "abc"); + assert.equal(cfg.model, "gpt-fast"); + assert.ok(cfg.timeoutMs > 0); + } finally { + for (const [k, v] of [[ENV.BASE_URL, prev.base], [ENV.API_KEY, prev.key], [ENV.MODEL, prev.model]]) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + } +}); + +test("complete: not_configured when no API key", async () => { + const prev = process.env[ENV.API_KEY]; + delete process.env[ENV.API_KEY]; + try { + const res = await complete({ system: "hi", user: "hi" }); + assert.equal(res.ok, false); + assert.equal(res.code, "not_configured"); + } finally { + if (prev !== undefined) process.env[ENV.API_KEY] = prev; + } +}); + +test("complete: no_model when key is set but model isn't", async () => { + const prev = { key: process.env[ENV.API_KEY], model: process.env[ENV.MODEL] }; + process.env[ENV.API_KEY] = "x"; + delete process.env[ENV.MODEL]; + try { + const res = await complete({ system: "s", user: "u" }); + assert.equal(res.ok, false); + assert.equal(res.code, "no_model"); + } finally { + if (prev.key === undefined) delete process.env[ENV.API_KEY]; + else process.env[ENV.API_KEY] = prev.key; + if (prev.model !== undefined) process.env[ENV.MODEL] = prev.model; + } +}); + +test("tryParseJson: parses naked, fenced, and embedded JSON", () => { + assert.deepEqual(tryParseJson('{"a":1}'), { a: 1 }); + assert.deepEqual(tryParseJson('```json\n{"a":2}\n```'), { a: 2 }); + assert.deepEqual(tryParseJson('prose before {"a":3} prose after'), { a: 3 }); + assert.equal(tryParseJson("nope"), null); + assert.equal(tryParseJson(""), null); +}); + +test("complete: real fetch path uses Bearer header and POSTs JSON", async () => { + // Stub global fetch to capture the request. + const calls = []; + const stub = async (url, opts) => { + calls.push({ url, opts }); + return { + ok: true, + json: async () => ({ + choices: [{ message: { content: JSON.stringify({ verdict: "loop", action: "wander" }) } }], + }), + }; + }; + const origFetch = globalThis.fetch; + globalThis.fetch = stub; + const prev = { key: process.env[ENV.API_KEY], model: process.env[ENV.MODEL], base: process.env[ENV.BASE_URL] }; + process.env[ENV.API_KEY] = "secret-123"; + process.env[ENV.MODEL] = "gpt-fast"; + process.env[ENV.BASE_URL] = "https://example/v1"; + try { + const res = await complete({ system: "be terse", user: "what now?", json: true }); + assert.equal(res.ok, true); + assert.deepEqual(res.text, { verdict: "loop", action: "wander" }); + assert.equal(calls.length, 1); + assert.equal(calls[0].url, "https://example/v1/chat/completions"); + assert.equal(calls[0].opts.method, "POST"); + assert.equal(calls[0].opts.headers["Authorization"], "Bearer secret-123"); + const sent = JSON.parse(calls[0].opts.body); + assert.equal(sent.model, "gpt-fast"); + assert.equal(sent.messages[0].role, "system"); + assert.equal(sent.messages[1].role, "user"); + assert.equal(sent.response_format.type, "json_object"); + } finally { + globalThis.fetch = origFetch; + for (const [k, v] of [[ENV.API_KEY, prev.key], [ENV.MODEL, prev.model], [ENV.BASE_URL, prev.base]]) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + } +}); + +test("complete: network error surfaces as code=network_error", async () => { + const origFetch = globalThis.fetch; + globalThis.fetch = async () => { throw new Error("boom"); }; + const prev = { key: process.env[ENV.API_KEY], model: process.env[ENV.MODEL] }; + process.env[ENV.API_KEY] = "x"; + process.env[ENV.MODEL] = "m"; + try { + const res = await complete({ system: "s", user: "u" }); + assert.equal(res.ok, false); + assert.equal(res.code, "network_error"); + assert.match(res.detail, /boom/); + } finally { + globalThis.fetch = origFetch; + if (prev.key === undefined) delete process.env[ENV.API_KEY]; + else process.env[ENV.API_KEY] = prev.key; + if (prev.model === undefined) delete process.env[ENV.MODEL]; + else process.env[ENV.MODEL] = prev.model; + } +}); + +test("complete: http error surfaces as http_", async () => { + const origFetch = globalThis.fetch; + globalThis.fetch = async () => ({ ok: false, status: 401, text: async () => "bad key" }); + const prev = { key: process.env[ENV.API_KEY], model: process.env[ENV.MODEL] }; + process.env[ENV.API_KEY] = "x"; + process.env[ENV.MODEL] = "m"; + try { + const res = await complete({ system: "s", user: "u" }); + assert.equal(res.ok, false); + assert.equal(res.code, "http_401"); + } finally { + globalThis.fetch = origFetch; + if (prev.key === undefined) delete process.env[ENV.API_KEY]; + else process.env[ENV.API_KEY] = prev.key; + if (prev.model === undefined) delete process.env[ENV.MODEL]; + else process.env[ENV.MODEL] = prev.model; + } +}); diff --git a/runtime/skill-registry.js b/runtime/skill-registry.js new file mode 100644 index 0000000..ab8ec2c --- /dev/null +++ b/runtime/skill-registry.js @@ -0,0 +1,68 @@ +// Single source of truth for "what skill ids are real" — exported separately +// from skills/index.js so coach/advice.js, coach/postmortem.js, coach/reflect.js, +// and coach/fast-advisor.js can all consult the SAME live registry without +// circular imports through runSkill. +// +// Why this exists: in v0.2.x Pi (the LLM coach) routinely fabricated +// skill ids that never existed — "relocate.surface", "choose.safe.surface", +// "survive.shelter", "gather.visible_log". Of 47 Pi-extracted lessons, +// 0 were ever applied because normalisePreferSkill() couldn't map them +// to anything real. The fix is two-pronged: (a) hand Pi the real registry +// in its system prompt so it doesn't have to guess; (b) reject anything +// not in the registry at the consult() boundary. + +import { listSkills } from "./skills/index.js"; + +let _cache = null; + +function rebuild() { + const all = listSkills(); + const byId = new Map(); + const byNamespace = new Map(); + for (const s of all) { + byId.set(s.id, s); + const ns = s.id.split(".")[0] || "misc"; + if (!byNamespace.has(ns)) byNamespace.set(ns, []); + byNamespace.get(ns).push(s); + } + _cache = { all, byId, byNamespace }; + return _cache; +} + +function get() { return _cache ?? rebuild(); } + +export function listSkillIds() { + return Array.from(get().byId.keys()); +} + +export function isRegistered(id) { + if (!id || typeof id !== "string") return false; + return get().byId.has(id); +} + +export function describeSkill(id) { + return get().byId.get(id) ?? null; +} + +// Human-readable block to drop into LLM system prompts. Groups by +// namespace, lists "id — title (timeoutMs)". Capped at ~2KB to stay +// well within the model's instruction window. +export function skillRegistryPrompt({ limit = 2000 } = {}) { + const { byNamespace } = get(); + const namespaces = Array.from(byNamespace.keys()).sort(); + const lines = ["Valid skill ids (USE ONLY THESE for avoid_skill / prefer_skill):"]; + for (const ns of namespaces) { + const skills = byNamespace.get(ns).sort((a, b) => a.id.localeCompare(b.id)); + lines.push(` ${ns}:`); + for (const s of skills) { + lines.push(` - ${s.id} — ${s.title ?? s.id}`); + } + } + lines.push(""); + lines.push("If no listed skill fits, set the field to null. NEVER invent new ids."); + const text = lines.join("\n"); + return text.length > limit ? text.slice(0, limit - 4) + "\n..." : text; +} + +// For tests / hot-reload scenarios. +export function _resetForTest() { _cache = null; } diff --git a/runtime/skill-registry.test.js b/runtime/skill-registry.test.js new file mode 100644 index 0000000..45bbd82 --- /dev/null +++ b/runtime/skill-registry.test.js @@ -0,0 +1,65 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { + listSkillIds, + isRegistered, + describeSkill, + skillRegistryPrompt, +} from "./skill-registry.js"; + +test("registry: lists at least the known v0.2 skill set", () => { + const ids = listSkillIds(); + assert.ok(ids.length >= 20, `expected 20+ skills, got ${ids.length}`); + for (const must of [ + "gather.logs", + "survive.eat", + "survive.sleep", + "survive.flee", + "survive.pillar-up", + "explore.far", + "explore.wander", + "recovery.tunnel-out", + "village.choose-base", + "village.build-shelter", + ]) { + assert.ok(ids.includes(must), `registry missing ${must}`); + } +}); + +test("isRegistered: true for real, false for hallucinated", () => { + assert.equal(isRegistered("survive.flee"), true); + assert.equal(isRegistered("relocate.surface"), false); + assert.equal(isRegistered("choose.safe.surface"), false); + assert.equal(isRegistered("gather.visible_log"), false); + assert.equal(isRegistered("survive.shelter"), false); + assert.equal(isRegistered("tunnel-out"), false, "missing recovery. prefix"); + assert.equal(isRegistered(null), false); + assert.equal(isRegistered(""), false); + assert.equal(isRegistered(42), false); +}); + +test("describeSkill: returns shape for known id", () => { + const s = describeSkill("survive.pillar-up"); + assert.ok(s, "expected description"); + assert.equal(s.id, "survive.pillar-up"); + assert.ok(typeof s.timeoutMs === "number" && s.timeoutMs > 0); +}); + +test("registryPrompt: contains the real ids grouped by namespace", () => { + const txt = skillRegistryPrompt(); + assert.match(txt, /Valid skill ids/); + assert.match(txt, /survive:/); + assert.match(txt, /- survive\.flee/); + assert.match(txt, /- recovery\.tunnel-out/); + assert.match(txt, /NEVER invent/); + // must NOT contain hallucinated ids + assert.doesNotMatch(txt, /relocate\.surface/); + assert.doesNotMatch(txt, /survive\.shelter[^-]/); +}); + +test("registryPrompt: respects limit parameter", () => { + const short = skillRegistryPrompt({ limit: 200 }); + assert.ok(short.length <= 200, `expected <=200, got ${short.length}`); + assert.ok(short.endsWith("...")); +});