From c7eab06f22ecf20c704462f87d0e8ab30f2a7b45 Mon Sep 17 00:00:00 2001 From: Yuriy Mayatnikov Date: Mon, 25 May 2026 22:35:01 +0300 Subject: [PATCH] feat(runtime): stuck-incident detector + skill metrics + edit scope (Phase 6) (#17) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 6 of plans/autonomous-survival-bot-prd.md. Expand the self-improvement loop so the bot can spot and report no-progress stagnation, not just exception-class failures. New: - runtime/stuck-incident.js: detector fires a structured proposal when the same noProgressReason persists past 5 min (cooldown 30 min). Body includes runtimeState, milestone, suggested skill, slim snapshot, last action result, per-skill success/failure metrics and a forbidden-paths list. Pure module — caller (bot.js) writes the proposal. - runtime/skill-metrics.js: in-memory per-skill ok/fail counters surfaced on snapshot.skillMetrics for the TUI and the incident body. - runtime/stuck-incident.test.js: 6 tests covering null reason, threshold gating, cooldown, reason change resetting the timer, body composition and metrics snapshot. Wiring: - runtime/state-store.js: writeProposal accepts {editScope: string[]} and persists it in the frontmatter; readProposalEditScope() reads it back so future auto-patch.js can refuse cherry-picks that touch other areas. - runtime/bot.js: tick() invokes the stuck detector each tick, records skill ok/fail via skillMetrics, stamps snapshot.skillMetrics and writes the stuck proposal via writeProposal({editScope}). dispatchAction now records into skillMetrics for both the resolved-result and the exception path. npm test now 46/46. Co-authored-by: Yuriy Mayatnikov Co-authored-by: Claude Opus 4.7 (1M context) --- docs/runtime.md | 23 ++++++ package.json | 2 +- runtime/bot.js | 33 +++++++++ runtime/skill-metrics.js | 28 ++++++++ runtime/state-store.js | 38 ++++++++-- runtime/stuck-incident.js | 123 +++++++++++++++++++++++++++++++++ runtime/stuck-incident.test.js | 86 +++++++++++++++++++++++ 7 files changed, 325 insertions(+), 8 deletions(-) create mode 100644 runtime/skill-metrics.js create mode 100644 runtime/stuck-incident.js create mode 100644 runtime/stuck-incident.test.js diff --git a/docs/runtime.md b/docs/runtime.md index c6da70d..2ecb6e6 100644 --- a/docs/runtime.md +++ b/docs/runtime.md @@ -246,6 +246,29 @@ in-process. The package is **not** a default dep — install it explicitly (`npm i prismarine-viewer`) before enabling. If missing, the runtime logs a warning and continues. +### Self-improvement v2 (Phase 6) + +Two classes of proposals now land in `state//proposals/`: + +1. **Bug-class failures** — same-label action returns `{ok: false}` 5× + in a row, dominated by `bug` (TypeError, "Cannot read properties") + or persistent timeout. Handled by the older tracker in `bot.js`. +2. **Stuck incidents** — `noProgressReason` stays the same for ≥5 min + without a productive dispatch. Handled by + `runtime/stuck-incident.js`. The proposal body includes the + runtimeState, milestone, suggested skill, slim snapshot, last + result, and per-skill success/failure metrics. + +Both kinds now persist an **`editScope`** in their frontmatter — an +array of repo-relative path prefixes the auto-patcher is allowed to +modify. `state-store.readProposalEditScope(filename)` reads it back; +hooking `scripts/auto-patch.js` to refuse cherry-picks that touch +other areas is the remaining follow-up. + +Per-skill metrics live in memory only (best-effort) but are surfaced +on `snapshot.skillMetrics = { [skillId]: { ok, fail, lastTs } }` so +the TUI can show which skills are reliable and which keep failing. + ### Social layer (Phase 5) Inbound MC chat is classified via `runtime/social/intent.js` into one diff --git a/package.json b/package.json index 274a5ed..ac91ce9 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "tui": "tsx tui/tui.tsx", "propose:apply": "node scripts/propose-apply.js", "stop": "bash scripts/stop.sh", - "test": "node --test runtime/skills/contract.test.js runtime/skills/groups.test.js runtime/curriculum.test.js runtime/social/social.test.js" + "test": "node --test runtime/skills/contract.test.js runtime/skills/groups.test.js runtime/curriculum.test.js runtime/social/social.test.js runtime/stuck-incident.test.js" }, "dependencies": { "dotenv": "^16.4.5", diff --git a/runtime/bot.js b/runtime/bot.js index 978bf35..d6c0802 100644 --- a/runtime/bot.js +++ b/runtime/bot.js @@ -44,6 +44,8 @@ import { nextMilestone as nextCurriculumMilestone } from "./curriculum.js"; import { classifyIntent, INTENTS } from "./social/intent.js"; import { generateReply } from "./social/reply.js"; import { createChatMemory } from "./social/memory.js"; +import { createStuckIncidentDetector } from "./stuck-incident.js"; +import { createSkillMetrics } from "./skill-metrics.js"; fs.mkdirSync(stateDir, { recursive: true }); const JOINED_FLAG = path.join(stateDir, "joined-before.flag"); @@ -68,6 +70,8 @@ let lastEscalationAt = 0; // future Telegram/diary surfaces) can answer "what is the bot doing and why // isn't it doing more?" without parsing the log stream. const noProgress = createNoProgressDetector(); +const stuckIncident = createStuckIncidentDetector(); +const skillMetrics = createSkillMetrics(); let lastResult = null; // { label, ok, code, detail, ts } let lastFailureAt = 0; let lastPlanReadAt = 0; @@ -177,6 +181,7 @@ function dispatchAction(fn, label, opts = {}) { detail: res?.detail, ts: Date.now(), }; + skillMetrics.record(label, ok); if (!ok) { lastFailureAt = lastResult.ts; recordFailure(label, res?.detail); @@ -202,6 +207,7 @@ function dispatchAction(fn, label, opts = {}) { detail: String(e?.message ?? e), ts: Date.now(), }; + skillMetrics.record(label, false); lastFailureAt = lastResult.ts; recordFailure(label, String(e?.message ?? e)); }) @@ -604,11 +610,38 @@ function tick() { lastSnapshot.lastResult = lastResult; lastSnapshot.noProgressReason = noProgressReason; lastSnapshot.failuresByCode = failuresByCode(); + lastSnapshot.skillMetrics = skillMetrics.snapshot(); lastSnapshot.lastEscalation = lastEscalationAt ? { ts: lastEscalationAt, ageMs: now - lastEscalationAt } : null; lastSnapshot.reflexPaused = reflexPaused; + // Stuck-incident detector: file a structured proposal when the same + // no-progress reason persists past the threshold. Kept separate from + // the existing bug-class failure tracker — they target different + // classes of breakage. The detector enforces its own cooldown so we + // don't spam the proposals dir. + const stuck = stuckIncident.check({ + snapshot: lastSnapshot, + lastResult, + metrics: lastSnapshot.skillMetrics, + now, + }); + if (stuck?.fire) { + try { + const { filename } = writeProposal({ + kind: stuck.kind, + summary: stuck.summary, + body: stuck.body, + editScope: stuck.editScope, + }); + warn("stuck", `filed ${filename}: ${stuck.summary}`); + appendDiary(`stuck-proposal filed: ${filename} (${stuck.summary})`); + } catch (e) { + warn("stuck", `writeProposal failed: ${e.message}`); + } + } + ipc?.broadcast(EVENT_TYPES.STATUS, lastSnapshot); } else { lastSnapshot = { connected: false }; diff --git a/runtime/skill-metrics.js b/runtime/skill-metrics.js new file mode 100644 index 0000000..d28abb2 --- /dev/null +++ b/runtime/skill-metrics.js @@ -0,0 +1,28 @@ +// Per-skill ok/fail counters. Aggregated for the lifetime of the bot +// process (best-effort persistence is left for a future iteration — +// today's counters reset on restart, which keeps the data store +// simple while still being useful for incident bodies and the TUI). + +export function createSkillMetrics() { + const counts = new Map(); // id → { ok, fail, lastTs } + + function record(id, ok) { + const cur = counts.get(id) ?? { ok: 0, fail: 0, lastTs: 0 }; + if (ok) cur.ok++; + else cur.fail++; + cur.lastTs = Date.now(); + counts.set(id, cur); + } + + function snapshot() { + const out = {}; + for (const [id, m] of counts) out[id] = { ...m }; + return out; + } + + function reset() { + counts.clear(); + } + + return { record, snapshot, reset }; +} diff --git a/runtime/state-store.js b/runtime/state-store.js index a58f725..800dc36 100644 --- a/runtime/state-store.js +++ b/runtime/state-store.js @@ -134,25 +134,49 @@ function slugify(s) { .slice(0, 60); } -export function writeProposal({ kind, summary, body }) { +// editScope (optional): array of repo-relative path prefixes the +// auto-patcher is allowed to modify. Read back from the frontmatter +// so scripts/auto-patch.js can refuse cherry-picks that touch other +// areas. Falls back to the historical default (`runtime/`) when +// absent for backwards compatibility with older proposals. +export function writeProposal({ kind, summary, body, editScope }) { const stamp = new Date().toISOString().replace(/[:.]/g, "-").slice(0, 19); const filename = `${stamp}-${slugify(kind)}.md`; const filePath = path.join(PROPOSALS_DIR, filename); - const content = [ + const frontmatter = [ "---", `kind: ${kind}`, `ts: ${new Date().toISOString()}`, `summary: ${JSON.stringify(summary)}`, "approved: false", - "---", - "", - body, - "", - ].join("\n"); + ]; + if (Array.isArray(editScope) && editScope.length) { + frontmatter.push(`editScope: ${JSON.stringify(editScope)}`); + } + frontmatter.push("---", ""); + const content = frontmatter.concat([body, ""]).join("\n"); fs.writeFileSync(filePath, content); return { filePath, filename }; } +// Best-effort parse of editScope from a proposal markdown body's +// frontmatter. Returns null when not specified. +export function readProposalEditScope(filename, { approved = false } = {}) { + const dir = approved ? PROPOSALS_APPROVED_DIR : PROPOSALS_DIR; + const filePath = path.join(dir, filename); + let raw; + try { raw = fs.readFileSync(filePath, "utf8"); } catch { return null; } + const m = raw.match(/^---\n([\s\S]*?)\n---/); + if (!m) return null; + const scopeLine = m[1].split("\n").find((l) => l.startsWith("editScope:")); + if (!scopeLine) return null; + try { + return JSON.parse(scopeLine.slice("editScope:".length).trim()); + } catch { + return null; + } +} + export function listProposals({ approved = false } = {}) { const dir = approved ? PROPOSALS_APPROVED_DIR : PROPOSALS_DIR; try { diff --git a/runtime/stuck-incident.js b/runtime/stuck-incident.js new file mode 100644 index 0000000..988b775 --- /dev/null +++ b/runtime/stuck-incident.js @@ -0,0 +1,123 @@ +// Stuck-incident detector. Today's failure-tracker in bot.js fires +// proposals on repeated *exception-class* failures (TypeError, timeout). +// This module fires on "no-progress" stagnation — the bot is healthy and +// the reflex loop hasn't crashed, but a single reason code (e.g. +// no_food_source, planner_empty) keeps coming back tick after tick. +// +// When the same reason persists past STUCK_THRESHOLD_MS we build a +// proposal body summarising the situation, including: +// - the no-progress reason +// - the active milestone + suggested skill +// - a slim snapshot +// - the last skill result +// - per-skill success/failure metrics (if available) +// - the allowed edit scope ("runtime/skills/.js" plus tests) +// +// The caller (bot.js) is responsible for actually writing the proposal — +// this module just produces the body string when the threshold trips. + +const STUCK_THRESHOLD_MS = 5 * 60 * 1000; +const COOLDOWN_MS = 30 * 60 * 1000; + +export function createStuckIncidentDetector({ thresholdMs = STUCK_THRESHOLD_MS, cooldownMs = COOLDOWN_MS } = {}) { + let currentReason = null; + let firstSeenAt = 0; + let lastFiredAt = 0; + + function reset() { + currentReason = null; + firstSeenAt = 0; + } + + // Returns { fire: true, body, kind, editScope } when an incident should + // be filed this tick; null otherwise. + function check({ snapshot, lastResult, metrics, now = Date.now() }) { + const reason = snapshot?.noProgressReason ?? null; + if (!reason) { + reset(); + return null; + } + if (reason !== currentReason) { + currentReason = reason; + firstSeenAt = now; + return null; + } + if (now - firstSeenAt < thresholdMs) return null; + if (lastFiredAt > 0 && now - lastFiredAt < cooldownMs) return null; + lastFiredAt = now; + + const milestone = snapshot?.curriculum?.milestone; + const suggested = snapshot?.curriculum?.plan?.skillId; + const slim = { + runtimeState: snapshot.runtimeState, + noProgressReason: reason, + milestone: milestone?.title ?? null, + suggestedSkill: suggested ?? null, + position: snapshot.position, + health: snapshot.health, + food: snapshot.food, + isDay: snapshot.isDay, + inventoryCounts: Object.keys(snapshot.inventory ?? {}).length, + closestHostile: snapshot.closestHostile, + }; + + const editScope = suggested + ? [`runtime/skills/${suggested.replace(/\./g, "-")}.js`, `runtime/skills/`] + : ["runtime/skills/", "runtime/"]; + + const metricsLine = metrics && Object.keys(metrics).length + ? Object.entries(metrics) + .map(([id, m]) => `${id}: ok=${m.ok} fail=${m.fail}`) + .join(", ") + : "(none)"; + + const body = [ + `# Stuck on \`${reason}\``, + "", + `The runtime has reported the same no-progress reason for >${Math.round(thresholdMs / 60000)} min without a productive action.`, + "", + "## Current state", + "", + "```json", + JSON.stringify(slim, null, 2), + "```", + "", + "## Last action result", + "", + lastResult + ? `\`${lastResult.label}\` → ${lastResult.code ?? (lastResult.ok ? "ok" : "fail")}${lastResult.detail ? ` (${JSON.stringify(lastResult.detail).slice(0, 200)})` : ""}` + : "_(none recorded)_", + "", + "## Skill metrics so far", + "", + metricsLine, + "", + "## Suggested fix", + "", + suggested + ? `Improve \`${suggested}\` so the bot can clear the \`${reason}\` blocker. Touch only the listed files. Add or update tests under \`runtime/skills/\`.` + : `The curriculum has no suggested skill for this state. Either teach the curriculum a new milestone OR add a recovery skill that turns this reason code into a productive action.`, + "", + "## Edit scope (auto-patch must obey this)", + "", + editScope.map((p) => `- ${p}`).join("\n"), + "", + "## Forbidden", + "", + "- Don't touch `.env`, `state/`, `extensions/`, `tui/` unless the scope above includes them.", + "- Don't add new npm dependencies.", + "- Don't change git history (no `--amend`, no `git reset --hard`).", + "", + ].join("\n"); + + return { + fire: true, + kind: `stuck-${reason}`, + summary: `stuck on ${reason}${milestone ? ` (milestone: ${milestone.title})` : ""}`, + body, + editScope, + }; + } + + return { check, reset }; +} diff --git a/runtime/stuck-incident.test.js b/runtime/stuck-incident.test.js new file mode 100644 index 0000000..04d2311 --- /dev/null +++ b/runtime/stuck-incident.test.js @@ -0,0 +1,86 @@ +// Tests for runtime/stuck-incident.js and runtime/skill-metrics.js. + +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { createStuckIncidentDetector } from "./stuck-incident.js"; +import { createSkillMetrics } from "./skill-metrics.js"; + +function snap(reason, extras = {}) { + return { + connected: true, + noProgressReason: reason, + runtimeState: "idle", + inventory: {}, + curriculum: extras.curriculum ?? null, + position: { x: 0, y: 64, z: 0 }, + health: 20, + food: 18, + isDay: true, + ...extras, + }; +} + +test("returns null while reason is absent", () => { + const d = createStuckIncidentDetector({ thresholdMs: 100, cooldownMs: 1_000 }); + assert.equal(d.check({ snapshot: snap(null), now: 0 }), null); + assert.equal(d.check({ snapshot: snap(null), now: 50 }), null); +}); + +test("does not fire before threshold elapses", () => { + const d = createStuckIncidentDetector({ thresholdMs: 100, cooldownMs: 1_000 }); + assert.equal(d.check({ snapshot: snap("planner_empty"), now: 0 }), null); + assert.equal(d.check({ snapshot: snap("planner_empty"), now: 50 }), null); + assert.equal(d.check({ snapshot: snap("planner_empty"), now: 99 }), null); +}); + +test("fires after threshold and includes scope + body", () => { + const d = createStuckIncidentDetector({ thresholdMs: 100, cooldownMs: 1_000 }); + d.check({ snapshot: snap("no_food_source"), now: 0 }); + const out = d.check({ + snapshot: snap("no_food_source", { + curriculum: { milestone: { title: "Secure food" }, plan: { skillId: "survive.eat" } }, + }), + lastResult: { label: "survive.eat", ok: false, code: "no_food_source", detail: "no edible item" }, + metrics: { "survive.eat": { ok: 0, fail: 3 } }, + now: 200, + }); + assert.ok(out); + assert.equal(out.fire, true); + assert.equal(out.kind, "stuck-no_food_source"); + assert.match(out.summary, /no_food_source/); + assert.match(out.body, /Secure food/); + assert.match(out.body, /survive\.eat/); + assert.ok(out.editScope.some((p) => p.includes("survive-eat") || p.includes("survive.eat") || p === "runtime/skills/")); +}); + +test("changes in reason restart the timer (no premature fire)", () => { + const d = createStuckIncidentDetector({ thresholdMs: 100, cooldownMs: 1_000 }); + d.check({ snapshot: snap("planner_empty"), now: 0 }); + d.check({ snapshot: snap("planner_empty"), now: 80 }); + // reason changes — restart + assert.equal(d.check({ snapshot: snap("no_food_source"), now: 100 }), null); + // only the NEW reason's clock counts now + assert.equal(d.check({ snapshot: snap("no_food_source"), now: 150 }), null); +}); + +test("cooldown prevents back-to-back firings", () => { + const d = createStuckIncidentDetector({ thresholdMs: 50, cooldownMs: 1_000 }); + d.check({ snapshot: snap("planner_empty"), now: 0 }); + const first = d.check({ snapshot: snap("planner_empty"), now: 100 }); + assert.ok(first?.fire); + const second = d.check({ snapshot: snap("planner_empty"), now: 200 }); + assert.equal(second, null); +}); + +test("skill metrics record ok/fail and expose snapshot", () => { + const m = createSkillMetrics(); + m.record("gather.logs", true); + m.record("gather.logs", true); + m.record("gather.logs", false); + m.record("survive.eat", false); + const snap = m.snapshot(); + assert.equal(snap["gather.logs"].ok, 2); + assert.equal(snap["gather.logs"].fail, 1); + assert.equal(snap["survive.eat"].fail, 1); +});