feat(runtime): cmd:screenshot + cmd:force-incident + prismarine-viewer dep
Three live-verification surfaces on top of v0.1.0:
1. cmd:screenshot { reason?, frames? } → runtime/viewer.takeScreenshot
- Headless POV render via prismarine-viewer.headless to
state/<host>/screenshots/<ISO>-<reason>.mp4 (1 frame ≈ ~10 KB).
- Lazy-loads the heavy GL stack on first call so the bot doesn't pay
the cost on startup or in TUI-only sessions.
- Returns { ok, path, error } over IPC LOG event.
- Known limitation: needs node-canvas. node-canvas v3 (current npm
default) is incompatible with prismarine-viewer's API; v2 doesn't
build under Node 24 (node-pre-gyp fail). So today the feature is
wired and the IPC contract is stable, but the underlying render
fails fast with "createCanvas is not a function". A future cleanup
can either fork the renderer or pin a Node 20 toolchain.
2. cmd:force-incident { kind?, reason? } → filePostCritique path
- Operator-triggered demo of the critic → proposal → auto-improve →
auto-patch chain. Was previously only observable when the bot
genuinely got stuck. Now a single IPC call exercises the full
loop on demand.
- Verified live 2026-05-26: critic call returned a real, useful
critique ("attack zombie returns done while target is alive →
blocks gather.logs"), proposal landed with all sections including
the Critic block, auto-improve picked it up within 10 s.
3. prismarine-viewer + canvas added to dependencies so npm install
builds the deps once and the IPC surface is always available.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.7
Generated
+1550
-2
File diff suppressed because it is too large
Load Diff
@@ -19,6 +19,7 @@
|
||||
"test": "node --test runtime/skills/contract.test.js runtime/skills/groups.test.js runtime/skills/compat.test.js runtime/skills/recovery-tunnel-out.test.js runtime/curriculum.test.js runtime/social/social.test.js runtime/social/conversation.test.js runtime/stuck-incident.test.js runtime/compat.test.js runtime/reflex.test.js runtime/base-site.test.js runtime/locations.test.js runtime/watch-filter.test.js runtime/world-journal.test.js runtime/scenario-memory.test.js runtime/critic.test.js runtime/skill-library.test.js runtime/modes.test.js scripts/edit-scope.test.js scripts/lint-patch.test.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"canvas": "^3.2.3",
|
||||
"dotenv": "^16.4.5",
|
||||
"ink": "^7.0.4",
|
||||
"ink-text-input": "^6.0.0",
|
||||
@@ -31,6 +32,7 @@
|
||||
"mineflayer-pvp": "^1.3.2",
|
||||
"mineflayer-tool": "^1.2.0",
|
||||
"prismarine-item": "^1.18.0",
|
||||
"prismarine-viewer": "^1.33.0",
|
||||
"react": "^19.2.6",
|
||||
"vec3": "^0.2.0"
|
||||
},
|
||||
|
||||
+28
-1
@@ -47,6 +47,7 @@ import { classifyIntent, INTENTS } from "./social/intent.js";
|
||||
import { generateReply } from "./social/reply.js";
|
||||
import { createChatMemory } from "./social/memory.js";
|
||||
import { openConversation, peekConversation, listConversations } from "./social/conversation.js";
|
||||
import { takeScreenshot } from "./viewer.js";
|
||||
import { createStuckIncidentDetector, attachCritique } from "./stuck-incident.js";
|
||||
import { requestCritique } from "./critic.js";
|
||||
import { createSkillMetrics } from "./skill-metrics.js";
|
||||
@@ -956,7 +957,7 @@ function handleCommand(msg, send) {
|
||||
const { topic: topic_, text, intent, position } = msg.payload ?? {};
|
||||
if (!topic_ || !text) { send(EVENT_TYPES.ERROR, { source: "conv", text: "topic and text required" }); return; }
|
||||
try {
|
||||
const h = openConversation(topic_, { speaker: cfg.username });
|
||||
const h = openConversation(topic_, { speaker: config.username });
|
||||
const turn = h.append({ text, intent, position: position ?? lastSnapshot?.position });
|
||||
send(EVENT_TYPES.LOG, { ts: new Date().toISOString(), level: "info", source: "conv", text: `say to ${topic_}`, details: turn });
|
||||
} catch (e) { send(EVENT_TYPES.ERROR, { source: "conv", text: e.message }); }
|
||||
@@ -978,6 +979,32 @@ function handleCommand(msg, send) {
|
||||
} catch (e) { send(EVENT_TYPES.ERROR, { source: "conv", text: e.message }); }
|
||||
break;
|
||||
}
|
||||
case COMMAND_TYPES.SCREENSHOT: {
|
||||
const { reason, frames } = msg.payload ?? {};
|
||||
(async () => {
|
||||
if (!bot) { send(EVENT_TYPES.ERROR, { source: "viewer", text: "bot not connected" }); return; }
|
||||
const res = await takeScreenshot(bot, { reason: reason ?? "ipc", frames: frames ?? 1 });
|
||||
send(EVENT_TYPES.LOG, { ts: new Date().toISOString(), level: "info", source: "viewer", text: res.ok ? `screenshot ok` : `screenshot fail`, details: res });
|
||||
})();
|
||||
break;
|
||||
}
|
||||
case COMMAND_TYPES.FORCE_INCIDENT: {
|
||||
const { kind, reason } = msg.payload ?? {};
|
||||
(async () => {
|
||||
const fakeIncident = {
|
||||
kind: kind ?? "force-demo",
|
||||
summary: `forced incident: ${reason ?? "operator demo"}`,
|
||||
body: stuckIncident._renderFake
|
||||
? stuckIncident._renderFake({ snapshot: lastSnapshot, lastResult, reason: reason ?? "operator demo" })
|
||||
: `# Forced incident\n\nOperator triggered via cmd:force-incident.\n\n## Snapshot\n\n\`\`\`json\n${JSON.stringify(lastSnapshot ?? {}, null, 2).slice(0, 2000)}\n\`\`\`\n\n## Last action\n\n${lastResult ? `\`${lastResult.label}\` → ${lastResult.code}` : "_(none)_"}\n\n## Suggested fix\n\nReview the snapshot and propose a productive next skill, or document why no productive action is possible from this state.\n\n## Edit scope\n\n- runtime/skills/\n- runtime/reflex.js\n`,
|
||||
editScope: ["runtime/skills/", "runtime/reflex.js"],
|
||||
};
|
||||
send(EVENT_TYPES.LOG, { ts: new Date().toISOString(), level: "info", source: "force", text: `dispatching critic for ${fakeIncident.kind}` });
|
||||
await filePostCritique(fakeIncident, "force");
|
||||
send(EVENT_TYPES.LOG, { ts: new Date().toISOString(), level: "info", source: "force", text: `force-incident done — check state/proposals/` });
|
||||
})();
|
||||
break;
|
||||
}
|
||||
default:
|
||||
warn("ipc", `unknown command type: ${msg.type}`);
|
||||
}
|
||||
|
||||
@@ -31,6 +31,8 @@ export const COMMAND_TYPES = Object.freeze({
|
||||
CONV_SAY: "cmd:conv-say", // { topic, text, intent?, position? } append turn to a multi-agent topic
|
||||
CONV_RECENT: "cmd:conv-recent", // { topic, n? } read last n turns
|
||||
CONV_LIST: "cmd:conv-list", // list active conversation topics
|
||||
SCREENSHOT: "cmd:screenshot", // { reason?, frames? } headless POV render to state/screenshots/
|
||||
FORCE_INCIDENT: "cmd:force-incident", // { kind?, reason? } emulate a stuck-incident to demo critic + auto-improve pipeline
|
||||
});
|
||||
|
||||
export function encodeFrame(obj) {
|
||||
|
||||
+63
-6
@@ -1,11 +1,19 @@
|
||||
// Optional prismarine-viewer launch — local visual debugging surface.
|
||||
// Activated by setting VIEWER_PORT in .env (e.g. 3007). The dependency is
|
||||
// not required at runtime: if `prismarine-viewer` is not installed, this
|
||||
// module logs once and returns, so production deploys aren't forced to
|
||||
// carry the extra dep.
|
||||
// Two viewer surfaces:
|
||||
// 1. maybeStartViewer(bot) — long-running HTTP browser viewer at
|
||||
// VIEWER_PORT (optional, opt-in). Useful for live operator
|
||||
// debugging.
|
||||
// 2. takeScreenshot(bot, opts) — on-demand headless render of "what
|
||||
// the bot sees right now" → PNG-equivalent file under
|
||||
// state/<host>/screenshots/. Drives cmd:screenshot IPC and can be
|
||||
// embedded in proposal bodies.
|
||||
//
|
||||
// Both require prismarine-viewer (npm i prismarine-viewer). If missing,
|
||||
// each function logs and returns gracefully.
|
||||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { info, warn } from "./log.js";
|
||||
import { config } from "./config.js";
|
||||
import { config, stateDir } from "./config.js";
|
||||
|
||||
let started = false;
|
||||
|
||||
@@ -28,3 +36,52 @@ export async function maybeStartViewer(bot) {
|
||||
warn("viewer", `failed to start: ${e?.message ?? e}`);
|
||||
}
|
||||
}
|
||||
|
||||
// --- On-demand screenshot ---------------------------------------------
|
||||
|
||||
const SHOT_DIR = path.join(stateDir, "screenshots");
|
||||
|
||||
function ensureShotDir() {
|
||||
try { fs.mkdirSync(SHOT_DIR, { recursive: true }); } catch {}
|
||||
}
|
||||
|
||||
function shotSlug(reason) {
|
||||
const ts = new Date().toISOString().replace(/[:.]/g, "-");
|
||||
const r = String(reason ?? "manual").replace(/[^a-zA-Z0-9_-]+/g, "_").slice(0, 32);
|
||||
return `${ts}-${r}`;
|
||||
}
|
||||
|
||||
let headlessFn = null;
|
||||
async function loadHeadless() {
|
||||
if (headlessFn) return headlessFn;
|
||||
const mod = await import("prismarine-viewer");
|
||||
headlessFn = mod.headless ?? mod.default?.headless ?? null;
|
||||
if (!headlessFn) throw new Error("prismarine-viewer.headless export missing");
|
||||
return headlessFn;
|
||||
}
|
||||
|
||||
// Take one rendered frame from the bot's POV. prismarine-viewer's
|
||||
// `headless()` writes an mp4 (1 frame ≈ ~10KB), which we keep as-is —
|
||||
// any tool that can read mp4 (ffmpeg, mpv, QuickTime) will display it.
|
||||
// We accept this over rolling our own renderer.
|
||||
export async function takeScreenshot(bot, { reason = "manual", frames = 1, width = 512, height = 384, viewDistance = 6 } = {}) {
|
||||
if (!bot) return { ok: false, error: "no bot" };
|
||||
ensureShotDir();
|
||||
const outPath = path.join(SHOT_DIR, `${shotSlug(reason)}.mp4`);
|
||||
let hl;
|
||||
try { hl = await loadHeadless(); }
|
||||
catch (e) {
|
||||
warn("viewer", `headless not available: ${e.message}`);
|
||||
return { ok: false, error: `viewer load: ${e.message}` };
|
||||
}
|
||||
try {
|
||||
await hl(bot, { output: outPath, frames, width, height, viewDistance });
|
||||
info("viewer", `screenshot saved: ${outPath} (reason=${reason})`);
|
||||
return { ok: true, path: outPath, reason };
|
||||
} catch (e) {
|
||||
warn("viewer", `screenshot failed: ${e.message}`);
|
||||
return { ok: false, error: e.message };
|
||||
}
|
||||
}
|
||||
|
||||
export const _internal = { shotSlug, SHOT_DIR };
|
||||
|
||||
Reference in New Issue
Block a user