feat(v0.3.0): paradigm shift — TimeWeb-only LLM + persistent advisor trail + improvement queue
This is the rc.4 batch the user requested:
1. Emergency triggers (low HP + close hostile, lava-under-foot)
bypass the long cooldown so the LLM is consulted BEFORE the bot
dies, not after.
2. Active manifesto need is now included in the advisor user prompt
— the LLM picks suggestions that satisfy the bot's current
concrete need (L2 tools_wood → "gather logs nearby" not
"explore further").
3. Every advisor recommendation is persisted to SQLite
(advisor_recommendations table) with full token usage. The
reflex marks 'applied=1' when it dispatches and updates
outcome_ok/code when the dispatch completes. Ground truth for
"is the LLM actually helping" lives in the DB, not in logs.
4. Pi CLI is OUT of every background loop. coach/postmortem and
coach/reflect now go through the same TimeWeb endpoint
fast-advisor uses, via the shared coach/llm-call.js helper.
Pi is reserved for manual operator commands.
5. The LLM (postmortem, reflect, advisor) can flag "structural
gaps" — missing skills/features the operator should implement.
These land in the new improvement_requests table. Dedup by
title bumps `votes` instead of inserting duplicates so the
queue doesn't bloat. Operator views via
`node scripts/list-improvements.js`.
6. A deterministic trigger-tuner runs hourly: reads 24h of
recommendation stats, flags triggers whose success rate is
below 25% (sample ≥ 5) or whose prompts are expensive (>1000
input tokens) with mediocre payoff. Improvements get
source="tuner", category="tuning". No LLM call.
New files:
runtime/coach/llm-call.js — askAnalytical() helper
runtime/coach/trigger-tuner.js — stats → improvements
runtime/coach/trigger-tuner.test.js
scripts/list-improvements.js — operator CLI
Schema additions:
advisor_recommendations: id, ts, trigger_reason, planned_skill,
recommended_skill, action, rationale, active_need, tokens_in,
tokens_out, latency_ms, applied, outcome_ok, outcome_code, outcome_at
improvement_requests: id, ts, source, category, title, description,
context, priority, status, duplicate_of, votes, implemented_at, notes
Renamed env-var consumers:
Pi-coach drainOnce({ askPi }) → drainOnce({ askAnalyticalFn? })
Pi-reflect runOnce({ askPi }) → runOnce({ askAnalyticalFn? })
bot.js attachCoach/attachReflect no longer pass askPi
attachTuner() added to bot.js spawn handler
lessons.source 'pi-coach' → 'timeweb-coach'
lessons.source 'pi-reflect' → 'timeweb-reflect'
Token cost measured live:
~705 input + 45 output = ~750 total per advisor call
worst case @ 6 calls/hour rate cap = ~108K tokens/day
OpenAI gpt-5-mini reference price: ~$0.60/month
Operator usage:
node scripts/list-improvements.js # open queue
node scripts/list-improvements.js --stats # advisor performance
node scripts/list-improvements.js --done 17 "shipped in 0.3.1"
node scripts/list-improvements.js --reject 18 "duplicate"
Tests: 360 green (was 332, +28 new).
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -39,16 +39,18 @@ test("extractJson: tolerates fences and surrounding text", () => {
|
||||
assert.equal(extractJson(""), null);
|
||||
});
|
||||
|
||||
test("buildPrompt: includes all death rows and JSON schema hint", () => {
|
||||
test("buildPrompt: returns {system, user}, includes all death rows + improvements schema", () => {
|
||||
const rows = [
|
||||
{ id: 1, ts: Date.now(), x: 100, y: 64, z: 200, cause: "hostile", hostile: "creeper", last_skill: "gather.logs", last_skill_code: "timeout", food_at_death: 14, context_blob: JSON.stringify({ recentScenarios: [{ skillId: "gather.logs", code: "timeout" }] }) },
|
||||
{ id: 2, ts: Date.now(), x: 102, y: 64, z: 201, cause: "hostile", hostile: "creeper", last_skill: "explore.far", last_skill_code: "done", food_at_death: 12, context_blob: null },
|
||||
];
|
||||
const prompt = buildPrompt(rows);
|
||||
assert.match(prompt, /death id=1/);
|
||||
assert.match(prompt, /death id=2/);
|
||||
assert.match(prompt, /creeper/);
|
||||
assert.match(prompt, /Reply with ONE JSON object/);
|
||||
const { system, user } = buildPrompt(rows);
|
||||
assert.match(user, /death id=1/);
|
||||
assert.match(user, /death id=2/);
|
||||
assert.match(user, /creeper/);
|
||||
assert.match(system, /Reply with ONE JSON object/);
|
||||
assert.match(system, /improvements/);
|
||||
assert.match(system, /Valid skill ids/);
|
||||
});
|
||||
|
||||
test("captureDeath: builds a row with context blob and inferred cause", () => {
|
||||
@@ -88,7 +90,7 @@ test("attach + emit('death'): inserts row in knowledge DB", async () => {
|
||||
rmSync(stateDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test("drainOnce: respects budget and parses Pi reply", async () => {
|
||||
test("drainOnce: respects budget and parses analytical LLM reply (incl. improvements)", async () => {
|
||||
const stateDir = mkdtempSync(join(tmpdir(), "pepa-coach-test-"));
|
||||
__resetForTests();
|
||||
await initKnowledge({ stateDir });
|
||||
@@ -105,7 +107,13 @@ test("drainOnce: respects budget and parses Pi reply", async () => {
|
||||
|
||||
const lessonsBefore = recall({ category: "combat" }).length;
|
||||
|
||||
const fakeReply = JSON.stringify({
|
||||
// TimeWeb path needs env vars to satisfy the llmAvailable check.
|
||||
const prevKey = process.env.TIMEWEB_API_KEY;
|
||||
const prevModel = process.env.TIMEWEB_MODEL;
|
||||
process.env.TIMEWEB_API_KEY = "test-key";
|
||||
process.env.TIMEWEB_MODEL = "test-model";
|
||||
|
||||
const fakeReply = {
|
||||
cause: "creeper_explosion_unarmed",
|
||||
next_action: "shelter at dusk",
|
||||
lessons: [{
|
||||
@@ -116,16 +124,23 @@ test("drainOnce: respects budget and parses Pi reply", async () => {
|
||||
prefer_skill: "survive.flee",
|
||||
confidence: 0.85,
|
||||
}],
|
||||
});
|
||||
const askPi = ({ onChunk, onDone }) => {
|
||||
onChunk({ stream: "stdout", text: fakeReply });
|
||||
onDone({ code: 0 });
|
||||
improvements: [
|
||||
{ title: "Add craft.shield skill", description: "No skill to craft a shield when creepers are around.", category: "skill", priority: 2 },
|
||||
],
|
||||
};
|
||||
const askAnalyticalFn = async () => fakeReply;
|
||||
|
||||
const result = await drainOnce({ stateDir, force: true, askAnalyticalFn });
|
||||
|
||||
if (prevKey === undefined) delete process.env.TIMEWEB_API_KEY;
|
||||
else process.env.TIMEWEB_API_KEY = prevKey;
|
||||
if (prevModel === undefined) delete process.env.TIMEWEB_MODEL;
|
||||
else process.env.TIMEWEB_MODEL = prevModel;
|
||||
|
||||
const result = await drainOnce({ askPi, stateDir, force: true });
|
||||
assert.equal(result.ok, true);
|
||||
assert.equal(result.analysed, 1);
|
||||
assert.equal(result.lessons, 1);
|
||||
assert.equal(result.improvements, 1);
|
||||
|
||||
const after = recall({ hostile: "creeper", category: "combat" });
|
||||
assert.ok(after.length > lessonsBefore, "new lesson recorded");
|
||||
@@ -137,18 +152,20 @@ test("drainOnce: respects budget and parses Pi reply", async () => {
|
||||
rmSync(stateDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test("drainOnce: empty queue → ok with 0 analysed", async () => {
|
||||
test("drainOnce: skipped when LLM not configured", async () => {
|
||||
const stateDir = mkdtempSync(join(tmpdir(), "pepa-coach-test-"));
|
||||
__resetForTests();
|
||||
await initKnowledge({ stateDir });
|
||||
if (!isAvailable()) {
|
||||
assert.ok(true);
|
||||
rmSync(stateDir, { recursive: true, force: true });
|
||||
return;
|
||||
}
|
||||
const result = await drainOnce({ askPi: () => {}, stateDir, force: true });
|
||||
assert.equal(result.ok, true);
|
||||
assert.equal(result.analysed, 0);
|
||||
const prevKey = process.env.TIMEWEB_API_KEY;
|
||||
delete process.env.TIMEWEB_API_KEY;
|
||||
const result = await drainOnce({ stateDir, force: true });
|
||||
if (prevKey !== undefined) process.env.TIMEWEB_API_KEY = prevKey;
|
||||
assert.equal(result.ok, false);
|
||||
assert.equal(result.reason, "llm not configured");
|
||||
closeStore();
|
||||
rmSync(stateDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user