mayatnikovandClaude Opus 4.7
0d97ccccfa
This is the rc.4 batch the user requested:
1. Emergency triggers (low HP + close hostile, lava-under-foot)
bypass the long cooldown so the LLM is consulted BEFORE the bot
dies, not after.
2. Active manifesto need is now included in the advisor user prompt
— the LLM picks suggestions that satisfy the bot's current
concrete need (L2 tools_wood → "gather logs nearby" not
"explore further").
3. Every advisor recommendation is persisted to SQLite
(advisor_recommendations table) with full token usage. The
reflex marks 'applied=1' when it dispatches and updates
outcome_ok/code when the dispatch completes. Ground truth for
"is the LLM actually helping" lives in the DB, not in logs.
4. Pi CLI is OUT of every background loop. coach/postmortem and
coach/reflect now go through the same TimeWeb endpoint
fast-advisor uses, via the shared coach/llm-call.js helper.
Pi is reserved for manual operator commands.
5. The LLM (postmortem, reflect, advisor) can flag "structural
gaps" — missing skills/features the operator should implement.
These land in the new improvement_requests table. Dedup by
title bumps `votes` instead of inserting duplicates so the
queue doesn't bloat. Operator views via
`node scripts/list-improvements.js`.
6. A deterministic trigger-tuner runs hourly: reads 24h of
recommendation stats, flags triggers whose success rate is
below 25% (sample ≥ 5) or whose prompts are expensive (>1000
input tokens) with mediocre payoff. Improvements get
source="tuner", category="tuning". No LLM call.
New files:
runtime/coach/llm-call.js — askAnalytical() helper
runtime/coach/trigger-tuner.js — stats → improvements
runtime/coach/trigger-tuner.test.js
scripts/list-improvements.js — operator CLI
Schema additions:
advisor_recommendations: id, ts, trigger_reason, planned_skill,
recommended_skill, action, rationale, active_need, tokens_in,
tokens_out, latency_ms, applied, outcome_ok, outcome_code, outcome_at
improvement_requests: id, ts, source, category, title, description,
context, priority, status, duplicate_of, votes, implemented_at, notes
Renamed env-var consumers:
Pi-coach drainOnce({ askPi }) → drainOnce({ askAnalyticalFn? })
Pi-reflect runOnce({ askPi }) → runOnce({ askAnalyticalFn? })
bot.js attachCoach/attachReflect no longer pass askPi
attachTuner() added to bot.js spawn handler
lessons.source 'pi-coach' → 'timeweb-coach'
lessons.source 'pi-reflect' → 'timeweb-reflect'
Token cost measured live:
~705 input + 45 output = ~750 total per advisor call
worst case @ 6 calls/hour rate cap = ~108K tokens/day
OpenAI gpt-5-mini reference price: ~$0.60/month
Operator usage:
node scripts/list-improvements.js # open queue
node scripts/list-improvements.js --stats # advisor performance
node scripts/list-improvements.js --done 17 "shipped in 0.3.1"
node scripts/list-improvements.js --reject 18 "duplicate"
Tests: 360 green (was 332, +28 new).
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
99 lines
3.6 KiB
JavaScript
99 lines
3.6 KiB
JavaScript
import { test } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { mkdtempSync, rmSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
|
|
import { initKnowledge, isAvailable, listImprovements } from "../knowledge/index.js";
|
|
import { closeStore, __resetForTests } from "../knowledge/store.js";
|
|
import { runOnce, __testing } from "./trigger-tuner.js";
|
|
|
|
const { MIN_SAMPLE } = __testing;
|
|
|
|
async function bootstrap() {
|
|
const tmp = mkdtempSync(join(tmpdir(), "pepa-tuner-test-"));
|
|
__resetForTests();
|
|
await initKnowledge({ stateDir: tmp });
|
|
return tmp;
|
|
}
|
|
|
|
function cleanup(tmp) {
|
|
closeStore();
|
|
try { rmSync(tmp, { recursive: true, force: true }); } catch {}
|
|
}
|
|
|
|
test("runOnce: empty stats → ok with 0 flagged", async () => {
|
|
const tmp = await bootstrap();
|
|
if (!isAvailable()) { cleanup(tmp); return; }
|
|
const r = runOnce({ stats: [] });
|
|
assert.equal(r.ok, true);
|
|
assert.equal(r.flagged, 0);
|
|
cleanup(tmp);
|
|
});
|
|
|
|
test("runOnce: ignores small samples (below MIN_SAMPLE)", async () => {
|
|
const tmp = await bootstrap();
|
|
if (!isAvailable()) { cleanup(tmp); return; }
|
|
const stats = [
|
|
{ trigger_reason: "wedged_60s", total: 2, applied: 2, succeeded: 0, failed: 2, avg_in: 700, avg_out: 40, avg_latency_ms: 5000 },
|
|
];
|
|
const r = runOnce({ stats });
|
|
assert.equal(r.flagged, 0, "applied=2 is below MIN_SAMPLE; skipped");
|
|
cleanup(tmp);
|
|
});
|
|
|
|
test("runOnce: flags low success-rate trigger as improvement", async () => {
|
|
const tmp = await bootstrap();
|
|
if (!isAvailable()) { cleanup(tmp); return; }
|
|
const stats = [
|
|
{ trigger_reason: "wedged_60s", total: 10, applied: 10, succeeded: 1, failed: 9, avg_in: 700, avg_out: 40, avg_latency_ms: 5000 },
|
|
];
|
|
const r = runOnce({ stats });
|
|
assert.equal(r.flagged, 1);
|
|
const requests = listImprovements({ source: "tuner" });
|
|
assert.ok(requests.some((req) => req.title.includes("wedged_60s") && req.title.includes("low success")));
|
|
cleanup(tmp);
|
|
});
|
|
|
|
test("runOnce: flags expensive prompt with mediocre payoff", async () => {
|
|
const tmp = await bootstrap();
|
|
if (!isAvailable()) { cleanup(tmp); return; }
|
|
const stats = [
|
|
{ trigger_reason: "repeat_4_explore.far", total: 10, applied: 10, succeeded: 4, failed: 6, avg_in: 1500, avg_out: 50, avg_latency_ms: 7000 },
|
|
];
|
|
const r = runOnce({ stats });
|
|
assert.equal(r.flagged, 1);
|
|
const requests = listImprovements({ source: "tuner", category: "tuning" });
|
|
assert.ok(requests.some((req) => req.title.includes("expensive")));
|
|
cleanup(tmp);
|
|
});
|
|
|
|
test("runOnce: healthy trigger does NOT get flagged", async () => {
|
|
const tmp = await bootstrap();
|
|
if (!isAvailable()) { cleanup(tmp); return; }
|
|
const stats = [
|
|
{ trigger_reason: "emergency_hp4_creeper@3", total: 8, applied: 8, succeeded: 7, failed: 1, avg_in: 700, avg_out: 40, avg_latency_ms: 5000 },
|
|
];
|
|
const r = runOnce({ stats });
|
|
assert.equal(r.flagged, 0);
|
|
cleanup(tmp);
|
|
});
|
|
|
|
test("runOnce: re-running with same low-success stats bumps votes, not row count", async () => {
|
|
const tmp = await bootstrap();
|
|
if (!isAvailable()) { cleanup(tmp); return; }
|
|
const stats = [
|
|
{ trigger_reason: "wedged_unique_label", total: 10, applied: 10, succeeded: 1, failed: 9, avg_in: 700, avg_out: 40, avg_latency_ms: 5000 },
|
|
];
|
|
runOnce({ stats });
|
|
runOnce({ stats });
|
|
const requests = listImprovements({ source: "tuner" }).filter((r) => r.title.includes("wedged_unique_label"));
|
|
assert.equal(requests.length, 1, "single row for the same title");
|
|
assert.ok(requests[0].votes >= 2, "votes bumped on re-flagging");
|
|
cleanup(tmp);
|
|
});
|
|
|
|
test("MIN_SAMPLE constant is reasonable", () => {
|
|
assert.ok(MIN_SAMPLE >= 3 && MIN_SAMPLE <= 10);
|
|
});
|