feat(v0.3.0): paradigm shift — TimeWeb-only LLM + persistent advisor trail + improvement queue

This is the rc.4 batch the user requested:

  1. Emergency triggers (low HP + close hostile, lava-under-foot)
     bypass the long cooldown so the LLM is consulted BEFORE the bot
     dies, not after.
  2. Active manifesto need is now included in the advisor user prompt
     — the LLM picks suggestions that satisfy the bot's current
     concrete need (L2 tools_wood → "gather logs nearby" not
     "explore further").
  3. Every advisor recommendation is persisted to SQLite
     (advisor_recommendations table) with full token usage. The
     reflex marks 'applied=1' when it dispatches and updates
     outcome_ok/code when the dispatch completes. Ground truth for
     "is the LLM actually helping" lives in the DB, not in logs.
  4. Pi CLI is OUT of every background loop. coach/postmortem and
     coach/reflect now go through the same TimeWeb endpoint
     fast-advisor uses, via the shared coach/llm-call.js helper.
     Pi is reserved for manual operator commands.
  5. The LLM (postmortem, reflect, advisor) can flag "structural
     gaps" — missing skills/features the operator should implement.
     These land in the new improvement_requests table. Dedup by
     title bumps `votes` instead of inserting duplicates so the
     queue doesn't bloat. Operator views via
     `node scripts/list-improvements.js`.
  6. A deterministic trigger-tuner runs hourly: reads 24h of
     recommendation stats, flags triggers whose success rate is
     below 25% (sample ≥ 5) or whose prompts are expensive (>1000
     input tokens) with mediocre payoff. Improvements get
     source="tuner", category="tuning". No LLM call.

New files:
  runtime/coach/llm-call.js        — askAnalytical() helper
  runtime/coach/trigger-tuner.js   — stats → improvements
  runtime/coach/trigger-tuner.test.js
  scripts/list-improvements.js     — operator CLI

Schema additions:
  advisor_recommendations: id, ts, trigger_reason, planned_skill,
    recommended_skill, action, rationale, active_need, tokens_in,
    tokens_out, latency_ms, applied, outcome_ok, outcome_code, outcome_at
  improvement_requests: id, ts, source, category, title, description,
    context, priority, status, duplicate_of, votes, implemented_at, notes

Renamed env-var consumers:
  Pi-coach drainOnce({ askPi })   → drainOnce({ askAnalyticalFn? })
  Pi-reflect runOnce({ askPi })   → runOnce({ askAnalyticalFn? })
  bot.js attachCoach/attachReflect no longer pass askPi
  attachTuner() added to bot.js spawn handler
  lessons.source 'pi-coach'   → 'timeweb-coach'
  lessons.source 'pi-reflect' → 'timeweb-reflect'

Token cost measured live:
  ~705 input + 45 output = ~750 total per advisor call
  worst case @ 6 calls/hour rate cap = ~108K tokens/day
  OpenAI gpt-5-mini reference price: ~$0.60/month

Operator usage:
  node scripts/list-improvements.js                # open queue
  node scripts/list-improvements.js --stats        # advisor performance
  node scripts/list-improvements.js --done 17 "shipped in 0.3.1"
  node scripts/list-improvements.js --reject 18 "duplicate"

Tests: 360 green (was 332, +28 new).

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
2026-05-27 19:12:48 +03:00
co-authored by Claude Opus 4.7
parent bc381b2a4b
commit 0d97ccccfa
18 changed files with 1188 additions and 197 deletions
+60 -6
View File
@@ -21,6 +21,7 @@
import { advise, isAvailable as advisorAvailable } from "./fast-advisor.js";
import { isRegistered } from "../skill-registry.js";
import { insertRecommendation } from "../knowledge/index.js";
import { info, warn } from "../log.js";
const TRIGGER_COOLDOWN_MS = 90_000;
@@ -28,6 +29,11 @@ const RECOMMENDATION_TTL_MS = 60_000;
const WEDGED_THRESHOLD_MS = 60_000;
const REPEAT_THRESHOLD = 4;
const PREEMPT_WINDOW_MS = 30_000;
// Emergency triggers — bypass cooldown because waiting another 90s
// when the bot is about to die is not useful.
const EMERGENCY_HP = 6;
const EMERGENCY_HOSTILE_DIST = 8;
const EMERGENCY_COOLDOWN_MS = 20_000;
let _lastTriggerAt = 0;
let _inFlight = false;
@@ -57,9 +63,6 @@ export function tickAdvisor(ctx, { plannedSkillId } = {}) {
if (_inFlight) return { fired: false, reason: "in_flight" };
const now = Date.now();
if (now - _lastTriggerAt < TRIGGER_COOLDOWN_MS) {
return { fired: false, reason: "cooldown" };
}
// Drop a recommendation that's already aged out.
if (ctx.advisorRecommendation && now - ctx.advisorRecommendation.at > RECOMMENDATION_TTL_MS) {
@@ -69,18 +72,42 @@ export function tickAdvisor(ctx, { plannedSkillId } = {}) {
const reason = detectTrigger(ctx, now, plannedSkillId);
if (!reason) return { fired: false, reason: "no_trigger" };
// Emergency triggers use a much shorter cooldown — waiting 90s with
// HP=4 and a creeper at 3 blocks is exactly when we MUST hit the LLM.
const isEmergency = reason.startsWith("emergency_");
const cooldownMs = isEmergency ? EMERGENCY_COOLDOWN_MS : TRIGGER_COOLDOWN_MS;
if (now - _lastTriggerAt < cooldownMs) {
return { fired: false, reason: "cooldown" };
}
_lastTriggerAt = now;
_inFlight = true;
const snapshot = ctx.snapshot ?? null;
const recentSkillIds = (ctx.recentSkillIds ?? []).slice(-8);
const activeNeed = ctx.activeNeed ?? null;
info("advisor-trigger", `firing because ${reason} (planned=${plannedSkillId ?? "?"})`);
info("advisor-trigger", `firing because ${reason} (planned=${plannedSkillId ?? "?"}, need=${activeNeed?.need?.id ?? "?"})`);
// Fire-and-forget. The promise's resolution writes ctx.advisorRecommendation.
advise({ snapshot, reason, recentSkillIds, lessonsTail: ctx.recentLessons ?? [], force: true })
advise({ snapshot, reason, recentSkillIds, lessonsTail: ctx.recentLessons ?? [], activeNeed, force: true })
.then((result) => {
_inFlight = false;
const needLabel = activeNeed
? `L${activeNeed.need.level} ${activeNeed.need.id}`
: null;
if (result.ok && result.action === "switch_skill" && isRegistered(result.skillId)) {
const recId = insertRecommendation({
triggerReason: reason,
plannedSkill: plannedSkillId ?? null,
recommendedSkill: result.skillId,
action: "switch_skill",
rationale: result.rationale,
activeNeed: needLabel,
tokensIn: result.usage?.in,
tokensOut: result.usage?.out,
latencyMs: result.latencyMs,
});
ctx.advisorRecommendation = {
id: recId,
at: Date.now(),
skillId: result.skillId,
action: "switch_skill",
@@ -89,9 +116,21 @@ export function tickAdvisor(ctx, { plannedSkillId } = {}) {
latencyMs: result.latencyMs,
usage: result.usage ?? null,
};
info("advisor-trigger", `recommendation cached: ${result.skillId} (${result.latencyMs}ms, in=${result.usage?.in ?? "?"}t/out=${result.usage?.out ?? "?"}t)`);
info("advisor-trigger", `recommendation cached: ${result.skillId} (${result.latencyMs}ms, in=${result.usage?.in ?? "?"}t/out=${result.usage?.out ?? "?"}t, db=${recId ?? "-"})`);
} else if (result.ok && (result.action === "wait" || result.action === "continue")) {
const recId = insertRecommendation({
triggerReason: reason,
plannedSkill: plannedSkillId ?? null,
recommendedSkill: null,
action: result.action,
rationale: result.rationale,
activeNeed: needLabel,
tokensIn: result.usage?.in,
tokensOut: result.usage?.out,
latencyMs: result.latencyMs,
});
ctx.advisorRecommendation = {
id: recId,
at: Date.now(),
action: result.action,
rationale: result.rationale,
@@ -113,6 +152,21 @@ export function tickAdvisor(ctx, { plannedSkillId } = {}) {
}
function detectTrigger(ctx, now, plannedSkillId) {
const snap = ctx.snapshot ?? {};
// 0. EMERGENCY: low HP + hostile near — call BEFORE the bot dies.
// Checked first so reason string starts with "emergency_" → bypasses
// the long trigger cooldown via the caller's isEmergency check.
const hp = snap.health ?? 20;
const hostile = snap.closestHostile;
if (hp <= EMERGENCY_HP && hostile && (hostile.distance ?? Infinity) <= EMERGENCY_HOSTILE_DIST) {
return `emergency_hp${Math.round(hp)}_${hostile.name ?? "hostile"}@${Math.round(hostile.distance)}`;
}
// 0b. EMERGENCY: drowning / lava / lethal fluid
if (snap.hazards?.footBlock === "lava") {
return "emergency_lava";
}
// 1. Wedged > threshold
if (ctx.lastSignificantMoveAt && (now - ctx.lastSignificantMoveAt) > WEDGED_THRESHOLD_MS) {
return `wedged_${Math.round((now - ctx.lastSignificantMoveAt) / 1000)}s`;
+28
View File
@@ -50,6 +50,34 @@ test("detectTrigger: returns null when nothing matches", () => {
assert.equal(r, null);
});
test("detectTrigger: low HP + hostile near → emergency_hp", () => {
const now = Date.now();
const r = detectTrigger({
recentSkillIds: [],
snapshot: { health: 4, closestHostile: { name: "creeper", distance: 3 } },
}, now, "gather.logs");
assert.match(r, /^emergency_hp4_creeper@3/);
});
test("detectTrigger: foot in lava → emergency_lava", () => {
const now = Date.now();
const r = detectTrigger({
recentSkillIds: [],
snapshot: { health: 18, hazards: { footBlock: "lava" } },
}, now, "explore.far");
assert.equal(r, "emergency_lava");
});
test("detectTrigger: emergency wins over wedged when both present", () => {
const now = Date.now();
const r = detectTrigger({
recentSkillIds: [],
snapshot: { health: 4, closestHostile: { name: "skeleton", distance: 5 } },
lastSignificantMoveAt: now - 120_000,
}, now, "x");
assert.match(r, /^emergency_/);
});
test("detectTrigger: wedged > 60s fires", () => {
const now = Date.now();
const r = detectTrigger(
+12 -2
View File
@@ -66,6 +66,7 @@ export async function advise({
reason = "unknown",
recentSkillIds = [],
lessonsTail = [],
activeNeed = null,
force = false,
} = {}) {
if (!isAvailable()) {
@@ -82,7 +83,7 @@ export async function advise({
}
const system = buildSystemPrompt();
const user = buildUserPrompt({ snapshot, reason, recentSkillIds, lessonsTail });
const user = buildUserPrompt({ snapshot, reason, recentSkillIds, lessonsTail, activeNeed });
_callTimes.push(now);
_lastCallAt = now;
@@ -163,16 +164,24 @@ function buildSystemPrompt() {
].join("\n");
}
function buildUserPrompt({ snapshot, reason, recentSkillIds, lessonsTail }) {
function buildUserPrompt({ snapshot, reason, recentSkillIds, lessonsTail, activeNeed }) {
const pos = snapshot?.position;
const inv = snapshot?.inventory ? Object.keys(snapshot.inventory).slice(0, 10).join(", ") : "(empty)";
const recent = (recentSkillIds ?? []).slice(-8).join(" → ") || "(none)";
const lessons = (lessonsTail ?? []).slice(0, 4).map((l) => ` - ${l.text ?? l}`).join("\n");
const needLine = activeNeed
? `L${activeNeed.need.level} ${activeNeed.need.id} (${activeNeed.need.title}) — manifesto wants ${activeNeed.skillId}`
: "(no active need)";
const hostile = snapshot?.closestHostile
? `${snapshot.closestHostile.name}@${snapshot.closestHostile.distance}b`
: "(none)";
return [
`Trigger: ${reason}`,
`Position: ${pos ? `(${Math.round(pos.x)}, ${Math.round(pos.y)}, ${Math.round(pos.z)})` : "?"}`,
`HP: ${snapshot?.health ?? "?"} food: ${snapshot?.food ?? "?"} day: ${snapshot?.isDay ? "yes" : "no"}`,
`Active need (Maslow ladder): ${needLine}`,
`Closest hostile: ${hostile}`,
`Active skill: ${snapshot?.activeSkill ?? "(idle)"}`,
`Recent dispatches: ${recent}`,
`Inventory keys: ${inv}`,
@@ -181,6 +190,7 @@ function buildUserPrompt({ snapshot, reason, recentSkillIds, lessonsTail }) {
"",
lessons ? `Relevant lessons:\n${lessons}\n` : "",
"What should the bot do RIGHT NOW? Return the JSON decision.",
"Prefer a skill that helps satisfy the active need unless an emergency forces another action.",
].filter(Boolean).join("\n");
}
+44
View File
@@ -0,0 +1,44 @@
// Shared helper for the slow-analytical coach loops (postmortem.js,
// reflect.js). Replaces the old askPi-based subprocess path with a
// direct TimeWeb / OpenAI-compatible HTTP call.
//
// Why this split: postmortem and reflect each took 5-15s via Pi CLI
// (with its own subprocess + auth + sometimes a fresh MC connection)
// and were rate-limited by the subscription. Now they take 5-15s via
// the same TimeWeb endpoint the fast-advisor uses — but they're
// analytical, NOT tactical, so they ask for a different prompt shape
// and a longer reply.
//
// The Pi CLI is no longer driven from background timers. It remains
// available for manual operator commands.
import { complete } from "../llm/provider.js";
import { warn } from "../log.js";
const ANALYTICAL_TIMEOUT_MS = 30_000;
/**
* askAnalytical({ system, user, json }) → text|object|null
*
* Higher-timeout, lower-temperature companion to fast-advisor's
* complete(). Returns just the parsed text/object on success or null
* on failure (so callers can keep their old "no reply" branch).
*
* Token usage is logged via the underlying provider — no extra
* accounting here.
*/
export async function askAnalytical({ system, user, json = true, timeoutMs } = {}) {
const res = await complete({
system,
user,
json,
timeoutMs: timeoutMs ?? ANALYTICAL_TIMEOUT_MS,
});
if (!res.ok) {
warn("coach-llm", `analytical call failed: ${res.code} (${res.detail})`);
return null;
}
return res.text;
}
export { ANALYTICAL_TIMEOUT_MS };
+59 -55
View File
@@ -25,19 +25,22 @@ import {
record as recordLesson,
poiNearby,
recordPOI,
createImprovementRequest,
} from "../knowledge/index.js";
import { isRegistered, skillRegistryPrompt } from "../skill-registry.js";
import { isAvailable as llmAvailable } from "../llm/provider.js";
import { askAnalytical } from "./llm-call.js";
import { info, warn } from "../log.js";
const COACH_INTERVAL_MS = 5 * 60 * 1000; // 5 min between coach passes
const COACH_BATCH_MAX = 8; // up to 8 deaths per Pi call
const COACH_PI_BUDGET_PER_HOUR = 3; // ≤ 3 Pi calls/hour
const COACH_BATCH_MAX = 8; // up to 8 deaths per LLM call
const COACH_BUDGET_PER_HOUR = 3; // ≤ 3 analytical LLM calls/hour
const COACH_COOLDOWN_MS = 12 * 60 * 1000; // 12 min between calls
const RECENT_CHAT_TAIL = 6;
const SCENARIO_TAIL = 12;
let _attached = null;
let _piCallTimes = [];
let _llmCallTimes = [];
let _coachTimer = null;
let _lastInventory = null;
@@ -76,17 +79,18 @@ export function attach(bot, ctx = {}) {
}
});
// Start the periodic Pi-coach drain loop.
if (ctx.askPi && !_coachTimer) {
// v0.3.0 — postmortem analysis runs through TimeWeb (the fast LLM
// provider). Pi CLI no longer drives this loop. The drain timer
// fires regardless of whether TimeWeb is configured; drainOnce()
// short-circuits when the LLM is unavailable.
if (!_coachTimer) {
_coachTimer = setInterval(() => {
drainOnce({ askPi: ctx.askPi, stateDir: ctx.stateDir }).catch((e) =>
drainOnce({ stateDir: ctx.stateDir }).catch((e) =>
warn("coach", `drain error: ${e?.message ?? e}`),
);
}, COACH_INTERVAL_MS);
_coachTimer.unref?.();
info("coach", `attached; drain every ${COACH_INTERVAL_MS / 1000}s`);
} else {
info("coach", "attached; Pi not provided, deaths captured without postmortem analysis");
info("coach", `attached; drain every ${COACH_INTERVAL_MS / 1000}s${llmAvailable() ? " (TimeWeb)" : " (LLM disabled — deaths captured only)"}`);
}
}
@@ -249,34 +253,29 @@ function readJournalNearby(stateDir, pos, radius) {
* Rate-limited: at most COACH_PI_BUDGET_PER_HOUR calls/hour, with
* COACH_COOLDOWN_MS gap between calls.
*/
export async function drainOnce({ askPi, stateDir, force = false } = {}) {
export async function drainOnce({ stateDir, force = false, askAnalyticalFn = askAnalytical } = {}) {
if (!knowledgeAvailable()) return { ok: false, reason: "knowledge unavailable" };
if (!askPi) return { ok: false, reason: "no askPi" };
if (!llmAvailable()) return { ok: false, reason: "llm not configured" };
const now = Date.now();
const hourAgo = now - 60 * 60 * 1000;
_piCallTimes = _piCallTimes.filter((t) => t > hourAgo);
if (!force && _piCallTimes.length >= COACH_PI_BUDGET_PER_HOUR) {
return { ok: false, reason: "hourly budget exhausted", calls: _piCallTimes.length };
_llmCallTimes = _llmCallTimes.filter((t) => t > hourAgo);
if (!force && _llmCallTimes.length >= COACH_BUDGET_PER_HOUR) {
return { ok: false, reason: "hourly budget exhausted", calls: _llmCallTimes.length };
}
if (!force && _piCallTimes.length > 0 && now - _piCallTimes[_piCallTimes.length - 1] < COACH_COOLDOWN_MS) {
if (!force && _llmCallTimes.length > 0 && now - _llmCallTimes[_llmCallTimes.length - 1] < COACH_COOLDOWN_MS) {
return { ok: false, reason: "cooldown" };
}
const pending = unanalysedDeaths({ limit: COACH_BATCH_MAX });
if (pending.length === 0) return { ok: true, analysed: 0 };
const prompt = buildPrompt(pending);
_piCallTimes.push(now);
const { system, user } = buildPrompt(pending);
_llmCallTimes.push(now);
const reply = await askPiOnce({ askPi, prompt });
if (!reply) return { ok: false, reason: "no reply" };
const parsed = extractJson(reply);
if (!parsed) {
warn("coach", "Pi reply was not parseable JSON");
return { ok: false, reason: "bad reply" };
}
const parsed = await askAnalyticalFn({ system, user, json: true });
if (!parsed) return { ok: false, reason: "no reply" };
const reply = typeof parsed === "string" ? parsed : JSON.stringify(parsed);
let lessonsCount = 0;
let rejectedPreferCount = 0;
@@ -312,7 +311,7 @@ export async function drainOnce({ askPi, stateDir, force = false } = {}) {
warn("coach", `dropped prefer_skill from ${rejectedPreferCount} lessons (not in registry)`);
}
// Write one postmortem per death; if Pi grouped them, share the same lesson.
// Write one postmortem per death; if grouped, share the same lesson.
const groupLesson = parsed.lessons?.[0]?.lesson ?? parsed.lesson ?? null;
for (const d of pending) {
insertPostmortem({
@@ -321,13 +320,31 @@ export async function drainOnce({ askPi, stateDir, force = false } = {}) {
lesson: groupLesson,
nextAction: parsed.next_action ?? null,
rawResponse: reply.slice(0, 4000),
source: "pi",
source: "timeweb",
});
markDeathAnalysed(d.id);
}
info("coach", `drain: analysed ${pending.length} deaths → ${lessonsCount} lessons`);
return { ok: true, analysed: pending.length, lessons: lessonsCount };
// v0.3.0 — record any improvement requests the LLM flagged. The
// LLM is encouraged to do this when the deaths point to a missing
// skill or feature; the operator reads scripts/list-improvements.js
// and decides what to implement.
let improvementsCount = 0;
for (const imp of asArray(parsed.improvements ?? [])) {
if (!imp?.title) continue;
createImprovementRequest({
source: "postmortem",
category: imp.category ?? "skill",
title: String(imp.title).slice(0, 120),
description: imp.description ?? null,
context: { death_ids: pending.map((d) => d.id), cause: parsed.cause },
priority: imp.priority ?? 3,
});
improvementsCount += 1;
}
info("coach", `drain: analysed ${pending.length} deaths → ${lessonsCount} lessons, ${improvementsCount} improvement requests`);
return { ok: true, analysed: pending.length, lessons: lessonsCount, improvements: improvementsCount };
}
// Mode names from runtime/modes.js (advice.js#MODE_TO_SKILL) — we accept
@@ -357,17 +374,14 @@ function buildPrompt(deaths) {
].filter(Boolean).join("\n");
}).join("\n\n");
return [
const system = [
"You are reviewing recent deaths of an autonomous Minecraft survival bot (pepa).",
"The bot is trying to gather wood, craft tools, build a small village, and survive nights.",
"It's currently dying repeatedly. Your job: extract 1-3 short, generalised lessons it can apply on respawn.",
"Your job: extract 1-3 short, generalised lessons + flag any missing-skill gaps.",
"",
skillRegistryPrompt({ limit: 1800 }),
"",
"DEATHS:",
summary,
"",
"Reply with ONE JSON object (no prose, no markdown fences):",
"Reply with ONE JSON object (no markdown fences):",
'{ "cause": "<short>", "next_action": "<one-sentence directive>",',
' "lessons": [',
' { "lesson": "...", "category": "combat|pathing|crafting|survival|social",',
@@ -375,30 +389,20 @@ function buildPrompt(deaths) {
' "trigger_hostile": "<mob name or null>",',
' "avoid_skill": "<registered skill id to NOT dispatch, or null>",',
' "prefer_skill": "<registered skill id to use instead, or null>",',
' "confidence": 0.7 }',
' ] }',
' "confidence": 0.7 } ],',
' "improvements": [',
' { "title": "<≤80 chars: what skill/feature is missing>",',
' "description": "<why current registry doesn\'t cover this; concrete example>",',
' "category": "skill|tuning|perception|planning|social|other",',
' "priority": 1 } ] }',
"",
"Keep each lesson under 30 words. Be specific (e.g., \"attack creeper with fists\" rather than \"don't fight\").",
"Keep each lesson under 30 words. Be specific.",
"CRITICAL: avoid_skill and prefer_skill MUST be one of the registered ids above, or null.",
"Use 'improvements' ONLY when a death is plausibly caused by the bot lacking a skill that doesn't exist in the registry (e.g. 'no skill to craft iron armor'). Skip it otherwise.",
].join("\n");
}
function askPiOnce({ askPi, prompt }) {
return new Promise((resolve) => {
let buf = "";
try {
askPi({
prompt,
onChunk: ({ stream, text }) => {
if (stream === "stdout") buf += text;
},
onDone: () => resolve(buf),
});
} catch (e) {
warn("coach", `askPi failed: ${e?.message ?? e}`);
resolve(null);
}
});
const user = `DEATHS:\n${summary}`;
return { system, user };
}
function extractJson(text) {
+35 -18
View File
@@ -39,16 +39,18 @@ test("extractJson: tolerates fences and surrounding text", () => {
assert.equal(extractJson(""), null);
});
test("buildPrompt: includes all death rows and JSON schema hint", () => {
test("buildPrompt: returns {system, user}, includes all death rows + improvements schema", () => {
const rows = [
{ id: 1, ts: Date.now(), x: 100, y: 64, z: 200, cause: "hostile", hostile: "creeper", last_skill: "gather.logs", last_skill_code: "timeout", food_at_death: 14, context_blob: JSON.stringify({ recentScenarios: [{ skillId: "gather.logs", code: "timeout" }] }) },
{ id: 2, ts: Date.now(), x: 102, y: 64, z: 201, cause: "hostile", hostile: "creeper", last_skill: "explore.far", last_skill_code: "done", food_at_death: 12, context_blob: null },
];
const prompt = buildPrompt(rows);
assert.match(prompt, /death id=1/);
assert.match(prompt, /death id=2/);
assert.match(prompt, /creeper/);
assert.match(prompt, /Reply with ONE JSON object/);
const { system, user } = buildPrompt(rows);
assert.match(user, /death id=1/);
assert.match(user, /death id=2/);
assert.match(user, /creeper/);
assert.match(system, /Reply with ONE JSON object/);
assert.match(system, /improvements/);
assert.match(system, /Valid skill ids/);
});
test("captureDeath: builds a row with context blob and inferred cause", () => {
@@ -88,7 +90,7 @@ test("attach + emit('death'): inserts row in knowledge DB", async () => {
rmSync(stateDir, { recursive: true, force: true });
});
test("drainOnce: respects budget and parses Pi reply", async () => {
test("drainOnce: respects budget and parses analytical LLM reply (incl. improvements)", async () => {
const stateDir = mkdtempSync(join(tmpdir(), "pepa-coach-test-"));
__resetForTests();
await initKnowledge({ stateDir });
@@ -105,7 +107,13 @@ test("drainOnce: respects budget and parses Pi reply", async () => {
const lessonsBefore = recall({ category: "combat" }).length;
const fakeReply = JSON.stringify({
// TimeWeb path needs env vars to satisfy the llmAvailable check.
const prevKey = process.env.TIMEWEB_API_KEY;
const prevModel = process.env.TIMEWEB_MODEL;
process.env.TIMEWEB_API_KEY = "test-key";
process.env.TIMEWEB_MODEL = "test-model";
const fakeReply = {
cause: "creeper_explosion_unarmed",
next_action: "shelter at dusk",
lessons: [{
@@ -116,16 +124,23 @@ test("drainOnce: respects budget and parses Pi reply", async () => {
prefer_skill: "survive.flee",
confidence: 0.85,
}],
});
const askPi = ({ onChunk, onDone }) => {
onChunk({ stream: "stdout", text: fakeReply });
onDone({ code: 0 });
improvements: [
{ title: "Add craft.shield skill", description: "No skill to craft a shield when creepers are around.", category: "skill", priority: 2 },
],
};
const askAnalyticalFn = async () => fakeReply;
const result = await drainOnce({ stateDir, force: true, askAnalyticalFn });
if (prevKey === undefined) delete process.env.TIMEWEB_API_KEY;
else process.env.TIMEWEB_API_KEY = prevKey;
if (prevModel === undefined) delete process.env.TIMEWEB_MODEL;
else process.env.TIMEWEB_MODEL = prevModel;
const result = await drainOnce({ askPi, stateDir, force: true });
assert.equal(result.ok, true);
assert.equal(result.analysed, 1);
assert.equal(result.lessons, 1);
assert.equal(result.improvements, 1);
const after = recall({ hostile: "creeper", category: "combat" });
assert.ok(after.length > lessonsBefore, "new lesson recorded");
@@ -137,18 +152,20 @@ test("drainOnce: respects budget and parses Pi reply", async () => {
rmSync(stateDir, { recursive: true, force: true });
});
test("drainOnce: empty queue → ok with 0 analysed", async () => {
test("drainOnce: skipped when LLM not configured", async () => {
const stateDir = mkdtempSync(join(tmpdir(), "pepa-coach-test-"));
__resetForTests();
await initKnowledge({ stateDir });
if (!isAvailable()) {
assert.ok(true);
rmSync(stateDir, { recursive: true, force: true });
return;
}
const result = await drainOnce({ askPi: () => {}, stateDir, force: true });
assert.equal(result.ok, true);
assert.equal(result.analysed, 0);
const prevKey = process.env.TIMEWEB_API_KEY;
delete process.env.TIMEWEB_API_KEY;
const result = await drainOnce({ stateDir, force: true });
if (prevKey !== undefined) process.env.TIMEWEB_API_KEY = prevKey;
assert.equal(result.ok, false);
assert.equal(result.reason, "llm not configured");
closeStore();
rmSync(stateDir, { recursive: true, force: true });
});
+72 -64
View File
@@ -15,9 +15,11 @@
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { resolve } from "node:path";
import { isAvailable as knowledgeAvailable, record as recordLesson } from "../knowledge/index.js";
import { isAvailable as knowledgeAvailable, record as recordLesson, createImprovementRequest } from "../knowledge/index.js";
import { isRegistered, skillRegistryPrompt } from "../skill-registry.js";
import { pickActiveNeed } from "../manifesto/state.js";
import { isAvailable as llmAvailable } from "../llm/provider.js";
import { askAnalytical } from "./llm-call.js";
import { info, warn } from "../log.js";
// Mode-name allow-list, mirrors postmortem.js (advice.js maps them to
@@ -37,25 +39,25 @@ const HISTORY_TAIL_LINES = 80;
let _attached = null;
let _timer = null;
let _piCallTimes = [];
let _llmCallTimes = [];
export function attach({ bot, stateDir, askPi, getSnapshot, intervalMs = DEFAULT_INTERVAL_MS } = {}) {
export function attach({ bot, stateDir, getSnapshot, intervalMs = DEFAULT_INTERVAL_MS } = {}) {
if (_attached) {
warn("reflect", "attach called twice; ignoring");
return;
}
if (!stateDir || !askPi || !getSnapshot) {
info("reflect", "attach: missing stateDir/askPi/getSnapshot — disabled");
if (!stateDir || !getSnapshot) {
info("reflect", "attach: missing stateDir/getSnapshot — disabled");
return;
}
_attached = { bot, stateDir, askPi, getSnapshot };
_attached = { bot, stateDir, getSnapshot };
_timer = setInterval(() => {
runOnce({ stateDir, askPi, getSnapshot }).catch((e) =>
runOnce({ stateDir, getSnapshot }).catch((e) =>
warn("reflect", `tick err: ${e?.message ?? e}`),
);
}, intervalMs);
_timer.unref?.();
info("reflect", `attached; self-assess every ${Math.round(intervalMs / 60000)} min`);
info("reflect", `attached; self-assess every ${Math.round(intervalMs / 60000)} min${llmAvailable() ? " (TimeWeb)" : " (LLM disabled — will skip)"}`);
}
export function detach() {
@@ -64,12 +66,13 @@ export function detach() {
_attached = null;
}
export async function runOnce({ stateDir, askPi, getSnapshot, force = false } = {}) {
export async function runOnce({ stateDir, getSnapshot, force = false, askAnalyticalFn = askAnalytical } = {}) {
const now = Date.now();
const hourAgo = now - 3600_000;
_piCallTimes = _piCallTimes.filter((t) => t > hourAgo);
if (!force && _piCallTimes.length >= HOURLY_BUDGET) {
return { ok: false, reason: "budget exhausted", calls: _piCallTimes.length };
_llmCallTimes = _llmCallTimes.filter((t) => t > hourAgo);
if (!llmAvailable()) return { ok: false, reason: "llm not configured" };
if (!force && _llmCallTimes.length >= HOURLY_BUDGET) {
return { ok: false, reason: "budget exhausted", calls: _llmCallTimes.length };
}
const snap = getSnapshot();
@@ -79,17 +82,12 @@ export async function runOnce({ stateDir, askPi, getSnapshot, force = false } =
const plan = readPlan(stateDir);
const activeNeed = pickActiveNeed(snap);
const prompt = buildPrompt({ snap, journal, scenarios, diary, plan, activeNeed });
_piCallTimes.push(now);
const { system, user } = buildPrompt({ snap, journal, scenarios, diary, plan, activeNeed });
_llmCallTimes.push(now);
const reply = await askPiOnce({ askPi, prompt });
if (!reply) return { ok: false, reason: "no reply" };
const parsed = parseReply(reply);
if (!parsed) {
warn("reflect", "Pi reply not parseable as JSON");
return { ok: false, reason: "bad reply", raw: reply.slice(0, 200) };
}
const parsed = await askAnalyticalFn({ system, user, json: true });
if (!parsed || typeof parsed !== "object") return { ok: false, reason: "no reply" };
const reply = JSON.stringify(parsed);
const path = writeReflection(stateDir, parsed, reply);
let rejectedPrefer = 0;
@@ -112,15 +110,34 @@ export async function runOnce({ stateDir, askPi, getSnapshot, force = false } =
avoidSkill,
preferSkill,
confidence: clamp(Number(l.confidence) || 0.5, 0.1, 0.9),
source: "pi-reflect",
source: "timeweb-reflect",
sourceRef: path,
});
}
if (rejectedPrefer > 0) {
warn("reflect", `dropped prefer_skill from ${rejectedPrefer} reflection lessons (not in registry)`);
}
info("reflect", `verdict=${parsed.verdict ?? "?"} ${parsed.summary?.slice(0, 80) ?? ""} (${path ?? "no file"})`);
return { ok: true, verdict: parsed.verdict, summary: parsed.summary, lessons: parsed.lessons ?? [] };
// v0.3.0 — improvement requests from reflection. The LLM is asked
// to flag missing skills/features when self-reflection reveals a
// systemic gap (e.g. "I keep failing iron tools because there's no
// craft.iron-pickaxe skill").
let improvementsCount = 0;
for (const imp of asArray(parsed.improvements ?? [])) {
if (!imp?.title) continue;
createImprovementRequest({
source: "reflect",
category: imp.category ?? "skill",
title: String(imp.title).slice(0, 120),
description: imp.description ?? null,
context: { verdict: parsed.verdict, reflection_path: path },
priority: imp.priority ?? 3,
});
improvementsCount += 1;
}
info("reflect", `verdict=${parsed.verdict ?? "?"} ${parsed.summary?.slice(0, 80) ?? ""} lessons=${(parsed.lessons ?? []).length} improvements=${improvementsCount} (${path ?? "no file"})`);
return { ok: true, verdict: parsed.verdict, summary: parsed.summary, lessons: parsed.lessons ?? [], improvements: improvementsCount };
}
function readJournalTail(stateDir) {
@@ -162,12 +179,38 @@ function buildPrompt({ snap, journal, scenarios, diary, plan, activeNeed }) {
const needLine = activeNeed
? `L${activeNeed.need.level} ${activeNeed.need.id}${activeNeed.skillId} (${activeNeed.need.title})`
: "(satisfied through L10 / no active need)";
return [
const system = [
"You are pepa, an autonomous Minecraft survival bot, reflecting on your own progress.",
"Look at the last ~30 minutes of activity below. Answer honestly: are you actually making progress, or stuck in a loop?",
"Answer honestly: are you making progress, stuck in a loop, or facing a structural gap?",
"",
skillRegistryPrompt({ limit: 1800 }),
"",
"Reply with ONE JSON object (no markdown fences, no prose):",
'{',
' "verdict": "progress" | "loop" | "recovering" | "idle" | "emergency",',
' "summary": "<2-3 sentence honest assessment in Russian>",',
' "next_action": "<one-sentence directive>",',
' "lessons": [',
' { "lesson": "<≤30 words, generalised rule>",',
' "category": "combat|pathing|crafting|survival|self-improve",',
' "trigger_skill": "<skill id or null>",',
' "trigger_hostile": "<mob name or null>",',
' "avoid_skill": "<registered skill id or null>",',
' "prefer_skill": "<registered skill id or null>",',
' "confidence": 0.6 } ],',
' "improvements": [',
' { "title": "<≤80 chars: structural gap (e.g. \'No craft.iron-pickaxe skill\')>",',
' "description": "<concrete example showing why no registered skill helps>",',
' "category": "skill|tuning|perception|planning|social|other",',
' "priority": 1 } ]',
'}',
"",
"CRITICAL: avoid_skill and prefer_skill MUST be one of the registered ids above, or null.",
"Use 'improvements' ONLY when you identify a structural gap — a missing skill or feature that would unblock a class of situations. Skip it otherwise.",
].join("\n");
const user = [
"## Current state",
`- position: ${pos ? `(${Math.round(pos.x)}, ${Math.round(pos.y)}, ${Math.round(pos.z)})` : "?"}`,
`- hp: ${snap?.health ?? "?"} food: ${snap?.food ?? "?"} day: ${snap?.isDay ? "yes" : "no"}`,
@@ -198,28 +241,9 @@ function buildPrompt({ snap, journal, scenarios, diary, plan, activeNeed }) {
"```",
journal.slice(-20).join("\n"),
"```",
"",
"Reply with ONE JSON object (no markdown fences, no prose):",
'{',
' "verdict": "progress" | "loop" | "recovering" | "idle" | "emergency",',
' "summary": "<2-3 sentence honest assessment in Russian>",',
' "next_action": "<one-sentence directive for what to do next>",',
' "lessons": [',
' { "lesson": "<≤30 words, generalised rule>",',
' "category": "combat|pathing|crafting|survival|self-improve",',
' "trigger_skill": "<skill id or null>",',
' "trigger_hostile": "<mob name or null>",',
' "avoid_skill": "<registered skill id to avoid or null>",',
' "prefer_skill": "<registered skill id to use instead or null>",',
' "confidence": 0.6 }',
' ]',
'}',
"",
"If you're clearly in a loop (same activity, no inventory growth, same position), say so honestly.",
"If you're stuck in a bad terrain (deep pit, hostile-rich area), recommend choosing a new base.",
"Lessons should be SHORT and ACTIONABLE. Don't repeat lessons the dispatcher already learned.",
"CRITICAL: avoid_skill and prefer_skill MUST be one of the registered ids listed at the top of this prompt, or null. Do NOT invent new ids.",
].join("\n");
return { system, user };
}
function parseReply(text) {
@@ -275,22 +299,6 @@ function writeReflection(stateDir, parsed, raw) {
}
}
function askPiOnce({ askPi, prompt }) {
return new Promise((res) => {
let buf = "";
try {
askPi({
prompt,
onChunk: ({ stream, text }) => { if (stream === "stdout") buf += text; },
onDone: () => res(buf),
});
} catch (e) {
warn("reflect", `askPi: ${e?.message ?? e}`);
res(null);
}
});
}
function asArray(v) { return Array.isArray(v) ? v : v ? [v] : []; }
function clamp(v, lo, hi) { return Math.max(lo, Math.min(hi, v)); }
+89 -49
View File
@@ -4,14 +4,27 @@ import { mkdtempSync, rmSync, readdirSync, existsSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { initKnowledge, recall } from "../knowledge/index.js";
import { initKnowledge, recall, listImprovements } from "../knowledge/index.js";
import { closeStore, __resetForTests, isAvailable } from "../knowledge/store.js";
import { runOnce, __testing } from "./reflect.js";
const { buildPrompt, parseReply } = __testing;
test("buildPrompt: includes runtime state + plan + diary", () => {
const p = buildPrompt({
function withTimeWebEnv(fn) {
const prevKey = process.env.TIMEWEB_API_KEY;
const prevModel = process.env.TIMEWEB_MODEL;
process.env.TIMEWEB_API_KEY = "test-key";
process.env.TIMEWEB_MODEL = "test-model";
return Promise.resolve(fn()).finally(() => {
if (prevKey === undefined) delete process.env.TIMEWEB_API_KEY;
else process.env.TIMEWEB_API_KEY = prevKey;
if (prevModel === undefined) delete process.env.TIMEWEB_MODEL;
else process.env.TIMEWEB_MODEL = prevModel;
});
}
test("buildPrompt: returns {system, user} with state, plan, diary, improvement schema", () => {
const { system, user } = buildPrompt({
snap: {
position: { x: 600, y: 64, z: 200 },
health: 4, food: 6, isDay: false,
@@ -26,15 +39,17 @@ test("buildPrompt: includes runtime state + plan + diary", () => {
scenarios: ['{"skillId":"explore.far","code":"wedged"}'],
diary: "13:00 spawned\n13:05 died",
plan: "1. Gather 16 logs\n2. Craft pickaxe",
activeNeed: null,
});
assert.match(p, /position: \(600, 64, 200\)/);
assert.match(p, /hp: 4 food: 6/);
assert.match(p, /emergency/);
assert.match(p, /Gather 16 logs/);
assert.match(p, /Reply with ONE JSON object/);
assert.match(user, /position: \(600, 64, 200\)/);
assert.match(user, /hp: 4 food: 6/);
assert.match(user, /emergency/);
assert.match(user, /Gather 16 logs/);
assert.match(system, /Reply with ONE JSON object/);
assert.match(system, /improvements/);
});
test("parseReply: extracts JSON from various Pi outputs", () => {
test("parseReply: extracts JSON from various LLM outputs", () => {
assert.deepEqual(parseReply('{"verdict":"loop","summary":"stuck"}'), { verdict: "loop", summary: "stuck" });
assert.deepEqual(parseReply('```json\n{"verdict":"progress"}\n```'), { verdict: "progress" });
const longReply = 'I see... your situation. Here is my JSON:\n{"verdict":"emergency","summary":"hp critical","lessons":[]}\nDone.';
@@ -43,7 +58,7 @@ test("parseReply: extracts JSON from various Pi outputs", () => {
assert.equal(parseReply(""), null);
});
test("runOnce: writes reflection file + records lessons", async () => {
test("runOnce: writes reflection file + records lessons + improvement requests", async () => {
const tmp = mkdtempSync(join(tmpdir(), "pepa-reflect-test-"));
__resetForTests();
await initKnowledge({ stateDir: tmp });
@@ -52,39 +67,64 @@ test("runOnce: writes reflection file + records lessons", async () => {
return;
}
const fakeReply = JSON.stringify({
verdict: "loop",
summary: "Бот ходит по кругу, ничего не добывает.",
next_action: "выбрать новое место под базу",
lessons: [{
lesson: "В этой точке постоянные смерти — искать новое место.",
category: "survival",
prefer_skill: "village.choose-base",
confidence: 0.7,
}],
});
const askPi = ({ onChunk, onDone }) => {
onChunk({ stream: "stdout", text: fakeReply });
onDone({ code: 0 });
};
const getSnapshot = () => ({
position: { x: 0, y: 64, z: 0 },
health: 8, food: 10, isDay: true,
runtimeState: "working",
inventory: {},
await withTimeWebEnv(async () => {
const fakeReply = {
verdict: "loop",
summary: "Бот ходит по кругу, ничего не добывает.",
next_action: "выбрать новое место под базу",
lessons: [{
lesson: "В этой точке постоянные смерти — искать новое место.",
category: "survival",
prefer_skill: "village.choose-base",
confidence: 0.7,
}],
improvements: [
{ title: "Add craft.iron-pickaxe skill", description: "Bot mines iron but cannot craft a tier-3 pickaxe.", category: "skill", priority: 2 },
],
};
const askAnalyticalFn = async () => fakeReply;
const getSnapshot = () => ({
position: { x: 0, y: 64, z: 0 },
health: 8, food: 10, isDay: true,
runtimeState: "working",
inventory: {},
});
const result = await runOnce({ stateDir: tmp, getSnapshot, force: true, askAnalyticalFn });
assert.equal(result.ok, true);
assert.equal(result.verdict, "loop");
assert.equal(result.improvements, 1);
const reflectionsDir = join(tmp, "reflections");
assert.ok(existsSync(reflectionsDir));
const files = readdirSync(reflectionsDir);
assert.ok(files.length >= 1, `expected ≥1 reflection file, got ${files.length}`);
const lessons = recall({ category: "survival" });
assert.ok(lessons.some((l) => l.source === "timeweb-reflect"), "lesson recorded with source=timeweb-reflect");
const improvements = listImprovements({ source: "reflect" });
assert.ok(improvements.some((r) => r.title === "Add craft.iron-pickaxe skill"));
});
const result = await runOnce({ stateDir: tmp, askPi, getSnapshot, force: true });
assert.equal(result.ok, true);
assert.equal(result.verdict, "loop");
closeStore();
try { rmSync(tmp, { recursive: true, force: true }); } catch {}
});
const reflectionsDir = join(tmp, "reflections");
assert.ok(existsSync(reflectionsDir));
const files = readdirSync(reflectionsDir);
assert.ok(files.length >= 1, `expected ≥1 reflection file, got ${files.length}`);
const lessons = recall({ category: "survival" });
assert.ok(lessons.some((l) => l.source === "pi-reflect"), "lesson recorded with source=pi-reflect");
test("runOnce: skipped when LLM not configured", async () => {
const tmp = mkdtempSync(join(tmpdir(), "pepa-reflect-test-"));
__resetForTests();
await initKnowledge({ stateDir: tmp });
if (!isAvailable()) {
try { rmSync(tmp, { recursive: true, force: true }); } catch {}
return;
}
const prevKey = process.env.TIMEWEB_API_KEY;
delete process.env.TIMEWEB_API_KEY;
const res = await runOnce({ stateDir: tmp, getSnapshot: () => ({}), force: true });
if (prevKey !== undefined) process.env.TIMEWEB_API_KEY = prevKey;
assert.equal(res.ok, false);
assert.equal(res.reason, "llm not configured");
closeStore();
try { rmSync(tmp, { recursive: true, force: true }); } catch {}
@@ -98,15 +138,15 @@ test("runOnce: budget exhausted → ok=false", async () => {
try { rmSync(tmp, { recursive: true, force: true }); } catch {}
return;
}
const askPi = ({ onDone }) => onDone({ code: 0 });
const getSnapshot = () => ({});
// Fire 2 forced calls to exhaust budget; 3rd without force should fail.
await runOnce({ stateDir: tmp, askPi, getSnapshot, force: true });
await runOnce({ stateDir: tmp, askPi, getSnapshot, force: true });
const res = await runOnce({ stateDir: tmp, askPi, getSnapshot, force: false });
assert.equal(res.ok, false);
assert.match(res.reason ?? "", /budget|reply/);
await withTimeWebEnv(async () => {
const askAnalyticalFn = async () => ({ verdict: "ok" });
const getSnapshot = () => ({});
await runOnce({ stateDir: tmp, getSnapshot, force: true, askAnalyticalFn });
await runOnce({ stateDir: tmp, getSnapshot, force: true, askAnalyticalFn });
const res = await runOnce({ stateDir: tmp, getSnapshot, force: false, askAnalyticalFn });
assert.equal(res.ok, false);
assert.match(res.reason ?? "", /budget|reply/);
});
closeStore();
try { rmSync(tmp, { recursive: true, force: true }); } catch {}
});
+103
View File
@@ -0,0 +1,103 @@
// Trigger tuner — periodic statistical sanity-check over the
// advisor_recommendations table.
//
// Replaces the old Pi-reflect "analyse your own pattern" loop with a
// deterministic local computation: no LLM call, no subscription, just
// SQL. Every TUNE_INTERVAL_MS the tuner reads the last 24h of
// recommendations, groups by trigger_reason, and flags two failure
// modes as improvement_requests for the operator:
//
// 1. low-success trigger: a trigger that fires often (≥ MIN_SAMPLE)
// but lands a successful outcome < SUCCESS_FLOOR of the time.
// The threshold probably needs tuning, or the prompt isn't giving
// the LLM the right hint.
// 2. expensive trigger: trigger averages > EXPENSIVE_TOKENS input
// tokens but its success rate is mediocre. Could mean the prompt
// includes context the LLM doesn't actually use.
//
// The tuner deduplicates via createImprovementRequest's votes mechanism
// — re-flagging the same gap just bumps the counter, not the row count.
import {
isAvailable as knowledgeAvailable,
recommendationStats,
createImprovementRequest,
} from "../knowledge/index.js";
import { info, warn } from "../log.js";
const TUNE_INTERVAL_MS = 60 * 60 * 1000; // 1 hour
const MIN_SAMPLE = 5;
const SUCCESS_FLOOR = 0.25;
const EXPENSIVE_TOKENS = 1000;
const EXPENSIVE_SUCCESS_CEILING = 0.5;
let _timer = null;
export function attach({ intervalMs = TUNE_INTERVAL_MS } = {}) {
if (_timer) {
warn("tuner", "attach called twice; ignoring");
return;
}
_timer = setInterval(() => {
runOnce().catch((e) => warn("tuner", `tick err: ${e?.message ?? e}`));
}, intervalMs);
_timer.unref?.();
info("tuner", `attached; tune every ${Math.round(intervalMs / 60000)} min`);
}
export function detach() {
if (_timer) clearInterval(_timer);
_timer = null;
}
export function runOnce({ stats = null } = {}) {
if (!knowledgeAvailable()) return { ok: false, reason: "knowledge unavailable" };
const rows = stats ?? recommendationStats({ sinceHours: 24 });
if (!rows.length) return { ok: true, flagged: 0, reason: "no data" };
const flagged = [];
for (const row of rows) {
const sample = (row.applied ?? 0);
if (sample < MIN_SAMPLE) continue;
const succ = row.succeeded ?? 0;
const successRate = sample === 0 ? 0 : succ / sample;
// 1. Low success → tune the trigger
if (successRate < SUCCESS_FLOOR) {
const title = `Trigger "${row.trigger_reason}" has low success rate`;
createImprovementRequest({
source: "tuner",
category: "tuning",
title,
description: `Over the last 24h, ${sample} applied recommendations from trigger ${row.trigger_reason} produced only ${succ} successful outcomes (${(successRate * 100).toFixed(0)}%). Consider tightening the trigger condition, improving the prompt, or adjusting the threshold.`,
context: { stats: row },
priority: 2,
});
flagged.push({ kind: "low_success", trigger: row.trigger_reason, sample, succ });
continue;
}
// 2. Expensive prompt with mediocre payoff
const avgIn = row.avg_in ?? 0;
if (avgIn > EXPENSIVE_TOKENS && successRate < EXPENSIVE_SUCCESS_CEILING) {
const title = `Trigger "${row.trigger_reason}" prompt is expensive`;
createImprovementRequest({
source: "tuner",
category: "tuning",
title,
description: `Trigger ${row.trigger_reason} averages ${Math.round(avgIn)} input tokens but lands successful outcomes only ${(successRate * 100).toFixed(0)}% of the time (${succ}/${sample}). The prompt may include context the model doesn't use — consider trimming.`,
context: { stats: row },
priority: 4,
});
flagged.push({ kind: "expensive_prompt", trigger: row.trigger_reason, avgIn });
}
}
if (flagged.length > 0) {
info("tuner", `flagged ${flagged.length} improvement(s) from ${rows.length} trigger group(s)`);
}
return { ok: true, flagged: flagged.length, items: flagged, groups: rows.length };
}
// Test exports
export const __testing = { TUNE_INTERVAL_MS, MIN_SAMPLE, SUCCESS_FLOOR, EXPENSIVE_TOKENS };
+98
View File
@@ -0,0 +1,98 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { initKnowledge, isAvailable, listImprovements } from "../knowledge/index.js";
import { closeStore, __resetForTests } from "../knowledge/store.js";
import { runOnce, __testing } from "./trigger-tuner.js";
const { MIN_SAMPLE } = __testing;
async function bootstrap() {
const tmp = mkdtempSync(join(tmpdir(), "pepa-tuner-test-"));
__resetForTests();
await initKnowledge({ stateDir: tmp });
return tmp;
}
function cleanup(tmp) {
closeStore();
try { rmSync(tmp, { recursive: true, force: true }); } catch {}
}
test("runOnce: empty stats → ok with 0 flagged", async () => {
const tmp = await bootstrap();
if (!isAvailable()) { cleanup(tmp); return; }
const r = runOnce({ stats: [] });
assert.equal(r.ok, true);
assert.equal(r.flagged, 0);
cleanup(tmp);
});
test("runOnce: ignores small samples (below MIN_SAMPLE)", async () => {
const tmp = await bootstrap();
if (!isAvailable()) { cleanup(tmp); return; }
const stats = [
{ trigger_reason: "wedged_60s", total: 2, applied: 2, succeeded: 0, failed: 2, avg_in: 700, avg_out: 40, avg_latency_ms: 5000 },
];
const r = runOnce({ stats });
assert.equal(r.flagged, 0, "applied=2 is below MIN_SAMPLE; skipped");
cleanup(tmp);
});
test("runOnce: flags low success-rate trigger as improvement", async () => {
const tmp = await bootstrap();
if (!isAvailable()) { cleanup(tmp); return; }
const stats = [
{ trigger_reason: "wedged_60s", total: 10, applied: 10, succeeded: 1, failed: 9, avg_in: 700, avg_out: 40, avg_latency_ms: 5000 },
];
const r = runOnce({ stats });
assert.equal(r.flagged, 1);
const requests = listImprovements({ source: "tuner" });
assert.ok(requests.some((req) => req.title.includes("wedged_60s") && req.title.includes("low success")));
cleanup(tmp);
});
test("runOnce: flags expensive prompt with mediocre payoff", async () => {
const tmp = await bootstrap();
if (!isAvailable()) { cleanup(tmp); return; }
const stats = [
{ trigger_reason: "repeat_4_explore.far", total: 10, applied: 10, succeeded: 4, failed: 6, avg_in: 1500, avg_out: 50, avg_latency_ms: 7000 },
];
const r = runOnce({ stats });
assert.equal(r.flagged, 1);
const requests = listImprovements({ source: "tuner", category: "tuning" });
assert.ok(requests.some((req) => req.title.includes("expensive")));
cleanup(tmp);
});
test("runOnce: healthy trigger does NOT get flagged", async () => {
const tmp = await bootstrap();
if (!isAvailable()) { cleanup(tmp); return; }
const stats = [
{ trigger_reason: "emergency_hp4_creeper@3", total: 8, applied: 8, succeeded: 7, failed: 1, avg_in: 700, avg_out: 40, avg_latency_ms: 5000 },
];
const r = runOnce({ stats });
assert.equal(r.flagged, 0);
cleanup(tmp);
});
test("runOnce: re-running with same low-success stats bumps votes, not row count", async () => {
const tmp = await bootstrap();
if (!isAvailable()) { cleanup(tmp); return; }
const stats = [
{ trigger_reason: "wedged_unique_label", total: 10, applied: 10, succeeded: 1, failed: 9, avg_in: 700, avg_out: 40, avg_latency_ms: 5000 },
];
runOnce({ stats });
runOnce({ stats });
const requests = listImprovements({ source: "tuner" }).filter((r) => r.title.includes("wedged_unique_label"));
assert.equal(requests.length, 1, "single row for the same title");
assert.ok(requests[0].votes >= 2, "votes bumped on re-flagging");
cleanup(tmp);
});
test("MIN_SAMPLE constant is reasonable", () => {
assert.ok(MIN_SAMPLE >= 3 && MIN_SAMPLE <= 10);
});