pepa-pi-bot/runtime/supervisor.js

// Supervisor: forks runtime/bot.js as a child process and restarts it
// when the child exits with a "reload requested" code (RELOAD_EXIT_CODE).
// Also watches runtime/*.js — when a file changes, signals the child to
// reload itself by exiting with the same code.
//
// True hot module reload in Node ESM is fragile (caches, open sockets,
// mineflayer client state). Restart-on-change is the same outcome with
// none of the gotchas: the only thing the bot loses is the MC TCP
// connection, which it would reconnect anyway after a Mineflayer kick.
//
// Run via `npm run bot`. Falls back to plain `node runtime/bot.js` via
// `npm run bot:bare` if you want to skip the supervisor.

import { spawn, spawnSync } from "node:child_process";
import fs from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";

import { stateDir } from "./config.js";
import { isWatchableJs } from "./watch-filter.js";

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const RUNTIME_DIR = __dirname;
const REPO_ROOT = path.resolve(RUNTIME_DIR, "..");
const BOT_ENTRY = path.join(RUNTIME_DIR, "bot.js");

export const RELOAD_EXIT_CODE = 42;
const WATCH_DEBOUNCE_MS = 800;
const MAX_RESTARTS_PER_MINUTE = 5;
const AUTO_PATCH_LOCK = path.join(REPO_ROOT, "state", "auto-patch.lock");
const LOCK_POLL_MS = 2_000;
const LOCK_POST_GRACE_MS = 1_500;
// When a recent auto-patch breaks the bot, we roll it back. "Recent" =
// landed on main within the last ROLLBACK_FRESHNESS_MS. The signal is
// MAX_RESTARTS_PER_MINUTE exceeded — i.e. the patch reliably crashes.
const ROLLBACK_FRESHNESS_MS = 15 * 60_000;
const MAX_ROLLBACKS = 3; // hard ceiling per supervisor lifetime

// Pidfile prevents two supervisors from racing on the same MC nickname. The
// Minecraft server refuses the second login ("Игрок с данным никнеймом уже
// играет на сервере") and the loser enters a kick-reconnect loop forever.
// Observed live 2026-05-25 when a smoke-test bot stayed alive in the
// background after its operator window was closed.
const PID_FILE = path.join(stateDir, "supervisor.pid");

let child = null;
let restartingDueToWatch = false;
const restartTimestamps = [];
let rollbackCount = 0;

function lastCommitAgeMs() {
	const res = spawnSync("git", ["log", "-1", "--format=%ct", "HEAD"], { cwd: REPO_ROOT, encoding: "utf8" });
	if (res.status !== 0) return Number.POSITIVE_INFINITY;
	const epochSec = Number.parseInt(res.stdout.trim(), 10);
	if (!Number.isFinite(epochSec)) return Number.POSITIVE_INFINITY;
	return Date.now() - epochSec * 1000;
}

function lastCommitTouchedRuntime() {
	const res = spawnSync("git", ["diff", "--name-only", "HEAD~1..HEAD"], { cwd: REPO_ROOT, encoding: "utf8" });
	if (res.status !== 0) return false;
	return res.stdout.split("\n").some((f) => f.startsWith("runtime/"));
}

function rollbackLastCommit() {
	const sha = spawnSync("git", ["rev-parse", "HEAD"], { cwd: REPO_ROOT, encoding: "utf8" }).stdout.trim();
	console.log(`[supervisor] rolling back HEAD (${sha.slice(0, 8)})`);
	const reset = spawnSync("git", ["reset", "--hard", "HEAD~1"], {
		cwd: REPO_ROOT,
		encoding: "utf8",
		stdio: "inherit",
	});
	if (reset.status !== 0) {
		console.error(`[supervisor] git reset failed — operator intervention needed`);
		return false;
	}
	rollbackCount++;
	return true;
}

function nowMs() {
	return Date.now();
}

function isProcessAlive(pid) {
	try {
		// Signal 0 doesn't kill — just probes existence + permission.
		process.kill(pid, 0);
		return true;
	} catch (e) {
		return e.code === "EPERM"; // exists but we don't own it
	}
}

// True iff scripts/auto-patch.js is editing runtime/*.js right now. While the
// lock is held, file-watch restarts are deferred — otherwise we kill the
// child mid-write and load a half-saved file with a SyntaxError. Observed
// 2026-05-26: Pi wrote a valid patch but the supervisor caught it between
// two consecutive writes and crash-looped.
function autoPatchLockHeld() {
	try {
		const pid = Number.parseInt(fs.readFileSync(AUTO_PATCH_LOCK, "utf8").trim(), 10);
		return Number.isFinite(pid) && isProcessAlive(pid);
	} catch {
		return false;
	}
}

function runtimeFileParses(absPath) {
	const res = spawnSync(process.execPath, ["--check", absPath], { encoding: "utf8" });
	return res.status === 0;
}

function acquireLock() {
	try {
		const existing = Number.parseInt(fs.readFileSync(PID_FILE, "utf8").trim(), 10);
		if (Number.isFinite(existing) && existing !== process.pid && isProcessAlive(existing)) {
			console.error(
				`[supervisor] another supervisor (pid=${existing}) is already running. ` +
					`Stop it first ('kill ${existing}') or delete ${PID_FILE} if it's stale.`,
			);
			process.exit(2);
		}
	} catch (e) {
		if (e.code !== "ENOENT") {
			console.error(`[supervisor] could not read pidfile: ${e.message}`);
		}
	}
	fs.mkdirSync(stateDir, { recursive: true });
	fs.writeFileSync(PID_FILE, String(process.pid));
}

function releaseLock() {
	try {
		const pid = Number.parseInt(fs.readFileSync(PID_FILE, "utf8").trim(), 10);
		if (pid === process.pid) fs.unlinkSync(PID_FILE);
	} catch {}
}

function spawnChild() {
	child = spawn(process.execPath, [BOT_ENTRY], {
		stdio: "inherit",
		env: { ...process.env, PEPA_SUPERVISED: "1" },
	});

	child.on("exit", (code, signal) => {
		console.log(`[supervisor] child exited code=${code} signal=${signal}`);
		const wantsRestart = code === RELOAD_EXIT_CODE || restartingDueToWatch;
		// Capture BEFORE clearing — watch-triggered restarts are intentional
		// and must not be counted toward the crash-loop rollback threshold.
		const isWatchRestart = restartingDueToWatch;
		restartingDueToWatch = false;
		if (!wantsRestart) {
			// Clean exit (SIGINT/SIGTERM bubble) or crash — don't relaunch.
			process.exit(code ?? 0);
		}
		// Rate-limit restarts so a crash loop doesn't burn CPU. Watch-triggered
		// restarts don't count — burned a working main once (2026-05-26) when
		// edits to runtime/*.test.js looked like a crash loop and rolled back
		// the scheduler PR.
		if (!isWatchRestart) {
			const now = nowMs();
			restartTimestamps.push(now);
			while (restartTimestamps.length && now - restartTimestamps[0] > 60_000) restartTimestamps.shift();
			if (restartTimestamps.length > MAX_RESTARTS_PER_MINUTE) {
				// Crash loop. If the last commit is young AND touched runtime/, it
				// probably broke us — roll it back and try once more.
				const ageMs = lastCommitAgeMs();
				if (
					ageMs < ROLLBACK_FRESHNESS_MS &&
					lastCommitTouchedRuntime() &&
					rollbackCount < MAX_ROLLBACKS &&
					rollbackLastCommit()
				) {
					console.log(`[supervisor] auto-rollback ${rollbackCount}/${MAX_ROLLBACKS} applied; restart counters reset`);
					restartTimestamps.length = 0;
					setTimeout(spawnChild, 500);
					return;
				}
				console.error(`[supervisor] too many restarts (${restartTimestamps.length} in 60s) — giving up`);
				process.exit(1);
			}
		}
		console.log(`[supervisor] restarting in 500ms…`);
		setTimeout(spawnChild, 500);
	});

	child.on("error", (err) => {
		console.error(`[supervisor] failed to spawn child: ${err.message}`);
		process.exit(1);
	});
}

let debounceTimer = null;
let pendingRestartFile = null;
function scheduleRestart(filename) {
	pendingRestartFile = filename;
	if (debounceTimer) clearTimeout(debounceTimer);
	debounceTimer = setTimeout(tryRestart, WATCH_DEBOUNCE_MS);
}
function tryRestart() {
	const filename = pendingRestartFile;
	if (!filename) return;
	if (autoPatchLockHeld()) {
		debounceTimer = setTimeout(tryRestart, LOCK_POLL_MS);
		return;
	}
	// Lock just dropped (or was never held). Pause briefly to let any final
	// write settle, then syntax-check the file before killing the child — a
	// half-written file would crash-loop the bot.
	debounceTimer = setTimeout(() => {
		const abs = path.join(RUNTIME_DIR, filename);
		if (fs.existsSync(abs) && !runtimeFileParses(abs)) {
			console.log(`[supervisor] ${filename} has syntax errors — waiting`);
			debounceTimer = setTimeout(tryRestart, LOCK_POLL_MS);
			return;
		}
		pendingRestartFile = null;
		console.log(`[supervisor] ${filename} changed — restarting child`);
		restartingDueToWatch = true;
		child?.kill("SIGTERM");
	}, LOCK_POST_GRACE_MS);
}
function watchRuntime() {
	// recursive:true so edits to runtime/skills/*.js and runtime/social/*.js
	// also restart the child. macOS + Linux support recursive fs.watch on
	// Node 20+.
	const watcher = fs.watch(RUNTIME_DIR, { recursive: true }, (eventType, filename) => {
		if (!isWatchableJs(filename)) return;
		scheduleRestart(filename);
	});
	watcher.on("error", (err) => {
		console.error(`[supervisor] watcher error: ${err.message}`);
	});
}

// Forward signals to the child, then exit ourselves once it has.
for (const sig of ["SIGINT", "SIGTERM"]) {
	process.on(sig, () => {
		console.log(`[supervisor] forwarding ${sig} to child`);
		releaseLock();
		if (!child) process.exit(0);
		child.once("exit", () => process.exit(0));
		child.kill(sig);
		// hard cap in case the child hangs
		setTimeout(() => process.exit(1), 5000).unref();
	});
}

// Best-effort lock release on any other exit path (uncaught, exit 1, etc).
process.on("exit", releaseLock);

acquireLock();
console.log(`[supervisor] starting (pid=${process.pid}); watching ${RUNTIME_DIR} for *.js changes`);
spawnChild();
watchRuntime();