fix(auto-patch): lockfile-coordinated supervisor restarts
When Pi writes a multi-file runtime patch, the supervisor's file watcher can fire between two consecutive writes, kill the bot mid-edit, and load a half-saved file with a SyntaxError. Loop until the operator stops it. scripts/auto-patch.js now creates state/auto-patch.lock with its PID right after the branch checkout (before spawning pi -p), and removes it on every exit path. runtime/supervisor.js defers any watch-triggered restart while the lock holder is alive, polling every 2 s; once the lock drops it waits 1.5 s for the final write to settle, then runs \`node --check\` on the changed file and only restarts if it parses. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
+52
-6
@@ -28,6 +28,9 @@ const BOT_ENTRY = path.join(RUNTIME_DIR, "bot.js");
|
||||
export const RELOAD_EXIT_CODE = 42;
|
||||
const WATCH_DEBOUNCE_MS = 800;
|
||||
const MAX_RESTARTS_PER_MINUTE = 5;
|
||||
const AUTO_PATCH_LOCK = path.join(REPO_ROOT, "state", "auto-patch.lock");
|
||||
const LOCK_POLL_MS = 2_000;
|
||||
const LOCK_POST_GRACE_MS = 1_500;
|
||||
// When a recent auto-patch breaks the bot, we roll it back. "Recent" =
|
||||
// landed on main within the last ROLLBACK_FRESHNESS_MS. The signal is
|
||||
// MAX_RESTARTS_PER_MINUTE exceeded — i.e. the patch reliably crashes.
|
||||
@@ -90,6 +93,25 @@ function isProcessAlive(pid) {
|
||||
}
|
||||
}
|
||||
|
||||
// True iff scripts/auto-patch.js is editing runtime/*.js right now. While the
|
||||
// lock is held, file-watch restarts are deferred — otherwise we kill the
|
||||
// child mid-write and load a half-saved file with a SyntaxError. Observed
|
||||
// 2026-05-26: Pi wrote a valid patch but the supervisor caught it between
|
||||
// two consecutive writes and crash-looped.
|
||||
function autoPatchLockHeld() {
|
||||
try {
|
||||
const pid = Number.parseInt(fs.readFileSync(AUTO_PATCH_LOCK, "utf8").trim(), 10);
|
||||
return Number.isFinite(pid) && isProcessAlive(pid);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function runtimeFileParses(absPath) {
|
||||
const res = spawnSync(process.execPath, ["--check", absPath], { encoding: "utf8" });
|
||||
return res.status === 0;
|
||||
}
|
||||
|
||||
function acquireLock() {
|
||||
try {
|
||||
const existing = Number.parseInt(fs.readFileSync(PID_FILE, "utf8").trim(), 10);
|
||||
@@ -171,18 +193,42 @@ function spawnChild() {
|
||||
}
|
||||
|
||||
let debounceTimer = null;
|
||||
let pendingRestartFile = null;
|
||||
function scheduleRestart(filename) {
|
||||
pendingRestartFile = filename;
|
||||
if (debounceTimer) clearTimeout(debounceTimer);
|
||||
debounceTimer = setTimeout(tryRestart, WATCH_DEBOUNCE_MS);
|
||||
}
|
||||
function tryRestart() {
|
||||
const filename = pendingRestartFile;
|
||||
if (!filename) return;
|
||||
if (autoPatchLockHeld()) {
|
||||
debounceTimer = setTimeout(tryRestart, LOCK_POLL_MS);
|
||||
return;
|
||||
}
|
||||
// Lock just dropped (or was never held). Pause briefly to let any final
|
||||
// write settle, then syntax-check the file before killing the child — a
|
||||
// half-written file would crash-loop the bot.
|
||||
debounceTimer = setTimeout(() => {
|
||||
const abs = path.join(RUNTIME_DIR, filename);
|
||||
if (fs.existsSync(abs) && !runtimeFileParses(abs)) {
|
||||
console.log(`[supervisor] ${filename} has syntax errors — waiting`);
|
||||
debounceTimer = setTimeout(tryRestart, LOCK_POLL_MS);
|
||||
return;
|
||||
}
|
||||
pendingRestartFile = null;
|
||||
console.log(`[supervisor] ${filename} changed — restarting child`);
|
||||
restartingDueToWatch = true;
|
||||
child?.kill("SIGTERM");
|
||||
}, LOCK_POST_GRACE_MS);
|
||||
}
|
||||
function watchRuntime() {
|
||||
// recursive:true so edits to runtime/skills/*.js and runtime/social/*.js
|
||||
// also restart the child. macOS + Linux support recursive fs.watch on
|
||||
// Node 20+.
|
||||
const watcher = fs.watch(RUNTIME_DIR, { recursive: true }, (eventType, filename) => {
|
||||
if (!isWatchableJs(filename)) return;
|
||||
if (debounceTimer) clearTimeout(debounceTimer);
|
||||
debounceTimer = setTimeout(() => {
|
||||
console.log(`[supervisor] ${filename} changed — restarting child`);
|
||||
restartingDueToWatch = true;
|
||||
child?.kill("SIGTERM");
|
||||
}, WATCH_DEBOUNCE_MS);
|
||||
scheduleRestart(filename);
|
||||
});
|
||||
watcher.on("error", (err) => {
|
||||
console.error(`[supervisor] watcher error: ${err.message}`);
|
||||
|
||||
@@ -26,6 +26,7 @@ import { parseEditScope, validateChangedFiles, effectiveScope } from "./edit-sco
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
const REPO_ROOT = path.resolve(__dirname, "..");
|
||||
const LOCK_FILE = path.join(REPO_ROOT, "state", "auto-patch.lock");
|
||||
|
||||
function log(level, msg) {
|
||||
const line = `${new Date().toISOString()} [auto-patch] [${level}] ${msg}`;
|
||||
@@ -37,8 +38,25 @@ function git(args, opts = {}) {
|
||||
return spawnSync("git", args, { cwd: REPO_ROOT, encoding: "utf8", ...opts });
|
||||
}
|
||||
|
||||
// Held while Pi is writing runtime/*.js so the supervisor's file watcher
|
||||
// doesn't kill the bot mid-write and reload a half-saved file with a
|
||||
// SyntaxError. See 2026-05-26 incident.
|
||||
function acquireLock() {
|
||||
fs.mkdirSync(path.dirname(LOCK_FILE), { recursive: true });
|
||||
fs.writeFileSync(LOCK_FILE, String(process.pid));
|
||||
}
|
||||
function releaseLock() {
|
||||
try {
|
||||
const pid = Number.parseInt(fs.readFileSync(LOCK_FILE, "utf8").trim(), 10);
|
||||
if (pid === process.pid) fs.unlinkSync(LOCK_FILE);
|
||||
} catch {}
|
||||
}
|
||||
process.on("exit", releaseLock);
|
||||
for (const sig of ["SIGINT", "SIGTERM"]) process.on(sig, () => { releaseLock(); process.exit(1); });
|
||||
|
||||
function exit(code, reason) {
|
||||
log(code === 0 ? "info" : "warn", `exit ${code}: ${reason}`);
|
||||
releaseLock();
|
||||
process.exit(code);
|
||||
}
|
||||
|
||||
@@ -100,6 +118,9 @@ git(["branch", "-D", branch]); // ignore error if absent
|
||||
const checkout = git(["checkout", "-b", branch]);
|
||||
if (checkout.status !== 0) exit(2, `cannot create branch ${branch}: ${checkout.stderr}`);
|
||||
|
||||
acquireLock();
|
||||
log("info", `acquired ${LOCK_FILE}`);
|
||||
|
||||
const scopeBullet = scope.map((p) => ` - \`${p}\``).join("\n");
|
||||
const prompt = [
|
||||
"You are patching the pepa-pi-bot repo to address an automatically-detected failure.",
|
||||
|
||||
Reference in New Issue
Block a user