fix(auto-patch): lockfile-coordinated supervisor restarts

When Pi writes a multi-file runtime patch, the supervisor's file watcher
can fire between two consecutive writes, kill the bot mid-edit, and load
a half-saved file with a SyntaxError. Loop until the operator stops it.

scripts/auto-patch.js now creates state/auto-patch.lock with its PID
right after the branch checkout (before spawning pi -p), and removes
it on every exit path. runtime/supervisor.js defers any watch-triggered
restart while the lock holder is alive, polling every 2 s; once the
lock drops it waits 1.5 s for the final write to settle, then runs
\`node --check\` on the changed file and only restarts if it parses.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-05-26 13:24:54 +03:00
co-authored by Claude Opus 4.7
parent 6909715f75
commit 69d1298fbd
2 changed files with 73 additions and 6 deletions
+52 -6
View File
@@ -28,6 +28,9 @@ const BOT_ENTRY = path.join(RUNTIME_DIR, "bot.js");
export const RELOAD_EXIT_CODE = 42;
const WATCH_DEBOUNCE_MS = 800;
const MAX_RESTARTS_PER_MINUTE = 5;
const AUTO_PATCH_LOCK = path.join(REPO_ROOT, "state", "auto-patch.lock");
const LOCK_POLL_MS = 2_000;
const LOCK_POST_GRACE_MS = 1_500;
// When a recent auto-patch breaks the bot, we roll it back. "Recent" =
// landed on main within the last ROLLBACK_FRESHNESS_MS. The signal is
// MAX_RESTARTS_PER_MINUTE exceeded — i.e. the patch reliably crashes.
@@ -90,6 +93,25 @@ function isProcessAlive(pid) {
}
}
// True iff scripts/auto-patch.js is editing runtime/*.js right now. While the
// lock is held, file-watch restarts are deferred — otherwise we kill the
// child mid-write and load a half-saved file with a SyntaxError. Observed
// 2026-05-26: Pi wrote a valid patch but the supervisor caught it between
// two consecutive writes and crash-looped.
function autoPatchLockHeld() {
try {
const pid = Number.parseInt(fs.readFileSync(AUTO_PATCH_LOCK, "utf8").trim(), 10);
return Number.isFinite(pid) && isProcessAlive(pid);
} catch {
return false;
}
}
function runtimeFileParses(absPath) {
const res = spawnSync(process.execPath, ["--check", absPath], { encoding: "utf8" });
return res.status === 0;
}
function acquireLock() {
try {
const existing = Number.parseInt(fs.readFileSync(PID_FILE, "utf8").trim(), 10);
@@ -171,18 +193,42 @@ function spawnChild() {
}
let debounceTimer = null;
let pendingRestartFile = null;
function scheduleRestart(filename) {
pendingRestartFile = filename;
if (debounceTimer) clearTimeout(debounceTimer);
debounceTimer = setTimeout(tryRestart, WATCH_DEBOUNCE_MS);
}
function tryRestart() {
const filename = pendingRestartFile;
if (!filename) return;
if (autoPatchLockHeld()) {
debounceTimer = setTimeout(tryRestart, LOCK_POLL_MS);
return;
}
// Lock just dropped (or was never held). Pause briefly to let any final
// write settle, then syntax-check the file before killing the child — a
// half-written file would crash-loop the bot.
debounceTimer = setTimeout(() => {
const abs = path.join(RUNTIME_DIR, filename);
if (fs.existsSync(abs) && !runtimeFileParses(abs)) {
console.log(`[supervisor] ${filename} has syntax errors — waiting`);
debounceTimer = setTimeout(tryRestart, LOCK_POLL_MS);
return;
}
pendingRestartFile = null;
console.log(`[supervisor] ${filename} changed — restarting child`);
restartingDueToWatch = true;
child?.kill("SIGTERM");
}, LOCK_POST_GRACE_MS);
}
function watchRuntime() {
// recursive:true so edits to runtime/skills/*.js and runtime/social/*.js
// also restart the child. macOS + Linux support recursive fs.watch on
// Node 20+.
const watcher = fs.watch(RUNTIME_DIR, { recursive: true }, (eventType, filename) => {
if (!isWatchableJs(filename)) return;
if (debounceTimer) clearTimeout(debounceTimer);
debounceTimer = setTimeout(() => {
console.log(`[supervisor] ${filename} changed — restarting child`);
restartingDueToWatch = true;
child?.kill("SIGTERM");
}, WATCH_DEBOUNCE_MS);
scheduleRestart(filename);
});
watcher.on("error", (err) => {
console.error(`[supervisor] watcher error: ${err.message}`);
+21
View File
@@ -26,6 +26,7 @@ import { parseEditScope, validateChangedFiles, effectiveScope } from "./edit-sco
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const REPO_ROOT = path.resolve(__dirname, "..");
const LOCK_FILE = path.join(REPO_ROOT, "state", "auto-patch.lock");
function log(level, msg) {
const line = `${new Date().toISOString()} [auto-patch] [${level}] ${msg}`;
@@ -37,8 +38,25 @@ function git(args, opts = {}) {
return spawnSync("git", args, { cwd: REPO_ROOT, encoding: "utf8", ...opts });
}
// Held while Pi is writing runtime/*.js so the supervisor's file watcher
// doesn't kill the bot mid-write and reload a half-saved file with a
// SyntaxError. See 2026-05-26 incident.
function acquireLock() {
fs.mkdirSync(path.dirname(LOCK_FILE), { recursive: true });
fs.writeFileSync(LOCK_FILE, String(process.pid));
}
function releaseLock() {
try {
const pid = Number.parseInt(fs.readFileSync(LOCK_FILE, "utf8").trim(), 10);
if (pid === process.pid) fs.unlinkSync(LOCK_FILE);
} catch {}
}
process.on("exit", releaseLock);
for (const sig of ["SIGINT", "SIGTERM"]) process.on(sig, () => { releaseLock(); process.exit(1); });
function exit(code, reason) {
log(code === 0 ? "info" : "warn", `exit ${code}: ${reason}`);
releaseLock();
process.exit(code);
}
@@ -100,6 +118,9 @@ git(["branch", "-D", branch]); // ignore error if absent
const checkout = git(["checkout", "-b", branch]);
if (checkout.status !== 0) exit(2, `cannot create branch ${branch}: ${checkout.stderr}`);
acquireLock();
log("info", `acquired ${LOCK_FILE}`);
const scopeBullet = scope.map((p) => ` - \`${p}\``).join("\n");
const prompt = [
"You are patching the pepa-pi-bot repo to address an automatically-detected failure.",