From 8232c28ba9ad43c6bf0bad92a9dcc4ed9fbfdbb2 Mon Sep 17 00:00:00 2001 From: RoboClaw Date: Sun, 27 Sep 2026 19:00:26 -0700 Subject: [PATCH] fix(ci): honor loaded stop policy in upgrade fixtures (#160000) Use the fixture's admitted loaded-unit stop budget for policy inspection and cleanup. Preserve loaded snapshots until reload, retain supervisor custody through failed policy reads, and report failed stops only after observed cleanup. Adapt supervisor test callers without weakening their behavior checks. Independent and ClawSweeper reviews resolved the outer-stop failure; installed-shim regressions cover policy loss, process extinction and uncertain settlement. Focused policy/cgroup and caller proofs passed. Full local suite setup limitations and exact-head hosted evidence are recorded in the PR. No product runtime changes or completed release qualification are claimed. Co-authored-by: RomneyDa <6581799+RomneyDa@users.noreply.github.com> --- .../lib/upgrade-survivor/systemd-fixture.mjs | 57 ++- .../upgrade-survivor/update-restart-auth.sh | 60 ++- test/scripts/docker-build-helper.test.ts | 84 +++- .../upgrade-survivor-stop-policy.test.ts | 468 ++++++++++++++++++ .../upgrade-survivor-systemd-cgroup.test.ts | 11 + test/scripts/upgrade-survivor-systemd.test.ts | 1 + 6 files changed, 646 insertions(+), 35 deletions(-) create mode 100644 test/scripts/upgrade-survivor-stop-policy.test.ts diff --git a/scripts/e2e/lib/upgrade-survivor/systemd-fixture.mjs b/scripts/e2e/lib/upgrade-survivor/systemd-fixture.mjs index bbb7311d9184..e73a41d1a5e8 100644 --- a/scripts/e2e/lib/upgrade-survivor/systemd-fixture.mjs +++ b/scripts/e2e/lib/upgrade-survivor/systemd-fixture.mjs @@ -135,7 +135,24 @@ function parseUnit(content) { if ([...directives.keys()].some((key) => !supported.has(key))) { fail(); } + // Generated units use whole seconds. Missing policy follows systemd's 90s + // default; zero/infinity disable the deadline. Reject unsupported spans and + // timer overflow instead of silently scheduling Node's 1ms overflow timer. + const stopSeconds = single("TimeoutStopSec") || "90"; + const unlimitedStop = stopSeconds === "infinity" || stopSeconds === "0"; + const stopTimeoutMs = unlimitedStop + ? Infinity + : /^\d+$/.test(stopSeconds) + ? Number(stopSeconds) * 1_000 + : Number.NaN; + if ( + !unlimitedStop && + (!Number.isSafeInteger(stopTimeoutMs) || stopTimeoutMs <= 0 || stopTimeoutMs > 2_147_483_647) + ) { + fail("Unsupported generated TimeoutStopSec policy."); + } return { + stopTimeoutMs, programArguments, workingDirectory, environment, @@ -157,8 +174,11 @@ function readUnit(reload = false, requireLoaded = false) { if (error.code !== "ENOENT") { throw error; } - if (!requireLoaded) { + if (reload) { fs.rmSync(loadedPath, { force: true }); + } else if (fs.existsSync(loadedPath)) { + // Ordinary status/command inspection cannot unload an admitted definition. + return { ...parseUnit(fs.readFileSync(loadedPath, "utf8")), reloadPending: true }; } return null; } @@ -227,7 +247,8 @@ function nativeRuntime() { ) : false; const unsettled = counts?.starting || supervisorPid || groupPid || populated; - const successful = !last || last.code === 0; + const stopFailed = counts?.stopFailed === true; + const successful = !stopFailed && (!last || last.code === 0); return { pid, // A manager draining descendants or awaiting restart is not a settled service. @@ -235,6 +256,7 @@ function nativeRuntime() { sub: pid ? "running" : unsettled ? "auto-restart" : "dead", generation: counts?.entered ?? 0, settled: !pid && !unsettled, + stopFailed, controlGroup: pid || unsettled ? paths.controlGroup || "" : "", restarts: counts?.restarts ?? 0, result: successful ? "success" : "exit-code", @@ -444,6 +466,16 @@ function run() { } return; } + if (operation === "check-stopped" && !args.length) { + const runtime = nativeRuntime(); + if (!runtime.settled) { + fail("Survivor service processes have not settled."); + } + if (runtime.stopFailed) { + fail("Survivor stop policy read failed; process cleanup completed."); + } + return; + } if (operation === "is-active" && !args.length) { const runtime = nativeRuntime(); process.exitCode = runtime.pid ? 0 : runtime.settled ? 3 : 1; @@ -480,6 +512,27 @@ function run() { readUnit(true); return; } + if (["stop-policy", "stop-timeout-ms"].includes(operation) && !args.length) { + // A running generation keeps its loaded policy even if an on-disk edit is + // invalid or removed. Only a successful reload replaces that snapshot. + const unit = fs.existsSync(loadedPath) + ? parseUnit(fs.readFileSync(loadedPath, "utf8")) + : readUnit(); + if (operation === "stop-policy") { + console.log(`LoadState=${unit ? "loaded" : "not-found"}`); + if (unit) { + console.log( + `TimeoutStopUSec=${unit.stopTimeoutMs === Infinity ? "infinity" : `${unit.stopTimeoutMs / 1_000}s`}`, + ); + } + } else { + if (!unit) { + fail("Cannot stop an absent fixture unit."); + } + console.log(unit.stopTimeoutMs); + } + return; + } if (operation === "load-state" && !args.length) { console.log(readUnit() ? "loaded" : "not-found"); return; diff --git a/scripts/e2e/lib/upgrade-survivor/update-restart-auth.sh b/scripts/e2e/lib/upgrade-survivor/update-restart-auth.sh index 949e92da6fc9..79c485f4709b 100644 --- a/scripts/e2e/lib/upgrade-survivor/update-restart-auth.sh +++ b/scripts/e2e/lib/upgrade-survivor/update-restart-auth.sh @@ -99,17 +99,33 @@ is_running() { } stop_gateway() { - local pid="" + local pid="" stop_policy_status=0 pid="$(cat "$pid_file" 2>/dev/null || true)" if [[ "$pid" =~ ^[0-9]+$ ]] && [ "$pid" -gt 1 ] && kill -0 "$pid" >/dev/null 2>&1; then + local stop_timeout_ms attempts=0 + stop_timeout_ms="$(node "$manager_script" stop-timeout-ms)" || { + stop_policy_status=$? + # No service budget is admitted: signal the existing fatal-cleanup owner + # and join using only the settlement allowance, not a substitute timeout. + stop_timeout_ms=0 + } kill "$pid" >/dev/null 2>&1 || true - # The supervisor gives its child 30s, so keep this outer deadline comfortably longer. - for _ in $(seq 1 350); do - is_running || break + # Leave the supervisor its loaded stop budget plus 5s to observe group exit. + while is_running; do + if [ "$stop_timeout_ms" != Infinity ] && + [ "$attempts" -ge "$(((stop_timeout_ms + 5000 + 99) / 100))" ]; then + break + fi sleep 0.1 + attempts=$((attempts + 1)) done - kill -9 "$pid" >/dev/null 2>&1 || true + if is_running; then + echo "Survivor supervisor has not settled; retaining process custody." >&2 + return 1 + fi fi + node "$manager_script" check-stopped || return "$?" + [ "$stop_policy_status" -eq 0 ] || return "$stop_policy_status" rm -f "$pid_file" "$supervisor_script" } @@ -125,16 +141,18 @@ start_gateway() { rm -f "${daemon_log}.exit.json" cat >"$supervisor_script" <<'SUPERVISOR' import fs from "node:fs"; -import { spawn } from "node:child_process"; +import { spawn, execFileSync } from "node:child_process"; +const managerScript = process.env.OPENCLAW_SYSTEMCTL_SHIM_MANAGER_SCRIPT; const command = process.env.OPENCLAW_SYSTEMCTL_SHIM_EXEC_START; const daemonLog = process.env.OPENCLAW_SYSTEMCTL_SHIM_DAEMON_LOG; -if (!command || !daemonLog) { +if (!command || !daemonLog || !managerScript) { process.exit(2); } const output = fs.openSync(daemonLog, "a"); const childEnv = { ...process.env }; +delete childEnv.OPENCLAW_SYSTEMCTL_SHIM_MANAGER_SCRIPT; delete childEnv.OPENCLAW_SYSTEMCTL_SHIM_EXEC_START; delete childEnv.OPENCLAW_SYSTEMCTL_SHIM_DAEMON_LOG; const managerEnv = JSON.parse(childEnv.OPENCLAW_SYSTEMCTL_SHIM_MANAGER_ENV); @@ -153,7 +171,6 @@ delete childEnv.OPENCLAW_COMPATIBILITY_HOST_VERSION; const restartDelayMs = 5_000; const restartWindowMs = 60_000; const restartBurst = 5; -const stopTimeoutMs = 30_000; const starts = []; let totalStarts = 0; let firstExit; @@ -161,12 +178,13 @@ let child; let activeGroupPid; let drainingGroupPid; let stopping = false; +let stopFailed = false; const publishRuntime = (pid, supervisorPid = process.pid) => { const file = `${daemonLog}.runtime.json`; // Both manager adapters observe the ExecStart child, not this synthetic manager. fs.writeFileSync(`${file}.pending`, JSON.stringify({ - pid, supervisorPid, groupPid: activeGroupPid ?? 0, + pid, supervisorPid, groupPid: activeGroupPid ?? 0, stopFailed, restarts: totalStarts - 1, entered: Number(process.hrtime.bigint() / 1000n), })); fs.renameSync(`${file}.pending`, file); @@ -178,7 +196,7 @@ const finish = () => { try { fs.closeSync(output); } catch {} - process.exit(0); + process.exit(stopFailed ? 1 : 0); }; const signalProcessGroup = (pid, signal) => { @@ -212,8 +230,21 @@ const drainProcessGroup = (pid, onStopped) => { if (activeGroupPid === pid) activeGroupPid = undefined; onStopped(); }; + // Read at stop, not launch: daemon-reload can repair a running unit's policy. + let stopTimeoutMs; + try { + stopTimeoutMs = Number(execFileSync(process.execPath, [managerScript, "stop-timeout-ms"], { encoding: "utf8" })); + } catch (error) { + // Broken fixture policy is fatal, not a new stop-budget default. Keep the + // supervisor alive until its owned group is gone, then report the policy failure. + stopFailed = true; + stopping = true; + fs.writeSync(output, `[systemctl-shim] stop policy read failed; cleaning up process group: ${String(error)}\n`); + publishRuntime(child?.pid ?? 0); + } signalProcessGroup(pid, "SIGTERM"); - const forceKill = setTimeout(() => { + if (stopFailed) signalProcessGroup(pid, "SIGKILL"); + const forceKill = stopFailed || stopTimeoutMs === Infinity ? undefined : setTimeout(() => { signalProcessGroup(pid, "SIGKILL"); // Signal delivery is not settlement; the existing observer must confirm exit. }, stopTimeoutMs); @@ -294,7 +325,8 @@ start(); SUPERVISOR # The manager must outlive the calling terminal, just like systemd. nohup alone # leaves Node in that terminal session and can strand its detached gateway. - OPENCLAW_SYSTEMCTL_SHIM_EXEC_START="$exec_start" \ + OPENCLAW_SYSTEMCTL_SHIM_MANAGER_SCRIPT="$manager_script" \ + OPENCLAW_SYSTEMCTL_SHIM_EXEC_START="$exec_start" \ OPENCLAW_SYSTEMCTL_SHIM_DAEMON_LOG="$daemon_log" \ OPENCLAW_SYSTEMCTL_SHIM_MANAGER_ENV="$manager_env" \ node --input-type=module - "$supervisor_script" "$pid_file" "${daemon_log}.bootstrap.log" <<'START_SUPERVISOR' @@ -388,6 +420,10 @@ case "$command" in exit 0 fi [ "$unit_name" = openclaw-gateway.service ] || exit 1 + if [ "$property" = LoadState,TimeoutStopUSec ]; then + node "$manager_script" stop-policy + exit 0 + fi # Published readers omit LoadState or ControlGroup; retain their exact queries. runtime_properties='Id,ActiveState,SubState,Result,NRestarts,StartLimitBurst,MainPID,ExecMainStatus,ExecMainCode,KillMode,TasksCurrent,MemoryCurrent' case "$property" in diff --git a/test/scripts/docker-build-helper.test.ts b/test/scripts/docker-build-helper.test.ts index 153f54041148..8f45fa81986a 100644 --- a/test/scripts/docker-build-helper.test.ts +++ b/test/scripts/docker-build-helper.test.ts @@ -566,6 +566,22 @@ function extractUpgradeSurvivorSupervisor(script: string): string { return source; } +// These process tests isolate supervision from unit parsing (covered by the +// systemd fixture suite), while exercising its real stop-policy subprocess call. +function writeUpgradeSurvivorStopPolicy(workDir: string, timeoutMs = 330_000): string { + const policyPath = join(workDir, "stop-policy-" + timeoutMs + ".mjs"); + writeFileSync( + policyPath, + [ + 'if (process.argv.length !== 3 || process.argv[2] !== "stop-timeout-ms") {', + ' throw new Error("Unexpected supervisor policy request");', + "}", + "process.stdout.write(" + JSON.stringify(String(timeoutMs)) + ");", + ].join("\n"), + ); + return policyPath; +} + function installUpgradeSurvivorSystemctlShim( prefix: string, env: NodeJS.ProcessEnv, @@ -645,7 +661,7 @@ async function forEachUpgradeSurvivorSystemctlShim( callback: (fixture: { pid: number; pidPath: string; - run: (procStat?: string) => number | null; + run: (procStat?: string, settled?: boolean) => number | null; readLog: () => string[]; scriptPath: string; }) => void | Promise, @@ -665,15 +681,22 @@ async function forEachUpgradeSurvivorSystemctlShim( } const pid = Number.parseInt(readFileSync(childPidPath, "utf8"), 10); writeFileSync(pidPath, `${pid}\n`); + const daemonLog = join(workDir, "gateway.log"); const fixtureEnv = { + HOME: workDir, OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_LOG: join(workDir, "systemctl.log"), OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_PID_FILE: pidPath, + OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_DAEMON_LOG: daemonLog, }; - const shimPath = installUpgradeSurvivorSystemctlShim( - workDir, - { HOME: workDir, ...fixtureEnv }, - scriptPath, + const unitDir = join(workDir, ".config/systemd/user"); + mkdirSync(unitDir, { recursive: true }); + writeFileSync( + join(unitDir, "openclaw-gateway.service"), + buildSystemdUnit({ + programArguments: [process.execPath, "gateway"], + }), ); + const shimPath = installUpgradeSurvivorSystemctlShim(workDir, fixtureEnv, scriptPath); writeExecutables(binDir, { cat: `#!/usr/bin/env bash case "\${1:-}" in @@ -690,7 +713,17 @@ printf 'wait\\n' >>"$OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_LOG" exit 97 `, }); - const run = (procStat?: string) => { + const run = (procStat?: string, settled = false) => { + // The synthetic /proc observation and manager custody describe the same + // state: a zombie has retired; unreadable/malformed state stays owned. + writeFileSync( + `${daemonLog}.runtime.json`, + JSON.stringify({ + pid: 0, + supervisorPid: settled ? 0 : pid, + groupPid: 0, + }), + ); writeFileSync(fixtureEnv.OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_LOG, ""); return spawnSync("bash", [shimPath, "--user", "stop", "openclaw-gateway.service"], { encoding: "utf8", @@ -4131,7 +4164,8 @@ printf '%s\n' "$status" >"$TMPDIR/status" 'if (key.startsWith("OPENCLAW_UPDATE_")) {', "delete childEnv.OPENCLAW_COMPATIBILITY_HOST_VERSION;", 'process.on("SIGTERM", stop);', - "const stopTimeoutMs = 30_000;", + 'OPENCLAW_SYSTEMCTL_SHIM_MANAGER_SCRIPT="$manager_script"', + '[managerScript, "stop-timeout-ms"]', "process.kill(-pid, signal);", 'signalProcessGroup(pid, "SIGTERM");', 'signalProcessGroup(pid, "SIGKILL");', @@ -4143,7 +4177,9 @@ printf '%s\n' "$status" >"$TMPDIR/status" "const restartBurst = 5;", "if (starts.length >= restartBurst) {", "setTimeout(start, restartDelayMs);", - "for _ in $(seq 1 350)", + 'stop_timeout_ms="$(node "$manager_script" stop-timeout-ms)"', + "stop_timeout_ms + 5000 + 99", + 'node "$manager_script" check-stopped', ]); } for (const script of [runner, publishedRunner]) { @@ -4156,8 +4192,13 @@ printf '%s\n' "$status" >"$TMPDIR/status" async () => { await forEachUpgradeSurvivorSystemctlShim(({ pid, run, readLog, scriptPath }) => { const procTail = Array.from({ length: 49 }, (_, field) => field + 1).join(" "); - expect(run(`${pid} (gateway (old) worker) Z ${procTail}`), scriptPath).toBe(0); - expect(readLog()).toEqual(["--user stop openclaw-gateway.service", "proc-stat-read"]); + expect(run(`${pid} (gateway (old) worker) Z ${procTail}`, true), scriptPath).toBe(0); + expect(readLog()).toEqual([ + "--user stop openclaw-gateway.service", + "proc-stat-read", + "proc-stat-read", + ]); + expect(isProcessRunning(pid)).toBe(true); }); }, ); @@ -4862,6 +4903,7 @@ ${storage === "wal" ? 'process.kill(process.pid, "SIGKILL");' : ""}`, ...process.env, OPENCLAW_SYSTEMCTL_SHIM_DAEMON_LOG: logPath, OPENCLAW_SYSTEMCTL_SHIM_MANAGER_ENV: "{}", + OPENCLAW_SYSTEMCTL_SHIM_MANAGER_SCRIPT: writeUpgradeSurvivorStopPolicy(workDir), OPENCLAW_SYSTEMCTL_SHIM_EXEC_START: `${shellQuote(process.execPath)} ${shellQuote(childPath)}`, }, stdio: "ignore", @@ -5400,6 +5442,7 @@ exit 0 COUNT_FILE: countPath, OPENCLAW_SYSTEMCTL_SHIM_DAEMON_LOG: logPath, OPENCLAW_SYSTEMCTL_SHIM_MANAGER_ENV: "{}", + OPENCLAW_SYSTEMCTL_SHIM_MANAGER_SCRIPT: writeUpgradeSurvivorStopPolicy(workDir), OPENCLAW_SYSTEMCTL_SHIM_EXEC_START: command, }, stdio: "ignore", @@ -5422,10 +5465,7 @@ exit 0 const supervisorPath = join(workDir, `graceful-supervisor-${index}.mjs`); const statePath = join(workDir, `graceful-state-${index}`); const logPath = join(workDir, `graceful-daemon-${index}.log`); - const source = extractUpgradeSurvivorSupervisor(script).replace( - "const stopTimeoutMs = 30_000;", - "const stopTimeoutMs = 200;", - ); + const source = extractUpgradeSurvivorSupervisor(script); writeFileSync(supervisorPath, source); const command = @@ -5435,6 +5475,7 @@ exit 0 ...process.env, OPENCLAW_SYSTEMCTL_SHIM_DAEMON_LOG: logPath, OPENCLAW_SYSTEMCTL_SHIM_MANAGER_ENV: "{}", + OPENCLAW_SYSTEMCTL_SHIM_MANAGER_SCRIPT: writeUpgradeSurvivorStopPolicy(workDir, 200), OPENCLAW_SYSTEMCTL_SHIM_EXEC_START: command, STATE_FILE: statePath, }, @@ -5487,6 +5528,7 @@ process.exit(starts === 1 ? 1 : 78); OPENCLAW_CLAWHUB_URL: "http://127.0.0.1:43123", OPENCLAW_SYSTEMCTL_SHIM_DAEMON_LOG: logPath, OPENCLAW_SYSTEMCTL_SHIM_MANAGER_ENV: "{}", + OPENCLAW_SYSTEMCTL_SHIM_MANAGER_SCRIPT: writeUpgradeSurvivorStopPolicy(workDir), OPENCLAW_SYSTEMCTL_SHIM_EXEC_START: `${shellQuote(process.execPath)} ${shellQuote(gatewayPath)}`, URLS_FILE: urlsPath, }, @@ -5539,10 +5581,7 @@ setInterval(() => {}, 1_000); const statePath = join(workDir, `process-group-state-${index}`); const descendantPidPath = join(workDir, `process-group-descendant-${index}.pid`); const logPath = join(workDir, `process-group-daemon-${index}.log`); - const source = extractUpgradeSurvivorSupervisor(script).replace( - "const stopTimeoutMs = 30_000;", - "const stopTimeoutMs = 200;", - ); + const source = extractUpgradeSurvivorSupervisor(script); writeFileSync(supervisorPath, source); const supervisor = spawn(process.execPath, [supervisorPath], { @@ -5552,6 +5591,7 @@ setInterval(() => {}, 1_000); DESCENDANT_SCRIPT: descendantPath, OPENCLAW_SYSTEMCTL_SHIM_DAEMON_LOG: logPath, OPENCLAW_SYSTEMCTL_SHIM_MANAGER_ENV: "{}", + OPENCLAW_SYSTEMCTL_SHIM_MANAGER_SCRIPT: writeUpgradeSurvivorStopPolicy(workDir, 200), OPENCLAW_SYSTEMCTL_SHIM_EXEC_START: `${shellQuote(process.execPath)} ${shellQuote(gatewayPath)}`, STATE_FILE: statePath, }, @@ -5625,9 +5665,10 @@ if (starts === 1) { const descendantPidPath = join(workDir, `restart-group-descendant-${index}.pid`); const replacementPath = join(workDir, `restart-group-replacement-${index}`); const logPath = join(workDir, `restart-group-daemon-${index}.log`); - const source = extractUpgradeSurvivorSupervisor(script) - .replace("const restartDelayMs = 5_000;", "const restartDelayMs = 5;") - .replace("const stopTimeoutMs = 30_000;", "const stopTimeoutMs = 200;"); + const source = extractUpgradeSurvivorSupervisor(script).replace( + "const restartDelayMs = 5_000;", + "const restartDelayMs = 5;", + ); writeFileSync(supervisorPath, source); const supervisor = spawn(process.execPath, [supervisorPath], { @@ -5637,6 +5678,7 @@ if (starts === 1) { DESCENDANT_SCRIPT: descendantPath, OPENCLAW_SYSTEMCTL_SHIM_DAEMON_LOG: logPath, OPENCLAW_SYSTEMCTL_SHIM_MANAGER_ENV: "{}", + OPENCLAW_SYSTEMCTL_SHIM_MANAGER_SCRIPT: writeUpgradeSurvivorStopPolicy(workDir, 200), OPENCLAW_SYSTEMCTL_SHIM_EXEC_START: `${shellQuote(process.execPath)} ${shellQuote(gatewayPath)}`, REPLACEMENT_FILE: replacementPath, STARTS_FILE: startsPath, diff --git a/test/scripts/upgrade-survivor-stop-policy.test.ts b/test/scripts/upgrade-survivor-stop-policy.test.ts new file mode 100644 index 000000000000..013928e429e5 --- /dev/null +++ b/test/scripts/upgrade-survivor-stop-policy.test.ts @@ -0,0 +1,468 @@ +import { spawnSync } from "node:child_process"; +import { existsSync, mkdirSync, readFileSync, rmSync, watch, writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import vm from "node:vm"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../helpers/temp-dir.js"; + +const tempDirs = useAutoCleanupTempDirTracker(afterEach); +const owner = resolve("scripts/e2e/lib/upgrade-survivor/update-restart-auth.sh"); +function fixture() { + const home = tempDirs.make("survivor-stop-policy-"); + const env = { HOME: home, npm_config_prefix: home, PATH: process.env.PATH }; + const installed = spawnSync( + "bash", + [ + "-c", + 'set -euo pipefail; source "$1"; install_update_restart_systemctl_shim', + "fixture", + owner, + ], + { env, encoding: "utf8" }, + ); + expect(installed.status, installed.stderr).toBe(0); + const unit = join(home, ".config/systemd/user/openclaw-gateway.service"); + mkdirSync(join(home, ".config/systemd/user"), { recursive: true }); + const systemctl = (...args: string[]) => + spawnSync(join(home, "bin/systemctl"), ["--user", ...args], { env, encoding: "utf8" }); + const manager = (...args: string[]) => + spawnSync(process.execPath, [join(home, "bin/systemd-fixture.mjs"), ...args], { + env, + encoding: "utf8", + }); + return { home, env, unit, systemctl, manager }; +} + +function waitForFixtureState(directory: string, settled: () => boolean) { + return new Promise((complete, reject) => { + const inspect = () => { + try { + if (!settled()) { + return; + } + } catch { + return; + } + watcher.close(); + clearTimeout(deadline); + complete(); + }; + const watcher = watch(directory, inspect); + const deadline = setTimeout(() => { + watcher.close(); + reject(new Error("fixture state did not settle")); + }, 5_000); + inspect(); + }); +} + +describe.skipIf(process.platform === "win32")("survivor loaded stop policy", () => { + it("reports the loaded stop policy until daemon-reload", () => { + const { unit, systemctl, manager } = fixture(); + const query = () => + systemctl( + "show", + "openclaw-gateway.service", + "--no-page", + "--property", + "LoadState,TimeoutStopUSec", + ); + expect(query().stdout).toBe("LoadState=not-found\n"); + const content = "[Service]\nExecStart=/usr/bin/true\nTimeoutStopSec=330\n"; + writeFileSync(unit, content); + expect(systemctl("daemon-reload").status).toBe(0); + expect(query()).toMatchObject({ + status: 0, + stdout: "LoadState=loaded\nTimeoutStopUSec=330s\n", + }); + expect(manager("stop-timeout-ms").stdout).toBe("330000\n"); + writeFileSync(unit, content.replace("TimeoutStopSec=330", "TimeoutStopSec=30")); + expect(query().stdout).toContain("TimeoutStopUSec=330s"); + expect(manager("stop-timeout-ms").stdout).toBe("330000\n"); + expect(systemctl("daemon-reload").status).toBe(0); + expect(query().stdout).toContain("TimeoutStopUSec=30s"); + expect(manager("stop-timeout-ms").stdout).toBe("30000\n"); + writeFileSync(unit, content.replace("TimeoutStopSec=330", "TimeoutStopSec=invalid")); + expect(systemctl("daemon-reload").status).not.toBe(0); + expect(query().stdout).toContain("TimeoutStopUSec=30s"); + expect(manager("stop-timeout-ms").stdout).toBe("30000\n"); + rmSync(unit); + const runtime = systemctl( + "show", + "openclaw-gateway.service", + "--property", + "Id,LoadState,ActiveState,SubState,Result,NRestarts,StartLimitBurst,MainPID,ExecMainStatus,ExecMainCode,KillMode,TasksCurrent,MemoryCurrent", + ); + expect(runtime.status, runtime.stderr).toBe(0); + expect(runtime.stdout).toContain("LoadState=loaded"); + expect(query().stdout).toContain("TimeoutStopUSec=30s"); + expect(systemctl("daemon-reload").status).toBe(0); + expect(query().stdout).toBe("LoadState=not-found\n"); + expect(manager("stop-timeout-ms").status).not.toBe(0); + }); + + it.each([ + { policy: "", value: "90000", display: "90s" }, + { policy: "TimeoutStopSec=0", value: "Infinity", display: "infinity" }, + { policy: "TimeoutStopSec=infinity", value: "Infinity", display: "infinity" }, + ])("handles generated stop policy $policy deliberately", ({ policy, value, display }) => { + const { unit, systemctl, manager } = fixture(); + writeFileSync(unit, "[Service]\nExecStart=/usr/bin/true\n" + policy + "\n"); + expect(systemctl("daemon-reload").status).toBe(0); + expect(manager("stop-timeout-ms")).toMatchObject({ status: 0, stdout: value + "\n" }); + expect( + systemctl("show", "openclaw-gateway.service", "--property=LoadState,TimeoutStopUSec").stdout, + ).toBe("LoadState=loaded\nTimeoutStopUSec=" + display + "\n"); + }); + + it.each(["-1", "bogus", "1ms", "2147484", "9".repeat(400), "30\nTimeoutStopSec=330"])( + "refuses unsupported stop policy %j at load", + (policy) => { + const { unit, systemctl } = fixture(); + writeFileSync(unit, "[Service]\nExecStart=/usr/bin/true\nTimeoutStopSec=" + policy + "\n"); + expect(systemctl("daemon-reload").status).not.toBe(0); + expect(existsSync(unit + ".loaded-unit")).toBe(false); + }, + ); + + // Execute the unchanged supervisor body with native boundaries substituted. The + // virtual clock advances policy-sized deadlines without launching or killing PIDs. + it.each([30, 330, "infinity"] as const)( + "uses loaded %s seconds for supervisor and outer stop", + (seconds) => { + const f = fixture(); + writeFileSync(f.unit, "[Service]\nExecStart=/usr/bin/true\nTimeoutStopSec=" + seconds + "\n"); + expect(f.systemctl("daemon-reload").status).toBe(0); + // An un-reloaded edit must not affect either consumer. + writeFileSync(f.unit, "[Service]\nExecStart=/usr/bin/true\nTimeoutStopSec=1\n"); + if (seconds === 330) { + writeFileSync(f.unit, "[Service]\nExecStart=/usr/bin/true\nTimeoutStopSec=30\n"); + expect(f.systemctl("daemon-reload").status).toBe(0); + } + const source = readFileSync(owner, "utf8"); + const body = source.split("<<'SUPERVISOR'\n")[1]?.split("\nSUPERVISOR")[0]; + expect(body).toBeDefined(); + const signals: Array = []; + const handlers = new Map void>(); + const timers = new Map void }>(); + let now = 0; + let serial = 0; + let alive = true; + const exit = vi.fn(); + const fs = { + openSync: () => 1, + closeSync: () => {}, + writeFileSync: () => {}, + renameSync: () => {}, + writeSync: () => {}, + }; + vm.runInNewContext(body!.replace(/^import .*;\n/gm, ""), { + fs, + spawn: () => ({ pid: 42, on: () => {}, once: () => {} }), + execFileSync: (_binary: string, args: string[]) => { + expect(args).toEqual([join(f.home, "bin/systemd-fixture.mjs"), "stop-timeout-ms"]); + const result = f.manager("stop-timeout-ms"); + expect(result.status, result.stderr).toBe(0); + return result.stdout; + }, + process: { + pid: 43, + execPath: process.execPath, + env: { + OPENCLAW_SYSTEMCTL_SHIM_MANAGER_SCRIPT: join(f.home, "bin/systemd-fixture.mjs"), + OPENCLAW_SYSTEMCTL_SHIM_EXEC_START: "/usr/bin/true", + OPENCLAW_SYSTEMCTL_SHIM_DAEMON_LOG: "/fixture/log", + OPENCLAW_SYSTEMCTL_SHIM_MANAGER_ENV: "{}", + }, + hrtime: { bigint: () => 1n }, + exit, + on: (signal: string, handler: () => void) => handlers.set(signal, handler), + kill: (pid: number, signal: string | number) => { + expect(pid).toBe(-42); + if (signal === 0) { + if (!alive) { + throw Object.assign(new Error("gone"), { code: "ESRCH" }); + } + } else { + signals.push(signal); + } + }, + }, + setTimeout: (callback: () => void, delay: number) => { + const id = ++serial; + timers.set(id, { at: now + delay, callback }); + return id; + }, + clearTimeout: (id: number) => timers.delete(id), + }); + const advance = (until: number) => { + while (true) { + const next = [...timers].toSorted((a, b) => a[1].at - b[1].at)[0]; + if (!next || next[1].at > until) { + break; + } + now = next[1].at; + timers.delete(next[0]); + next[1].callback(); + } + now = until; + }; + if (seconds === 330) { + // Repair the manager policy after supervisor launch, before its stop. + writeFileSync(f.unit, "[Service]\nExecStart=/usr/bin/true\nTimeoutStopSec=330\n"); + expect(f.systemctl("daemon-reload").status).toBe(0); + writeFileSync(f.unit, "[Service]\nExecStart=/usr/bin/true\nTimeoutStopSec=1\n"); + } + handlers.get(seconds === 30 ? "SIGINT" : "SIGTERM")!(); + const deadline = seconds === "infinity" ? 400_000 : seconds * 1_000; + advance(deadline - 1); + expect(signals).toEqual(["SIGTERM"]); + expect(exit).not.toHaveBeenCalled(); + advance(deadline); + expect(signals).toEqual(seconds === "infinity" ? ["SIGTERM"] : ["SIGTERM", "SIGKILL"]); + // Sending SIGKILL is not extinction; settlement waits for the group observer. + expect(exit).not.toHaveBeenCalled(); + alive = false; + advance(deadline + 25); + expect(exit).toHaveBeenCalledExactlyOnceWith(0); + expect(timers.size).toBe(0); + + const stop = source.match(/stop_gateway\(\) \{[\s\S]*?\n\}/)?.[0]; + expect(stop).toBeDefined(); + const pidFile = join(f.home, "pid"); + writeFileSync(pidFile, "999999\n"); + const outer = spawnSync( + "bash", + [ + "-c", + [ + "set -euo pipefail", + stop, + "pid_file=$1; supervisor_script=$1.supervisor; manager_script=$2", + "ticks=0; kill() { :; }; sleep() { ticks=$((ticks + 1)); }", + "is_running() { [ $ticks -lt 4000 ]; }", + "status=0; stop_gateway || status=$?; printf '%s' $ticks; exit $status", + ].join("\n"), + "fixture", + pidFile, + join(f.home, "bin/systemd-fixture.mjs"), + ], + { env: f.env, encoding: "utf8" }, + ); + expect(outer.status, outer.stderr).toBe(seconds === "infinity" ? 0 : 1); + if (seconds !== "infinity") { + expect(outer.stderr).toContain("retaining process custody"); + expect(existsSync(pidFile)).toBe(true); + } + expect(Number(outer.stdout)).toBe( + seconds === "infinity" ? 4000 : (seconds * 1_000 + 5_000) / 100, + ); + }, + ); + + it.each(["signal", "child-close"])( + "retains custody when the policy read fails during %s drain", + (entry) => { + const source = readFileSync(owner, "utf8"); + const body = source.split("<<'SUPERVISOR'\n")[1]?.split("\nSUPERVISOR")[0]; + expect(body).toBeDefined(); + const handlers = new Map void>(); + let closeChild: ((code: number, signal: string | null) => void) | undefined; + const timers: Array<() => void> = []; + const signals: Array = []; + let alive = true; + const exit = vi.fn(); + const log = vi.fn(); + const files = new Map(); + const spawn = vi.fn(() => ({ + pid: 42, + on: () => {}, + once: (_event: string, callback: (code: number, signal: string | null) => void) => { + closeChild = callback; + }, + })); + vm.runInNewContext(body!.replace(/^import .*;\n/gm, ""), { + fs: { + openSync: () => 1, + closeSync: () => {}, + writeSync: log, + writeFileSync: (file: string, contents: string) => files.set(file, contents), + renameSync: (from: string, to: string) => files.set(to, files.get(from)!), + }, + spawn, + execFileSync: () => { + throw new Error("policy read failed"); + }, + process: { + pid: 43, + execPath: process.execPath, + cwd: () => "/fixture", + env: { + OPENCLAW_SYSTEMCTL_SHIM_MANAGER_SCRIPT: "/fixture/manager.mjs", + OPENCLAW_SYSTEMCTL_SHIM_EXEC_START: "/usr/bin/true", + OPENCLAW_SYSTEMCTL_SHIM_DAEMON_LOG: "/fixture/log", + OPENCLAW_SYSTEMCTL_SHIM_MANAGER_ENV: "{}", + }, + hrtime: { bigint: () => 1n }, + exit, + on: (signal: string, callback: () => void) => handlers.set(signal, callback), + kill: (pid: number, signal: string | number) => { + expect(pid).toBe(-42); + if (signal === 0) { + if (!alive) { + throw Object.assign(new Error("gone"), { code: "ESRCH" }); + } + } else { + signals.push(signal); + } + }, + }, + setTimeout: (callback: () => void) => { + timers.push(callback); + return timers.length; + }, + clearTimeout: () => {}, + }); + const beginDrain = () => + entry === "signal" ? handlers.get("SIGTERM")!() : closeChild!(0, null); + expect(beginDrain).not.toThrow(); + expect(signals).toEqual(["SIGTERM", "SIGKILL"]); + expect(log.mock.calls.flat().join(" ")).toContain("stop policy read failed"); + expect(exit).not.toHaveBeenCalled(); + expect(JSON.parse(files.get("/fixture/log.runtime.json")!)).toMatchObject({ + groupPid: 42, + supervisorPid: 43, + }); + // A failed forced kill cannot turn into a clean exit or restart. + timers.shift()!(); + expect(exit).not.toHaveBeenCalled(); + alive = false; + timers.shift()!(); + expect(exit).toHaveBeenCalledExactlyOnceWith(1); + expect(spawn).toHaveBeenCalledOnce(); + const runtime = JSON.parse(files.get("/fixture/log.runtime.json")!); + expect(runtime).toMatchObject({ pid: 0, groupPid: 0, supervisorPid: 0, stopFailed: true }); + const f = fixture(); + writeFileSync( + join(f.home, "bin/systemctl-shim-gateway.log.runtime.json"), + JSON.stringify(runtime), + ); + const stopped = f.systemctl("stop", "openclaw-gateway.service"); + expect(stopped.status).toBe(1); + expect(stopped.stderr).toContain("stop policy read failed"); + }, + ); + + it.each([false, true])( + "joins the installed outer stop after removed-unit reload=%s", + async (removed) => { + const f = fixture(); + const bin = join(f.home, "bin"); + const ready = join(f.home, "ready"); + const program = join(f.home, "child.mjs"); + const runtimeFile = join(bin, "systemctl-shim-gateway.log.runtime.json"); + const runtime = () => JSON.parse(readFileSync(runtimeFile, "utf8")); + writeFileSync( + program, + 'import fs from "node:fs"; process.on("SIGTERM", () => ' + + (removed ? "{}" : "process.exit(0)") + + "); fs.writeFileSync(" + + JSON.stringify(ready) + + ", String(process.pid)); setInterval(() => {}, 1000);", + ); + writeFileSync( + f.unit, + [ + "[Service]", + "ExecStart=" + JSON.stringify(process.execPath) + " " + JSON.stringify(program), + "TimeoutStopSec=330", + "", + ].join("\n"), + ); + try { + const started = f.systemctl("start", "openclaw-gateway.service"); + expect(started.status, started.stderr).toBe(0); + await waitForFixtureState( + f.home, + () => existsSync(ready) && readFileSync(ready, "utf8").length > 0, + ); + const pid = Number(readFileSync(ready, "utf8")); + if (removed) { + rmSync(f.unit); + expect(f.systemctl("daemon-reload").status).toBe(0); + expect(existsSync(f.unit + ".loaded-unit")).toBe(false); + } + const stopped = f.systemctl("stop", "openclaw-gateway.service"); + expect(stopped.status, stopped.stderr).toBe(removed ? 1 : 0); + expect(runtime()).toMatchObject({ + pid: 0, + groupPid: 0, + supervisorPid: 0, + stopFailed: removed, + }); + expect(() => process.kill(-pid, 0)).toThrow(); + if (removed) { + expect(stopped.stderr).toContain("stop policy read failed"); + } + } finally { + if (existsSync(runtimeFile)) { + const owned = runtime(); + // Failed proof must still retire only this fixture's published processes. + if (owned.supervisorPid) { + try { + process.kill(owned.supervisorPid, "SIGTERM"); + } catch {} + } + if (owned.groupPid) { + try { + process.kill(-owned.groupPid, "SIGKILL"); + } catch {} + } + await waitForFixtureState(bin, () => { + const observed = runtime(); + return observed.pid === 0 && observed.supervisorPid === 0 && observed.groupPid === 0; + }); + } + } + }, + ); + + it.each([true, false])( + "keeps outer policy lookup failure nonzero with supervisor active=%s", + (active) => { + const f = fixture(); + const stop = readFileSync(owner, "utf8").match(/stop_gateway\(\) \{[\s\S]*?\n\}/)?.[0]; + expect(stop).toBeDefined(); + const pid = join(f.home, "pid"); + const supervisor = join(f.home, "supervisor"); + const signals = join(f.home, "signals"); + writeFileSync(pid, "424242\n"); + writeFileSync(supervisor, "owned"); + const result = spawnSync( + "bash", + [ + "-c", + [ + "set -euo pipefail", + stop, + "pid_file=$1; supervisor_script=$2; signal_log=$3; manager_script=fixture-manager", + 'ticks=0; kill() { printf "%s\\n" "$*" >> "$signal_log"; }; sleep() { ticks=$((ticks+1)); }', + 'node() { [ "$2" != stop-timeout-ms ] || return 17; [ "$2" = check-stopped ]; }', + active ? "is_running() { return 0; }" : "is_running() { return 1; }", + 'status=0; stop_gateway || status=$?; printf "%s %s\\n" "$status" "$ticks"', + ].join("\n"), + "fixture", + pid, + supervisor, + signals, + ], + { env: f.env, encoding: "utf8" }, + ); + expect(result.status, result.stderr).toBe(0); + expect(result.stdout.trim()).toBe(active ? "1 50" : "17 0"); + expect(readFileSync(signals, "utf8").trim().split("\n")).toEqual(["-0 424242", "424242"]); + expect(existsSync(pid)).toBe(true); + expect(existsSync(supervisor)).toBe(true); + }, + ); +}); diff --git a/test/scripts/upgrade-survivor-systemd-cgroup.test.ts b/test/scripts/upgrade-survivor-systemd-cgroup.test.ts index d19521f77081..308cb165668e 100644 --- a/test/scripts/upgrade-survivor-systemd-cgroup.test.ts +++ b/test/scripts/upgrade-survivor-systemd-cgroup.test.ts @@ -83,4 +83,15 @@ it.each([ await import("../../scripts/e2e/lib/upgrade-survivor/systemd-fixture.mjs"); expect(error).not.toHaveBeenCalled(); expect(JSON.parse(String(log.mock.calls.at(-1)?.[0]))).toEqual({ type: "s", data: row.expected }); + + vi.resetModules(); + process.exitCode = 0; + process.argv = ["node", "fixture", "check-stopped"]; + await import("../../scripts/e2e/lib/upgrade-survivor/systemd-fixture.mjs"); + expect(process.exitCode).toBe(row.pid || row.populated ? 1 : 0); + if (row.pid || row.populated) { + expect(error).toHaveBeenCalledWith("Survivor service processes have not settled."); + } else { + expect(error).not.toHaveBeenCalled(); + } }); diff --git a/test/scripts/upgrade-survivor-systemd.test.ts b/test/scripts/upgrade-survivor-systemd.test.ts index 27a27dd99c5f..1a5d483ba1a0 100644 --- a/test/scripts/upgrade-survivor-systemd.test.ts +++ b/test/scripts/upgrade-survivor-systemd.test.ts @@ -290,6 +290,7 @@ fs.existsSync = (file) => file === "/sys/fs/cgroup/openclaw-gateway.service/cgro ); expect(await readSystemdServiceRuntime(env)).toMatchObject({ status: "unknown" }); rmSync(unit); + expect(systemctl("daemon-reload").status).toBe(0); expect(await readSystemdServiceExecStart(env, { requireEffective: true })).toBeNull(); expect(await readSystemdServiceRuntime(env)).toMatchObject({ status: "stopped",