diff --git a/scripts/e2e/lib/upgrade-survivor/systemd-fixture.mjs b/scripts/e2e/lib/upgrade-survivor/systemd-fixture.mjs index bbb7311d9184..e73a41d1a5e8 100644 --- a/scripts/e2e/lib/upgrade-survivor/systemd-fixture.mjs +++ b/scripts/e2e/lib/upgrade-survivor/systemd-fixture.mjs @@ -135,7 +135,24 @@ function parseUnit(content) { if ([...directives.keys()].some((key) => !supported.has(key))) { fail(); } + // Generated units use whole seconds. Missing policy follows systemd's 90s + // default; zero/infinity disable the deadline. Reject unsupported spans and + // timer overflow instead of silently scheduling Node's 1ms overflow timer. + const stopSeconds = single("TimeoutStopSec") || "90"; + const unlimitedStop = stopSeconds === "infinity" || stopSeconds === "0"; + const stopTimeoutMs = unlimitedStop + ? Infinity + : /^\d+$/.test(stopSeconds) + ? Number(stopSeconds) * 1_000 + : Number.NaN; + if ( + !unlimitedStop && + (!Number.isSafeInteger(stopTimeoutMs) || stopTimeoutMs <= 0 || stopTimeoutMs > 2_147_483_647) + ) { + fail("Unsupported generated TimeoutStopSec policy."); + } return { + stopTimeoutMs, programArguments, workingDirectory, environment, @@ -157,8 +174,11 @@ function readUnit(reload = false, requireLoaded = false) { if (error.code !== "ENOENT") { throw error; } - if (!requireLoaded) { + if (reload) { fs.rmSync(loadedPath, { force: true }); + } else if (fs.existsSync(loadedPath)) { + // Ordinary status/command inspection cannot unload an admitted definition. + return { ...parseUnit(fs.readFileSync(loadedPath, "utf8")), reloadPending: true }; } return null; } @@ -227,7 +247,8 @@ function nativeRuntime() { ) : false; const unsettled = counts?.starting || supervisorPid || groupPid || populated; - const successful = !last || last.code === 0; + const stopFailed = counts?.stopFailed === true; + const successful = !stopFailed && (!last || last.code === 0); return { pid, // A manager draining descendants or awaiting restart is not a settled service. @@ -235,6 +256,7 @@ function nativeRuntime() { sub: pid ? "running" : unsettled ? "auto-restart" : "dead", generation: counts?.entered ?? 0, settled: !pid && !unsettled, + stopFailed, controlGroup: pid || unsettled ? paths.controlGroup || "" : "", restarts: counts?.restarts ?? 0, result: successful ? "success" : "exit-code", @@ -444,6 +466,16 @@ function run() { } return; } + if (operation === "check-stopped" && !args.length) { + const runtime = nativeRuntime(); + if (!runtime.settled) { + fail("Survivor service processes have not settled."); + } + if (runtime.stopFailed) { + fail("Survivor stop policy read failed; process cleanup completed."); + } + return; + } if (operation === "is-active" && !args.length) { const runtime = nativeRuntime(); process.exitCode = runtime.pid ? 0 : runtime.settled ? 3 : 1; @@ -480,6 +512,27 @@ function run() { readUnit(true); return; } + if (["stop-policy", "stop-timeout-ms"].includes(operation) && !args.length) { + // A running generation keeps its loaded policy even if an on-disk edit is + // invalid or removed. Only a successful reload replaces that snapshot. + const unit = fs.existsSync(loadedPath) + ? parseUnit(fs.readFileSync(loadedPath, "utf8")) + : readUnit(); + if (operation === "stop-policy") { + console.log(`LoadState=${unit ? "loaded" : "not-found"}`); + if (unit) { + console.log( + `TimeoutStopUSec=${unit.stopTimeoutMs === Infinity ? "infinity" : `${unit.stopTimeoutMs / 1_000}s`}`, + ); + } + } else { + if (!unit) { + fail("Cannot stop an absent fixture unit."); + } + console.log(unit.stopTimeoutMs); + } + return; + } if (operation === "load-state" && !args.length) { console.log(readUnit() ? "loaded" : "not-found"); return; diff --git a/scripts/e2e/lib/upgrade-survivor/update-restart-auth.sh b/scripts/e2e/lib/upgrade-survivor/update-restart-auth.sh index 949e92da6fc9..79c485f4709b 100644 --- a/scripts/e2e/lib/upgrade-survivor/update-restart-auth.sh +++ b/scripts/e2e/lib/upgrade-survivor/update-restart-auth.sh @@ -99,17 +99,33 @@ is_running() { } stop_gateway() { - local pid="" + local pid="" stop_policy_status=0 pid="$(cat "$pid_file" 2>/dev/null || true)" if [[ "$pid" =~ ^[0-9]+$ ]] && [ "$pid" -gt 1 ] && kill -0 "$pid" >/dev/null 2>&1; then + local stop_timeout_ms attempts=0 + stop_timeout_ms="$(node "$manager_script" stop-timeout-ms)" || { + stop_policy_status=$? + # No service budget is admitted: signal the existing fatal-cleanup owner + # and join using only the settlement allowance, not a substitute timeout. + stop_timeout_ms=0 + } kill "$pid" >/dev/null 2>&1 || true - # The supervisor gives its child 30s, so keep this outer deadline comfortably longer. - for _ in $(seq 1 350); do - is_running || break + # Leave the supervisor its loaded stop budget plus 5s to observe group exit. + while is_running; do + if [ "$stop_timeout_ms" != Infinity ] && + [ "$attempts" -ge "$(((stop_timeout_ms + 5000 + 99) / 100))" ]; then + break + fi sleep 0.1 + attempts=$((attempts + 1)) done - kill -9 "$pid" >/dev/null 2>&1 || true + if is_running; then + echo "Survivor supervisor has not settled; retaining process custody." >&2 + return 1 + fi fi + node "$manager_script" check-stopped || return "$?" + [ "$stop_policy_status" -eq 0 ] || return "$stop_policy_status" rm -f "$pid_file" "$supervisor_script" } @@ -125,16 +141,18 @@ start_gateway() { rm -f "${daemon_log}.exit.json" cat >"$supervisor_script" <<'SUPERVISOR' import fs from "node:fs"; -import { spawn } from "node:child_process"; +import { spawn, execFileSync } from "node:child_process"; +const managerScript = process.env.OPENCLAW_SYSTEMCTL_SHIM_MANAGER_SCRIPT; const command = process.env.OPENCLAW_SYSTEMCTL_SHIM_EXEC_START; const daemonLog = process.env.OPENCLAW_SYSTEMCTL_SHIM_DAEMON_LOG; -if (!command || !daemonLog) { +if (!command || !daemonLog || !managerScript) { process.exit(2); } const output = fs.openSync(daemonLog, "a"); const childEnv = { ...process.env }; +delete childEnv.OPENCLAW_SYSTEMCTL_SHIM_MANAGER_SCRIPT; delete childEnv.OPENCLAW_SYSTEMCTL_SHIM_EXEC_START; delete childEnv.OPENCLAW_SYSTEMCTL_SHIM_DAEMON_LOG; const managerEnv = JSON.parse(childEnv.OPENCLAW_SYSTEMCTL_SHIM_MANAGER_ENV); @@ -153,7 +171,6 @@ delete childEnv.OPENCLAW_COMPATIBILITY_HOST_VERSION; const restartDelayMs = 5_000; const restartWindowMs = 60_000; const restartBurst = 5; -const stopTimeoutMs = 30_000; const starts = []; let totalStarts = 0; let firstExit; @@ -161,12 +178,13 @@ let child; let activeGroupPid; let drainingGroupPid; let stopping = false; +let stopFailed = false; const publishRuntime = (pid, supervisorPid = process.pid) => { const file = `${daemonLog}.runtime.json`; // Both manager adapters observe the ExecStart child, not this synthetic manager. fs.writeFileSync(`${file}.pending`, JSON.stringify({ - pid, supervisorPid, groupPid: activeGroupPid ?? 0, + pid, supervisorPid, groupPid: activeGroupPid ?? 0, stopFailed, restarts: totalStarts - 1, entered: Number(process.hrtime.bigint() / 1000n), })); fs.renameSync(`${file}.pending`, file); @@ -178,7 +196,7 @@ const finish = () => { try { fs.closeSync(output); } catch {} - process.exit(0); + process.exit(stopFailed ? 1 : 0); }; const signalProcessGroup = (pid, signal) => { @@ -212,8 +230,21 @@ const drainProcessGroup = (pid, onStopped) => { if (activeGroupPid === pid) activeGroupPid = undefined; onStopped(); }; + // Read at stop, not launch: daemon-reload can repair a running unit's policy. + let stopTimeoutMs; + try { + stopTimeoutMs = Number(execFileSync(process.execPath, [managerScript, "stop-timeout-ms"], { encoding: "utf8" })); + } catch (error) { + // Broken fixture policy is fatal, not a new stop-budget default. Keep the + // supervisor alive until its owned group is gone, then report the policy failure. + stopFailed = true; + stopping = true; + fs.writeSync(output, `[systemctl-shim] stop policy read failed; cleaning up process group: ${String(error)}\n`); + publishRuntime(child?.pid ?? 0); + } signalProcessGroup(pid, "SIGTERM"); - const forceKill = setTimeout(() => { + if (stopFailed) signalProcessGroup(pid, "SIGKILL"); + const forceKill = stopFailed || stopTimeoutMs === Infinity ? undefined : setTimeout(() => { signalProcessGroup(pid, "SIGKILL"); // Signal delivery is not settlement; the existing observer must confirm exit. }, stopTimeoutMs); @@ -294,7 +325,8 @@ start(); SUPERVISOR # The manager must outlive the calling terminal, just like systemd. nohup alone # leaves Node in that terminal session and can strand its detached gateway. - OPENCLAW_SYSTEMCTL_SHIM_EXEC_START="$exec_start" \ + OPENCLAW_SYSTEMCTL_SHIM_MANAGER_SCRIPT="$manager_script" \ + OPENCLAW_SYSTEMCTL_SHIM_EXEC_START="$exec_start" \ OPENCLAW_SYSTEMCTL_SHIM_DAEMON_LOG="$daemon_log" \ OPENCLAW_SYSTEMCTL_SHIM_MANAGER_ENV="$manager_env" \ node --input-type=module - "$supervisor_script" "$pid_file" "${daemon_log}.bootstrap.log" <<'START_SUPERVISOR' @@ -388,6 +420,10 @@ case "$command" in exit 0 fi [ "$unit_name" = openclaw-gateway.service ] || exit 1 + if [ "$property" = LoadState,TimeoutStopUSec ]; then + node "$manager_script" stop-policy + exit 0 + fi # Published readers omit LoadState or ControlGroup; retain their exact queries. runtime_properties='Id,ActiveState,SubState,Result,NRestarts,StartLimitBurst,MainPID,ExecMainStatus,ExecMainCode,KillMode,TasksCurrent,MemoryCurrent' case "$property" in diff --git a/test/scripts/docker-build-helper.test.ts b/test/scripts/docker-build-helper.test.ts index 153f54041148..8f45fa81986a 100644 --- a/test/scripts/docker-build-helper.test.ts +++ b/test/scripts/docker-build-helper.test.ts @@ -566,6 +566,22 @@ function extractUpgradeSurvivorSupervisor(script: string): string { return source; } +// These process tests isolate supervision from unit parsing (covered by the +// systemd fixture suite), while exercising its real stop-policy subprocess call. +function writeUpgradeSurvivorStopPolicy(workDir: string, timeoutMs = 330_000): string { + const policyPath = join(workDir, "stop-policy-" + timeoutMs + ".mjs"); + writeFileSync( + policyPath, + [ + 'if (process.argv.length !== 3 || process.argv[2] !== "stop-timeout-ms") {', + ' throw new Error("Unexpected supervisor policy request");', + "}", + "process.stdout.write(" + JSON.stringify(String(timeoutMs)) + ");", + ].join("\n"), + ); + return policyPath; +} + function installUpgradeSurvivorSystemctlShim( prefix: string, env: NodeJS.ProcessEnv, @@ -645,7 +661,7 @@ async function forEachUpgradeSurvivorSystemctlShim( callback: (fixture: { pid: number; pidPath: string; - run: (procStat?: string) => number | null; + run: (procStat?: string, settled?: boolean) => number | null; readLog: () => string[]; scriptPath: string; }) => void | Promise, @@ -665,15 +681,22 @@ async function forEachUpgradeSurvivorSystemctlShim( } const pid = Number.parseInt(readFileSync(childPidPath, "utf8"), 10); writeFileSync(pidPath, `${pid}\n`); + const daemonLog = join(workDir, "gateway.log"); const fixtureEnv = { + HOME: workDir, OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_LOG: join(workDir, "systemctl.log"), OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_PID_FILE: pidPath, + OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_DAEMON_LOG: daemonLog, }; - const shimPath = installUpgradeSurvivorSystemctlShim( - workDir, - { HOME: workDir, ...fixtureEnv }, - scriptPath, + const unitDir = join(workDir, ".config/systemd/user"); + mkdirSync(unitDir, { recursive: true }); + writeFileSync( + join(unitDir, "openclaw-gateway.service"), + buildSystemdUnit({ + programArguments: [process.execPath, "gateway"], + }), ); + const shimPath = installUpgradeSurvivorSystemctlShim(workDir, fixtureEnv, scriptPath); writeExecutables(binDir, { cat: `#!/usr/bin/env bash case "\${1:-}" in @@ -690,7 +713,17 @@ printf 'wait\\n' >>"$OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_LOG" exit 97 `, }); - const run = (procStat?: string) => { + const run = (procStat?: string, settled = false) => { + // The synthetic /proc observation and manager custody describe the same + // state: a zombie has retired; unreadable/malformed state stays owned. + writeFileSync( + `${daemonLog}.runtime.json`, + JSON.stringify({ + pid: 0, + supervisorPid: settled ? 0 : pid, + groupPid: 0, + }), + ); writeFileSync(fixtureEnv.OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_LOG, ""); return spawnSync("bash", [shimPath, "--user", "stop", "openclaw-gateway.service"], { encoding: "utf8", @@ -4131,7 +4164,8 @@ printf '%s\n' "$status" >"$TMPDIR/status" 'if (key.startsWith("OPENCLAW_UPDATE_")) {', "delete childEnv.OPENCLAW_COMPATIBILITY_HOST_VERSION;", 'process.on("SIGTERM", stop);', - "const stopTimeoutMs = 30_000;", + 'OPENCLAW_SYSTEMCTL_SHIM_MANAGER_SCRIPT="$manager_script"', + '[managerScript, "stop-timeout-ms"]', "process.kill(-pid, signal);", 'signalProcessGroup(pid, "SIGTERM");', 'signalProcessGroup(pid, "SIGKILL");', @@ -4143,7 +4177,9 @@ printf '%s\n' "$status" >"$TMPDIR/status" "const restartBurst = 5;", "if (starts.length >= restartBurst) {", "setTimeout(start, restartDelayMs);", - "for _ in $(seq 1 350)", + 'stop_timeout_ms="$(node "$manager_script" stop-timeout-ms)"', + "stop_timeout_ms + 5000 + 99", + 'node "$manager_script" check-stopped', ]); } for (const script of [runner, publishedRunner]) { @@ -4156,8 +4192,13 @@ printf '%s\n' "$status" >"$TMPDIR/status" async () => { await forEachUpgradeSurvivorSystemctlShim(({ pid, run, readLog, scriptPath }) => { const procTail = Array.from({ length: 49 }, (_, field) => field + 1).join(" "); - expect(run(`${pid} (gateway (old) worker) Z ${procTail}`), scriptPath).toBe(0); - expect(readLog()).toEqual(["--user stop openclaw-gateway.service", "proc-stat-read"]); + expect(run(`${pid} (gateway (old) worker) Z ${procTail}`, true), scriptPath).toBe(0); + expect(readLog()).toEqual([ + "--user stop openclaw-gateway.service", + "proc-stat-read", + "proc-stat-read", + ]); + expect(isProcessRunning(pid)).toBe(true); }); }, ); @@ -4862,6 +4903,7 @@ ${storage === "wal" ? 'process.kill(process.pid, "SIGKILL");' : ""}`, ...process.env, OPENCLAW_SYSTEMCTL_SHIM_DAEMON_LOG: logPath, OPENCLAW_SYSTEMCTL_SHIM_MANAGER_ENV: "{}", + OPENCLAW_SYSTEMCTL_SHIM_MANAGER_SCRIPT: writeUpgradeSurvivorStopPolicy(workDir), OPENCLAW_SYSTEMCTL_SHIM_EXEC_START: `${shellQuote(process.execPath)} ${shellQuote(childPath)}`, }, stdio: "ignore", @@ -5400,6 +5442,7 @@ exit 0 COUNT_FILE: countPath, OPENCLAW_SYSTEMCTL_SHIM_DAEMON_LOG: logPath, OPENCLAW_SYSTEMCTL_SHIM_MANAGER_ENV: "{}", + OPENCLAW_SYSTEMCTL_SHIM_MANAGER_SCRIPT: writeUpgradeSurvivorStopPolicy(workDir), OPENCLAW_SYSTEMCTL_SHIM_EXEC_START: command, }, stdio: "ignore", @@ -5422,10 +5465,7 @@ exit 0 const supervisorPath = join(workDir, `graceful-supervisor-${index}.mjs`); const statePath = join(workDir, `graceful-state-${index}`); const logPath = join(workDir, `graceful-daemon-${index}.log`); - const source = extractUpgradeSurvivorSupervisor(script).replace( - "const stopTimeoutMs = 30_000;", - "const stopTimeoutMs = 200;", - ); + const source = extractUpgradeSurvivorSupervisor(script); writeFileSync(supervisorPath, source); const command = @@ -5435,6 +5475,7 @@ exit 0 ...process.env, OPENCLAW_SYSTEMCTL_SHIM_DAEMON_LOG: logPath, OPENCLAW_SYSTEMCTL_SHIM_MANAGER_ENV: "{}", + OPENCLAW_SYSTEMCTL_SHIM_MANAGER_SCRIPT: writeUpgradeSurvivorStopPolicy(workDir, 200), OPENCLAW_SYSTEMCTL_SHIM_EXEC_START: command, STATE_FILE: statePath, }, @@ -5487,6 +5528,7 @@ process.exit(starts === 1 ? 1 : 78); OPENCLAW_CLAWHUB_URL: "http://127.0.0.1:43123", OPENCLAW_SYSTEMCTL_SHIM_DAEMON_LOG: logPath, OPENCLAW_SYSTEMCTL_SHIM_MANAGER_ENV: "{}", + OPENCLAW_SYSTEMCTL_SHIM_MANAGER_SCRIPT: writeUpgradeSurvivorStopPolicy(workDir), OPENCLAW_SYSTEMCTL_SHIM_EXEC_START: `${shellQuote(process.execPath)} ${shellQuote(gatewayPath)}`, URLS_FILE: urlsPath, }, @@ -5539,10 +5581,7 @@ setInterval(() => {}, 1_000); const statePath = join(workDir, `process-group-state-${index}`); const descendantPidPath = join(workDir, `process-group-descendant-${index}.pid`); const logPath = join(workDir, `process-group-daemon-${index}.log`); - const source = extractUpgradeSurvivorSupervisor(script).replace( - "const stopTimeoutMs = 30_000;", - "const stopTimeoutMs = 200;", - ); + const source = extractUpgradeSurvivorSupervisor(script); writeFileSync(supervisorPath, source); const supervisor = spawn(process.execPath, [supervisorPath], { @@ -5552,6 +5591,7 @@ setInterval(() => {}, 1_000); DESCENDANT_SCRIPT: descendantPath, OPENCLAW_SYSTEMCTL_SHIM_DAEMON_LOG: logPath, OPENCLAW_SYSTEMCTL_SHIM_MANAGER_ENV: "{}", + OPENCLAW_SYSTEMCTL_SHIM_MANAGER_SCRIPT: writeUpgradeSurvivorStopPolicy(workDir, 200), OPENCLAW_SYSTEMCTL_SHIM_EXEC_START: `${shellQuote(process.execPath)} ${shellQuote(gatewayPath)}`, STATE_FILE: statePath, }, @@ -5625,9 +5665,10 @@ if (starts === 1) { const descendantPidPath = join(workDir, `restart-group-descendant-${index}.pid`); const replacementPath = join(workDir, `restart-group-replacement-${index}`); const logPath = join(workDir, `restart-group-daemon-${index}.log`); - const source = extractUpgradeSurvivorSupervisor(script) - .replace("const restartDelayMs = 5_000;", "const restartDelayMs = 5;") - .replace("const stopTimeoutMs = 30_000;", "const stopTimeoutMs = 200;"); + const source = extractUpgradeSurvivorSupervisor(script).replace( + "const restartDelayMs = 5_000;", + "const restartDelayMs = 5;", + ); writeFileSync(supervisorPath, source); const supervisor = spawn(process.execPath, [supervisorPath], { @@ -5637,6 +5678,7 @@ if (starts === 1) { DESCENDANT_SCRIPT: descendantPath, OPENCLAW_SYSTEMCTL_SHIM_DAEMON_LOG: logPath, OPENCLAW_SYSTEMCTL_SHIM_MANAGER_ENV: "{}", + OPENCLAW_SYSTEMCTL_SHIM_MANAGER_SCRIPT: writeUpgradeSurvivorStopPolicy(workDir, 200), OPENCLAW_SYSTEMCTL_SHIM_EXEC_START: `${shellQuote(process.execPath)} ${shellQuote(gatewayPath)}`, REPLACEMENT_FILE: replacementPath, STARTS_FILE: startsPath, diff --git a/test/scripts/upgrade-survivor-stop-policy.test.ts b/test/scripts/upgrade-survivor-stop-policy.test.ts new file mode 100644 index 000000000000..013928e429e5 --- /dev/null +++ b/test/scripts/upgrade-survivor-stop-policy.test.ts @@ -0,0 +1,468 @@ +import { spawnSync } from "node:child_process"; +import { existsSync, mkdirSync, readFileSync, rmSync, watch, writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import vm from "node:vm"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../helpers/temp-dir.js"; + +const tempDirs = useAutoCleanupTempDirTracker(afterEach); +const owner = resolve("scripts/e2e/lib/upgrade-survivor/update-restart-auth.sh"); +function fixture() { + const home = tempDirs.make("survivor-stop-policy-"); + const env = { HOME: home, npm_config_prefix: home, PATH: process.env.PATH }; + const installed = spawnSync( + "bash", + [ + "-c", + 'set -euo pipefail; source "$1"; install_update_restart_systemctl_shim', + "fixture", + owner, + ], + { env, encoding: "utf8" }, + ); + expect(installed.status, installed.stderr).toBe(0); + const unit = join(home, ".config/systemd/user/openclaw-gateway.service"); + mkdirSync(join(home, ".config/systemd/user"), { recursive: true }); + const systemctl = (...args: string[]) => + spawnSync(join(home, "bin/systemctl"), ["--user", ...args], { env, encoding: "utf8" }); + const manager = (...args: string[]) => + spawnSync(process.execPath, [join(home, "bin/systemd-fixture.mjs"), ...args], { + env, + encoding: "utf8", + }); + return { home, env, unit, systemctl, manager }; +} + +function waitForFixtureState(directory: string, settled: () => boolean) { + return new Promise((complete, reject) => { + const inspect = () => { + try { + if (!settled()) { + return; + } + } catch { + return; + } + watcher.close(); + clearTimeout(deadline); + complete(); + }; + const watcher = watch(directory, inspect); + const deadline = setTimeout(() => { + watcher.close(); + reject(new Error("fixture state did not settle")); + }, 5_000); + inspect(); + }); +} + +describe.skipIf(process.platform === "win32")("survivor loaded stop policy", () => { + it("reports the loaded stop policy until daemon-reload", () => { + const { unit, systemctl, manager } = fixture(); + const query = () => + systemctl( + "show", + "openclaw-gateway.service", + "--no-page", + "--property", + "LoadState,TimeoutStopUSec", + ); + expect(query().stdout).toBe("LoadState=not-found\n"); + const content = "[Service]\nExecStart=/usr/bin/true\nTimeoutStopSec=330\n"; + writeFileSync(unit, content); + expect(systemctl("daemon-reload").status).toBe(0); + expect(query()).toMatchObject({ + status: 0, + stdout: "LoadState=loaded\nTimeoutStopUSec=330s\n", + }); + expect(manager("stop-timeout-ms").stdout).toBe("330000\n"); + writeFileSync(unit, content.replace("TimeoutStopSec=330", "TimeoutStopSec=30")); + expect(query().stdout).toContain("TimeoutStopUSec=330s"); + expect(manager("stop-timeout-ms").stdout).toBe("330000\n"); + expect(systemctl("daemon-reload").status).toBe(0); + expect(query().stdout).toContain("TimeoutStopUSec=30s"); + expect(manager("stop-timeout-ms").stdout).toBe("30000\n"); + writeFileSync(unit, content.replace("TimeoutStopSec=330", "TimeoutStopSec=invalid")); + expect(systemctl("daemon-reload").status).not.toBe(0); + expect(query().stdout).toContain("TimeoutStopUSec=30s"); + expect(manager("stop-timeout-ms").stdout).toBe("30000\n"); + rmSync(unit); + const runtime = systemctl( + "show", + "openclaw-gateway.service", + "--property", + "Id,LoadState,ActiveState,SubState,Result,NRestarts,StartLimitBurst,MainPID,ExecMainStatus,ExecMainCode,KillMode,TasksCurrent,MemoryCurrent", + ); + expect(runtime.status, runtime.stderr).toBe(0); + expect(runtime.stdout).toContain("LoadState=loaded"); + expect(query().stdout).toContain("TimeoutStopUSec=30s"); + expect(systemctl("daemon-reload").status).toBe(0); + expect(query().stdout).toBe("LoadState=not-found\n"); + expect(manager("stop-timeout-ms").status).not.toBe(0); + }); + + it.each([ + { policy: "", value: "90000", display: "90s" }, + { policy: "TimeoutStopSec=0", value: "Infinity", display: "infinity" }, + { policy: "TimeoutStopSec=infinity", value: "Infinity", display: "infinity" }, + ])("handles generated stop policy $policy deliberately", ({ policy, value, display }) => { + const { unit, systemctl, manager } = fixture(); + writeFileSync(unit, "[Service]\nExecStart=/usr/bin/true\n" + policy + "\n"); + expect(systemctl("daemon-reload").status).toBe(0); + expect(manager("stop-timeout-ms")).toMatchObject({ status: 0, stdout: value + "\n" }); + expect( + systemctl("show", "openclaw-gateway.service", "--property=LoadState,TimeoutStopUSec").stdout, + ).toBe("LoadState=loaded\nTimeoutStopUSec=" + display + "\n"); + }); + + it.each(["-1", "bogus", "1ms", "2147484", "9".repeat(400), "30\nTimeoutStopSec=330"])( + "refuses unsupported stop policy %j at load", + (policy) => { + const { unit, systemctl } = fixture(); + writeFileSync(unit, "[Service]\nExecStart=/usr/bin/true\nTimeoutStopSec=" + policy + "\n"); + expect(systemctl("daemon-reload").status).not.toBe(0); + expect(existsSync(unit + ".loaded-unit")).toBe(false); + }, + ); + + // Execute the unchanged supervisor body with native boundaries substituted. The + // virtual clock advances policy-sized deadlines without launching or killing PIDs. + it.each([30, 330, "infinity"] as const)( + "uses loaded %s seconds for supervisor and outer stop", + (seconds) => { + const f = fixture(); + writeFileSync(f.unit, "[Service]\nExecStart=/usr/bin/true\nTimeoutStopSec=" + seconds + "\n"); + expect(f.systemctl("daemon-reload").status).toBe(0); + // An un-reloaded edit must not affect either consumer. + writeFileSync(f.unit, "[Service]\nExecStart=/usr/bin/true\nTimeoutStopSec=1\n"); + if (seconds === 330) { + writeFileSync(f.unit, "[Service]\nExecStart=/usr/bin/true\nTimeoutStopSec=30\n"); + expect(f.systemctl("daemon-reload").status).toBe(0); + } + const source = readFileSync(owner, "utf8"); + const body = source.split("<<'SUPERVISOR'\n")[1]?.split("\nSUPERVISOR")[0]; + expect(body).toBeDefined(); + const signals: Array = []; + const handlers = new Map void>(); + const timers = new Map void }>(); + let now = 0; + let serial = 0; + let alive = true; + const exit = vi.fn(); + const fs = { + openSync: () => 1, + closeSync: () => {}, + writeFileSync: () => {}, + renameSync: () => {}, + writeSync: () => {}, + }; + vm.runInNewContext(body!.replace(/^import .*;\n/gm, ""), { + fs, + spawn: () => ({ pid: 42, on: () => {}, once: () => {} }), + execFileSync: (_binary: string, args: string[]) => { + expect(args).toEqual([join(f.home, "bin/systemd-fixture.mjs"), "stop-timeout-ms"]); + const result = f.manager("stop-timeout-ms"); + expect(result.status, result.stderr).toBe(0); + return result.stdout; + }, + process: { + pid: 43, + execPath: process.execPath, + env: { + OPENCLAW_SYSTEMCTL_SHIM_MANAGER_SCRIPT: join(f.home, "bin/systemd-fixture.mjs"), + OPENCLAW_SYSTEMCTL_SHIM_EXEC_START: "/usr/bin/true", + OPENCLAW_SYSTEMCTL_SHIM_DAEMON_LOG: "/fixture/log", + OPENCLAW_SYSTEMCTL_SHIM_MANAGER_ENV: "{}", + }, + hrtime: { bigint: () => 1n }, + exit, + on: (signal: string, handler: () => void) => handlers.set(signal, handler), + kill: (pid: number, signal: string | number) => { + expect(pid).toBe(-42); + if (signal === 0) { + if (!alive) { + throw Object.assign(new Error("gone"), { code: "ESRCH" }); + } + } else { + signals.push(signal); + } + }, + }, + setTimeout: (callback: () => void, delay: number) => { + const id = ++serial; + timers.set(id, { at: now + delay, callback }); + return id; + }, + clearTimeout: (id: number) => timers.delete(id), + }); + const advance = (until: number) => { + while (true) { + const next = [...timers].toSorted((a, b) => a[1].at - b[1].at)[0]; + if (!next || next[1].at > until) { + break; + } + now = next[1].at; + timers.delete(next[0]); + next[1].callback(); + } + now = until; + }; + if (seconds === 330) { + // Repair the manager policy after supervisor launch, before its stop. + writeFileSync(f.unit, "[Service]\nExecStart=/usr/bin/true\nTimeoutStopSec=330\n"); + expect(f.systemctl("daemon-reload").status).toBe(0); + writeFileSync(f.unit, "[Service]\nExecStart=/usr/bin/true\nTimeoutStopSec=1\n"); + } + handlers.get(seconds === 30 ? "SIGINT" : "SIGTERM")!(); + const deadline = seconds === "infinity" ? 400_000 : seconds * 1_000; + advance(deadline - 1); + expect(signals).toEqual(["SIGTERM"]); + expect(exit).not.toHaveBeenCalled(); + advance(deadline); + expect(signals).toEqual(seconds === "infinity" ? ["SIGTERM"] : ["SIGTERM", "SIGKILL"]); + // Sending SIGKILL is not extinction; settlement waits for the group observer. + expect(exit).not.toHaveBeenCalled(); + alive = false; + advance(deadline + 25); + expect(exit).toHaveBeenCalledExactlyOnceWith(0); + expect(timers.size).toBe(0); + + const stop = source.match(/stop_gateway\(\) \{[\s\S]*?\n\}/)?.[0]; + expect(stop).toBeDefined(); + const pidFile = join(f.home, "pid"); + writeFileSync(pidFile, "999999\n"); + const outer = spawnSync( + "bash", + [ + "-c", + [ + "set -euo pipefail", + stop, + "pid_file=$1; supervisor_script=$1.supervisor; manager_script=$2", + "ticks=0; kill() { :; }; sleep() { ticks=$((ticks + 1)); }", + "is_running() { [ $ticks -lt 4000 ]; }", + "status=0; stop_gateway || status=$?; printf '%s' $ticks; exit $status", + ].join("\n"), + "fixture", + pidFile, + join(f.home, "bin/systemd-fixture.mjs"), + ], + { env: f.env, encoding: "utf8" }, + ); + expect(outer.status, outer.stderr).toBe(seconds === "infinity" ? 0 : 1); + if (seconds !== "infinity") { + expect(outer.stderr).toContain("retaining process custody"); + expect(existsSync(pidFile)).toBe(true); + } + expect(Number(outer.stdout)).toBe( + seconds === "infinity" ? 4000 : (seconds * 1_000 + 5_000) / 100, + ); + }, + ); + + it.each(["signal", "child-close"])( + "retains custody when the policy read fails during %s drain", + (entry) => { + const source = readFileSync(owner, "utf8"); + const body = source.split("<<'SUPERVISOR'\n")[1]?.split("\nSUPERVISOR")[0]; + expect(body).toBeDefined(); + const handlers = new Map void>(); + let closeChild: ((code: number, signal: string | null) => void) | undefined; + const timers: Array<() => void> = []; + const signals: Array = []; + let alive = true; + const exit = vi.fn(); + const log = vi.fn(); + const files = new Map(); + const spawn = vi.fn(() => ({ + pid: 42, + on: () => {}, + once: (_event: string, callback: (code: number, signal: string | null) => void) => { + closeChild = callback; + }, + })); + vm.runInNewContext(body!.replace(/^import .*;\n/gm, ""), { + fs: { + openSync: () => 1, + closeSync: () => {}, + writeSync: log, + writeFileSync: (file: string, contents: string) => files.set(file, contents), + renameSync: (from: string, to: string) => files.set(to, files.get(from)!), + }, + spawn, + execFileSync: () => { + throw new Error("policy read failed"); + }, + process: { + pid: 43, + execPath: process.execPath, + cwd: () => "/fixture", + env: { + OPENCLAW_SYSTEMCTL_SHIM_MANAGER_SCRIPT: "/fixture/manager.mjs", + OPENCLAW_SYSTEMCTL_SHIM_EXEC_START: "/usr/bin/true", + OPENCLAW_SYSTEMCTL_SHIM_DAEMON_LOG: "/fixture/log", + OPENCLAW_SYSTEMCTL_SHIM_MANAGER_ENV: "{}", + }, + hrtime: { bigint: () => 1n }, + exit, + on: (signal: string, callback: () => void) => handlers.set(signal, callback), + kill: (pid: number, signal: string | number) => { + expect(pid).toBe(-42); + if (signal === 0) { + if (!alive) { + throw Object.assign(new Error("gone"), { code: "ESRCH" }); + } + } else { + signals.push(signal); + } + }, + }, + setTimeout: (callback: () => void) => { + timers.push(callback); + return timers.length; + }, + clearTimeout: () => {}, + }); + const beginDrain = () => + entry === "signal" ? handlers.get("SIGTERM")!() : closeChild!(0, null); + expect(beginDrain).not.toThrow(); + expect(signals).toEqual(["SIGTERM", "SIGKILL"]); + expect(log.mock.calls.flat().join(" ")).toContain("stop policy read failed"); + expect(exit).not.toHaveBeenCalled(); + expect(JSON.parse(files.get("/fixture/log.runtime.json")!)).toMatchObject({ + groupPid: 42, + supervisorPid: 43, + }); + // A failed forced kill cannot turn into a clean exit or restart. + timers.shift()!(); + expect(exit).not.toHaveBeenCalled(); + alive = false; + timers.shift()!(); + expect(exit).toHaveBeenCalledExactlyOnceWith(1); + expect(spawn).toHaveBeenCalledOnce(); + const runtime = JSON.parse(files.get("/fixture/log.runtime.json")!); + expect(runtime).toMatchObject({ pid: 0, groupPid: 0, supervisorPid: 0, stopFailed: true }); + const f = fixture(); + writeFileSync( + join(f.home, "bin/systemctl-shim-gateway.log.runtime.json"), + JSON.stringify(runtime), + ); + const stopped = f.systemctl("stop", "openclaw-gateway.service"); + expect(stopped.status).toBe(1); + expect(stopped.stderr).toContain("stop policy read failed"); + }, + ); + + it.each([false, true])( + "joins the installed outer stop after removed-unit reload=%s", + async (removed) => { + const f = fixture(); + const bin = join(f.home, "bin"); + const ready = join(f.home, "ready"); + const program = join(f.home, "child.mjs"); + const runtimeFile = join(bin, "systemctl-shim-gateway.log.runtime.json"); + const runtime = () => JSON.parse(readFileSync(runtimeFile, "utf8")); + writeFileSync( + program, + 'import fs from "node:fs"; process.on("SIGTERM", () => ' + + (removed ? "{}" : "process.exit(0)") + + "); fs.writeFileSync(" + + JSON.stringify(ready) + + ", String(process.pid)); setInterval(() => {}, 1000);", + ); + writeFileSync( + f.unit, + [ + "[Service]", + "ExecStart=" + JSON.stringify(process.execPath) + " " + JSON.stringify(program), + "TimeoutStopSec=330", + "", + ].join("\n"), + ); + try { + const started = f.systemctl("start", "openclaw-gateway.service"); + expect(started.status, started.stderr).toBe(0); + await waitForFixtureState( + f.home, + () => existsSync(ready) && readFileSync(ready, "utf8").length > 0, + ); + const pid = Number(readFileSync(ready, "utf8")); + if (removed) { + rmSync(f.unit); + expect(f.systemctl("daemon-reload").status).toBe(0); + expect(existsSync(f.unit + ".loaded-unit")).toBe(false); + } + const stopped = f.systemctl("stop", "openclaw-gateway.service"); + expect(stopped.status, stopped.stderr).toBe(removed ? 1 : 0); + expect(runtime()).toMatchObject({ + pid: 0, + groupPid: 0, + supervisorPid: 0, + stopFailed: removed, + }); + expect(() => process.kill(-pid, 0)).toThrow(); + if (removed) { + expect(stopped.stderr).toContain("stop policy read failed"); + } + } finally { + if (existsSync(runtimeFile)) { + const owned = runtime(); + // Failed proof must still retire only this fixture's published processes. + if (owned.supervisorPid) { + try { + process.kill(owned.supervisorPid, "SIGTERM"); + } catch {} + } + if (owned.groupPid) { + try { + process.kill(-owned.groupPid, "SIGKILL"); + } catch {} + } + await waitForFixtureState(bin, () => { + const observed = runtime(); + return observed.pid === 0 && observed.supervisorPid === 0 && observed.groupPid === 0; + }); + } + } + }, + ); + + it.each([true, false])( + "keeps outer policy lookup failure nonzero with supervisor active=%s", + (active) => { + const f = fixture(); + const stop = readFileSync(owner, "utf8").match(/stop_gateway\(\) \{[\s\S]*?\n\}/)?.[0]; + expect(stop).toBeDefined(); + const pid = join(f.home, "pid"); + const supervisor = join(f.home, "supervisor"); + const signals = join(f.home, "signals"); + writeFileSync(pid, "424242\n"); + writeFileSync(supervisor, "owned"); + const result = spawnSync( + "bash", + [ + "-c", + [ + "set -euo pipefail", + stop, + "pid_file=$1; supervisor_script=$2; signal_log=$3; manager_script=fixture-manager", + 'ticks=0; kill() { printf "%s\\n" "$*" >> "$signal_log"; }; sleep() { ticks=$((ticks+1)); }', + 'node() { [ "$2" != stop-timeout-ms ] || return 17; [ "$2" = check-stopped ]; }', + active ? "is_running() { return 0; }" : "is_running() { return 1; }", + 'status=0; stop_gateway || status=$?; printf "%s %s\\n" "$status" "$ticks"', + ].join("\n"), + "fixture", + pid, + supervisor, + signals, + ], + { env: f.env, encoding: "utf8" }, + ); + expect(result.status, result.stderr).toBe(0); + expect(result.stdout.trim()).toBe(active ? "1 50" : "17 0"); + expect(readFileSync(signals, "utf8").trim().split("\n")).toEqual(["-0 424242", "424242"]); + expect(existsSync(pid)).toBe(true); + expect(existsSync(supervisor)).toBe(true); + }, + ); +}); diff --git a/test/scripts/upgrade-survivor-systemd-cgroup.test.ts b/test/scripts/upgrade-survivor-systemd-cgroup.test.ts index d19521f77081..308cb165668e 100644 --- a/test/scripts/upgrade-survivor-systemd-cgroup.test.ts +++ b/test/scripts/upgrade-survivor-systemd-cgroup.test.ts @@ -83,4 +83,15 @@ it.each([ await import("../../scripts/e2e/lib/upgrade-survivor/systemd-fixture.mjs"); expect(error).not.toHaveBeenCalled(); expect(JSON.parse(String(log.mock.calls.at(-1)?.[0]))).toEqual({ type: "s", data: row.expected }); + + vi.resetModules(); + process.exitCode = 0; + process.argv = ["node", "fixture", "check-stopped"]; + await import("../../scripts/e2e/lib/upgrade-survivor/systemd-fixture.mjs"); + expect(process.exitCode).toBe(row.pid || row.populated ? 1 : 0); + if (row.pid || row.populated) { + expect(error).toHaveBeenCalledWith("Survivor service processes have not settled."); + } else { + expect(error).not.toHaveBeenCalled(); + } }); diff --git a/test/scripts/upgrade-survivor-systemd.test.ts b/test/scripts/upgrade-survivor-systemd.test.ts index 27a27dd99c5f..1a5d483ba1a0 100644 --- a/test/scripts/upgrade-survivor-systemd.test.ts +++ b/test/scripts/upgrade-survivor-systemd.test.ts @@ -290,6 +290,7 @@ fs.existsSync = (file) => file === "/sys/fs/cgroup/openclaw-gateway.service/cgro ); expect(await readSystemdServiceRuntime(env)).toMatchObject({ status: "unknown" }); rmSync(unit); + expect(systemctl("daemon-reload").status).toBe(0); expect(await readSystemdServiceExecStart(env, { requireEffective: true })).toBeNull(); expect(await readSystemdServiceRuntime(env)).toMatchObject({ status: "stopped",