mirror of
https://github.com/openclaw/openclaw.git
synced 2026-09-28 14:12:28 +08:00
fix(tooling): enforce Linux process-tree memory limits (#160024)
Add opt-in aggregate Linux process-tree memory containment to the managed tooling runner. Own the invocation-specific scope through verified cleanup before releasing its resource claim; preserve uncapped caller behavior. Validated real-kernel OOM, descendant cleanup, signals, cancellation and packaged launcher behavior; focused lifecycle/preload tests and exact-head CI/security gates passed. Related: #160010.
This commit is contained in:
Vincent Koc
@@ -8,6 +8,7 @@ import type {
|
||||
StdioOptions,
|
||||
} from "node:child_process";
|
||||
import { constants as osConstants, tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { Writable, type Readable } from "node:stream";
|
||||
import { buildCmdExeCommandLine, resolveWindowsCmdExePath } from "../windows-cmd-helpers.mjs";
|
||||
import type { ManagedWindowsJob } from "./managed-windows-job.mts";
|
||||
@@ -74,7 +75,9 @@ type ManagedCommandOptions = {
|
||||
comSpec?: string;
|
||||
};
|
||||
|
||||
type RunManagedCommandOptions = ManagedCommandOptions & {
|
||||
export type RunManagedCommandOptions = ManagedCommandOptions & {
|
||||
memoryLimitBytes?: number;
|
||||
onMemoryScope?: (unit: string) => void;
|
||||
timeoutMs?: number;
|
||||
timeoutKillGraceMs?: number;
|
||||
signalKillGraceMs?: number;
|
||||
@@ -450,22 +453,108 @@ export async function waitForManagedProcessGroupExit(
|
||||
}
|
||||
|
||||
/** Run a child command while forwarding termination signals to its process group. */
|
||||
export async function runManagedCommand({
|
||||
stdio = "inherit",
|
||||
platform = process.platform,
|
||||
timeoutMs,
|
||||
timeoutKillGraceMs,
|
||||
signalKillGraceMs,
|
||||
timeoutForceKillOnLeaderExit = false,
|
||||
requireProcessTreeExit = false,
|
||||
runTaskkill = spawnSync,
|
||||
onReady,
|
||||
signal,
|
||||
abortKillGraceMs,
|
||||
cleanupDrainTimeoutMs,
|
||||
onSignal,
|
||||
...commandOptions
|
||||
}: RunManagedCommandOptions) {
|
||||
export async function runManagedCommand(options: RunManagedCommandOptions): Promise<number> {
|
||||
const { memoryLimitBytes } = options;
|
||||
if (memoryLimitBytes !== undefined) {
|
||||
if (!Number.isSafeInteger(memoryLimitBytes) || memoryLimitBytes <= 0) {
|
||||
throw new Error("Managed command memory limit must be a positive integer");
|
||||
}
|
||||
const platform = options.platform ?? process.platform;
|
||||
if (platform === "linux") {
|
||||
if (
|
||||
Array.isArray(options.stdio) &&
|
||||
(options.stdio.length > 3 || options.stdio.includes("ipc"))
|
||||
) {
|
||||
throw new Error(
|
||||
"Linux memory-limited commands do not support IPC or extra stdio descriptors",
|
||||
);
|
||||
}
|
||||
// Preserve spawn's input snapshot while the Linux containment module loads.
|
||||
const command = {
|
||||
...options,
|
||||
args: options.args?.slice(),
|
||||
cwd: path.resolve(options.cwd ?? process.cwd()),
|
||||
env: { ...(options.env ?? process.env) },
|
||||
stdio: Array.isArray(options.stdio) ? [...options.stdio] : options.stdio,
|
||||
};
|
||||
const { runLinuxMemoryCommand } = await import("./managed-memory.mts");
|
||||
// The cgroup owner can prove extinction after a process-group cleanup failure.
|
||||
// Its resource claim therefore outlives the inner runner's weaker observation.
|
||||
const env = command.env;
|
||||
const releaseClaim = findVitestResourceOwner(
|
||||
env.TMPDIR || env.TMP || env.TEMP || tmpdir(),
|
||||
)?.claim();
|
||||
let joined = true;
|
||||
let leafActive = false;
|
||||
let receivedSignal: NodeJS.Signals | undefined;
|
||||
// The leaf owns delivery and grace. Keep the outer owner alive after it
|
||||
// exits, while the cgroup still joins detached members and releases claims.
|
||||
const rememberSignal = (received: NodeJS.Signals) => {
|
||||
receivedSignal ??= received;
|
||||
if (!leafActive) {
|
||||
command.onSignal?.(received);
|
||||
}
|
||||
};
|
||||
installSignalHandlers();
|
||||
managedChildren.add(rememberSignal);
|
||||
try {
|
||||
const status = await runLinuxMemoryCommand(command, async (scopedCommand) => {
|
||||
leafActive = true;
|
||||
try {
|
||||
return await runManagedCommandInner(scopedCommand, false);
|
||||
} finally {
|
||||
leafActive = false;
|
||||
}
|
||||
});
|
||||
if (receivedSignal) {
|
||||
return signalExitCode(receivedSignal);
|
||||
}
|
||||
if (command.signal?.aborted) {
|
||||
throw Object.assign(new Error("Managed command aborted"), { code: "ABORT_ERR" });
|
||||
}
|
||||
return status;
|
||||
} catch (error) {
|
||||
joined = !hasUnjoinedWork(error);
|
||||
throw error;
|
||||
} finally {
|
||||
try {
|
||||
if (joined) {
|
||||
releaseClaim?.();
|
||||
}
|
||||
} finally {
|
||||
managedChildren.delete(rememberSignal);
|
||||
removeSignalHandlersIfIdle();
|
||||
}
|
||||
}
|
||||
}
|
||||
throw new Error(
|
||||
"Semantic checks require verified kernel memory containment. Run this command through Crabbox or a memory-limited Linux VM; native containment is not qualified on this platform.",
|
||||
);
|
||||
}
|
||||
return await runManagedCommandInner(options);
|
||||
}
|
||||
|
||||
async function runManagedCommandInner(
|
||||
{
|
||||
stdio = "inherit",
|
||||
platform = process.platform,
|
||||
memoryLimitBytes: _memoryLimitBytes,
|
||||
onMemoryScope: _onMemoryScope,
|
||||
timeoutMs,
|
||||
timeoutKillGraceMs,
|
||||
signalKillGraceMs,
|
||||
timeoutForceKillOnLeaderExit = false,
|
||||
requireProcessTreeExit = false,
|
||||
runTaskkill = spawnSync,
|
||||
onReady,
|
||||
signal,
|
||||
abortKillGraceMs,
|
||||
cleanupDrainTimeoutMs,
|
||||
onSignal,
|
||||
...commandOptions
|
||||
}: RunManagedCommandOptions,
|
||||
claimResources = true,
|
||||
) {
|
||||
if (platform === "win32" && requireProcessTreeExit) {
|
||||
throw Object.assign(
|
||||
new Error("Strict managed process-tree verification is not supported on Windows"),
|
||||
@@ -506,9 +595,11 @@ export async function runManagedCommand({
|
||||
const loading = loadManagedChildSpawner(platform);
|
||||
const spawnManagedChild = typeof loading === "function" ? loading : await loading;
|
||||
signal?.throwIfAborted();
|
||||
let releaseClaim = findVitestResourceOwner(
|
||||
commandEnv.TMPDIR || commandEnv.TMP || commandEnv.TEMP || tmpdir(),
|
||||
)?.claim();
|
||||
let releaseClaim = claimResources
|
||||
? findVitestResourceOwner(
|
||||
commandEnv.TMPDIR || commandEnv.TMP || commandEnv.TEMP || tmpdir(),
|
||||
)?.claim()
|
||||
: undefined;
|
||||
const releaseOwnership = () => {
|
||||
releaseClaim?.();
|
||||
releaseClaim = undefined;
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
import { extname } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
export const managedMemoryEntrypoint = {
|
||||
currentModuleUrl: import.meta.url,
|
||||
sourceWorkerName: "managed-memory-launcher",
|
||||
sourceExtension: ".mts",
|
||||
distWorkerPath: "tooling/managed-memory-launcher.js",
|
||||
} as const;
|
||||
|
||||
/** Resolve the standalone launcher in source checkouts and packaged tooling. */
|
||||
export function resolveManagedMemoryEntrypointUrl(): URL {
|
||||
const current = new URL(import.meta.url);
|
||||
const distIndex = current.pathname.lastIndexOf("/dist/");
|
||||
return distIndex < 0
|
||||
? new URL(`./managed-memory-launcher${extname(fileURLToPath(current))}`, current)
|
||||
: new URL(
|
||||
current.pathname.slice(0, distIndex + 6) + managedMemoryEntrypoint.distWorkerPath,
|
||||
current,
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { signalExitCode } from "./managed-child-process.mts";
|
||||
import { hasLinuxMemoryContainment } from "./process-memory.mts";
|
||||
|
||||
const [rawLimit, scope, shell, bin, ...args] = process.argv.slice(2);
|
||||
const maxBytes = Number(rawLimit);
|
||||
if (
|
||||
!scope ||
|
||||
!bin ||
|
||||
(shell !== "true" && shell !== "false") ||
|
||||
!Number.isSafeInteger(maxBytes) ||
|
||||
maxBytes <= 0 ||
|
||||
!hasLinuxMemoryContainment(maxBytes, {}, scope)
|
||||
) {
|
||||
console.error(
|
||||
"[memory] The owned cgroup has no verified memory/swap limit; semantic command was not started. Use a cgroup-v2 host with a user systemd manager or a bounded Crabbox.",
|
||||
);
|
||||
process.exitCode = 75;
|
||||
} else {
|
||||
const env: NodeJS.ProcessEnv = {
|
||||
...process.env,
|
||||
NODE_OPTIONS: process.env.OPENCLAW_MANAGED_NODE_OPTIONS ?? "",
|
||||
};
|
||||
delete env.OPENCLAW_MANAGED_NODE_OPTIONS;
|
||||
// The outer cgroup owner owns descendants and resource receipts, including OOM.
|
||||
// Keep both processes in the outer group: it owns signal delivery and grace.
|
||||
const child = spawn(bin, args, { env, stdio: "inherit", shell: shell === "true" });
|
||||
let received: NodeJS.Signals | undefined;
|
||||
let exitSignal: NodeJS.Signals | undefined;
|
||||
const remember = (signal: NodeJS.Signals) => {
|
||||
received ??= signal;
|
||||
};
|
||||
const handlers = (["SIGINT", "SIGTERM", "SIGHUP"] as const).map(
|
||||
(signal) => [signal, () => remember(signal)] as const,
|
||||
);
|
||||
for (const [signal, handler] of handlers) {
|
||||
process.on(signal, handler);
|
||||
}
|
||||
process.exitCode = await new Promise<number>((resolve) => {
|
||||
child.once("error", (error) => {
|
||||
console.error(error);
|
||||
resolve(75);
|
||||
});
|
||||
child.once("exit", (status, signal) => {
|
||||
exitSignal = received ?? signal ?? undefined;
|
||||
resolve(exitSignal ? signalExitCode(exitSignal) : (status ?? 75));
|
||||
});
|
||||
});
|
||||
for (const [signal, handler] of handlers) {
|
||||
process.off(signal, handler);
|
||||
}
|
||||
// The outer owner distinguishes a signal from numeric 143 to let surviving
|
||||
// descendants drain gracefully. Removing a signal listener restores its default
|
||||
// disposition in libuv, including Node's special SIGPIPE/SIGUSR1 dispositions.
|
||||
if (exitSignal) {
|
||||
if (exitSignal !== "SIGKILL" && exitSignal !== "SIGSTOP") {
|
||||
const reset = () => {};
|
||||
process.on(exitSignal, reset);
|
||||
process.off(exitSignal, reset);
|
||||
}
|
||||
process.kill(process.pid, exitSignal);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { hasUnjoinedWork, type RunManagedCommandOptions } from "./managed-child-process.mts";
|
||||
import { resolveManagedMemoryEntrypointUrl } from "./managed-memory-entrypoint.mts";
|
||||
|
||||
/** systemd owns the cgroup; the managed child owner still owns signals and output. */
|
||||
export async function runLinuxMemoryCommand(
|
||||
options: RunManagedCommandOptions,
|
||||
run: (options: RunManagedCommandOptions) => Promise<number>,
|
||||
) {
|
||||
const { memoryLimitBytes, onMemoryScope, ...command } = options;
|
||||
// Scope names belong to this invocation. A caller-selected name can race
|
||||
// creation and let a failed contender stop another command during cleanup.
|
||||
const unit = "openclaw-check-" + randomUUID() + ".scope";
|
||||
options.signal?.throwIfAborted();
|
||||
const control = (args: string[]) =>
|
||||
spawnSync("systemctl", ["--user", ...args, unit], {
|
||||
encoding: "utf8",
|
||||
timeout: 5_000,
|
||||
killSignal: "SIGKILL",
|
||||
env: options.env,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
// Qualification precedes creation: unavailable backends must not strand admission.
|
||||
const initial = control(["show", "--property=LoadState"]);
|
||||
if (initial.error || !initial.stdout?.includes("LoadState=not-found")) {
|
||||
throw new Error(
|
||||
"[memory] A cgroup-v2 systemd user manager is required. Use a bounded Crabbox worker.",
|
||||
);
|
||||
}
|
||||
onMemoryScope?.(unit);
|
||||
options.signal?.throwIfAborted();
|
||||
const env = { ...(options.env ?? process.env) };
|
||||
// The trusted launcher verifies kernel limits before restoring workload preloads.
|
||||
env.OPENCLAW_MANAGED_NODE_OPTIONS = env.NODE_OPTIONS ?? "";
|
||||
delete env.NODE_OPTIONS;
|
||||
const cleanupScope = async (failure: unknown) => {
|
||||
// Inner cleanup has finished its grace period. A stop-client timeout does not
|
||||
// escalate systemd's longer stop timer, so kill any remaining owned descendants.
|
||||
control(["kill", "--kill-whom=all", "--signal=SIGKILL"]);
|
||||
control(["stop"]);
|
||||
const deadline = Date.now() + 5_000;
|
||||
let empty: boolean;
|
||||
do {
|
||||
const state = control([
|
||||
"show",
|
||||
"--property=LoadState",
|
||||
"--property=ActiveState",
|
||||
"--property=ControlGroup",
|
||||
]);
|
||||
const fields = Object.fromEntries(
|
||||
(state.stdout ?? "")
|
||||
.trim()
|
||||
.split("\n")
|
||||
.map((line) => line.split("=")),
|
||||
);
|
||||
empty = !state.error && fields.LoadState === "not-found";
|
||||
if (!state.error && fields.ControlGroup?.endsWith("/" + unit)) {
|
||||
try {
|
||||
empty = /^populated 0$/mu.test(
|
||||
fs.readFileSync(
|
||||
path.join("/sys/fs/cgroup", fields.ControlGroup, "cgroup.events"),
|
||||
"utf8",
|
||||
),
|
||||
);
|
||||
} catch (error) {
|
||||
empty = Boolean(
|
||||
error &&
|
||||
typeof error === "object" &&
|
||||
"code" in error &&
|
||||
error.code === "ENOENT" &&
|
||||
fields.ActiveState === "inactive",
|
||||
);
|
||||
}
|
||||
}
|
||||
if (empty) {
|
||||
break;
|
||||
}
|
||||
await delay(50);
|
||||
} while (Date.now() < deadline);
|
||||
// A failed scope can still contain descendants after a SIGKILL timeout.
|
||||
// Only kernel extinction or manager-confirmed removal releases admission.
|
||||
if (!empty) {
|
||||
throw Object.assign(
|
||||
new Error("Memory scope cleanup could not be verified: " + unit, { cause: failure }),
|
||||
{
|
||||
code: "EPROCESSGROUP_CLEANUP_FAILED",
|
||||
processTreeState: "indeterminate",
|
||||
},
|
||||
);
|
||||
}
|
||||
if (hasUnjoinedWork(failure)) {
|
||||
// The cgroup includes detached descendants that escaped the inner process group.
|
||||
// Preserve failure, but replace its superseded cleanup receipt after extinction.
|
||||
throw Object.assign(new Error(failure instanceof Error ? failure.message : String(failure)), {
|
||||
code: "EPROCESSGROUP_CLEANUP_FAILED",
|
||||
processTreeState: "terminated",
|
||||
});
|
||||
}
|
||||
};
|
||||
let failure: unknown;
|
||||
try {
|
||||
return await run({
|
||||
...command,
|
||||
// Cgroup ownership starts cleanup at launcher exit, even when a detached
|
||||
// descendant still holds output open and the caller supplied no deadline.
|
||||
requireProcessTreeExit: true,
|
||||
bin: "systemd-run",
|
||||
shell: false,
|
||||
env,
|
||||
args: [
|
||||
"--user",
|
||||
"--scope",
|
||||
"--collect",
|
||||
"--quiet",
|
||||
"--expand-environment=no",
|
||||
"--unit=" + unit,
|
||||
"--property=MemoryMax=" + memoryLimitBytes,
|
||||
"--property=MemorySwapMax=0",
|
||||
"--property=OOMPolicy=kill",
|
||||
"--",
|
||||
process.execPath,
|
||||
fileURLToPath(resolveManagedMemoryEntrypointUrl()),
|
||||
String(memoryLimitBytes),
|
||||
unit,
|
||||
String(options.shell ?? false),
|
||||
options.bin,
|
||||
...(options.args ?? []),
|
||||
],
|
||||
});
|
||||
} catch (error) {
|
||||
failure = error;
|
||||
throw error;
|
||||
} finally {
|
||||
await cleanupScope(failure);
|
||||
}
|
||||
}
|
||||
@@ -522,3 +522,44 @@ export function readProcessMemoryCapacity(params: MemoryLimitParams) {
|
||||
: Math.min(cgroupMemory.limitBytes, physicalLimitBytes);
|
||||
return { ...cgroupMemory, capacityBytes, limitBytes, availableBytes: hostAvailableBytes };
|
||||
}
|
||||
|
||||
/** Verify actual kernel containment, rather than accepting an environment or manager claim. */
|
||||
export function hasLinuxMemoryContainment(
|
||||
maxBytes: number,
|
||||
params: MemoryLimitParams = {},
|
||||
scope?: string,
|
||||
) {
|
||||
if ((params.platform ?? process.platform) !== "linux") {
|
||||
return false;
|
||||
}
|
||||
const resolved = resolveCgroupMemoryLimitPaths(params);
|
||||
if (
|
||||
!resolved.sawObservedV2Mapping ||
|
||||
resolved.cgroupRecordReadFailed ||
|
||||
resolved.sawUnresolvedCgroupLimit
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
const files = params.fs ?? fs;
|
||||
return resolved.paths.some((file) => {
|
||||
if (path.basename(file) !== "memory.max") {
|
||||
return false;
|
||||
}
|
||||
if (scope && path.basename(path.dirname(file)) !== scope) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
const limit = parseCgroupMemoryLimitBytes(files.readFileSync(file, "utf8"));
|
||||
const directory = path.dirname(file);
|
||||
return (
|
||||
limit !== null &&
|
||||
limit > 0 &&
|
||||
limit <= maxBytes &&
|
||||
files.readFileSync(path.join(directory, "memory.swap.max"), "utf8").trim() === "0" &&
|
||||
files.readFileSync(path.join(directory, "memory.oom.group"), "utf8").trim() === "1"
|
||||
);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { runtimeProcessEntrypoints } from "../../src/infra/runtime-process-entrypoints.ts";
|
||||
import { managedMemoryEntrypoint } from "./managed-memory-entrypoint.mts";
|
||||
import { managedWindowsJobEntrypoint } from "./managed-windows-job-entrypoint.mts";
|
||||
|
||||
export function createRuntimeProcessBuildEntries(
|
||||
@@ -26,6 +27,7 @@ export function createRuntimeProcessBuildEntries(
|
||||
export const runtimeProcessCoreEntrypoints = [
|
||||
...Object.values(runtimeProcessEntrypoints),
|
||||
managedWindowsJobEntrypoint,
|
||||
managedMemoryEntrypoint,
|
||||
];
|
||||
export const runtimeProcessCoreBuildEntries = createRuntimeProcessBuildEntries(
|
||||
runtimeProcessCoreEntrypoints,
|
||||
@@ -33,6 +35,7 @@ export const runtimeProcessCoreBuildEntries = createRuntimeProcessBuildEntries(
|
||||
|
||||
// Keep small helper processes out of the shared runtime bundle.
|
||||
export const standaloneRuntimeProcessBuildEntries = createRuntimeProcessBuildEntries([
|
||||
managedMemoryEntrypoint,
|
||||
runtimeProcessEntrypoints.sqliteReadOnly,
|
||||
runtimeProcessEntrypoints.sqliteSourceRevision,
|
||||
runtimeProcessEntrypoints.stateRead,
|
||||
|
||||
@@ -34,7 +34,15 @@ describe("jsdom native API boundary", () => {
|
||||
await environment.teardown(globalThis);
|
||||
}
|
||||
`,
|
||||
path.resolve("ui/package.json"),
|
||||
path.join(
|
||||
path.dirname(
|
||||
process
|
||||
.getBuiltinModule("module")
|
||||
.createRequire(path.resolve("ui/package.json"))
|
||||
.resolve("vitest/package.json"),
|
||||
),
|
||||
"dist/workers/forks.js",
|
||||
),
|
||||
],
|
||||
{ encoding: "utf8" },
|
||||
);
|
||||
|
||||
@@ -47,6 +47,7 @@ function expectedHarnessSparseCheckoutArgs(linux: boolean) {
|
||||
"/scripts/lib/pnpm-lockfile-documents.mjs",
|
||||
"/scripts/ios-screenshot-evidence.mjs",
|
||||
"/scripts/lib/direct-run.mjs",
|
||||
"/scripts/ci-static-step.sh",
|
||||
...(linux
|
||||
? [
|
||||
"/scripts/lib/release-upgrade-baseline.mjs",
|
||||
@@ -391,6 +392,7 @@ it.concurrent.each([
|
||||
const evidenceScripts = {
|
||||
"scripts/ios-screenshot-evidence.mjs": "workflow evidence script\n",
|
||||
"scripts/lib/direct-run.mjs": "workflow direct-run script\n",
|
||||
"scripts/ci-static-step.sh": "workflow static-step script\n",
|
||||
};
|
||||
const nodeSetupScripts = {
|
||||
"scripts/lib/pnpm-lockfile-documents.mjs": readFileSync(
|
||||
|
||||
@@ -472,6 +472,11 @@ function runCheckShardFixture(options: {
|
||||
mkdirSync(fakeBin);
|
||||
if (typeCheck) {
|
||||
mkdirSync(path.join(root, "scripts"));
|
||||
mkdirSync(path.join(root, ".ci-harness/scripts"), { recursive: true });
|
||||
copyFileSync(
|
||||
new URL("../../scripts/ci-static-step.sh", import.meta.url),
|
||||
path.join(root, ".ci-harness/scripts/ci-static-step.sh"),
|
||||
);
|
||||
writeFileSync(
|
||||
path.join(root, "scripts/run-tsgo-core-test-shards.mts"),
|
||||
options.types?.stripeSupport === false ? "// legacy runner\n" : "// --stripe\n",
|
||||
|
||||
@@ -53,12 +53,15 @@ export function waitForFile(file) {
|
||||
"lib/tsx-cli-shim.mjs",
|
||||
"lib/local-check-runtime.mts",
|
||||
"lib/check-limits.mts",
|
||||
"lib/ci-static-check-evidence.mjs",
|
||||
"lib/direct-run.mjs",
|
||||
"lib/dist-artifact-ownership.mts",
|
||||
"lib/dist-artifact-lock.mts",
|
||||
"lib/record-shared.mjs",
|
||||
"lib/failed-trailer.mts",
|
||||
"lib/managed-child-process.mts",
|
||||
"lib/managed-memory.mts",
|
||||
"lib/managed-memory-entrypoint.mts",
|
||||
"lib/vitest-resource-ownership.mts",
|
||||
"lib/windows-taskkill.mjs",
|
||||
"lib/repo-root.mjs",
|
||||
|
||||
@@ -0,0 +1,208 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { expect, it } from "vitest";
|
||||
import {
|
||||
hasUnjoinedWork,
|
||||
runManagedCommand,
|
||||
signalExitCode,
|
||||
} from "../../scripts/lib/managed-child-process.mts";
|
||||
import { hasSemanticTestBackend } from "./native-boundary-fixture.js";
|
||||
|
||||
const available = process.platform === "linux" && hasSemanticTestBackend();
|
||||
|
||||
it.runIf(available)(
|
||||
"contains aggregate native allocations and joins the whole cgroup after OOM",
|
||||
async ({ signal }) => {
|
||||
let memoryScope = "";
|
||||
const allocate =
|
||||
"const a=[];for(let i=0;i<16;i++)a.push(Buffer.alloc(8*1024**2,1));setInterval(()=>{},1000)";
|
||||
const code = await runManagedCommand({
|
||||
bin: process.execPath,
|
||||
args: [
|
||||
"-e",
|
||||
[
|
||||
"const {spawn}=require('node:child_process');",
|
||||
"for(let i=0;i<2;i++)spawn(process.execPath,['-e'," +
|
||||
JSON.stringify(allocate) +
|
||||
"],{stdio:'inherit'});",
|
||||
"setInterval(()=>{},1000);",
|
||||
].join("\n"),
|
||||
],
|
||||
memoryLimitBytes: 256 * 1024 ** 2,
|
||||
onMemoryScope(unit) {
|
||||
memoryScope = unit;
|
||||
},
|
||||
timeoutMs: 15_000,
|
||||
requireProcessTreeExit: true,
|
||||
signal,
|
||||
});
|
||||
expect(code).toBe(137);
|
||||
const state = spawnSync("systemctl", ["--user", "show", "--property=LoadState", memoryScope], {
|
||||
encoding: "utf8",
|
||||
timeout: 5_000,
|
||||
});
|
||||
expect(state.stdout).toContain("LoadState=not-found");
|
||||
},
|
||||
25_000,
|
||||
);
|
||||
|
||||
it.runIf(available).for([false, true])(
|
||||
"returns the workload result through a verified bounded launcher (shell: %s)",
|
||||
{ timeout: 20_000 },
|
||||
async (shell, { signal }) => {
|
||||
let output = "";
|
||||
const code = await runManagedCommand({
|
||||
bin: shell ? "printf 'bounded\\n'; exit 7" : process.execPath,
|
||||
args: shell ? [] : ["-e", "process.stdin.pipe(process.stdout);process.exitCode=7;"],
|
||||
shell,
|
||||
memoryLimitBytes: 256 * 1024 ** 2,
|
||||
timeoutMs: 10_000,
|
||||
requireProcessTreeExit: true,
|
||||
signal,
|
||||
stdio: shell ? ["ignore", "pipe", "pipe"] : "pipe",
|
||||
onReady(child) {
|
||||
child.stdout!.on("data", (chunk) => {
|
||||
output += String(chunk);
|
||||
});
|
||||
child.stdin?.end("bounded\n");
|
||||
},
|
||||
});
|
||||
expect(code).toBe(7);
|
||||
expect(output).toBe("bounded\n");
|
||||
},
|
||||
);
|
||||
|
||||
it.runIf(available).for(["SIGPIPE", "SIGUSR1"] as const)(
|
||||
"preserves native %s exit status through Node's special signal disposition",
|
||||
{ timeout: 15_000 },
|
||||
async (signal, { signal: abortSignal }) => {
|
||||
let exitSignal: NodeJS.Signals | null | undefined;
|
||||
const code = await runManagedCommand({
|
||||
bin: "/bin/sh",
|
||||
args: ["-c", `kill -${signal.slice(3)} $$`],
|
||||
memoryLimitBytes: 256 * 1024 ** 2,
|
||||
timeoutMs: 10_000,
|
||||
signal: abortSignal,
|
||||
stdio: "ignore",
|
||||
onReady(child) {
|
||||
child.once("exit", (_code, received) => {
|
||||
exitSignal = received;
|
||||
});
|
||||
},
|
||||
});
|
||||
expect(code).toBe(signalExitCode(signal));
|
||||
expect(exitSignal).toBe(signal);
|
||||
},
|
||||
);
|
||||
|
||||
it.runIf(available)(
|
||||
"keeps one cancellation signal and the caller's longer cleanup grace",
|
||||
async ({ signal }) => {
|
||||
const abort = new AbortController();
|
||||
let output = "";
|
||||
const result = runManagedCommand({
|
||||
bin: process.execPath,
|
||||
args: [
|
||||
"-e",
|
||||
[
|
||||
"let signals=0;process.on('SIGTERM',()=>{",
|
||||
"if(++signals>1)process.exit(9);process.stdout.write('term\\n');",
|
||||
// This must exceed the removed launcher's independent five-second timer.
|
||||
"setTimeout(()=>process.stdout.write('drained\\n',()=>process.exit(0)),5500)});",
|
||||
"setInterval(()=>{},1000);console.log('ready');",
|
||||
].join("\n"),
|
||||
],
|
||||
memoryLimitBytes: 256 * 1024 ** 2,
|
||||
timeoutMs: 15_000,
|
||||
abortKillGraceMs: 7_000,
|
||||
signal: AbortSignal.any([signal, abort.signal]),
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
onReady(child) {
|
||||
child.stdout!.on("data", (chunk) => {
|
||||
output += String(chunk);
|
||||
if (output.includes("ready\n")) {
|
||||
abort.abort();
|
||||
}
|
||||
});
|
||||
},
|
||||
});
|
||||
await expect(result).rejects.toMatchObject({ code: "ABORT_ERR" });
|
||||
expect(output).toBe("ready\nterm\ndrained\n");
|
||||
},
|
||||
20_000,
|
||||
);
|
||||
|
||||
it.runIf(available)(
|
||||
"joins a detached pipe holder with default cleanup options and no deadline",
|
||||
async ({ signal }) => {
|
||||
let memoryScope = "";
|
||||
let failure: unknown;
|
||||
let status: number | undefined;
|
||||
try {
|
||||
status = await runManagedCommand({
|
||||
bin: process.execPath,
|
||||
args: [
|
||||
"-e",
|
||||
[
|
||||
"const leaf=\"process.on('SIGTERM',()=>{});setInterval(()=>{},1000);process.send('ready');process.disconnect()\";",
|
||||
"const child=require('node:child_process').spawn(process.execPath,['-e',leaf],{detached:true,stdio:['ignore','inherit','inherit','ipc']});",
|
||||
"child.once('message',()=>process.exit(0));",
|
||||
].join("\n"),
|
||||
],
|
||||
memoryLimitBytes: 256 * 1024 ** 2,
|
||||
onMemoryScope(unit) {
|
||||
memoryScope = unit;
|
||||
},
|
||||
signal,
|
||||
});
|
||||
} catch (error) {
|
||||
failure = error;
|
||||
}
|
||||
expect(failure !== undefined || (status !== undefined && status !== 0)).toBe(true);
|
||||
expect(hasUnjoinedWork(failure)).toBe(false);
|
||||
const state = spawnSync("systemctl", ["--user", "show", "--property=LoadState", memoryScope], {
|
||||
encoding: "utf8",
|
||||
timeout: 5_000,
|
||||
});
|
||||
expect(state.stdout).toContain("LoadState=not-found");
|
||||
},
|
||||
30_000,
|
||||
);
|
||||
|
||||
it.runIf(available)(
|
||||
"preserves a workload signal exit so surviving descendants drain gracefully",
|
||||
async ({ signal }) => {
|
||||
let output = "";
|
||||
let exitSignal: NodeJS.Signals | null | undefined;
|
||||
const leaf =
|
||||
"process.on('SIGTERM',()=>process.stdout.write('drained\\n',()=>process.exit(0)));setInterval(()=>{},1000);process.send('ready');process.disconnect()";
|
||||
const code = await runManagedCommand({
|
||||
bin: process.execPath,
|
||||
args: [
|
||||
"-e",
|
||||
[
|
||||
"const child=require('node:child_process').spawn(process.execPath,['-e'," +
|
||||
JSON.stringify(leaf) +
|
||||
"],{stdio:['ignore','inherit','inherit','ipc']});",
|
||||
"child.once('message',()=>process.kill(process.pid,'SIGTERM'));",
|
||||
].join("\n"),
|
||||
],
|
||||
memoryLimitBytes: 256 * 1024 ** 2,
|
||||
timeoutMs: 10_000,
|
||||
requireProcessTreeExit: true,
|
||||
signal,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
onReady(child) {
|
||||
child.stdout!.on("data", (chunk) => {
|
||||
output += String(chunk);
|
||||
});
|
||||
child.once("exit", (_code, received) => {
|
||||
exitSignal = received;
|
||||
});
|
||||
},
|
||||
});
|
||||
expect(code).toBe(143);
|
||||
expect(exitSignal).toBe("SIGTERM");
|
||||
expect(output).toBe("drained\n");
|
||||
},
|
||||
20_000,
|
||||
);
|
||||
@@ -0,0 +1,389 @@
|
||||
import type { StdioOptions } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
import { runManagedCommand } from "../../scripts/lib/managed-child-process.mts";
|
||||
import { runLinuxMemoryCommand } from "../../scripts/lib/managed-memory.mts";
|
||||
import { hasLinuxMemoryContainment } from "../../scripts/lib/process-memory.mts";
|
||||
import { createDeferred } from "../helpers/promise.js";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
control: vi.fn(),
|
||||
uuid: vi.fn(() => "abcd"),
|
||||
resourceOwner: vi.fn(),
|
||||
}));
|
||||
vi.mock("node:child_process", () => ({ spawnSync: mocks.control }));
|
||||
vi.mock("node:crypto", () => ({ randomUUID: mocks.uuid }));
|
||||
vi.mock("../../scripts/lib/vitest-resource-ownership.mts", () => ({
|
||||
findVitestResourceOwner: mocks.resourceOwner,
|
||||
}));
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
mocks.resourceOwner.mockReset();
|
||||
});
|
||||
|
||||
const command = {
|
||||
bin: "fixture",
|
||||
memoryLimitBytes: 256 * 1024 ** 2,
|
||||
};
|
||||
const memoryScope = "openclaw-check-abcd.scope";
|
||||
const response = (stdout: string) => ({ stdout, stderr: "", status: 0 });
|
||||
|
||||
it("removes signal ownership even when resource receipt release fails", async () => {
|
||||
const memory = await import("../../scripts/lib/managed-memory.mts");
|
||||
vi.spyOn(memory, "runLinuxMemoryCommand").mockResolvedValue(0);
|
||||
const error = new Error("receipt release failed");
|
||||
mocks.resourceOwner.mockReturnValue({
|
||||
claim: () => () => {
|
||||
throw error;
|
||||
},
|
||||
});
|
||||
const previousListeners = new Set(process.listeners("SIGTERM"));
|
||||
await expect(runManagedCommand({ ...command, platform: "linux" })).rejects.toBe(error);
|
||||
expect(new Set(process.listeners("SIGTERM"))).toEqual(previousListeners);
|
||||
});
|
||||
|
||||
it.for([
|
||||
{ kind: "abort", uncertain: false },
|
||||
{ kind: "signal", uncertain: false },
|
||||
{ kind: "abort", uncertain: true },
|
||||
{ kind: "signal", uncertain: true },
|
||||
])(
|
||||
"owns $kind cancellation until cgroup cleanup settles (uncertain=$uncertain)",
|
||||
async (params) => {
|
||||
const enteredCleanup = createDeferred<void>();
|
||||
const cleanup = createDeferred<number>();
|
||||
const memory = await import("../../scripts/lib/managed-memory.mts");
|
||||
vi.spyOn(memory, "runLinuxMemoryCommand").mockImplementation(async () => {
|
||||
enteredCleanup.resolve();
|
||||
return await cleanup.promise;
|
||||
});
|
||||
const release = vi.fn();
|
||||
mocks.resourceOwner.mockReturnValue({ claim: () => release });
|
||||
const abort = new AbortController();
|
||||
const previousListeners = new Set(process.listeners("SIGTERM"));
|
||||
const onSignal = vi.fn();
|
||||
const result = runManagedCommand({
|
||||
...command,
|
||||
platform: "linux",
|
||||
signal: abort.signal,
|
||||
onSignal,
|
||||
});
|
||||
let settled = false;
|
||||
void result.then(
|
||||
() => {
|
||||
settled = true;
|
||||
},
|
||||
() => {
|
||||
settled = true;
|
||||
},
|
||||
);
|
||||
await enteredCleanup.promise;
|
||||
if (params.kind === "abort") {
|
||||
abort.abort();
|
||||
} else {
|
||||
// Invoke only this command's listener; never signal the shared test process.
|
||||
const handler = process
|
||||
.listeners("SIGTERM")
|
||||
.find((listener) => !previousListeners.has(listener));
|
||||
expect(handler).toBeTypeOf("function");
|
||||
handler!("SIGTERM");
|
||||
expect(onSignal).toHaveBeenCalledExactlyOnceWith("SIGTERM");
|
||||
}
|
||||
await Promise.resolve();
|
||||
expect(settled).toBe(false);
|
||||
expect(release).not.toHaveBeenCalled();
|
||||
if (params.uncertain) {
|
||||
const failure = Object.assign(new Error("scope still populated"), {
|
||||
processTreeState: "live",
|
||||
});
|
||||
cleanup.reject(failure);
|
||||
await expect(result).rejects.toBe(failure);
|
||||
expect(release).not.toHaveBeenCalled();
|
||||
} else {
|
||||
cleanup.resolve(0);
|
||||
if (params.kind === "abort") {
|
||||
await expect(result).rejects.toMatchObject({ code: "ABORT_ERR" });
|
||||
} else {
|
||||
await expect(result).resolves.toBe(143);
|
||||
}
|
||||
expect(release).toHaveBeenCalledOnce();
|
||||
}
|
||||
expect(new Set(process.listeners("SIGTERM"))).toEqual(previousListeners);
|
||||
},
|
||||
);
|
||||
|
||||
it("snapshots Linux command inputs before loading containment", async () => {
|
||||
const memory = await import("../../scripts/lib/managed-memory.mts");
|
||||
const run = vi.spyOn(memory, "runLinuxMemoryCommand").mockResolvedValue(0);
|
||||
const args = ["original"];
|
||||
const env = { TMPDIR: process.cwd(), VALUE: "original" };
|
||||
const stdio: StdioOptions = ["ignore", "pipe", "pipe"];
|
||||
const cwd = process.cwd();
|
||||
const result = runManagedCommand({ ...command, args, env, stdio, cwd: ".", platform: "linux" });
|
||||
vi.spyOn(process, "cwd").mockReturnValue(path.dirname(cwd));
|
||||
args[0] = "mutated";
|
||||
env.VALUE = "mutated";
|
||||
stdio[1] = "ignore";
|
||||
await expect(result).resolves.toBe(0);
|
||||
expect(run).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
args: ["original"],
|
||||
env: { ...env, VALUE: "original" },
|
||||
cwd,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
}),
|
||||
expect.any(Function),
|
||||
);
|
||||
});
|
||||
|
||||
it.for<StdioOptions>(["inherit", "pipe", "ignore", [0, 1, 2], [null, "pipe"]])(
|
||||
"preserves standard stdio %j",
|
||||
async (stdio) => {
|
||||
const memory = await import("../../scripts/lib/managed-memory.mts");
|
||||
const run = vi.spyOn(memory, "runLinuxMemoryCommand").mockResolvedValue(0);
|
||||
await runManagedCommand({ ...command, stdio, platform: "linux" });
|
||||
expect(run).toHaveBeenCalledWith(expect.objectContaining({ stdio }), expect.any(Function));
|
||||
},
|
||||
);
|
||||
|
||||
it.for<StdioOptions>([
|
||||
["ignore", "pipe", "ipc"],
|
||||
["ignore", "pipe", "pipe", "pipe"],
|
||||
])("rejects unsupported stdio %j before entering containment", async (stdio) => {
|
||||
const memory = await import("../../scripts/lib/managed-memory.mts");
|
||||
const run = vi.spyOn(memory, "runLinuxMemoryCommand");
|
||||
await expect(runManagedCommand({ ...command, stdio, platform: "linux" })).rejects.toThrow(
|
||||
"do not support IPC or extra stdio descriptors",
|
||||
);
|
||||
expect(run).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([undefined, 1_001])(
|
||||
"keeps wall deadline ownership in the managed runner (%s)",
|
||||
async (timeoutMs) => {
|
||||
mocks.control.mockReturnValue(response("LoadState=not-found\n"));
|
||||
const run = vi.fn(async (_options: Parameters<typeof runLinuxMemoryCommand>[0]) => 0);
|
||||
await runLinuxMemoryCommand({ ...command, timeoutMs }, run);
|
||||
expect(
|
||||
run.mock.calls[0]?.[0]?.args?.filter((arg) => arg.startsWith("--property=RuntimeMaxSec=")),
|
||||
).toEqual([]);
|
||||
expect(run).toHaveBeenCalledWith(expect.objectContaining({ requireProcessTreeExit: true }));
|
||||
},
|
||||
);
|
||||
|
||||
it("generates separate cleanup identities for concurrent invocations", async () => {
|
||||
mocks.control.mockReset().mockReturnValue(response("LoadState=not-found\n"));
|
||||
mocks.uuid.mockReturnValueOnce("aaaa").mockReturnValueOnce("bbbb");
|
||||
const scopes: string[] = [];
|
||||
const run = vi.fn(async () => 0);
|
||||
await Promise.all(
|
||||
[0, 1].map(() =>
|
||||
runLinuxMemoryCommand({ ...command, onMemoryScope: (unit) => scopes.push(unit) }, run),
|
||||
),
|
||||
);
|
||||
expect(scopes).toEqual(["openclaw-check-aaaa.scope", "openclaw-check-bbbb.scope"]);
|
||||
expect(run.mock.calls).toHaveLength(2);
|
||||
for (const unit of scopes) {
|
||||
expect(mocks.control).toHaveBeenCalledWith(
|
||||
"systemctl",
|
||||
["--user", "kill", "--kill-whom=all", "--signal=SIGKILL", unit],
|
||||
expect.any(Object),
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("refuses an existing generated scope without launching or stopping it", async () => {
|
||||
mocks.control.mockReset().mockReturnValue(response("LoadState=loaded\n"));
|
||||
const run = vi.fn();
|
||||
await expect(runLinuxMemoryCommand(command, run)).rejects.toThrow("user manager");
|
||||
expect(run).not.toHaveBeenCalled();
|
||||
expect(mocks.control).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it.each(["darwin", "win32"] as const)(
|
||||
"refuses an unqualified %s cap before starting a workload",
|
||||
async (platform) => {
|
||||
mocks.control.mockClear();
|
||||
await expect(runManagedCommand({ ...command, platform })).rejects.toThrow("not qualified");
|
||||
expect(mocks.control).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it("forces remaining scope members before waiting for systemd cleanup", async () => {
|
||||
mocks.control.mockReset().mockReturnValue(response("LoadState=not-found\n"));
|
||||
await runLinuxMemoryCommand(command, async () => 0);
|
||||
expect(mocks.control.mock.calls.map((call) => call[1])).toEqual([
|
||||
["--user", "show", "--property=LoadState", memoryScope],
|
||||
["--user", "kill", "--kill-whom=all", "--signal=SIGKILL", memoryScope],
|
||||
["--user", "stop", memoryScope],
|
||||
[
|
||||
"--user",
|
||||
"show",
|
||||
"--property=LoadState",
|
||||
"--property=ActiveState",
|
||||
"--property=ControlGroup",
|
||||
memoryScope,
|
||||
],
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not start work when the systemd user manager is unavailable", async () => {
|
||||
mocks.control.mockReturnValue({ error: new Error("no user bus"), stdout: "", status: 1 });
|
||||
const run = vi.fn();
|
||||
await expect(runLinuxMemoryCommand(command, run)).rejects.toThrow("user manager");
|
||||
expect(run).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("preserves a pre-spawn failure without claiming an unjoined scope", async () => {
|
||||
mocks.control.mockReturnValue(response("LoadState=not-found\n"));
|
||||
const error = Object.assign(new Error("systemd-run missing"), { code: "ENOENT" });
|
||||
await expect(
|
||||
runLinuxMemoryCommand(command, async () => {
|
||||
throw error;
|
||||
}),
|
||||
).rejects.toBe(error);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["1", false],
|
||||
["0", true],
|
||||
])(
|
||||
"requires kernel extinction before releasing a failed scope (populated=%s)",
|
||||
async (populated, empty) => {
|
||||
let now = 0;
|
||||
vi.spyOn(Date, "now").mockImplementation(() => (now += 6_000));
|
||||
mocks.control
|
||||
.mockReset()
|
||||
.mockReturnValue(
|
||||
response(
|
||||
"LoadState=loaded\nActiveState=failed\nControlGroup=/user.slice/openclaw-check-abcd.scope\n",
|
||||
),
|
||||
)
|
||||
.mockReturnValueOnce(response("LoadState=not-found\n"));
|
||||
vi.spyOn(fs, "readFileSync").mockReturnValue("populated " + populated + "\n");
|
||||
const run = runLinuxMemoryCommand(command, async () => 137);
|
||||
if (empty) {
|
||||
await expect(run).resolves.toBe(137);
|
||||
} else {
|
||||
await expect(run).rejects.toMatchObject({
|
||||
code: "EPROCESSGROUP_CLEANUP_FAILED",
|
||||
processTreeState: "indeterminate",
|
||||
});
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it.each([undefined, "ENOENT"])(
|
||||
"preserves failure %s when descendant cleanup is uncertain",
|
||||
async (code) => {
|
||||
let now = 0;
|
||||
vi.spyOn(Date, "now").mockImplementation(() => (now += 6_000));
|
||||
mocks.control
|
||||
.mockReset()
|
||||
.mockReturnValue(response("LoadState=loaded\nActiveState=failed\n"))
|
||||
.mockReturnValueOnce(response("LoadState=not-found\n"));
|
||||
const original = Object.assign(new Error("workload failed"), { code });
|
||||
await expect(
|
||||
runLinuxMemoryCommand(command, async () => {
|
||||
throw original;
|
||||
}),
|
||||
).rejects.toMatchObject({ cause: original });
|
||||
},
|
||||
);
|
||||
|
||||
it("passes literal arguments and restores preloads only inside the bounded launcher", async () => {
|
||||
mocks.control.mockReturnValue(response("LoadState=not-found\n"));
|
||||
const run = vi.fn(async () => 0);
|
||||
await expect(
|
||||
runLinuxMemoryCommand(
|
||||
{ ...command, args: ["$LITERAL"], env: { NODE_OPTIONS: "--require=workload" } },
|
||||
run,
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
expect(run).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
args: expect.arrayContaining([
|
||||
"--expand-environment=no",
|
||||
"--property=MemoryMax=268435456",
|
||||
"--property=MemorySwapMax=0",
|
||||
"--property=OOMPolicy=kill",
|
||||
"$LITERAL",
|
||||
]),
|
||||
env: { OPENCLAW_MANAGED_NODE_OPTIONS: "--require=workload" },
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it.each([undefined, null])(
|
||||
"retains admission when a cleanup probe has no output (%s)",
|
||||
async (stdout) => {
|
||||
let now = 0;
|
||||
vi.spyOn(Date, "now").mockImplementation(() => (now += 6_000));
|
||||
mocks.control
|
||||
.mockReset()
|
||||
.mockReturnValue({
|
||||
error: Object.assign(new Error("spawn failed"), { code: "ENOMEM" }),
|
||||
stdout,
|
||||
})
|
||||
.mockReturnValueOnce(response("LoadState=not-found\n"));
|
||||
await expect(runLinuxMemoryCommand(command, async () => 0)).rejects.toMatchObject({
|
||||
code: "EPROCESSGROUP_CLEANUP_FAILED",
|
||||
processTreeState: "indeterminate",
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
{ scope: "openclaw-check-abcd.scope", swap: "0", group: "1", expected: true },
|
||||
{ scope: "other.scope", swap: "0", group: "1", expected: false },
|
||||
{ scope: "openclaw-check-abcd.scope", swap: "max", group: "1", expected: false },
|
||||
{ scope: "openclaw-check-abcd.scope", swap: "0", group: "0", expected: false },
|
||||
])(
|
||||
"qualifies the owned kernel cgroup only: $scope/$swap/$group",
|
||||
({ scope, swap, group, expected }) => {
|
||||
const files: Record<string, string> = {
|
||||
"/proc/self/cgroup": "0::/user.slice/" + scope + "\n",
|
||||
"/proc/self/mountinfo": "29 23 0:26 / /sys/fs/cgroup rw - cgroup2 cgroup rw\n",
|
||||
["/sys/fs/cgroup/user.slice/" + scope + "/memory.max"]: "268435456",
|
||||
["/sys/fs/cgroup/user.slice/" + scope + "/memory.swap.max"]: swap,
|
||||
["/sys/fs/cgroup/user.slice/" + scope + "/memory.oom.group"]: group,
|
||||
"/sys/fs/cgroup/user.slice/memory.max": "268435456",
|
||||
"/sys/fs/cgroup/user.slice/memory.swap.max": "0",
|
||||
"/sys/fs/cgroup/user.slice/memory.oom.group": "1",
|
||||
};
|
||||
expect(
|
||||
hasLinuxMemoryContainment(
|
||||
command.memoryLimitBytes,
|
||||
{
|
||||
platform: "linux",
|
||||
fs: {
|
||||
readFileSync(file) {
|
||||
if (!(file in files)) {
|
||||
throw Object.assign(new Error(file), { code: "ENOENT" });
|
||||
}
|
||||
return files[file]!;
|
||||
},
|
||||
},
|
||||
},
|
||||
memoryScope,
|
||||
),
|
||||
).toBe(expected);
|
||||
},
|
||||
);
|
||||
|
||||
it("records cgroup extinction after an inner process-group cleanup failure", async () => {
|
||||
mocks.control.mockReturnValue(response("LoadState=not-found\n"));
|
||||
const original = Object.assign(new Error("detached descendant held output"), {
|
||||
processTreeState: "live",
|
||||
});
|
||||
const result = runLinuxMemoryCommand(command, async () => {
|
||||
throw original;
|
||||
});
|
||||
await expect(result).rejects.toMatchObject({
|
||||
message: original.message,
|
||||
code: "EPROCESSGROUP_CLEANUP_FAILED",
|
||||
processTreeState: "terminated",
|
||||
});
|
||||
await expect(result).rejects.not.toHaveProperty("cause");
|
||||
});
|
||||
@@ -5,6 +5,23 @@ import path from "node:path";
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
|
||||
/** Availability only; integration assertions still verify the actual kernel scope. */
|
||||
export function hasSemanticTestBackend(): boolean {
|
||||
if (process.platform !== "linux") return false;
|
||||
try {
|
||||
return (
|
||||
fs
|
||||
.readFileSync("/sys/fs/cgroup/cgroup.controllers", "utf8")
|
||||
.split(/\s+/u)
|
||||
.includes("memory") &&
|
||||
spawnSync("systemctl", ["--user", "show", "--property=Version"], { timeout: 5_000 })
|
||||
.status === 0
|
||||
);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** Native receipts and default libraries must belong to the fixture's own install. */
|
||||
export function materializeNativeCompiler(rootDir: string) {
|
||||
const root = fs.realpathSync.native(rootDir);
|
||||
|
||||
@@ -246,6 +246,7 @@ describe("tsdown config", () => {
|
||||
requireStandaloneRuntimeGraph("agents/harness/native-hook-relay-client.worker"),
|
||||
requireStandaloneRuntimeGraph("process/spawn-broker/worker"),
|
||||
requireStandaloneRuntimeGraph("state/openclaw-state-lease-heartbeat.worker"),
|
||||
requireStandaloneRuntimeGraph("tooling/managed-memory-launcher"),
|
||||
]);
|
||||
|
||||
for (const config of configs) {
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { afterEach, expect, it } from "vitest";
|
||||
import { runManagedCommand } from "../../scripts/lib/managed-child-process.mts";
|
||||
import { resolveTestNodeExecPath } from "../../src/test-utils/node-process.js";
|
||||
import { createFixtureLifetime } from "../helpers/fixture-lifetime.js";
|
||||
|
||||
const lifetime = createFixtureLifetime();
|
||||
afterEach(() => lifetime.cleanup());
|
||||
const preload = new URL("../vitest/vitest.jsdom-preload.mts", import.meta.url).href;
|
||||
|
||||
async function run(root: string, args: string[], signal: AbortSignal) {
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
const code = await lifetime.track(
|
||||
runManagedCommand({
|
||||
bin: resolveTestNodeExecPath(),
|
||||
args: ["--no-warnings", `--import=${preload}`, ...args],
|
||||
cwd: root,
|
||||
signal,
|
||||
timeoutMs: 10_000,
|
||||
requireProcessTreeExit: true,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
onReady(child) {
|
||||
child.stdout!.on("data", (chunk) => {
|
||||
stdout += String(chunk);
|
||||
});
|
||||
child.stderr!.on("data", (chunk) => {
|
||||
stderr += String(chunk);
|
||||
});
|
||||
},
|
||||
}),
|
||||
);
|
||||
return { code, stdout, stderr };
|
||||
}
|
||||
|
||||
it("leaves inherited fork and thread preloads inert outside Vitest workers", ({ signal }) =>
|
||||
lifetime.run(async () => {
|
||||
const root = lifetime.createTempDir("openclaw-preload-descendants-");
|
||||
const child = path.join(root, "child.mjs");
|
||||
fs.writeFileSync(
|
||||
child,
|
||||
`
|
||||
import { parentPort } from 'node:worker_threads';
|
||||
const flags = process.execArgv.includes('--no-warnings');
|
||||
if (parentPort) parentPort.postMessage({ kind: 'thread', flags });
|
||||
else process.send({ kind: 'fork', flags }, () => process.disconnect());
|
||||
`,
|
||||
);
|
||||
const parent = path.join(root, "parent.mjs");
|
||||
fs.writeFileSync(
|
||||
parent,
|
||||
`
|
||||
import { fork } from 'node:child_process';
|
||||
import { Worker } from 'node:worker_threads';
|
||||
const wait = child => new Promise((resolve, reject) => {
|
||||
let message;
|
||||
child.once('message', value => { message = value; });
|
||||
child.once('error', reject);
|
||||
child.once('exit', code => code === 0 ? resolve(message) : reject(new Error('exit ' + code)));
|
||||
});
|
||||
const forked = fork(${JSON.stringify(child)}, [], { stdio: ['ignore', 'ignore', 'inherit', 'ipc'] });
|
||||
const thread = new Worker(new URL(${JSON.stringify(pathToFileURL(child).href)}));
|
||||
console.log(JSON.stringify(await Promise.all([wait(forked), wait(thread)])));
|
||||
`,
|
||||
);
|
||||
const result = await run(root, [parent], signal);
|
||||
expect(result.code, result.stderr).toBe(0);
|
||||
expect(JSON.parse(result.stdout)).toEqual([
|
||||
{ kind: "fork", flags: true },
|
||||
{ kind: "thread", flags: true },
|
||||
]);
|
||||
}));
|
||||
|
||||
it("does not require an entrypoint for ordinary eval commands", ({ signal }) =>
|
||||
lifetime.run(async () => {
|
||||
const result = await run(
|
||||
lifetime.createTempDir("openclaw-preload-eval-"),
|
||||
["--eval", "process.stdout.write('ok')"],
|
||||
signal,
|
||||
);
|
||||
expect(result).toEqual({ code: 0, stdout: "ok", stderr: "" });
|
||||
}));
|
||||
|
||||
it("keeps runtime resolution failures fatal for recognized Vitest workers", ({ signal }) =>
|
||||
lifetime.run(async () => {
|
||||
const root = lifetime.createTempDir("openclaw-preload-invalid-worker-");
|
||||
const entrypoint = path.join(root, "vitest/dist/workers/forks.js");
|
||||
fs.mkdirSync(path.dirname(entrypoint), { recursive: true });
|
||||
fs.writeFileSync(entrypoint, "throw new Error('worker must not start');");
|
||||
const result = await run(root, [entrypoint], signal);
|
||||
expect(result.code).not.toBe(0);
|
||||
expect(result.stderr).toContain("Cannot find module 'vitest/runtime'");
|
||||
expect(result.stderr).not.toContain("worker must not start");
|
||||
}));
|
||||
@@ -1,6 +1,11 @@
|
||||
import { installJsdomEnvironmentAdapter } from "../jsdom-compat.mts";
|
||||
|
||||
// Match the worker's Vitest instance, including package-local pnpm peer graphs.
|
||||
const require = process.getBuiltinModule("module").createRequire(process.argv[1]!);
|
||||
const { builtinEnvironments }: typeof import("vitest/runtime") = require("vitest/runtime");
|
||||
installJsdomEnvironmentAdapter(builtinEnvironments.jsdom);
|
||||
// Vitest 5 starts these four packaged workers. Descendants inherit execArgv,
|
||||
// but ordinary forks/threads must not load a test runtime or alter their IPC.
|
||||
const entrypoint = process.argv[1]?.replaceAll("\\", "/");
|
||||
if (/\/vitest\/dist\/workers\/(?:forks|threads|vmForks|vmThreads)\.js$/u.test(entrypoint ?? "")) {
|
||||
// Match the active worker's instance, including package-local pnpm peer graphs.
|
||||
const require = process.getBuiltinModule("module").createRequire(process.argv[1]!);
|
||||
const { builtinEnvironments }: typeof import("vitest/runtime") = require("vitest/runtime");
|
||||
installJsdomEnvironmentAdapter(builtinEnvironments.jsdom);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user