fix(test): release compiler shard resources after cancellation (#159454)

* fix(test): join compiler shards before releasing artifact ownership

* test: use shared deferred helper for shard cancellation

docs/help/testing/suites.md

@@ -141,6 +141,9 @@ Native dependency policy:
       the remaining value must use plain decimal digits without leading zeros,
       so values such as `1e5` or `007` are rejected. Unset the variable to
       disable the watchdog.
+      Canceling a compiler shard batch joins each compiler, including forced
+      termination when needed, before releasing checkout artifact ownership so
+      the next build or check can proceed.
 
   </Accordion>
 

scripts/run-tsgo-core-test-shards.mts

@@ -6,12 +6,8 @@ import { existsSync, realpathSync } from "node:fs";
 import path from "node:path";
 import type { CoreTsgoGraph } from "./check-tsgo-core-boundary.mts";
 import { isDirectRunUrl } from "./lib/direct-run.mjs";
-import {
-  distArtifactEntryArgs,
-  withDistArtifactOwnership,
-} from "./lib/dist-artifact-ownership.mts";
+import { withDistArtifactOwnership } from "./lib/dist-artifact-ownership.mts";
 import { resolveLocalCheckEnv } from "./lib/local-check-runtime.mts";
-import { runManagedCommand } from "./lib/managed-child-process.mts";
 import { resolveRepoRoot } from "./lib/repo-root.mjs";
 import {
   selectTsgoCoreTestShards,
@@ -22,31 +18,28 @@ import {
   TSGO_CORE_TEST_SHARDS,
   selectTsgoCoreTestStripe,
 } from "./lib/tsgo-core-test-shards.mts";
+import { prepareTsgoCommand, runPreparedTsgoCommand } from "./run-tsgo.mts";
 
 const repoRoot = resolveRepoRoot(import.meta.url);
-function runShard(config: string, env: NodeJS.ProcessEnv): Promise<number> {
-  return runManagedCommand({
-    bin: process.execPath,
-    shell: false,
-    args: distArtifactEntryArgs(
-      path.join(repoRoot, "scripts/run-tsgo.mts"),
-      // These graphs have no project references. Project mode rechecks root
-      // membership even when a restored build-info file is newer than a new root.
-      [
-        "-p",
-        config,
-        "--incremental",
-        // The package command pins this config's cache at the repository root.
-        ...(config === "test/tsconfig/tsconfig.test.root.json"
-          ? ["--tsBuildInfoFile", ".artifacts/tsgo-cache/test-root.tsbuildinfo"]
-          : []),
-      ],
-      { native: true },
-    ),
-    cwd: repoRoot,
+async function runShard(config: string, env: NodeJS.ProcessEnv): Promise<number> {
+  const command = prepareTsgoCommand(
+    // These graphs have no project references. Project mode rechecks root
+    // membership even when a restored build-info file is newer than a new root.
+    [
+      "-p",
+      config,
+      "--incremental",
+      // The package command pins this config's cache at the repository root.
+      ...(config === "test/tsconfig/tsconfig.test.root.json"
+        ? ["--tsBuildInfoFile", ".artifacts/tsgo-cache/test-root.tsbuildinfo"]
+        : []),
+    ],
     env,
-    requireProcessTreeExit: process.platform !== "win32",
-  });
+    repoRoot,
+  );
+  // One owner joins each native compiler. A second wrapper's escalation deadline
+  // can kill the cleanup owner before it releases its artifact claim.
+  return command ? await runPreparedTsgoCommand(command) : 0;
 }
 
 /** Runs selected canonical graphs under the same output and child-process owner. */

scripts/run-tsgo.mts

@@ -80,6 +80,32 @@ export function prepareTsgoCommand(
   };
 }
 
+/** The caller holds artifact ownership until this compiler and its output are joined. */
+export async function runPreparedTsgoCommand(
+  command: NonNullable<ReturnType<typeof prepareTsgoCommand>>,
+): Promise<number> {
+  try {
+    const tsBuildInfoFile = readFlagValue(command.args, "--tsBuildInfoFile");
+    if (tsBuildInfoFile) {
+      fs.mkdirSync(path.dirname(path.resolve(command.cwd, tsBuildInfoFile)), { recursive: true });
+    }
+    // Managed cleanup forwards SIGTERM before bounded SIGKILL escalation, then
+    // joins the compiler group and output before reporting a timeout.
+    return await runManagedCommand({
+      ...command,
+      requireProcessTreeExit: process.platform !== "win32",
+    });
+  } catch (error) {
+    if ((error as { code?: string } | undefined)?.code !== "ETIMEDOUT") {
+      throw error;
+    }
+    console.error(
+      `[tsgo] no completion after ${command.timeoutMs}ms; killed the tsgo process tree. Raise OPENCLAW_TSGO_TIMEOUT_MS for intentionally longer builds, or unset it to disable the watchdog.`,
+    );
+    return 1;
+  }
+}
+
 async function main(): Promise<void> {
   let command: ReturnType<typeof prepareTsgoCommand>;
   try {
@@ -92,31 +118,11 @@ async function main(): Promise<void> {
   if (!command) {
     return;
   }
-  try {
-    // Preflight must refuse or skip before installed bootstrap dependencies load.
-    // Output mutation and the compiler still remain inside their checkout owner.
-    const { withDistArtifactOwnership } = await import("./lib/dist-artifact-ownership.mts");
-    await withDistArtifactOwnership(command.cwd, async () => {
-      const tsBuildInfoFile = readFlagValue(command.args, "--tsBuildInfoFile");
-      if (tsBuildInfoFile) {
-        fs.mkdirSync(path.dirname(path.resolve(command.cwd, tsBuildInfoFile)), { recursive: true });
-      }
-      // Managed cleanup forwards SIGTERM before bounded SIGKILL escalation, then
-      // joins the compiler group and output before reporting a timeout.
-      process.exitCode = await runManagedCommand({
-        ...command,
-        requireProcessTreeExit: process.platform !== "win32",
-      });
-    });
-  } catch (error) {
-    if ((error as { code?: string } | undefined)?.code !== "ETIMEDOUT") {
-      throw error;
-    }
-    console.error(
-      `[tsgo] no completion after ${command.timeoutMs}ms; killed the tsgo process tree. Raise OPENCLAW_TSGO_TIMEOUT_MS for intentionally longer builds, or unset it to disable the watchdog.`,
-    );
-    process.exitCode = 1;
-  }
+  // Preflight must refuse or skip before installed bootstrap dependencies load.
+  const { withDistArtifactOwnership } = await import("./lib/dist-artifact-ownership.mts");
+  process.exitCode = await withDistArtifactOwnership(command.cwd, () =>
+    runPreparedTsgoCommand(command),
+  );
 }
 
 if (isDirectRunUrl(process.argv[1], import.meta.url)) {

test/scripts/dist-artifact-fixture.ts

@@ -0,0 +1,48 @@
+import fs from "node:fs";
+import path from "node:path";
+import { materializeNativeCompiler } from "./native-boundary-fixture.js";
+
+function write(root: string, relative: string, content: string) {
+  const target = path.join(root, relative);
+  fs.mkdirSync(path.dirname(target), { recursive: true });
+  fs.writeFileSync(target, content);
+  return target;
+}
+
+export function installDistArtifactScripts(
+  root: string,
+  scripts: string[],
+  { compiler = true, dependencies = ["tsx", "@openclaw/fs-safe"] } = {},
+) {
+  const sourceRoot = process.cwd();
+  // Keep the checkpoint launcher when installCompiler already owns this toolchain.
+  if (compiler && !fs.existsSync(path.join(root, "node_modules/typescript/package.json"))) {
+    materializeNativeCompiler(root);
+  }
+  for (const script of ["tsx.mjs", ...scripts]) {
+    write(
+      root,
+      `scripts/${script}`,
+      fs.readFileSync(path.join(sourceRoot, "scripts", script), "utf8"),
+    );
+  }
+  for (const file of [
+    "scripts/lib",
+    "scripts/windows-cmd-helpers.mjs",
+    "packages/normalization-core/src",
+    "packages/normalization-core/package.json",
+  ]) {
+    fs.mkdirSync(path.dirname(path.join(root, file)), { recursive: true });
+    fs.cpSync(path.join(sourceRoot, file), path.join(root, file), { recursive: true });
+  }
+  write(root, "scripts/lib/plugin-sdk-entrypoints.json", '["qa-channel-protocol"]');
+  fs.mkdirSync(path.join(root, "node_modules"), { recursive: true });
+  for (const name of dependencies) {
+    fs.mkdirSync(path.dirname(path.join(root, "node_modules", name)), { recursive: true });
+    fs.symlinkSync(
+      path.join(sourceRoot, "node_modules", name),
+      path.join(root, "node_modules", name),
+      process.platform === "win32" ? "junction" : "dir",
+    );
+  }
+}

test/scripts/dist-artifact-ownership.test.ts

@@ -19,6 +19,7 @@ import { createVitestResourceOwner } from "../../scripts/lib/vitest-resource-own
 import { resolveTestNodeExecPath } from "../../src/test-utils/node-process.js";
 import { createFixtureLifetime } from "../helpers/fixture-lifetime.js";
 import { waitForDead } from "../helpers/process-wait.js";
+import { installDistArtifactScripts as installScripts } from "./dist-artifact-fixture.js";
 import {
   materializeNativeCompiler,
   overrideNativeFixtureExecutable,
@@ -100,43 +101,6 @@ function installBuildCheckpoint(root: string, checkpoint: string) {
   write(root, "pnpm.cjs", 'import("./node_modules/tsdown/dist/run.mjs");\n');
 }
 
-function installScripts(
-  root: string,
-  scripts: string[],
-  { compiler = true, dependencies = ["tsx", "@openclaw/fs-safe"] } = {},
-) {
-  // Keep the checkpoint launcher when installCompiler already owns this toolchain.
-  if (compiler && !fs.existsSync(path.join(root, "node_modules/typescript/package.json"))) {
-    materializeNativeCompiler(root);
-  }
-  for (const script of ["tsx.mjs", ...scripts]) {
-    write(
-      root,
-      `scripts/${script}`,
-      fs.readFileSync(path.join(sourceRoot, "scripts", script), "utf8"),
-    );
-  }
-  for (const file of [
-    "scripts/lib",
-    "scripts/windows-cmd-helpers.mjs",
-    "packages/normalization-core/src",
-    "packages/normalization-core/package.json",
-  ]) {
-    fs.mkdirSync(path.dirname(path.join(root, file)), { recursive: true });
-    fs.cpSync(path.join(sourceRoot, file), path.join(root, file), { recursive: true });
-  }
-  write(root, "scripts/lib/plugin-sdk-entrypoints.json", '["qa-channel-protocol"]');
-  fs.mkdirSync(path.join(root, "node_modules"), { recursive: true });
-  for (const name of dependencies) {
-    fs.mkdirSync(path.dirname(path.join(root, "node_modules", name)), { recursive: true });
-    fs.symlinkSync(
-      path.join(sourceRoot, "node_modules", name),
-      path.join(root, "node_modules", name),
-      process.platform === "win32" ? "junction" : "dir",
-    );
-  }
-}
-
 function withProcesses(...args: Parameters<typeof runWithProcesses>) {
   return fixture.run(() => runWithProcesses(...args));
 }
@@ -315,7 +279,12 @@ describe("native check launchers in paths with spaces", () => {
         const root = createCheckout("openclaw check launchers ");
         installScripts(
           root,
-          ["run-tsgo-core-test-shards.mts", "run-oxlint.mts", "run-oxlint-shards.mts"],
+          [
+            "run-tsgo-core-test-shards.mts",
+            "run-tsgo.mts",
+            "run-oxlint.mts",
+            "run-oxlint-shards.mts",
+          ],
           {
             compiler: false,
             dependencies: ["tsx", "@openclaw/fs-safe", "json5", "p-map", "koffi"],
@@ -325,17 +294,17 @@ describe("native check launchers in paths with spaces", () => {
         write(root, nativeJob, fs.readFileSync(path.join(sourceRoot, nativeJob), "utf8"));
         const compiler = script === "run-tsgo-core-test-shards.mts";
         const workload = compiler
-          ? "run-tsgo.mts"
-          : "prepare-extension-package-boundary-artifacts.mts";
+          ? "node_modules/typescript/compiler.mjs"
+          : "scripts/prepare-extension-package-boundary-artifacts.mts";
         const observed = path.join(root, "child.json");
         const settled = path.join(root, "child-settled");
         const consumed = path.join(root, "lint-consumed");
         // Keep the real CLI, artifact handoff and managed process owner. Only the
         // terminal compiler/preparation workload waits at this completion barrier.
-        write(
+        const workloadPath = write(
           root,
-          `scripts/${workload}`,
-          `
+          workload,
+          `#!/usr/bin/env node
         import fs from 'node:fs';
         import { createRequire } from 'node:module';
         const require = createRequire(import.meta.url);
@@ -348,6 +317,13 @@ describe("native check launchers in paths with spaces", () => {
         process.exitCode = ${exitCode};
       `,
         );
+        if (compiler) {
+          fs.chmodSync(workloadPath, 0o755);
+          overrideNativeFixtureExecutable(root, workloadPath);
+          if (process.platform === "win32") {
+            write(root, `${workload}.cmd`, `@"${testNodeExecPath}" "%~dp0compiler.mjs" %*\r\n`);
+          }
+        }
         const lint = write(
           root,
           "node_modules/.bin/oxlint",
@@ -373,14 +349,16 @@ describe("native check launchers in paths with spaces", () => {
         const child: { argv: string[]; pid: number } = JSON.parse(
           fs.readFileSync(observed, "utf8"),
         );
-        expect(child.argv).toEqual(
+        expect(compiler ? child.argv.slice(0, 3) : child.argv).toEqual(
           compiler
             ? ["-p", TSGO_CORE_TEST_SHARDS[0].config, "--incremental"]
             : ["--mode=package-boundary"],
         );
         const lock = resolveDistArtifactLockPath(root);
         expect(fs.existsSync(path.join(lock, "owner.json"))).toBe(true);
-        expect(fs.readdirSync(lock)).toContain(`child-${child.pid}`);
+        if (!compiler) {
+          expect(fs.readdirSync(lock)).toContain(`child-${child.pid}`);
+        }
         expect(fs.existsSync(settled)).toBe(false);
         expect(fs.existsSync(consumed)).toBe(false);
         gate.write("continue");

test/scripts/tsgo-shard-cancellation.test.ts

@@ -0,0 +1,117 @@
+import type { ChildProcess } from "node:child_process";
+import fs from "node:fs";
+import path from "node:path";
+import { createInterface } from "node:readline";
+import { pathToFileURL } from "node:url";
+import { afterEach, expect, it } from "vitest";
+import { withDistArtifactOwnership } from "../../scripts/lib/dist-artifact-ownership.mts";
+import { runManagedCommand } from "../../scripts/lib/managed-child-process.mts";
+import { TSGO_CORE_TEST_SHARDS } from "../../scripts/lib/tsgo-core-test-shards.mts";
+import { createDeferredCore } from "../../src/shared/deferred.js";
+import { resolveTestNodeExecPath } from "../../src/test-utils/node-process.js";
+import { createFixtureLifetime } from "../helpers/fixture-lifetime.js";
+import { isProcessAlive, waitForDead } from "../helpers/process-wait.js";
+import { installDistArtifactScripts } from "./dist-artifact-fixture.js";
+import { overrideNativeFixtureExecutable } from "./native-boundary-fixture.js";
+
+const lifetime = createFixtureLifetime();
+afterEach(() => lifetime.cleanup());
+
+it.runIf(process.platform !== "win32")(
+  "releases shard artifacts after canceling a compiler that requires forced termination",
+  ({ signal }) =>
+    lifetime.run(async () => {
+      const root = fs.realpathSync(lifetime.createTempDir("openclaw-cancel-shards-"));
+      fs.writeFileSync(path.join(root, "package.json"), '{"type":"module"}');
+      fs.writeFileSync(path.join(root, "pnpm-workspace.yaml"), "packages: []\n");
+      installDistArtifactScripts(root, ["run-tsgo-core-test-shards.mts", "run-tsgo.mts"], {
+        compiler: false,
+        dependencies: ["@openclaw/fs-safe"],
+      });
+      const compiler = path.join(root, "compiler.cjs");
+      fs.writeFileSync(
+        compiler,
+        `#!${resolveTestNodeExecPath()}
+process.on('SIGTERM', () => {});
+process.stdin.resume();
+console.log(JSON.stringify({ pid: process.pid }));
+`,
+      );
+      fs.chmodSync(compiler, 0o755);
+      overrideNativeFixtureExecutable(root, compiler);
+      const clock = path.join(root, "supervisor-clock.mjs");
+      // Accelerate every supervisor's existing grace period equally. Readiness and
+      // process completion still use real pipes and OS signals, without sleeps.
+      fs.writeFileSync(
+        clock,
+        `const now = Date.now.bind(Date), start = now();
+Date.now = () => start + (now() - start) * 10;
+`,
+      );
+      const ready = createDeferredCore<{ pid: number }>();
+      let child: ChildProcess | undefined;
+      let output = "";
+      let compilerPid: number | undefined;
+      const completion = lifetime.track(
+        runManagedCommand({
+          bin: resolveTestNodeExecPath(),
+          args: [
+            path.join(root, "scripts/run-tsgo-core-test-shards.mts"),
+            "--stripe",
+            `1/${TSGO_CORE_TEST_SHARDS.length}`,
+          ],
+          cwd: root,
+          env: {
+            ...process.env,
+            NODE_OPTIONS: `--import=${pathToFileURL(clock).href}`,
+          },
+          stdio: ["pipe", "pipe", "pipe"],
+          signal,
+          requireProcessTreeExit: true,
+          onReady: (started) => {
+            child = started;
+            createInterface({ input: started.stdout! }).once("line", (line) => {
+              ready.resolve(JSON.parse(line) as { pid: number });
+            });
+            started.stderr!.on("data", (chunk) => {
+              output += String(chunk);
+            });
+          },
+        }),
+      );
+      try {
+        const compilerReady = await Promise.race([
+          ready.promise,
+          completion.then((code) => {
+            throw new Error(`Shards exited before compiler readiness: ${code}\n${output}`);
+          }),
+        ]);
+        compilerPid = compilerReady.pid;
+        if (!child) {
+          throw new Error("Missing shard process");
+        }
+        child.kill("SIGTERM");
+        expect(await completion, output).toBe(143);
+        expect(() => process.kill(compilerReady.pid, 0)).toThrow();
+        expect(fs.readdirSync(path.join(root, ".artifacts/dist-artifacts.lock"))).toEqual([]);
+        await withDistArtifactOwnership(root, async () => {
+          fs.writeFileSync(path.join(root, "next-check"), "acquired");
+        });
+        expect(fs.readFileSync(path.join(root, "next-check"), "utf8")).toBe("acquired");
+      } finally {
+        await lifetime.verifyCleanup(async () => {
+          child?.stdin?.end();
+          try {
+            await completion;
+          } finally {
+            if (compilerPid !== undefined) {
+              if (isProcessAlive(compilerPid)) {
+                process.kill(compilerPid, "SIGKILL");
+              }
+              await waitForDead(compilerPid, 2_000);
+            }
+          }
+        });
+      }
+    }),
+);