From 8831478021e4b5120eb0ce79367d8792e3967417 Mon Sep 17 00:00:00 2001 From: Rob Lourens Date: Thu, 6 Aug 2026 07:15:51 -0700 Subject: [PATCH] agentHost: run E2E suites in parallel (#329314) * agentHost: run E2E suites in parallel Run the deterministic conformance and provider entrypoints concurrently while preserving serial server reuse within each suite. Integrate the parallel runner into coverage and full integration runs.\n\n(Written by Copilot)\n\nCo-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * agentHost: harden parallel E2E runner Preserve Windows argument boundaries through a PowerShell wrapper and fail coverage runs when a worker does not emit protocol-surface observations. (Written by Copilot) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * agentHost: stabilize terminal clear E2E test Wait for a marker that appears only in command output, not in the shell's echoed input, before clearing terminal state. (Written by Copilot) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/skills/integration-tests/SKILL.md | 1 + package.json | 1 + scripts/agent-host-e2e-coverage.ts | 12 +- scripts/test-agent-host-e2e-child.ps1 | 13 + scripts/test-agent-host-e2e.ts | 320 ++++++++++++++++++ scripts/test-integration.bat | 6 +- scripts/test-integration.sh | 5 +- .../agentHost/test/node/e2e/README.md | 14 + .../node/e2e/suites/stateOperationsSuite.ts | 2 +- test/unit/electron/index.js | 6 +- test/unit/electron/renderer.js | 6 +- 11 files changed, 370 insertions(+), 16 deletions(-) create mode 100644 scripts/test-agent-host-e2e-child.ps1 create mode 100644 scripts/test-agent-host-e2e.ts diff --git a/.github/skills/integration-tests/SKILL.md b/.github/skills/integration-tests/SKILL.md index e8428fac8986..349c54b59bb2 100644 --- a/.github/skills/integration-tests/SKILL.md +++ b/.github/skills/integration-tests/SKILL.md @@ -16,6 +16,7 @@ Integration tests in VS Code are split into two categories: - **Windows:** `.\scripts\test-integration.bat [options]` When run **without filters**, both scripts execute all node.js integration tests followed by all extension host tests. +The deterministic Agent Host E2E entrypoints are parallelized across isolated test processes during the node.js phase, then excluded from the remaining serial node.js run. When run **with `--run` or `--runGlob`** (without `--suite`), only the node.js integration tests are run and the filter is applied. Extension host tests are skipped since these filters are node.js-specific. diff --git a/package.json b/package.json index 728767a25f7f..d8dd35651d26 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "test-node": "mocha test/unit/node/index.js --delay --ui=tdd --timeout=5000 --exit", "test-extension": "vscode-test", "test-build-scripts": "cd build && npm run test", + "test-agent-host-e2e": "node scripts/test-agent-host-e2e.ts", "test-agent-host-e2e-coverage": "node scripts/agent-host-e2e-coverage.ts", "check-cyclic-dependencies": "node build/lib/checkCyclicDependencies.ts out", "preinstall": "node build/npm/preinstall.ts", diff --git a/scripts/agent-host-e2e-coverage.ts b/scripts/agent-host-e2e-coverage.ts index 6a24977b96ed..d1dd4a2c39ff 100644 --- a/scripts/agent-host-e2e-coverage.ts +++ b/scripts/agent-host-e2e-coverage.ts @@ -46,12 +46,6 @@ const incompatibleFlags = [ 'AGENT_HOST_UPDATE_SNAPSHOTS', ]; -/** - * Every entrypoint that makes up the portable suite: the conformance tier - * (registered once) plus one parity entrypoint per provider. - */ -const e2eGlob = '**/agentHost/test/node/e2e/{providers/*AgentHostE2E,conformance/*}.integrationTest.js'; - function main(): void { validateEnvironment(); @@ -70,8 +64,7 @@ function main(): void { AGENT_HOST_RECORD_PROTOCOL_SURFACE: '1', AGENT_HOST_PROTOCOL_SURFACE_OUT: observedSurfacePath, }; - const testScript = join(repoRoot, 'scripts', process.platform === 'win32' ? 'test-integration.bat' : 'test-integration.sh'); - run(testScript, ['--runGlob', e2eGlob], testEnvironment, process.platform === 'win32'); + run(process.execPath, [join(repoRoot, 'scripts', 'test-agent-host-e2e.ts')], testEnvironment); const rawFiles = readdirSync(rawCoveragePath).filter(file => file.endsWith('.json')); if (rawFiles.length === 0) { @@ -115,11 +108,10 @@ function validateEnvironment(): void { } } -function run(command: string, args: readonly string[], environment: NodeJS.ProcessEnv, shell = false): void { +function run(command: string, args: readonly string[], environment: NodeJS.ProcessEnv): void { const result = spawnSync(command, args, { cwd: repoRoot, env: environment, - shell, stdio: 'inherit', }); if (result.error) { diff --git a/scripts/test-agent-host-e2e-child.ps1 b/scripts/test-agent-host-e2e-child.ps1 new file mode 100644 index 000000000000..6218f5d6fe06 --- /dev/null +++ b/scripts/test-agent-host-e2e-child.ps1 @@ -0,0 +1,13 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +param( + [Parameter(Mandatory = $true)] + [string]$TestScript, + + [Parameter(ValueFromRemainingArguments = $true)] + [string[]]$TestArguments +) + +& $TestScript @TestArguments +exit $LASTEXITCODE diff --git a/scripts/test-agent-host-e2e.ts b/scripts/test-agent-host-e2e.ts new file mode 100644 index 000000000000..4eb17885cf1c --- /dev/null +++ b/scripts/test-agent-host-e2e.ts @@ -0,0 +1,320 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +const childProcess: typeof import('child_process') = require('child_process'); +const fs: typeof import('fs') = require('fs'); +const os: typeof import('os') = require('os'); +const path: typeof import('path') = require('path'); +const { spawn, spawnSync } = childProcess; +const { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } = fs; +const { availableParallelism, cpus } = os; +const { basename, dirname, extname, join, resolve } = path; + +const repoRoot = resolve(__dirname, '..'); +const testScript = join(repoRoot, 'scripts', process.platform === 'win32' ? 'test-integration.bat' : 'test-integration.sh'); +const windowsTestWrapper = join(repoRoot, 'scripts', 'test-agent-host-e2e-child.ps1'); +const incompatibleFlags = [ + 'AGENT_HOST_REPLAY_RECORD', + 'AGENT_HOST_UPDATE_AHP_SNAPSHOTS', + 'AGENT_HOST_UPDATE_SNAPSHOTS', +]; + +interface ISuite { + readonly id: string; + readonly label: string; + readonly file: string; +} + +interface IRunResult { + readonly suite: ISuite; + readonly succeeded: boolean; + readonly durationSeconds: number; + readonly failure?: string; +} + +interface IObservedSurface { + readonly commands: readonly string[]; + readonly notifications: readonly string[]; + readonly actions: readonly string[]; +} + +const suites: readonly ISuite[] = [ + { + id: 'conformance', + label: 'Conformance', + file: 'src/vs/platform/agentHost/test/node/e2e/conformance/agentHostConformance.integrationTest.ts', + }, + { + id: 'claude', + label: 'Claude', + file: 'src/vs/platform/agentHost/test/node/e2e/providers/claudeAgentHostE2E.integrationTest.ts', + }, + { + id: 'codex', + label: 'Codex', + file: 'src/vs/platform/agentHost/test/node/e2e/providers/codexAgentHostE2E.integrationTest.ts', + }, + { + id: 'copilot', + label: 'Copilot', + file: 'src/vs/platform/agentHost/test/node/e2e/providers/copilotAgentHostE2E.integrationTest.ts', + }, +]; + +async function main(): Promise { + validateEnvironment(); + const { jobs, forwardedArgs } = parseArguments(process.argv.slice(2)); + prepareTestRuntime(); + + const startedAt = process.hrtime.bigint(); + const surfaceOutputs = prepareSurfaceOutputs(); + const results: IRunResult[] = []; + let nextSuite = 0; + + const workers = Array.from({ length: jobs }, async () => { + while (nextSuite < suites.length) { + const suiteIndex = nextSuite++; + const suite = suites[suiteIndex]; + results[suiteIndex] = await runSuite(suite, forwardedArgs, surfaceOutputs.get(suite.id)); + } + }); + await Promise.all(workers); + + const failures = results.filter(result => !result.succeeded); + if (surfaceOutputs.size > 0 && failures.length === 0) { + mergeSurfaceOutputs(surfaceOutputs); + } + + const durationSeconds = elapsedSeconds(startedAt); + console.log(`\nAgent Host E2E suites completed in ${durationSeconds.toFixed(1)}s (${jobs} parallel ${jobs === 1 ? 'worker' : 'workers'}).`); + for (const result of results) { + console.log(` ${result.succeeded ? 'PASS' : 'FAIL'} ${result.suite.label}: ${result.durationSeconds.toFixed(1)}s`); + } + if (failures.length > 0) { + process.exitCode = 1; + } +} + +function validateEnvironment(): void { + const enabledFlags = incompatibleFlags.filter(flag => process.env[flag] === '1'); + if (enabledFlags.length > 0) { + throw new Error(`Parallel Agent Host E2E runs only support deterministic replay; unset ${enabledFlags.join(', ')}`); + } +} + +function parseArguments(args: readonly string[]): { jobs: number; forwardedArgs: readonly string[] } { + const forwardedArgs: string[] = []; + let requestedJobs: string | undefined = process.env['AGENT_HOST_E2E_JOBS']; + + for (let index = 0; index < args.length; index++) { + const argument = args[index]; + if (argument === '--jobs') { + requestedJobs = args[++index]; + if (!requestedJobs) { + throw new Error('--jobs requires a value'); + } + } else if (argument.startsWith('--jobs=')) { + requestedJobs = argument.slice('--jobs='.length); + } else { + forwardedArgs.push(argument); + } + } + + const ownedArguments = ['--run', '--runGlob', '--glob', '--runGrep', '--testSplit']; + const conflictingArgument = forwardedArgs.find(argument => ownedArguments.some(owned => argument === owned || argument.startsWith(`${owned}=`))); + if (conflictingArgument) { + throw new Error(`${conflictingArgument} is managed by the Agent Host E2E runner`); + } + + const defaultJobs = Math.min(suites.length, availableParallelism?.() ?? cpus().length); + const jobs = requestedJobs === undefined ? defaultJobs : Number(requestedJobs); + if (!Number.isInteger(jobs) || jobs < 1) { + throw new Error(`Invalid Agent Host E2E worker count: ${requestedJobs}`); + } + return { jobs: Math.min(jobs, suites.length), forwardedArgs }; +} + +function prepareTestRuntime(): void { + const environment = { ...process.env }; + delete environment.ELECTRON_RUN_AS_NODE; + + if (!existsSync(join(repoRoot, 'node_modules'))) { + runSync(process.platform === 'win32' ? 'npm.cmd' : 'npm', ['install'], environment); + } + if (process.env['VSCODE_SKIP_PRELAUNCH'] !== '1') { + runSync(process.platform === 'win32' ? 'npm.cmd' : 'npm', ['run', 'electron'], environment); + } +} + +function runSync(command: string, args: readonly string[], environment: NodeJS.ProcessEnv): void { + const result = spawnSync(command, args, { + cwd: repoRoot, + env: environment, + stdio: 'inherit', + }); + if (result.error) { + throw result.error; + } + if (result.status !== 0) { + const reason = result.signal ? `signal ${result.signal}` : `code ${result.status}`; + throw new Error(`${command} exited with ${reason}`); + } +} + +async function runSuite(suite: ISuite, forwardedArgs: readonly string[], surfaceOutput: string | undefined): Promise { + console.log(`Starting Agent Host E2E — ${suite.label}`); + const startedAt = process.hrtime.bigint(); + const environment = { + ...process.env, + VSCODE_SKIP_PRELAUNCH: '1', + ...(surfaceOutput ? { AGENT_HOST_PROTOCOL_SURFACE_OUT: surfaceOutput } : {}), + }; + delete environment.ELECTRON_RUN_AS_NODE; + + return new Promise(resolveResult => { + const testArguments = ['--run', suite.file, ...suiteArguments(forwardedArgs, suite)]; + const child = process.platform === 'win32' + ? spawn(join(process.env['SYSTEMROOT'] ?? 'C:\\Windows', 'System32', 'WindowsPowerShell', 'v1.0', 'powershell.exe'), [ + '-NoLogo', + '-NoProfile', + '-NonInteractive', + '-ExecutionPolicy', 'Bypass', + '-File', windowsTestWrapper, + testScript, + ...testArguments, + ], { + cwd: repoRoot, + env: environment, + stdio: ['ignore', 'pipe', 'pipe'], + }) + : spawn(testScript, testArguments, { + cwd: repoRoot, + env: environment, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let output = ''; + child.stdout.setEncoding('utf8'); + child.stderr.setEncoding('utf8'); + child.stdout.on('data', chunk => output += chunk); + child.stderr.on('data', chunk => output += chunk); + child.on('error', error => { + resolveResult({ + suite, + succeeded: false, + durationSeconds: elapsedSeconds(startedAt), + failure: error.message, + }); + }); + child.on('close', (code, signal) => { + const succeeded = code === 0; + const failure = succeeded ? undefined : signal ? `signal ${signal}` : `code ${code}`; + console.log(`\n===== Agent Host E2E — ${suite.label} =====`); + process.stdout.write(output); + if (!output.endsWith('\n')) { + process.stdout.write('\n'); + } + resolveResult({ + suite, + succeeded, + durationSeconds: elapsedSeconds(startedAt), + failure, + }); + }); + }).then(result => { + if (result.failure) { + console.error(`Agent Host E2E — ${suite.label} failed with ${result.failure}`); + } + return result; + }); +} + +function suiteArguments(args: readonly string[], suite: ISuite): readonly string[] { + const result = [...args]; + const tfsIndex = result.indexOf('--tfs'); + if (tfsIndex >= 0 && result[tfsIndex + 1]) { + result[tfsIndex + 1] = `${result[tfsIndex + 1]} ${suite.label}`; + } + return result; +} + +function prepareSurfaceOutputs(): ReadonlyMap { + if (process.env['AGENT_HOST_RECORD_PROTOCOL_SURFACE'] !== '1') { + return new Map(); + } + + const combinedOutput = process.env['AGENT_HOST_PROTOCOL_SURFACE_OUT'] + ?? join(repoRoot, '.build', 'agent-host-e2e-coverage', 'protocol-surface', 'observed.json'); + const extension = extname(combinedOutput); + const stem = basename(combinedOutput, extension); + const outputs = new Map(); + for (const suite of suites) { + const output = join(dirname(combinedOutput), `${stem}-${suite.id}${extension}`); + rmSync(output, { force: true }); + outputs.set(suite.id, output); + } + return outputs; +} + +function mergeSurfaceOutputs(outputs: ReadonlyMap): void { + const combinedOutput = process.env['AGENT_HOST_PROTOCOL_SURFACE_OUT'] + ?? join(repoRoot, '.build', 'agent-host-e2e-coverage', 'protocol-surface', 'observed.json'); + const commands = new Set(); + const notifications = new Set(); + const actions = new Set(); + + for (const output of outputs.values()) { + if (!existsSync(output)) { + if (process.env['AGENT_HOST_E2E_COVERAGE'] === '1') { + throw new Error(`Missing protocol surface observations from ${output}`); + } + continue; + } + const observed = readObservedSurface(output); + observed.commands.forEach(command => commands.add(command)); + observed.notifications.forEach(notification => notifications.add(notification)); + observed.actions.forEach(action => actions.add(action)); + rmSync(output, { force: true }); + } + + mkdirSync(dirname(combinedOutput), { recursive: true }); + writeFileSync(combinedOutput, `${JSON.stringify({ + commands: [...commands].sort(), + notifications: [...notifications].sort(), + actions: [...actions].sort(), + }, undefined, '\t')}\n`); +} + +function readObservedSurface(file: string): IObservedSurface { + const value: unknown = JSON.parse(readFileSync(file, 'utf8')); + if (!isRecord(value) + || !isStringArray(value.commands) + || !isStringArray(value.notifications) + || !isStringArray(value.actions) + ) { + throw new Error(`Invalid protocol surface observations in ${file}`); + } + return { + commands: value.commands, + notifications: value.notifications, + actions: value.actions, + }; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null; +} + +function isStringArray(value: unknown): value is readonly string[] { + return Array.isArray(value) && value.every(entry => typeof entry === 'string'); +} + +function elapsedSeconds(startedAt: bigint): number { + return Number(process.hrtime.bigint() - startedAt) / 1_000_000_000; +} + +main().catch(error => { + console.error(error); + process.exitCode = 1; +}); diff --git a/scripts/test-integration.bat b/scripts/test-integration.bat index fbbbcf2490f7..f69e52db0bb3 100644 --- a/scripts/test-integration.bat +++ b/scripts/test-integration.bat @@ -35,6 +35,7 @@ if defined SHOW_HELP ( echo. echo Runs integration tests. When no filters are given, all integration tests echo ^(node.js integration tests + extension host tests^) are run. + echo Agent Host E2E entrypoints run in parallel before the remaining node.js tests. echo. echo --run and --runGlob select which node.js integration test files to load. echo Extension host tests are skipped when these options are used. @@ -128,7 +129,10 @@ if defined RUN_GLOB ( ) else if defined RUN_FILE ( call .\scripts\test.bat %* ) else ( - call .\scripts\test.bat --runGlob **\*.integrationTest.js %* + call node .\scripts\test-agent-host-e2e.ts %* + if errorlevel 1 exit /b 1 + set VSCODE_SKIP_PRELAUNCH=1 + call .\scripts\test.bat --runGlob **\*.integrationTest.js --excludeRunGlob "**/agentHost/test/node/e2e/{providers/*AgentHostE2E,conformance/*}.integrationTest.js" %* ) if %errorlevel% neq 0 exit /b %errorlevel% :skip_nodejs_tests diff --git a/scripts/test-integration.sh b/scripts/test-integration.sh index 926f89aa2cb8..6cebad2e422d 100755 --- a/scripts/test-integration.sh +++ b/scripts/test-integration.sh @@ -17,6 +17,7 @@ RUN_GLOB="" GREP_PATTERN="" SUITE_FILTER="" HELP=false +AGENT_HOST_E2E_GLOB="**/agentHost/test/node/e2e/{providers/*AgentHostE2E,conformance/*}.integrationTest.js" while [[ $# -gt 0 ]]; do case "$1" in @@ -58,6 +59,7 @@ if $HELP; then echo "" echo "Runs integration tests. When no filters are given, all integration tests" echo "(node.js integration tests + extension host tests) are run." + echo "Agent Host E2E entrypoints run in parallel before the remaining node.js tests." echo "" echo "--run and --runGlob select which node.js integration test files to load." echo "Extension host tests are skipped when these options are used." @@ -171,7 +173,8 @@ if [[ -z "$SUITE_FILTER" ]]; then echo "### node.js integration tests" echo if [[ -z "$RUN_GLOB" && -z "$RUN_FILE" ]]; then - ./scripts/test.sh --runGlob "**/*.integrationTest.js" "${EXTRA_ARGS[@]}" + node ./scripts/test-agent-host-e2e.ts "${EXTRA_ARGS[@]}" + VSCODE_SKIP_PRELAUNCH=1 ./scripts/test.sh --runGlob "**/*.integrationTest.js" --excludeRunGlob "$AGENT_HOST_E2E_GLOB" "${EXTRA_ARGS[@]}" else ./scripts/test.sh "${EXTRA_ARGS[@]}" fi diff --git a/src/vs/platform/agentHost/test/node/e2e/README.md b/src/vs/platform/agentHost/test/node/e2e/README.md index 5ff94e146e39..9bcda5c98fe9 100644 --- a/src/vs/platform/agentHost/test/node/e2e/README.md +++ b/src/vs/platform/agentHost/test/node/e2e/README.md @@ -13,6 +13,9 @@ They do this by recording the model traffic once (against real CAPI) into commit ## TL;DR ```bash +# Run the complete deterministic suite in parallel. +npm run test-agent-host-e2e + # Replay (default): deterministic, tokenless. This is what CI runs. ./scripts/test-integration.sh --run src/vs/platform/agentHost/test/node/e2e/providers/copilotAgentHostE2E.integrationTest.ts @@ -185,9 +188,18 @@ A mismatch fails the test as `[capi-replay] N model request mismatch(es)` and pr Replay is the default — no setup, no token: ```bash +# Run conformance and all provider suites in parallel. +npm run test-agent-host-e2e + +# Limit parallelism when machine resources are constrained. +npm run test-agent-host-e2e -- --jobs 2 + +# Run one provider. ./scripts/test-integration.sh --run src/vs/platform/agentHost/test/node/e2e/providers/copilotAgentHostE2E.integrationTest.ts ``` +The complete-suite runner starts one test process per entrypoint and runs up to four concurrently. `AGENT_HOST_E2E_JOBS` or `--jobs` can lower the worker count. Recording and snapshot-update modes remain per-provider commands so they never make concurrent writes or real CAPI requests. + Provider availability: - **Copilot** (`copilotcli`) — always enabled (the CLI is a dev dependency). @@ -206,6 +218,8 @@ The lease also owns a fresh suite data directory. Every server it starts uses th - **Shared** (the default in replay, for every provider) — reuse a server + proxy across tests, swapping the per-test fixture and reconnecting a fresh client. The lease recycles after 25 model-backed tests or 40 total tests, whichever comes first. The model cap bounds provider-process load; the total cap bounds host-owned terminals, watchers, subscriptions, and other resource accumulation in host-only suites. +The complete-suite runner parallelizes above this lease: conformance, Claude, Codex, and Copilot each run in an isolated test process with their own server lease. Tests within one entrypoint stay serial and continue sharing servers, preserving the lifecycle and fixture-window invariants while letting the four independent entrypoints overlap. + The swap is what makes sharing cheap: the proxy is an `http.Server` running **inside the test process**, so `CapiReplayProxy.resetForReplay(fixturePath)` is a plain in-process method call — no IPC, no re-fork. It reloads the replay buckets and clears the cache-miss log while keeping the **same proxy URL**, so the long-lived agent host (forked against that URL) keeps talking to the same proxy and just receives the next fixture's recorded responses. Per-test state must be reset there rather than read from the proxy's constructor options, which belong to whichever test started the shared server. Teardown calls `assertNoReplayMismatches()` to verify a test's traffic *without* stopping the server (vs `stop()`, which verifies then closes); the suite's `suiteTeardown` closes it via `close()`. **The one invariant: a shared-server test must not leave a turn in flight.** Because one server serves multiple tests, each test's request/response traffic must land inside its own fixture window. If a test returns mid-turn, the SDK's continuation HTTP call fires *after* the fixture is swapped for the next test, landing in that test's window as an unrecorded call. In replay, failure to drain to `turnComplete` is fatal. Direct live recording may use an explicitly bounded best-effort drain because provider latency is not deterministic. diff --git a/src/vs/platform/agentHost/test/node/e2e/suites/stateOperationsSuite.ts b/src/vs/platform/agentHost/test/node/e2e/suites/stateOperationsSuite.ts index d53a54b270ba..fb883481c6d4 100644 --- a/src/vs/platform/agentHost/test/node/e2e/suites/stateOperationsSuite.ts +++ b/src/vs/platform/agentHost/test/node/e2e/suites/stateOperationsSuite.ts @@ -411,7 +411,7 @@ export function defineStateOperationsTests(context: IAgentHostE2ETestContext): v context.client.dispatch({ channel: terminalUri, clientSeq: 1, - action: { type: ActionType.TerminalInput, data: 'node -p "\'CLEAR_MARKER\'"\r' }, + action: { type: ActionType.TerminalInput, data: 'node -p "\'CLEAR_\'+\'MARKER\'"\r' }, }); let streamedOutput = ''; await context.client.waitForNotification(n => { diff --git a/test/unit/electron/index.js b/test/unit/electron/index.js index 16863fcc2281..1a3c9ae18f27 100644 --- a/test/unit/electron/index.js +++ b/test/unit/electron/index.js @@ -31,6 +31,7 @@ const minimist = require('minimist'); * grep: string; * run: string | string[]; * runGlob: string; + * excludeRunGlob: string; * testSplit: string; * dev: boolean; * reporter: string; @@ -48,7 +49,7 @@ const minimist = require('minimist'); * }} */ const args = minimist(process.argv.slice(2), { - string: ['grep', 'run', 'runGlob', 'reporter', 'reporter-options', 'waitServer', 'timeout', 'crash-reporter-directory', 'tfs', 'coveragePath', 'coverageFormats', 'testSplit'], + string: ['grep', 'run', 'runGlob', 'excludeRunGlob', 'reporter', 'reporter-options', 'waitServer', 'timeout', 'crash-reporter-directory', 'tfs', 'coveragePath', 'coverageFormats', 'testSplit'], boolean: ['build', 'coverage', 'help', 'dev', 'per-test-coverage'], alias: { 'grep': ['g', 'f'], @@ -72,6 +73,7 @@ Options: --grep, -g, -f only run tests matching --run only run tests from --runGlob, --glob, --runGrep only run tests matching +--excludeRunGlob exclude tests matching from --runGlob --testSplit / split tests into parts and run the th part --build run with build output (out-build) --coverage generate coverage report @@ -127,7 +129,7 @@ if (crashReporterDirectory) { } if (!args.dev) { - app.setPath('userData', path.join(tmpdir(), `vscode-tests-${Date.now()}`)); + app.setPath('userData', path.join(tmpdir(), `vscode-tests-${Date.now()}-${process.pid}`)); } function deserializeSuite(suite) { diff --git a/test/unit/electron/renderer.js b/test/unit/electron/renderer.js index 718cf9dd8d7f..299c3ee36f73 100644 --- a/test/unit/electron/renderer.js +++ b/test/unit/electron/renderer.js @@ -170,7 +170,11 @@ async function loadTestModules(opts) { } const pattern = opts.runGlob || _tests_glob; - const files = await globAsync(pattern, { cwd: loadFn._out }); + let files = await globAsync(pattern, { cwd: loadFn._out }); + if (opts.excludeRunGlob) { + const excludedFiles = new Set(await globAsync(opts.excludeRunGlob, { cwd: loadFn._out })); + files = files.filter(file => !excludedFiles.has(file)); + } let modules = files.map(file => file.replace(/\.js$/, '')); if (opts.testSplit) { const [i, n] = opts.testSplit.split('/').map(Number);