Retry suite-level runtime failures (#58631)

Summary:
Pull Request resolved: https://github.com/react/react-native/pull/58631

Fantom's CI retries use Jest's `--onlyFailures` mode. Jest 29's default sequencer caches suite-level runtime errors as passing because they contain no failed test cases, so retries select no suites.

See https://github.com/react/react-native/actions/runs/35630015753/job/106440689782?pr=58624 of an example of this, where retries didn't actually do anything

Add a Fantom-specific sequencer that marks runtime-error suites as failed only for cache bookkeeping. This preserves the original test reporting while ensuring targeted retries rerun the crashed suite.

Changelog: [Internal]

Reviewed By: cipolleschi

Differential Revision: D121063351

fbshipit-source-id: e24737cc247ea0e11aa04dac74d557a3f41f9db6

package.json

@@ -66,6 +66,7 @@
     "@electron/packager": "^20.0.0",
     "@expo/spawn-async": "^1.7.2",
     "@jest/create-cache-key-function": "^29.7.0",
+    "@jest/test-sequencer": "^29.7.0",
     "@microsoft/api-extractor": "^7.52.2",
     "@octokit/rest": "^22.0.0",
     "@react-native/metro-babel-transformer": "0.87.0-main",

private/react-native-fantom/config/FantomTestSequencer.js

@@ -0,0 +1,24 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @noflow
+ * @format
+ */
+
+'use strict';
+
+const prepareFantomTestResultsForCache = require('./prepareFantomTestResultsForCache');
+const TestSequencer = require('@jest/test-sequencer').default;
+
+class FantomTestSequencer extends TestSequencer {
+  cacheResults(tests, results) {
+    // Jest 29 treats suite-level runtime errors as passing in its retry cache
+    // because they have no failed test cases (https://github.com/jestjs/jest/issues/15382).
+    super.cacheResults(tests, prepareFantomTestResultsForCache(results));
+  }
+}
+
+module.exports = FantomTestSequencer;

private/react-native-fantom/config/__tests__/prepareFantomTestResultsForCache-test.js

@@ -0,0 +1,36 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @flow strict-local
+ * @format
+ */
+
+'use strict';
+
+const prepareFantomTestResultsForCache = require('../prepareFantomTestResultsForCache');
+
+describe('prepareFantomTestResultsForCache', () => {
+  it('marks suite-level runtime errors as failed for Jest retries', () => {
+    const runtimeFailure = {
+      numFailingTests: 0,
+      testExecError: new Error('Process exited with SIGSEGV'),
+    };
+    const passingResult = {numFailingTests: 0};
+    const assertionFailure = {numFailingTests: 1};
+    const results = {
+      testResults: [runtimeFailure, passingResult, assertionFailure],
+    };
+
+    const cacheResults = prepareFantomTestResultsForCache(results);
+
+    expect(cacheResults.testResults).toEqual([
+      {...runtimeFailure, numFailingTests: 1},
+      passingResult,
+      assertionFailure,
+    ]);
+    expect(results.testResults[0]).toBe(runtimeFailure);
+  });
+});

private/react-native-fantom/config/prepareFantomTestResultsForCache.js

@@ -0,0 +1,39 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @flow strict-local
+ * @format
+ */
+
+'use strict';
+
+/*::
+type FantomTestResult = {
+  readonly numFailingTests: number,
+  readonly testExecError?: ?unknown,
+  ...,
+};
+
+type FantomAggregatedResult = {
+  readonly testResults: ReadonlyArray<FantomTestResult>,
+  ...,
+};
+*/
+
+function prepareFantomTestResultsForCache(
+  results /*: FantomAggregatedResult */,
+) /*: FantomAggregatedResult */ {
+  return {
+    ...results,
+    testResults: results.testResults.map(testResult =>
+      testResult.testExecError != null && testResult.numFailingTests === 0
+        ? {...testResult, numFailingTests: 1}
+        : testResult,
+    ),
+  };
+}
+
+module.exports = prepareFantomTestResultsForCache;

scripts/fantom.sh

@@ -37,4 +37,7 @@ if [[ -n "$FANTOM_RUN_BENCHMARKS" ]]; then
   ARGS+=("--runInBand")
 fi
 
-yarn jest --config private/react-native-fantom/config/jest.config.js "${ARGS[@]}"
+yarn jest \
+  --config private/react-native-fantom/config/jest.config.js \
+  --testSequencer '<rootDir>/private/react-native-fantom/config/FantomTestSequencer.js' \
+  "${ARGS[@]}"