feat(test runner): conditional retries based on error (#43929)

Author: pavelfeldman

docs/src/test-api/class-fullproject.md

@@ -62,7 +62,7 @@ See [`property: TestProject.repeatEach`].
 * since: v1.10
 - type: <[int]>
 
-See [`property: TestProject.retries`].
+See [`property: TestProject.retries`]. When retries are configured as a list, this is the largest `count` in the list.
 
 ## property: FullProject.teardown
 * since: v1.34

docs/src/test-api/class-test.md

@@ -583,9 +583,11 @@ Execution mode. Learn more about the execution modes [here](../test-parallel.md)
 
 ### option: Test.describe.configure.retries
 * since: v1.28
-- `retries` <[int]>
+- `retries` <[int]|[Array]<[Object]>>
+  - `count` <[int]> The maximum number of retries given to a failed test when this entry applies.
+  - `condition` ?<[RegExp]> Regular expression matched against the error messages of the failed attempt. An entry without a condition always applies.
 
-The number of retries for each test.
+The number of retries for each test. Set to a list to make the number of retries depend on the error, see [`property: TestConfig.retries`].
 
 ### option: Test.describe.configure.timeout
 * since: v1.28

docs/src/test-api/class-testconfig.md

@@ -502,10 +502,14 @@ Whether to skip entries from `.gitignore` when searching for test files. By defa
 
 ## property: TestConfig.retries
 * since: v1.10
-- type: ?<[int]>
+- type: ?<[int]|[Array]<[Object]>>
+  - `count` <[int]> The maximum number of retries given to a failed test when this entry applies.
+  - `condition` ?<[RegExp]> Regular expression matched against the error messages of the failed attempt. An entry without a condition always applies.
 
 The maximum number of retry attempts given to failed tests. By default failing tests are not retried. Learn more about [test retries](../test-retries.md#retries).
 
+When set to a list, the number of retries depends on the error. After each failed attempt, Playwright takes the largest `count` among the entries that apply, and retries the test when it has run fewer times than that. The latest failed attempt decides: a test that first fails with a matching error and then with a different one only gets the retries of the latter. Worker crashes and timeouts are matched by their error message like any other failure.
+
 **Usage**
 
 ```js title="playwright.config.ts"
@@ -516,6 +520,19 @@ export default defineConfig({
 });
 ```
 
+Retry any failure once, and up to three times when the browser or the worker crashed:
+
+```js title="playwright.config.ts"
+import { defineConfig } from '@playwright/test';
+
+export default defineConfig({
+  retries: [
+    { count: 1 },
+    { count: 3, condition: /Target crashed|worker process exited unexpectedly/ },
+  ],
+});
+```
+
 ## property: TestConfig.retryStrategy
 * since: v1.62
 - type: "RetryStrategy" ?<"immediate"|"isolated">

docs/src/test-api/class-testproject.md

@@ -273,10 +273,14 @@ Whether to skip entries from `.gitignore` when searching for test files. By defa
 
 ## property: TestProject.retries
 * since: v1.10
-- type: ?<[int]>
+- type: ?<[int]|[Array]<[Object]>>
+  - `count` <[int]> The maximum number of retries given to a failed test when this entry applies.
+  - `condition` ?<[RegExp]> Regular expression matched against the error messages of the failed attempt. An entry without a condition always applies.
 
 The maximum number of retry attempts given to failed tests. Learn more about [test retries](../test-retries.md#retries).
 
+Set to a list to make the number of retries depend on the error, see [`property: TestConfig.retries`].
+
 Use [`method: Test.describe.configure`] to change the number of retries for a specific file or a group of tests.
 
 Use [`property: TestConfig.retries`] to change this option for all projects.

docs/src/test-reporter-api/class-testcase.md

@@ -72,7 +72,7 @@ Results for each run of this test.
 * since: v1.10
 - type: <[int]>
 
-The maximum number of retries given to this test in the configuration.
+The maximum number of retries given to this test in the configuration. With conditional retries, this is updated after each failed attempt to the number of retries that applies to that failure.
 
 Learn more about [test retries](../test-retries.md#retries).
 

docs/src/test-retries-js.md

@@ -127,6 +127,25 @@ test.describe(() => {
 });
 ```
 
+### Conditional retries
+
+The number of retries can depend on the error. Give a list of entries, each with a `count` and an optional `condition` regular expression. After each failed attempt, the largest `count` among the entries whose condition matches an error message of that attempt applies. An entry without a condition always applies.
+
+```js title="playwright.config.ts"
+import { defineConfig } from '@playwright/test';
+
+export default defineConfig({
+  retries: [
+    // Retry any failure once.
+    { count: 1 },
+    // Retry up to three times when the browser or the worker crashed.
+    { count: 3, condition: /Target crashed|worker process exited unexpectedly/ },
+  ],
+});
+```
+
+Worker crashes and timeouts are matched by their error message like any other failure. The latest failed attempt decides: a test that first crashes and then fails on an assertion only gets the retries of the assertion failure.
+
 ## Serial mode
 
 Use [`method: Test.describe.serial`] to group dependent tests to ensure they will always run together and in order. If one of the tests fails, all subsequent tests are skipped. All tests in the group are retried together.

packages/playwright/src/common/config.ts

@@ -20,10 +20,12 @@ import path from 'path';
 
 import { packageJSON } from '../package';
 import { getPackageJsonPath, mergeObjects, takeFirst } from '../util';
+import { maxRetries, normalizeRetries } from './retries';
 
 import type { Config, Fixtures, Metadata, PlaywrightWorkerOptions, Project, ReporterDescription } from '../../types/test';
 import type { TestRunnerPluginRegistration } from '../plugins';
 import type { ConfigCLIOverrides } from './ipc';
+import type { RetriesEntry } from './retries';
 import type { Location } from '../../types/testReporter';
 import type { FullConfig, FullProject } from '../../types/testReporter';
 
@@ -164,6 +166,7 @@ export class FullProjectInternal {
   readonly respectGitIgnore: boolean;
   readonly snapshotPathTemplate: string | undefined;
   readonly workers: number | undefined;
+  readonly retries: RetriesEntry[];
   readonly default: boolean;
   id = '';
   deps: FullProjectInternal[] = [];
@@ -183,14 +186,15 @@ export class FullProjectInternal {
     if (typeof configCLIOverrides.use?.trace === 'string' && typeof configTrace === 'object' && configTrace)
       use.trace = { ...configTrace, mode: configCLIOverrides.use.trace };
 
+    const retries = normalizeRetries(takeFirst<number | RetriesEntry[]>(configCLIOverrides.retries, projectConfig.retries, config.retries, 0));
     this.project = {
       grep: takeFirst(projectConfig.grep, config.grep, defaultGrep),
       grepInvert: takeFirst(projectConfig.grepInvert, config.grepInvert, null),
       outputDir: takeFirst(configCLIOverrides.outputDir, pathResolve(configDir, projectConfig.outputDir), pathResolve(configDir, config.outputDir), path.join(packageJsonDir, 'test-results')),
       // Note: we either apply the cli override for repeatEach or not, depending on whether the
       // project is top-level vs dependency. See collectProjectsAndTestFiles in loadUtils.
       repeatEach: takeFirst(projectConfig.repeatEach, config.repeatEach, 1),
-      retries: takeFirst(configCLIOverrides.retries, projectConfig.retries, config.retries, 0),
+      retries: maxRetries(retries),
       metadata: takeFirst(projectConfig.metadata, config.metadata, {}),
       name: takeFirst(projectConfig.name, config.name, ''),
       testDir,
@@ -208,6 +212,7 @@ export class FullProjectInternal {
       if (server.port !== undefined && server.url !== undefined)
         throw new Error(`Either 'port' or 'url' should be specified in projects[${this.project.name}].webServer.`);
     }
+    this.retries = retries;
     this.fullyParallel = takeFirst(configCLIOverrides.fullyParallel, projectConfig.fullyParallel, config.fullyParallel, undefined);
     this.expect = takeFirst(projectConfig.expect, config.expect, {});
     if (this.expect.toHaveScreenshot?.stylePath) {

packages/playwright/src/common/configLoader.ts

@@ -274,8 +274,19 @@ function validateProject(file: string, project: Project, title: string) {
   }
 
   if ('retries' in project && project.retries !== undefined) {
-    if (typeof project.retries !== 'number' || project.retries < 0)
-      throw errorWithFile(file, `${title}.retries must be a non-negative number`);
+    if (Array.isArray(project.retries)) {
+      for (let index = 0; index < project.retries.length; index++) {
+        const entry = project.retries[index];
+        if (!entry || typeof entry !== 'object')
+          throw errorWithFile(file, `${title}.retries[${index}] must be an object`);
+        if (typeof entry.count !== 'number' || entry.count < 0)
+          throw errorWithFile(file, `${title}.retries[${index}].count must be a non-negative number`);
+        if (entry.condition !== undefined && !isRegExp(entry.condition))
+          throw errorWithFile(file, `${title}.retries[${index}].condition must be a RegExp`);
+      }
+    } else if (typeof project.retries !== 'number' || project.retries < 0) {
+      throw errorWithFile(file, `${title}.retries must be a non-negative number or an array of { count, condition } objects`);
+    }
   }
 
   if ('testDir' in project && project.testDir !== undefined) {

packages/playwright/src/common/retries.ts

@@ -0,0 +1,47 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export type RetriesEntry = { count: number, condition?: RegExp };
+export type SerializedRetriesEntry = { count: number, condition?: { source: string, flags: string } };
+
+export function normalizeRetries(retries: number | RetriesEntry[]): RetriesEntry[] {
+  if (typeof retries === 'number')
+    return [{ count: retries }];
+  return retries.map(entry => ({ ...entry }));
+}
+
+export function maxRetries(entries: RetriesEntry[]): number {
+  return entries.reduce((max, entry) => Math.max(max, entry.count), 0);
+}
+
+export function computeRetries(entries: RetriesEntry[], errors: { message?: string }[]): number {
+  const matching = entries.filter(entry => !entry.condition || errors.some(error => !!(error.message ?? '').match(entry.condition!)));
+  return maxRetries(matching);
+}
+
+export function serializeRetries(entries: RetriesEntry[] | undefined): SerializedRetriesEntry[] | undefined {
+  return entries?.map(entry => ({
+    count: entry.count,
+    ...(entry.condition ? { condition: { source: entry.condition.source, flags: entry.condition.flags } } : {}),
+  }));
+}
+
+export function parseRetries(data: SerializedRetriesEntry[] | undefined): RetriesEntry[] | undefined {
+  return data?.map(entry => ({
+    count: entry.count,
+    ...(entry.condition ? { condition: new RegExp(entry.condition.source, entry.condition.flags) } : {}),
+  }));
+}

packages/playwright/src/common/suiteUtils.ts

@@ -20,8 +20,10 @@ import { calculateSha1 } from '@utils/crypto';
 import { toPosixPath } from '@utils/fileUtils';
 
 import { createFileMatcher, forceRegExp, parseLocationArg } from '../util';
+import { maxRetries } from './retries';
 
 import type { FullProjectInternal } from './config';
+import type { RetriesEntry } from './retries';
 import type { Suite, TestCase } from './test';
 import type { Matcher, TestCaseFilter } from '../util';
 
@@ -52,7 +54,7 @@ export function bindFileSuiteToProject(project: FullProjectInternal, suite: Suit
     test._projectId = project.id;
 
     // Inherit properties from parent suites.
-    let inheritedRetries: number | undefined;
+    let inheritedRetries: RetriesEntry[] | undefined;
     let inheritedTimeout: number | undefined;
     for (let parentSuite: Suite | undefined = suite; parentSuite; parentSuite = parentSuite.parent) {
       if (parentSuite._staticAnnotations.length)
@@ -66,7 +68,8 @@ export function bindFileSuiteToProject(project: FullProjectInternal, suite: Suit
       if (inheritedTimeout === undefined && parentSuite._timeout !== undefined)
         inheritedTimeout = parentSuite._timeout;
     }
-    test.retries = inheritedRetries ?? project.project.retries;
+    test._retries = inheritedRetries ?? project.retries;
+    test.retries = maxRetries(test._retries);
     test.timeout = inheritedTimeout ?? project.project.timeout;
 
     // Skip annotations imply skipped expectedStatus.