From 2d52e4b24129b1f4d36d85eab21266fd4da2e8db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliv=C3=A9r=20Falvai?= Date: Fri, 15 Nov 2024 09:22:26 +0100 Subject: [PATCH] Retry flaky tests by default (#253) * Retry flaky tests by default * Set default repetition mode in E2E tests --- README.md | 2 +- e2e/bitrise.yml | 4 ++-- step.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3c530b64..56383e9a 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,7 @@ Run tests with custom xcconfig file path: | `scheme` | Xcode Scheme name. The input value sets xcodebuild's `-scheme` option. | required | `$BITRISE_SCHEME` | | `destination` | Destination specifier describes the device to use as a destination. The input value sets xcodebuild's `-destination` option. In a CI environment, a Simulator device called `Bitrise iOS default` is already created. It is a compatible device with the selected Simulator runtime, pre-warmed for better performance. If a device with this name is not found (e.g. in a local dev environment), the first matching device will be selected. | required | `platform=iOS Simulator,name=Bitrise iOS default,OS=latest` | | `test_plan` | Run tests in a specific Test Plan associated with the Scheme. Leave this input empty to run the default Test Plan or Test Targets associated with the Scheme. The input value sets xcodebuild's `-testPlan` option. | | | -| `test_repetition_mode` | Determines how the tests will repeat. Available options: - `none`: Tests will never repeat. - `until_failure`: Tests will repeat until failure or up to maximum repetitions. - `retry_on_failure`: Only failed tests will repeat up to maximum repetitions. - `up_until_maximum_repetitions`: Tests will repeat up until maximum repetitions. The input value together with Maximum Test Repetitions (`maximum_test_repetitions`) input sets xcodebuild's `-run-tests-until-failure` / `-retry-tests-on-failure` or `-test-iterations` option. | | `none` | +| `test_repetition_mode` | Determines how the tests will repeat. Available options: - `none`: Tests will never repeat. - `until_failure`: Tests will repeat until failure or up to maximum repetitions. - `retry_on_failure`: Only failed tests will repeat up to maximum repetitions. - `up_until_maximum_repetitions`: Tests will repeat up until maximum repetitions. The input value together with Maximum Test Repetitions (`maximum_test_repetitions`) input sets xcodebuild's `-run-tests-until-failure` / `-retry-tests-on-failure` or `-test-iterations` option. | | `retry_on_failure` | | `maximum_test_repetitions` | The maximum number of times a test repeats based on the Test Repetition Mode (`test_repetition_mode`). Should be more than 1 if the Test Repetition Mode is other than `none`. The input value sets xcodebuild's `-test-iterations` option. | required | `3` | | `relaunch_tests_for_each_repetition` | If this input is set, tests will launch in a new process for each repetition. By default, tests launch in the same process for each repetition. The input value sets xcodebuild's `-test-repetition-relaunch-enabled` option. | | `no` | | `xcconfig_content` | Build settings to override the project's build settings, using xcodebuild's `-xcconfig` option. You can't define `-xcconfig` option in `Additional options for the xcodebuild command` if this input is set. If empty, no setting is changed. When set it can be either: 1. Existing `.xcconfig` file path. Example: `./ios-sample/ios-sample/Configurations/Dev.xcconfig` 2. The contents of a newly created temporary `.xcconfig` file. (This is the default.) Build settings must be separated by newline character (`\n`). Example: ``` COMPILER_INDEX_STORE_ENABLE = NO ONLY_ACTIVE_ARCH[config=Debug][sdk=*][arch=*] = YES ``` | | `COMPILER_INDEX_STORE_ENABLE = NO` | diff --git a/e2e/bitrise.yml b/e2e/bitrise.yml index 49cf5812..ca60084e 100644 --- a/e2e/bitrise.yml +++ b/e2e/bitrise.yml @@ -109,12 +109,12 @@ workflows: fi envman add --key COLLECT_SIM_DIAGNOSTICS --value ${COLLECT_SIM_DIAGNOSTICS-$COLLECT_SIM_DIAGNOSTICS_DEFAULT} - script: - title: Set TEST_REPETITION_MODE to 'none' if not set + title: Set TEST_REPETITION_MODE to 'retry_on_failure' if not set inputs: - content: |- #!/bin/env bash set -eo pipefail - envman add --key TEST_REPETITION_MODE --value ${TEST_REPETITION_MODE-none} + envman add --key TEST_REPETITION_MODE --value ${TEST_REPETITION_MODE-retry_on_failure} - script: title: Set MAXIMUM_TEST_REPETITIONS to '3' if not set inputs: diff --git a/step.yml b/step.yml index daeb87b3..5fffc9e9 100644 --- a/step.yml +++ b/step.yml @@ -90,7 +90,7 @@ inputs: # Test Repetition -- test_repetition_mode: none +- test_repetition_mode: retry_on_failure opts: category: Test Repetition title: Test Repetition Mode