Fail fast on intermediate Bitrise test failures

bitrise.yml

@@ -356,13 +356,13 @@ workflows:
           title: Execute general instrumentation tests
           timeout: 1200
           inputs:
-            - content: ./scripts/retry_with_emulator_cleanup.sh 3 ./gradlew -Pandroid.experimental.androidTest.numManagedDeviceShards=1 pixel2api33DebugAndroidTest -x :paymentsheet-example:pixel2api33BaseDebugAndroidTest -x :paymentsheet:pixel2api33DebugAndroidTest -x :example:pixel2api33DebugAndroidTest -x :financial-connections:pixel2api33DebugAndroidTest -x :financial-connections-example:pixel2api33DebugAndroidTest -x :camera-core:pixel2api33DebugAndroidTest -x :crypto-onramp-example:pixel2api33DebugAndroidTest -x :stripecardscan:pixel2api33DebugAndroidTest -x :stripecardscan-example:pixel2api33DebugAndroidTest --init-script build-configuration/instrumentation-test-init.gradle
+            - content: ./scripts/retry_with_emulator_cleanup.sh --fail-fast-tests 3 ./gradlew -Pandroid.experimental.androidTest.numManagedDeviceShards=1 pixel2api33DebugAndroidTest -x :paymentsheet-example:pixel2api33BaseDebugAndroidTest -x :paymentsheet:pixel2api33DebugAndroidTest -x :example:pixel2api33DebugAndroidTest -x :financial-connections:pixel2api33DebugAndroidTest -x :financial-connections-example:pixel2api33DebugAndroidTest -x :camera-core:pixel2api33DebugAndroidTest -x :crypto-onramp-example:pixel2api33DebugAndroidTest -x :stripecardscan:pixel2api33DebugAndroidTest -x :stripecardscan-example:pixel2api33DebugAndroidTest --init-script build-configuration/instrumentation-test-init.gradle
       - script@1:
           title: Execute PaymentSheet IME instrumentation tests
           is_always_run: true
           timeout: 1200
           inputs:
-            - content: ./scripts/retry_with_emulator_cleanup.sh 3 ./gradlew -Pandroid.testInstrumentationRunnerArguments.annotation=com.stripe.android.paymentsheet.RequiresIme :paymentsheet:pixel2api33imeDebugAndroidTest --init-script build-configuration/instrumentation-test-init.gradle
+            - content: ./scripts/retry_with_emulator_cleanup.sh --fail-fast-tests 3 ./gradlew -Pandroid.testInstrumentationRunnerArguments.annotation=com.stripe.android.paymentsheet.RequiresIme :paymentsheet:pixel2api33imeDebugAndroidTest --init-script build-configuration/instrumentation-test-init.gradle
       - bundle::export_instrumentation_test_results:
           title: Export general and IME instrumentation test results
           is_always_run: true
@@ -378,7 +378,7 @@ workflows:
           title: Execute instrumentation tests
           timeout: 1200
           inputs:
-            - content: ./scripts/retry_with_emulator_cleanup.sh 3 ./gradlew -Pandroid.experimental.androidTest.numManagedDeviceShards=6 -Pandroid.experimental.testOptions.managedDevices.maxConcurrentDevices=6 -Pandroid.testInstrumentationRunnerArguments.notAnnotation=com.stripe.android.paymentsheet.RequiresIme :paymentsheet:pixel2api33DebugAndroidTest --init-script build-configuration/instrumentation-test-init.gradle
+            - content: ./scripts/retry_with_emulator_cleanup.sh --fail-fast-tests 3 ./gradlew -Pandroid.experimental.androidTest.numManagedDeviceShards=6 -Pandroid.experimental.testOptions.managedDevices.maxConcurrentDevices=6 -Pandroid.testInstrumentationRunnerArguments.notAnnotation=com.stripe.android.paymentsheet.RequiresIme :paymentsheet:pixel2api33DebugAndroidTest --init-script build-configuration/instrumentation-test-init.gradle
       - bundle::export_instrumentation_test_results:
           title: Export PaymentSheet instrumentation test results
           inputs:
@@ -397,7 +397,7 @@ workflows:
           title: Execute PaymentSheet IME instrumentation tests
           timeout: 1200
           inputs:
-            - content: ./scripts/retry_with_emulator_cleanup.sh 3 ./gradlew -Pandroid.testInstrumentationRunnerArguments.annotation=com.stripe.android.paymentsheet.RequiresIme :paymentsheet:pixel2api33imeDebugAndroidTest --init-script build-configuration/instrumentation-test-init.gradle
+            - content: ./scripts/retry_with_emulator_cleanup.sh --fail-fast-tests 3 ./gradlew -Pandroid.testInstrumentationRunnerArguments.annotation=com.stripe.android.paymentsheet.RequiresIme :paymentsheet:pixel2api33imeDebugAndroidTest --init-script build-configuration/instrumentation-test-init.gradle
       - bundle::export_instrumentation_test_results:
           title: Export PaymentSheet IME instrumentation test results
           inputs:
@@ -421,7 +421,7 @@ workflows:
           title: Execute instrumentation tests
           timeout: 1200
           inputs:
-            - content: ./scripts/retry_with_emulator_cleanup.sh 3 ./gradlew :financial-connections:connectedAndroidTest :financial-connections-example:connectedAndroidTest :crypto-onramp-example:connectedDebugAndroidTest --init-script build-configuration/instrumentation-test-init.gradle --continue
+            - content: ./scripts/retry_with_emulator_cleanup.sh --fail-fast-tests 3 ./gradlew :financial-connections:connectedAndroidTest :financial-connections-example:connectedAndroidTest :crypto-onramp-example:connectedDebugAndroidTest --init-script build-configuration/instrumentation-test-init.gradle --continue
       - bundle::export_instrumentation_test_results:
           title: Export FC and Crypto instrumentation test results
           inputs:
@@ -449,7 +449,7 @@ workflows:
           title: Execute instrumentation tests
           timeout: 1200
           inputs:
-            - content: ./scripts/retry_with_emulator_cleanup.sh 3 ./gradlew :camera-core:connectedAndroidTest :stripecardscan:connectedAndroidTest :stripecardscan-example:connectedAndroidTest --init-script build-configuration/instrumentation-test-init.gradle
+            - content: ./scripts/retry_with_emulator_cleanup.sh --fail-fast-tests 3 ./gradlew :camera-core:connectedAndroidTest :stripecardscan:connectedAndroidTest :stripecardscan-example:connectedAndroidTest --init-script build-configuration/instrumentation-test-init.gradle
       - bundle::export_instrumentation_test_results:
           title: Export CardScan instrumentation test results
           inputs:
@@ -500,7 +500,7 @@ workflows:
                 # Keep the explicit class list so tests with dedicated workflows stay excluded.
                 CLASSES=$(python3 scripts/get_shard_test_classes.py --shard-index 0 --num-shards 1)
 
-                ./scripts/retry_with_emulator_cleanup.sh 3 ./gradlew -PSTRIPE_PAYMENTSHEET_EXAMPLE_SENTRY_DSN=$STRIPE_PAYMENTSHEET_EXAMPLE_SENTRY_DSN -Pandroid.experimental.androidTest.numManagedDeviceShards=$PAYMENTSHEET_E2E_DEVICE_SHARD_COUNT -Pandroid.experimental.testOptions.managedDevices.maxConcurrentDevices=$PAYMENTSHEET_E2E_DEVICE_SHARD_COUNT "-Pandroid.testInstrumentationRunnerArguments.class=$CLASSES" :paymentsheet-example:pixel2api33chromeBaseDebugAndroidTest --init-script build-configuration/instrumentation-test-init.gradle
+                ./scripts/retry_with_emulator_cleanup.sh --fail-fast-tests 3 ./gradlew -PSTRIPE_PAYMENTSHEET_EXAMPLE_SENTRY_DSN=$STRIPE_PAYMENTSHEET_EXAMPLE_SENTRY_DSN -Pandroid.experimental.androidTest.numManagedDeviceShards=$PAYMENTSHEET_E2E_DEVICE_SHARD_COUNT -Pandroid.experimental.testOptions.managedDevices.maxConcurrentDevices=$PAYMENTSHEET_E2E_DEVICE_SHARD_COUNT "-Pandroid.testInstrumentationRunnerArguments.class=$CLASSES" :paymentsheet-example:pixel2api33chromeBaseDebugAndroidTest --init-script build-configuration/instrumentation-test-init.gradle
       - bundle::export_instrumentation_test_results:
           title: Export PaymentSheet E2E test results
           inputs:

scripts/retry_with_emulator_cleanup.sh

@@ -1,10 +1,25 @@
 #!/bin/bash
 
 # Retry a command, killing unhealthy emulators between attempts.
-# Usage: ./scripts/retry_with_emulator_cleanup.sh <retries> <command...>
+# Usage: ./scripts/retry_with_emulator_cleanup.sh [--fail-fast-tests] <retries> <command...>
+
+FAIL_FAST_TESTS=false
+if [ "${1:-}" = "--fail-fast-tests" ]; then
+  FAIL_FAST_TESTS=true
+  shift
+fi
+
+FAIL_FAST_GRACE_SECONDS="${BITRISE_FAIL_FAST_GRACE_SECONDS:-10}"
 
 OUTPUT_LOG=$(mktemp)
-trap 'rm -f "$OUTPUT_LOG"' EXIT
+OUTPUT_PIPE=""
+cleanup_retry_files() {
+  rm -f "$OUTPUT_LOG"
+  if [ -n "$OUTPUT_PIPE" ]; then
+    rm -f "$OUTPUT_PIPE"
+  fi
+}
+trap cleanup_retry_files EXIT
 
 SOURCE_DIR="${BITRISE_SOURCE_DIR:-.}"
 SOURCE_DIR=$(cd "$SOURCE_DIR" && pwd)
@@ -19,6 +34,92 @@ function is_gradle_test_failure {
   grep -qF "There were failing tests." "$log_file"
 }
 
+function process_is_running {
+  local pid="$1"
+  if [ -r "/proc/$pid/stat" ]; then
+    local state
+    state=$(awk '{print $3}' "/proc/$pid/stat" 2>/dev/null)
+    [ -n "$state" ] && [ "$state" != "Z" ]
+  else
+    jobs -pr | grep -qE "(^|[[:space:]])$pid($|[[:space:]])"
+  fi
+}
+
+function stop_process_group {
+  local pid="$1"
+  echo "Stopping failed Gradle test process group $pid..."
+  kill -TERM -- "-$pid" 2>/dev/null || true
+  kill -TERM "$pid" 2>/dev/null || true
+
+  local remaining=$((FAIL_FAST_GRACE_SECONDS * 10))
+  while process_is_running "$pid" && [ "$remaining" -gt 0 ]; do
+    sleep 0.1
+    remaining=$((remaining - 1))
+  done
+
+  if process_is_running "$pid"; then
+    echo "Gradle test process group $pid did not stop gracefully; sending SIGKILL."
+    kill -KILL -- "-$pid" 2>/dev/null || true
+    kill -KILL "$pid" 2>/dev/null || true
+  fi
+}
+
+function stop_gradle_daemons {
+  local command_path="$1"
+  case "$(basename "$command_path")" in
+    gradlew|gradlew.bat)
+      echo "Stopping Gradle daemons after early test termination..."
+      if command -v timeout >/dev/null 2>&1; then
+        timeout 30 "$command_path" --stop >/dev/null 2>&1 || true
+      elif command -v gtimeout >/dev/null 2>&1; then
+        gtimeout 30 "$command_path" --stop >/dev/null 2>&1 || true
+      else
+        "$command_path" --stop >/dev/null 2>&1 || true
+      fi
+      ;;
+  esac
+}
+
+function run_with_live_test_failure_detection {
+  : > "$OUTPUT_LOG"
+  OUTPUT_PIPE=$(mktemp)
+  rm -f "$OUTPUT_PIPE"
+  mkfifo "$OUTPUT_PIPE"
+  tee "$OUTPUT_LOG" < "$OUTPUT_PIPE" &
+  local tee_pid=$!
+  if command -v setsid >/dev/null 2>&1; then
+    setsid "$@" > "$OUTPUT_PIPE" 2>&1 &
+  else
+    # macOS does not ship setsid. Job control gives this background command a
+    # process group so local tests exercise the same group-kill contract.
+    set -m
+    "$@" > "$OUTPUT_PIPE" 2>&1 &
+  fi
+  local command_pid=$!
+
+  while process_is_running "$command_pid"; do
+    if is_gradle_test_failure "$OUTPUT_LOG"; then
+      echo "Detected a Gradle test failure before the attempt completed."
+      stop_process_group "$command_pid"
+      stop_gradle_daemons "$1"
+      wait "$command_pid" 2>/dev/null || true
+      wait "$tee_pid" 2>/dev/null || true
+      rm -f "$OUTPUT_PIPE"
+      OUTPUT_PIPE=""
+      return 143
+    fi
+    sleep 0.1
+  done
+
+  local command_status
+  wait "$command_pid"
+  command_status=$?
+  wait "$tee_pid" 2>/dev/null || true
+  rm -f "$OUTPUT_PIPE"
+  OUTPUT_PIPE=""
+  return "$command_status"
+}
+
 function capture_attempt_results {
   local attempt="$1"
 
@@ -87,21 +188,27 @@ function retry {
 
   local count=0
   while true; do
-    "$@" 2>&1 | tee "$OUTPUT_LOG"
-    local exit=${PIPESTATUS[0]}
-    if [ $exit -eq 0 ]; then
+    local exit
+    if [ "$FAIL_FAST_TESTS" = true ] && [ $((count + 1)) -lt "$retries" ]; then
+      run_with_live_test_failure_detection "$@"
+      exit=$?
+    else
+      "$@" 2>&1 | tee "$OUTPUT_LOG"
+      exit=${PIPESTATUS[0]}
+    fi
+    if [ "$exit" -eq 0 ]; then
       return 0
     fi
-    count=$(($count + 1))
-    if [ $count -lt $retries ]; then
+    count=$((count + 1))
+    if [ "$count" -lt "$retries" ]; then
       capture_attempt_results "$count"
       echo "Retry $count/$retries exited $exit. Checking for known failures..."
       clear_corrupted_orchestrator_cache
       echo "Checking emulator health..."
       kill_unhealthy_emulators
     else
       echo "Retry $count/$retries exited $exit, no more retries left."
-      return $exit
+      return "$exit"
     fi
   done
 }

scripts/test_retry_with_emulator_cleanup.py

@@ -0,0 +1,172 @@
+#!/usr/bin/env python3
+import os
+import subprocess
+import tempfile
+import textwrap
+import time
+import unittest
+from pathlib import Path
+
+
+SCRIPT = Path(__file__).with_name("retry_with_emulator_cleanup.sh")
+
+
+class RetryWithEmulatorCleanupProcessTest(unittest.TestCase):
+    def setUp(self) -> None:
+        self.temp_dirs: list[tempfile.TemporaryDirectory[str]] = []
+
+    def tearDown(self) -> None:
+        for temp_dir in self.temp_dirs:
+            temp_dir.cleanup()
+
+    def run_wrapper(self, command: str, retries: int = 2, timeout: float = 8) -> tuple[subprocess.CompletedProcess[str], Path]:
+        temp_dir = tempfile.TemporaryDirectory()
+        self.temp_dirs.append(temp_dir)
+        root = Path(temp_dir.name)
+        fake_adb = root / "adb"
+        fake_adb.write_text("#!/bin/sh\nprintf 'List of devices attached\\n'\n", encoding="utf-8")
+        fake_adb.chmod(0o755)
+        env = os.environ.copy()
+        env["PATH"] = f"{root}:{env['PATH']}"
+        env["BITRISE_SOURCE_DIR"] = str(root)
+        env["BITRISE_RETRY_RESULTS_DIR"] = str(root / "retry-results")
+        env["BITRISE_FAIL_FAST_GRACE_SECONDS"] = "1"
+        started = time.monotonic()
+        result = subprocess.run(
+            ["bash", str(SCRIPT), "--fail-fast-tests", str(retries), "bash", "-c", textwrap.dedent(command)],
+            capture_output=True,
+            text=True,
+            env=env,
+            timeout=timeout,
+        )
+        result.elapsed = time.monotonic() - started
+        return result, root
+
+    def assert_process_stopped(self, pid_file: Path) -> None:
+        pid = int(pid_file.read_text(encoding="utf-8"))
+        for _ in range(20):
+            try:
+                os.kill(pid, 0)
+            except ProcessLookupError:
+                return
+            time.sleep(0.05)
+        self.fail(f"process {pid} is still running")
+
+    def test_intermediate_test_failure_is_terminated_and_retried(self) -> None:
+        result, root = self.run_wrapper(
+            """
+            attempt_file="$BITRISE_SOURCE_DIR/attempt"
+            attempt=$(($(cat "$attempt_file" 2>/dev/null || printf 0) + 1))
+            printf '%s' "$attempt" > "$attempt_file"
+            if [ "$attempt" -eq 1 ]; then
+              printf '%s\n' 'There were failing tests. See the report at: file:///tmp/report'
+              sleep 30 &
+              child=$!
+              printf '%s' "$child" > "$BITRISE_SOURCE_DIR/child-pid"
+              wait "$child"
+            fi
+            printf '%s\n' complete >> "$BITRISE_SOURCE_DIR/completions"
+            """
+        )
+
+        self.assertEqual(0, result.returncode, result.stderr)
+        self.assertLess(result.elapsed, 5, result.stdout)
+        self.assertEqual("2", (root / "attempt").read_text(encoding="utf-8"))
+        self.assertEqual("complete\n", (root / "completions").read_text(encoding="utf-8"))
+        self.assertIn("Stopping failed Gradle test process group", result.stdout)
+        self.assert_process_stopped(root / "child-pid")
+
+    def test_intermediate_junit_results_are_captured_before_retry(self) -> None:
+        result, root = self.run_wrapper(
+            """
+            attempt_file="$BITRISE_SOURCE_DIR/attempt"
+            attempt=$(($(cat "$attempt_file" 2>/dev/null || printf 0) + 1))
+            printf '%s' "$attempt" > "$attempt_file"
+            result_dir="$BITRISE_SOURCE_DIR/paymentsheet/build/outputs/androidTest-results/managedDevice/device-1"
+            mkdir -p "$result_dir"
+            if [ "$attempt" -eq 1 ]; then
+              printf '%s' '<testsuite><testcase classname="com.example.RetryTest" name="testFlaky"><failure>first failure</failure></testcase></testsuite>' > "$result_dir/TEST-RetryTest.xml"
+              printf '%s\n' 'There were failing tests. See the report at: file:///tmp/report'
+              sleep 0.3
+            else
+              printf '%s' '<testsuite><testcase classname="com.example.RetryTest" name="testFlaky" /></testsuite>' > "$result_dir/TEST-RetryTest.xml"
+            fi
+            """
+        )
+
+        self.assertEqual(0, result.returncode, result.stderr)
+        captured = [
+            path
+            for path in (root / "retry-results").rglob("TEST-RetryTest.xml")
+            if "attempt-1" in path.parts
+        ]
+        self.assertEqual(1, len(captured))
+        self.assertIn("first failure", captured[0].read_text(encoding="utf-8"))
+
+    def test_final_attempt_runs_to_completion(self) -> None:
+        result, root = self.run_wrapper(
+            """
+            printf '%s\n' 'There were failing tests. See the report at: file:///tmp/report'
+            sleep 0.4
+            printf '%s\n' complete > "$BITRISE_SOURCE_DIR/final-complete"
+            exit 0
+            """,
+            retries=1,
+        )
+
+        self.assertEqual(0, result.returncode, result.stderr)
+        self.assertTrue((root / "final-complete").is_file())
+        self.assertGreater(result.elapsed, 0.3, result.stdout)
+        self.assertNotIn("Stopping failed Gradle test process group", result.stdout)
+
+    def test_uncooperative_process_is_force_killed_after_grace_period(self) -> None:
+        result, root = self.run_wrapper(
+            """
+            attempt_file="$BITRISE_SOURCE_DIR/attempt"
+            attempt=$(($(cat "$attempt_file" 2>/dev/null || printf 0) + 1))
+            printf '%s' "$attempt" > "$attempt_file"
+            if [ "$attempt" -eq 1 ]; then
+              trap '' TERM
+              printf '%s\n' 'There were failing tests. See the report at: file:///tmp/report'
+              sleep 30 &
+              child=$!
+              printf '%s' "$child" > "$BITRISE_SOURCE_DIR/child-pid"
+              wait "$child"
+            fi
+            printf '%s\n' complete >> "$BITRISE_SOURCE_DIR/completions"
+            """
+        )
+
+        self.assertEqual(0, result.returncode, result.stderr)
+        self.assertLess(result.elapsed, 5, result.stdout)
+        self.assertIn("sending SIGKILL", result.stdout)
+        self.assert_process_stopped(root / "child-pid")
+
+    def test_non_test_failure_is_not_terminated_early(self) -> None:
+        result, root = self.run_wrapper(
+            """
+            printf '%s\n' 'Execution failed for task :compileDebugKotlin.'
+            sleep 0.4
+            printf '%s\n' complete >> "$BITRISE_SOURCE_DIR/completions"
+            exit 17
+            """
+        )
+
+        self.assertEqual(17, result.returncode)
+        self.assertEqual("complete\ncomplete\n", (root / "completions").read_text(encoding="utf-8"))
+        self.assertGreater(result.elapsed, 0.7, result.stdout)
+        self.assertNotIn("Stopping failed Gradle test process group", result.stdout)
+
+    def test_final_exit_status_is_preserved(self) -> None:
+        result, _ = self.run_wrapper(
+            """
+            exit 23
+            """,
+            retries=1,
+        )
+
+        self.assertEqual(23, result.returncode, result.stdout)
+
+
+if __name__ == "__main__":
+    unittest.main()