ci(sync-service): add oracle property test job with server restarts

Author: robacourt

.github/workflows/sync_service_tests.yml

@@ -169,6 +169,55 @@ jobs:
 
       - *upload_test_results_to_codecov
 
+  oracle_property_test_with_restarts:
+    name: 'Oracle property test with restarts (${{ matrix.restart_type }})'
+    runs-on: blacksmith-4vcpu-ubuntu-2404
+    defaults:
+      run:
+        working-directory: packages/sync-service
+    strategy:
+      fail-fast: false
+      matrix:
+        restart_type: [graceful, brutal]
+    env:
+      MIX_ENV: test
+      MIX_TARGET: application
+      POSTGRES_VERSION: '170000'
+      CODECOV_FLAGS: elixir,oracle-tests,sync-service,postgres-170000
+      CODECOV_TEST_RESULTS_FILES: ./junit/regular-test-junit-report.xml
+      CHECK_TIMEOUT: 60000
+      SHAPE_COUNT: 200
+      MUTATIONS_PER_TXN: 10
+      TXNS_PER_BATCH: 10
+      BATCH_COUNT: 50
+      SKIP_REPATCH_PREWARM: 'true'
+      RESTART_SERVER_EVERY: 3
+      RESTART_TYPE: ${{ matrix.restart_type }}
+      TEST_POOL_SIZE: 20
+    services:
+      postgres:
+        image: 'ghcr.io/${{ github.repository }}/postgres:17-alpine-logical'
+        env:
+          POSTGRES_PASSWORD: password
+        options: *postgres_health_check
+        ports:
+          - 54321:5432
+
+      pgbouncer: *pgbouncer_service
+    steps:
+      - *checkout_source
+      - *seed_database
+      - *setup_beam
+      - *cache_dependencies
+      - *cache_compiled_code
+      - *install_dependencies
+      - *compile_package
+
+      - name: Run oracle property test with ${{ matrix.restart_type }} server restarts
+        run: mix test --only oracle test/integration/oracle_property_test.exs
+
+      - *upload_test_results_to_codecov
+
   performance_test:
     name: 'Performance test, pg17'
     runs-on: blacksmith-4vcpu-ubuntu-2404

packages/sync-service/test/support/component_setup.ex

@@ -419,6 +419,13 @@ defmodule Support.ComponentSetup do
     [on_cleanup: on_cleanup]
   end
 
+  defp env_pool_size do
+    case System.get_env("TEST_POOL_SIZE") do
+      nil -> 2
+      value -> String.to_integer(value)
+    end
+  end
+
   def with_complete_stack(ctx) do
     stack_id = full_test_name(ctx)
 
@@ -731,7 +738,11 @@ defmodule Support.ComponentSetup do
          pool_opts: [
            backoff_type: :stop,
            max_restarts: 0,
-           pool_size: 2
+           # Default of 2 leaves a snapshot pool of 1 connection
+           # (Connection.Manager.pool_sizes/1), which under many-shape loads
+           # (e.g. the oracle property test) starves move-in queries into
+           # queue timeouts and 409 load-shedding. Override for such tests.
+           pool_size: Map.get(ctx, :db_pool_size, env_pool_size())
          ],
          tweaks: [
            registry_partitions: 1,