pull_request.yml

name: Pull Request

on:
  pull_request:
    types: [opened, reopened, synchronize]

permissions:
  contents: read

jobs:
  # Multiple parallel jobs:
  #  1. style: Runs all checks on the build (including compilation static analysis) except the tests
  #     This allows us to catch anything related to build and style faster, as we don't have to wait
  #     on tests to complete. It also allows the test jobs to continue even if there's a style failure.
  #  2. plan: Computes which subprojects (and their transitive dependents) are affected by this PR's
  #     changed files, using the subproject dependency graph exposed by the printDependentSubprojects
  #     Gradle task. This lets jobs 3 and 4 skip subprojects that can't have been affected by the
  #     diff, since those are the most expensive jobs in this workflow.
  #  3. tests: Runs the tests for a subset of subprojects in parallel. These are the most intensive
  #     subprojects, so separating them out allows us to run those parts of the build in parallel, and
  #     it also allows us to ensure they get run even if there are failures in other subprojects,
  #     giving us more insight into the types of test failures that a PR might have induced. The
  #     matrix is narrowed to the subprojects the `plan` job determined are affected; the whole job
  #     is skipped if none of them are.
  #  4. other-tests: Runs the rest of the tests. This tests all the remaining subprojects. Skipped
  #     entirely if the `plan` job determined none of these subprojects are affected.
  #  5. coverage: Merges the JaCoCo output of 3 and 4 and generates reports. Tolerates 3 and/or 4
  #     having been skipped or having skipped some matrix legs.

  style:
    runs-on: ubuntu-latest
    permissions:
      checks: write
      contents: read
      pull-requests: write
    timeout-minutes: 30
    steps:
      - name: Checkout sources
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      - name: Setup Base Environment
        uses: ./actions/setup-base-env
      - name: Run Gradle Build
        uses: ./actions/run-gradle
        with:
          gradle_command: build -x test -x destructiveTest -x scalarFallbackTest -PreleaseBuild=false -PpublishBuild=false -PspotbugsEnableHtmlReport

  plan:
    runs-on: ubuntu-latest
    permissions:
      contents: read
      pull-requests: read
    timeout-minutes: 5
    outputs:
      run_all: ${{ steps.plan.outputs.run_all }}
      affected: ${{ steps.plan.outputs.affected }}
      matrix: ${{ steps.plan.outputs.matrix }}
      run_other_tests: ${{ steps.plan.outputs.run_other_tests }}
    steps:
      - name: Checkout sources
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      - name: Setup Base Environment
        uses: ./actions/setup-base-env
      - name: Get Changed Files
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          gh pr diff ${{ github.event.pull_request.number }} --name-only | tee changed_files.txt
      - name: Get subproject dependency graph
        uses: ./actions/run-gradle
        with:
          gradle_command: printDependentSubprojects -PsubprojectDeps.output=subproject_deps.json
      - name: Compute plan
        id: plan
        env:
          # The subset of subprojects that should be executed in their own job by the `tests` matrix.
          # The `other-tests` and `coverage` steps below need to be kept in sync, the former to ensure that
          # no subproject is tested twice, and the latter to ensure that we collect all coverage reports.
          MATRIX_CANDIDATES: '["fdb-extensions","fdb-record-layer-core","fdb-record-layer-lucene","yaml-tests"]'
        run: |
          python3 build/affected_subprojects.py subproject_deps.json \
            --changed-files-file changed_files.txt \
            --matrix-candidates "$MATRIX_CANDIDATES" \
            --output subproject_plan.json >> "$GITHUB_STEP_SUMMARY"
          echo "run_all=$(jq -r '.run_all' subproject_plan.json)" >> "$GITHUB_OUTPUT"
          echo "affected=$(jq -c '.affected' subproject_plan.json)" >> "$GITHUB_OUTPUT"
          echo "matrix=$(jq -c '.matrix' subproject_plan.json)" >> "$GITHUB_OUTPUT"
          echo "run_other_tests=$(jq -c '.run_other_tests' subproject_plan.json)" >> "$GITHUB_OUTPUT"

  tests:
    needs: plan
    if: needs.plan.outputs.matrix != '[]'
    strategy:
      matrix:
        subproject: ${{ fromJSON(needs.plan.outputs.matrix) }}
    runs-on: ubuntu-latest
    permissions:
      checks: write
      contents: read
      pull-requests: write
    timeout-minutes: 40
    steps:
      - name: Checkout sources
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      - name: Setup Base Environment
        uses: ./actions/setup-base-env
      - name: Setup FDB
        uses: ./actions/setup-fdb
      - name: Compute extra tasks
        id: extra_tasks
        run: |
          # fdb-extensions also runs its scalar-fallback test task here (not in the style job, which
          # is test-free) so its JaCoCo execution data is produced alongside the other test tasks and
          # picked up by the coverage-data upload below.
          if [[ ${{ matrix.subproject }} == 'fdb-extensions' ]] ; then
            echo "task=:fdb-extensions:scalarFallbackTest" >> "$GITHUB_OUTPUT"
          fi
      - name: Run Gradle Test
        uses: ./actions/gradle-test
        with:
          gradle_command: :${{ matrix.subproject }}:jar :${{ matrix.subproject }}:test :${{ matrix.subproject }}:destructiveTest ${{ steps.extra_tasks.outputs.task }}
          gradle_args: -PreleaseBuild=false -PpublishBuild=false
          report_name: ${{ matrix.subproject }}-test-reports
      - name: Publish Coverage Data
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: ${{ matrix.subproject }}-coverage-data
          path: |
            **/.out/jacoco/*.exec
            **/.out/libs/*.jar
          include-hidden-files: true
          retention-days: 1

  other-tests:
    needs: plan
    if: needs.plan.outputs.run_other_tests == 'true'
    runs-on: ubuntu-latest
    permissions:
      checks: write
      contents: read
      pull-requests: write
    timeout-minutes: 60
    steps:
      - name: Checkout sources
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      - name: Setup Base Environment
        uses: ./actions/setup-base-env
      - name: Setup FDB
        uses: ./actions/setup-fdb
      - name: Run Gradle Test
        uses: ./actions/gradle-test
        with:
          gradle_command: >-
            jar
            test
            -x :fdb-extensions:test
            -x :fdb-record-layer-core:test
            -x :fdb-record-layer-lucene:test
            -x :yaml-tests:test
            destructiveTest
            -x :fdb-extensions:destructiveTest
            -x :fdb-record-layer-core:destructiveTest
            -x :fdb-record-layer-lucene:destructiveTest
            -x :yaml-tests:destructiveTest
          gradle_args: -PreleaseBuild=false -PpublishBuild=false
          report_name: other-test-reports
      - name: Publish Coverage Data
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: other-coverage-data
          path: |
            **/.out/jacoco/*.exec
            **/.out/libs/*.jar
          include-hidden-files: true
          retention-days: 1

  coverage:
    needs: [plan, tests, other-tests]
    if: >-
      always() &&
      needs.plan.result == 'success' &&
      (needs.tests.result == 'success' || needs.tests.result == 'skipped') &&
      (needs.other-tests.result == 'success' || needs.other-tests.result == 'skipped')
    runs-on: ubuntu-latest
    permissions:
      checks: write
      contents: read
      pull-requests: read
    timeout-minutes: 10
    steps:
      - name: Checkout HEAD sources
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          # Full history so the PR's base and head commits (and their merge-base) are available locally for
          # the git-based diff below — `gh pr diff` caps at 20000 lines and fails with HTTP 406 on large PRs.
          fetch-depth: 0
      - name: Setup Base Environment
        uses: ./actions/setup-base-env
      # It looks like, if you try to download them all as a pattern, the nested directories get stripped
      # so the coverage data (for e.g. lucene) does not end up in the appropirate subproject directory
      - name: 'Download lucene'
        if: contains(fromJSON(needs.plan.outputs.matrix), 'fdb-record-layer-lucene')
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: fdb-record-layer-lucene-coverage-data
      - name: 'Download extensions'
        if: contains(fromJSON(needs.plan.outputs.matrix), 'fdb-extensions')
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: fdb-extensions-coverage-data
      - name: 'Download core'
        if: contains(fromJSON(needs.plan.outputs.matrix), 'fdb-record-layer-core')
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: fdb-record-layer-core-coverage-data
      - name: 'Download yaml'
        if: contains(fromJSON(needs.plan.outputs.matrix), 'yaml-tests')
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: yaml-tests-coverage-data
      - name: 'Download other'
        if: needs.plan.outputs.run_other_tests == 'true'
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: other-coverage-data
      # If this run tested no subprojects there are no jars or execution data to merge, so
      # codeCoverageReport would be skipped and no coverage-report artifact would be produced.
      # Build all jars and seed empty execution data so codeCoverageReport still emits a
      # full-codebase 0% report. This must be a separate Gradle invocation from the report below:
      # codeCoverageReport only mustRunAfter (not dependsOn) the jar tasks and evaluates which
      # jars exist at configuration time, so the jars must already be on disk when it configures.
      - name: Build jars and seed empty coverage data
        if: needs.plan.outputs.run_all != 'true' && needs.plan.outputs.matrix == '[]' && needs.plan.outputs.run_other_tests != 'true'
        uses: ./actions/run-gradle
        with:
          gradle_command: jar createEmptyCoverageData
      - name: Run JaCoCo Report
        uses: ./actions/run-gradle
        with:
          gradle_command: codeCoverageReport
      - name: Check coverage report was generated
        id: coverage_report
        run: |
          # If we skipped all of the tests, we may not have a code coverage report file, so skip
          # the report if the file doesn't actually exist. However, while we're here, protect against
          # a case where we should have generated a report but didn't by also overriding should_exist
          # to true if any of the conditions that should have resulted in tests being run are true.
          if [[ -f '${{ github.workspace }}/.out/reports/jacoco/codeCoverageReport/codeCoverageReport.xml' \
                || '${{ needs.plan.outputs.run_all }}' == 'true' \
                || '${{ needs.plan.outputs.matrix }}' != '[]' \
                || '${{ needs.plan.outputs.run_other_tests }}' == 'true' ]] ; then
            echo "should_exist=true" >> "$GITHUB_OUTPUT"
          else
            echo "should_exist=false" >> "$GITHUB_OUTPUT"
            echo "### Coverage Note" >> "$GITHUB_STEP_SUMMARY"
            echo "No coverage report was generated: no subprojects were tested this run." >> "$GITHUB_STEP_SUMMARY"
          fi
      - name: Publish Coverage Report
        if: steps.coverage_report.outputs.should_exist == 'true'
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: coverage-report
          path: |
            ${{ github.workspace }}/.out/reports/jacoco/codeCoverageReport/
      - name: Note Skipped Subprojects
        if: steps.coverage_report.outputs.should_exist == 'true' && needs.plan.outputs.run_all != 'true'
        env:
          AFFECTED_SUBPROJECTS: ${{ needs.plan.outputs.affected }}
        run: |
          echo "### Coverage Note" >> "$GITHUB_STEP_SUMMARY"
          echo "Only subprojects affected by this PR's diff were tested this run: ${AFFECTED_SUBPROJECTS}" >> "$GITHUB_STEP_SUMMARY"
      - name: Get PR Diff
        if: steps.coverage_report.outputs.should_exist == 'true'
        run: |
          # `gh pr diff` uses the GitHub API .diff media type, which returns HTTP 406 once the diff exceeds
          # 20000 lines. Compute the same three-dot (merge-base..head) diff with git, which has no such limit;
          # fetch-depth: 0 on the checkout above ensures both commits and their merge-base are present.
          git diff "${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }}" > pr_diff.patch
      - name: Annotate Coverage
        if: steps.coverage_report.outputs.should_exist == 'true'
        run: |
          python3 build/coverage_annotations.py \
            --report .out/reports/jacoco/codeCoverageReport/codeCoverageReport.xml \
            --diff pr_diff.patch