CI: Address occasional ccache install hangs

Author: padelsbach

.github/actions/ccache-setup/action.yml

@@ -23,6 +23,16 @@ inputs:
     description: 'Per-job ccache max size (passed to ccache -M).'
     required: false
     default: '500M'
+  ghcr-debs-tag:
+    description: >
+      Bundle to install ccache from offline (see install-apt-deps).
+      'auto' resolves to the runner's ubuntu-<version>-minimal bundle,
+      which already carries ccache and which most callers pull earlier in
+      the same job, making the second pull free. 'none' skips the bundle
+      and uses apt - for callers whose job pulls a different bundle,
+      where fetching this one costs more than it saves.
+    required: false
+    default: 'auto'
   read-only:
     description: >
       When 'true', restore the cache but do NOT save it (no post-job
@@ -36,21 +46,52 @@ inputs:
 runs:
   using: 'composite'
   steps:
-    - name: Install ccache
+    - name: Check for ccache
+      id: check
       shell: bash
       run: |
         if command -v ccache >/dev/null 2>&1; then
           echo "ccache already installed: $(ccache --version | head -1)"
-        elif [ "${{ runner.os }}" = "Linux" ]; then
-          sudo apt-get update -q
-          sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
-            --no-install-recommends ccache
-        elif [ "${{ runner.os }}" = "macOS" ]; then
-          brew install ccache
-        else
-          echo "::error::ccache install not supported on ${{ runner.os }}"
-          exit 1
+          echo "present=true" >> "$GITHUB_OUTPUT"
+          exit 0
+        fi
+        echo "present=false" >> "$GITHUB_OUTPUT"
+        case "${{ runner.os }}" in
+          Linux|macOS) ;;
+          *) echo "::error::ccache install not supported on ${{ runner.os }}"
+             exit 1 ;;
+        esac
+        tag=""
+        if [ "${{ runner.os }}" = "Linux" ]; then
+          tag="${{ inputs.ghcr-debs-tag }}"
+          if [ "$tag" = "auto" ]; then
+            # os-release rather than lsb_release: no package needed.
+            . /etc/os-release
+            tag="ubuntu-$VERSION_ID-minimal"
+          elif [ "$tag" = "none" ]; then
+            tag=""
+          fi
         fi
+        echo "tag=$tag" >> "$GITHUB_OUTPUT"
+
+    # Offline from the ghcr bundle when it has ccache, apt otherwise -
+    # bounded either way. See install-apt-deps.
+    - name: Install ccache (Linux)
+      if: steps.check.outputs.present != 'true' && runner.os == 'Linux'
+      uses: ./.github/actions/install-apt-deps
+      with:
+        packages: ccache
+        no-install-recommends: 'true'
+        ghcr-debs-tag: ${{ steps.check.outputs.tag }}
+        # ccache is an optimization: never spend more of the job
+        # installing it than a cold compile would have cost.
+        apt-budget: '120'
+        apt-install-timeout: '120'
+
+    - name: Install ccache (macOS)
+      if: steps.check.outputs.present != 'true' && runner.os == 'macOS'
+      shell: bash
+      run: brew install ccache
 
     # read-only=false (default): restore + post-job save (the run_id in the
     # key never hits, so it always saves its contribution).

.github/actions/install-apt-deps/action.yml

@@ -12,6 +12,18 @@ inputs:
     description: 'Initial delay between retries (seconds, doubles each attempt)'
     required: false
     default: '5'
+  apt-budget:
+    description: >
+      Wall-clock budget (seconds) for retrying the apt install.
+    required: false
+    default: '600'
+  apt-install-timeout:
+    description: >
+      Per-attempt cap (seconds) on apt-get install. The default suits the
+      large bundles; callers installing one small package should pass a
+      much smaller value.
+    required: false
+    default: '600'
   no-install-recommends:
     description: 'Pass --no-install-recommends to apt-get install'
     required: false
@@ -85,13 +97,29 @@ runs:
           NO_REC="--no-install-recommends"
         fi
 
+        # A stalled mirror connection returns no error, so the loop below
+        # cannot see it - apt just waits out its timeout once per index
+        # file, and the runner fetches around twenty of them. Keep that
+        # per-connection timeout short so failing over to the next mirror
+        # is cheap, cap each apt-get so a wedged one is killed, and stop
+        # retrying once the budget is gone.
+        APT_OPTS=(-o Acquire::Retries=2 -o Acquire::http::Timeout=10
+                  -o Acquire::https::Timeout=10)
+        # Enough for a full failover of every index file at the timeout
+        # above, so a slow-but-recovering update is not cut short.
+        UPDATE_TIMEOUT=240
+        deadline=$((SECONDS + ${{ inputs.apt-budget }}))
+
         for i in $(seq 1 $RETRIES); do
-          if sudo apt-get update -q && \
-             sudo apt-get install -y $NO_REC ${{ inputs.packages }}; then
+          if sudo timeout -k 10 $UPDATE_TIMEOUT \
+               apt-get "${APT_OPTS[@]}" update -q && \
+             sudo timeout -k 10 ${{ inputs.apt-install-timeout }} \
+               apt-get "${APT_OPTS[@]}" install -y $NO_REC \
+               ${{ inputs.packages }}; then
             exit 0
           fi
-          if [ "$i" -eq "$RETRIES" ]; then
-            echo "::error::apt-get failed after $RETRIES attempts"
+          if [ "$i" -eq "$RETRIES" ] || [ "$SECONDS" -ge "$deadline" ]; then
+            echo "::error::apt-get failed after $i attempt(s)"
             exit 1
           fi
           echo "::warning::apt-get failed (attempt $i/$RETRIES), retrying in ${DELAY}s..."

.github/ci-deps/packages-ubuntu-22.04-minimal.txt

@@ -4,6 +4,7 @@
 autoconf
 automake
 build-essential
+ccache
 crossbuild-essential-arm64
 crossbuild-essential-armel
 crossbuild-essential-armhf

.github/workflows/fips-dev-no-post.yml

@@ -69,6 +69,9 @@ jobs:
           workflow-id: fips-dev-no-post
           read-only: ${{ github.event_name == 'pull_request' }}
           max-size: 500M
+          # This job pulls the linuxkm bundle, not -minimal; fetching the
+          # latter just for ccache would cost more than the apt path.
+          ghcr-debs-tag: none
 
       - name: Prepare target kernel for module builds
         run: |