Use apt lock timeout for prerequisites

scripts/run_performance_job.py

@@ -23,6 +23,12 @@
 from send_to_helix import PerfSendToHelixArgs, perf_send_to_helix
 
 DEFAULT_BUILD_CONFIG = "Release"
+APT_LOCK_TIMEOUT_OPTION = "-o DPkg::Lock::Timeout=120"
+
+
+def apt_command(arguments: str, *, executable: str = "apt-get") -> str:
+    return f"sudo {executable} {APT_LOCK_TIMEOUT_OPTION} {arguments}"
+
 
 def output_counters_for_crank(reports: list[Any]):
     print("#StartJobStatistics")
@@ -231,10 +237,8 @@ def get_pre_commands(
                     ]
                 else:
                     install_prerequisites += [
-                        'echo "** Waiting for dpkg to unlock (up to 2 minutes) **"',
-                        'timeout 2m bash -c \'while sudo fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1; do if [ -z "$printed" ]; then echo "Waiting for dpkg lock to be released... Lock is held by: $(ps -o cmd= -p $(sudo fuser /var/lib/dpkg/lock-frontend))"; printed=1; fi; echo "Waiting 5 seconds to check again"; sleep 5; done;\'',
-                        "sudo apt-get remove -y lttng-modules-dkms", # https://github.com/dotnet/runtime/pull/101142
-                        "sudo apt-get -y install python3-pip"
+                        apt_command("remove -y lttng-modules-dkms"),  # https://github.com/dotnet/runtime/pull/101142
+                        apt_command("-y install python3-pip")
                     ]
 
             install_prerequisites += [
@@ -272,8 +276,11 @@ def get_pre_commands(
                 ]
             else:
                 install_prerequisites += [
-                    "sudo apt-get update",
-                    "sudo apt -y install curl dirmngr apt-transport-https lsb-release ca-certificates"
+                    apt_command("update"),
+                    apt_command(
+                        "-y install curl dirmngr apt-transport-https lsb-release ca-certificates",
+                        executable="apt"
+                    )
                 ]
 
     # Set up everything needed for WASM runs (both Mono and CoreCLR)
@@ -294,19 +301,17 @@ def get_pre_commands(
         else:
             install_prerequisites += [
                 "export RestoreAdditionalProjectSources=$HELIX_CORRELATION_PAYLOAD/built-nugets",
-                'echo "** Waiting for dpkg to unlock (up to 2 minutes) **"',
-                'timeout 2m bash -c \'while sudo fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1; do if [ -z "$printed" ]; then echo "Waiting for dpkg lock to be released... Lock is held by: $(ps -o cmd= -p $(sudo fuser /var/lib/dpkg/lock-frontend))"; printed=1; fi; echo "Waiting 5 seconds to check again"; sleep 5; done;\'',
-                "sudo apt-get -y remove nodejs",
-                "sudo apt-get update",
-                "sudo apt-get install -y ca-certificates curl gnupg",
+                apt_command("-y remove nodejs"),
+                apt_command("update"),
+                apt_command("install -y ca-certificates curl gnupg"),
                 "sudo mkdir -p /etc/apt/keyrings",
                 "sudo rm -f /etc/apt/keyrings/nodesource.gpg",
                 "curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | sudo gpg --dearmor --batch -o /etc/apt/keyrings/nodesource.gpg",
                 "export NODE_MAJOR=18",
                 "echo \"deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main\" | sudo tee /etc/apt/sources.list.d/nodesource.list",
-                "sudo apt-get update",
-                "sudo apt autoremove -y",
-                "sudo apt-get install nodejs -y",
+                apt_command("update"),
+                apt_command("autoremove -y", executable="apt"),
+                apt_command("install nodejs -y"),
                 f"test -n \"{v8_version}\"",
                 "npm install --prefix $HELIX_WORKITEM_ROOT jsvu -g",
                 f"$HELIX_WORKITEM_ROOT/bin/jsvu --os=linux64 v8@{v8_version}",

scripts/tests/test_run_performance_job.py

@@ -0,0 +1,60 @@
+import pytest
+
+from scripts.run_performance_job import APT_LOCK_TIMEOUT_OPTION, get_pre_commands
+
+
+def get_generated_apt_commands(*, internal: bool, runtime_type: str) -> list[str]:
+    pre_commands = get_pre_commands(
+        os_group="linux",
+        os_distro="ubuntu",
+        internal=internal,
+        runtime_type=runtime_type,
+        codegen_type="jit",
+        build_config="Release",
+        v8_version="12.0.0",
+    )
+
+    return [
+        command.strip()
+        for pre_command in pre_commands
+        for command in pre_command.split(" && ")
+        if command.strip().startswith(("sudo apt ", "sudo apt-get "))
+    ]
+
+
+@pytest.mark.parametrize(
+    ("internal", "runtime_type", "expected_command_count"),
+    [
+        (True, "coreclr", 4),
+        (False, "wasm", 6),
+        (True, "wasm", 10),
+        (False, "wasm_coreclr", 6),
+    ],
+)
+def test_all_generated_apt_commands_use_lock_timeout(
+    internal: bool, runtime_type: str, expected_command_count: int
+):
+    apt_commands = get_generated_apt_commands(
+        internal=internal, runtime_type=runtime_type
+    )
+
+    assert len(apt_commands) == expected_command_count
+    assert all(
+        f" {APT_LOCK_TIMEOUT_OPTION} " in command for command in apt_commands
+    )
+
+
+def test_generated_prerequisites_do_not_poll_dpkg_lock():
+    pre_commands = get_pre_commands(
+        os_group="linux",
+        os_distro="ubuntu",
+        internal=True,
+        runtime_type="wasm",
+        codegen_type="jit",
+        build_config="Release",
+        v8_version="12.0.0",
+    )
+
+    prerequisites = "\n".join(pre_commands)
+    assert "fuser" not in prerequisites
+    assert "Waiting for dpkg" not in prerequisites