diff --git a/scripts/run_performance_job.py b/scripts/run_performance_job.py index e8026a71639..21c6c46d4a5 100644 --- a/scripts/run_performance_job.py +++ b/scripts/run_performance_job.py @@ -23,6 +23,12 @@ from send_to_helix import PerfSendToHelixArgs, perf_send_to_helix DEFAULT_BUILD_CONFIG = "Release" +APT_LOCK_TIMEOUT_OPTION = "-o DPkg::Lock::Timeout=120" + + +def apt_command(arguments: str, *, executable: str = "apt-get") -> str: + return f"sudo {executable} {APT_LOCK_TIMEOUT_OPTION} {arguments}" + def output_counters_for_crank(reports: list[Any]): print("#StartJobStatistics") @@ -231,10 +237,8 @@ def get_pre_commands( ] else: install_prerequisites += [ - 'echo "** Waiting for dpkg to unlock (up to 2 minutes) **"', - 'timeout 2m bash -c \'while sudo fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1; do if [ -z "$printed" ]; then echo "Waiting for dpkg lock to be released... Lock is held by: $(ps -o cmd= -p $(sudo fuser /var/lib/dpkg/lock-frontend))"; printed=1; fi; echo "Waiting 5 seconds to check again"; sleep 5; done;\'', - "sudo apt-get remove -y lttng-modules-dkms", # https://github.com/dotnet/runtime/pull/101142 - "sudo apt-get -y install python3-pip" + apt_command("remove -y lttng-modules-dkms"), # https://github.com/dotnet/runtime/pull/101142 + apt_command("-y install python3-pip") ] install_prerequisites += [ @@ -272,8 +276,11 @@ def get_pre_commands( ] else: install_prerequisites += [ - "sudo apt-get update", - "sudo apt -y install curl dirmngr apt-transport-https lsb-release ca-certificates" + apt_command("update"), + apt_command( + "-y install curl dirmngr apt-transport-https lsb-release ca-certificates", + executable="apt" + ) ] # Set up everything needed for WASM runs (both Mono and CoreCLR) @@ -294,19 +301,17 @@ def get_pre_commands( else: install_prerequisites += [ "export RestoreAdditionalProjectSources=$HELIX_CORRELATION_PAYLOAD/built-nugets", - 'echo "** Waiting for dpkg to unlock (up to 2 minutes) **"', - 'timeout 2m bash -c \'while sudo fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1; do if [ -z "$printed" ]; then echo "Waiting for dpkg lock to be released... Lock is held by: $(ps -o cmd= -p $(sudo fuser /var/lib/dpkg/lock-frontend))"; printed=1; fi; echo "Waiting 5 seconds to check again"; sleep 5; done;\'', - "sudo apt-get -y remove nodejs", - "sudo apt-get update", - "sudo apt-get install -y ca-certificates curl gnupg", + apt_command("-y remove nodejs"), + apt_command("update"), + apt_command("install -y ca-certificates curl gnupg"), "sudo mkdir -p /etc/apt/keyrings", "sudo rm -f /etc/apt/keyrings/nodesource.gpg", "curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | sudo gpg --dearmor --batch -o /etc/apt/keyrings/nodesource.gpg", "export NODE_MAJOR=18", "echo \"deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main\" | sudo tee /etc/apt/sources.list.d/nodesource.list", - "sudo apt-get update", - "sudo apt autoremove -y", - "sudo apt-get install nodejs -y", + apt_command("update"), + apt_command("autoremove -y", executable="apt"), + apt_command("install nodejs -y"), f"test -n \"{v8_version}\"", "npm install --prefix $HELIX_WORKITEM_ROOT jsvu -g", f"$HELIX_WORKITEM_ROOT/bin/jsvu --os=linux64 v8@{v8_version}", diff --git a/scripts/tests/test_run_performance_job.py b/scripts/tests/test_run_performance_job.py new file mode 100644 index 00000000000..edc5275f49a --- /dev/null +++ b/scripts/tests/test_run_performance_job.py @@ -0,0 +1,60 @@ +import pytest + +from scripts.run_performance_job import APT_LOCK_TIMEOUT_OPTION, get_pre_commands + + +def get_generated_apt_commands(*, internal: bool, runtime_type: str) -> list[str]: + pre_commands = get_pre_commands( + os_group="linux", + os_distro="ubuntu", + internal=internal, + runtime_type=runtime_type, + codegen_type="jit", + build_config="Release", + v8_version="12.0.0", + ) + + return [ + command.strip() + for pre_command in pre_commands + for command in pre_command.split(" && ") + if command.strip().startswith(("sudo apt ", "sudo apt-get ")) + ] + + +@pytest.mark.parametrize( + ("internal", "runtime_type", "expected_command_count"), + [ + (True, "coreclr", 4), + (False, "wasm", 6), + (True, "wasm", 10), + (False, "wasm_coreclr", 6), + ], +) +def test_all_generated_apt_commands_use_lock_timeout( + internal: bool, runtime_type: str, expected_command_count: int +): + apt_commands = get_generated_apt_commands( + internal=internal, runtime_type=runtime_type + ) + + assert len(apt_commands) == expected_command_count + assert all( + f" {APT_LOCK_TIMEOUT_OPTION} " in command for command in apt_commands + ) + + +def test_generated_prerequisites_do_not_poll_dpkg_lock(): + pre_commands = get_pre_commands( + os_group="linux", + os_distro="ubuntu", + internal=True, + runtime_type="wasm", + codegen_type="jit", + build_config="Release", + v8_version="12.0.0", + ) + + prerequisites = "\n".join(pre_commands) + assert "fuser" not in prerequisites + assert "Waiting for dpkg" not in prerequisites