[XPU][UT] Fix OOM and skip graph case

tests/conftest.py

@@ -1001,6 +1001,21 @@ def __init__(
             from tests.utils import wait_for_rocm_memory_to_settle
 
             wait_for_rocm_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
+        elif current_platform.is_xpu():
+            # The XPU/oneAPI runtime keeps ~1 GiB of context resident in the
+            # parent pytest process for its whole lifetime (grown by in-process
+            # HfRunner models), and distributed tests additionally allocate a
+            # CCL context in the engine subprocess. The default utilization of
+            # 0.92 leaves too little headroom for both, so lower it on XPU when
+            # the caller did not request an explicit value.
+            if "gpu_memory_utilization" not in kwargs:
+                kwargs["gpu_memory_utilization"] = 0.9
+            gpu_memory_utilization = kwargs["gpu_memory_utilization"]
+            # XPU (Level Zero) can also release device memory lazily after a
+            # previous engine shuts down, so wait before constructing LLM.
+            from tests.utils import wait_for_xpu_memory_to_settle
+
+            wait_for_xpu_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
 
         with init_ctx:
             self.llm = LLM(
@@ -1335,6 +1350,15 @@ def _wait_for_rocm_memory_release(self, gpu_memory_utilization: float) -> None:
         # wait is bounded so cleanup failures fail this test instead of hanging.
         wait_for_rocm_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
 
+    def _wait_for_xpu_memory_release(self, gpu_memory_utilization: float) -> None:
+        from tests.utils import wait_for_xpu_memory_to_settle
+
+        # V1 startup requires free_memory >= total * gpu_memory_utilization.
+        # XPU (Level Zero) releases device memory lazily after an engine shuts
+        # down, so wait for the complementary used-memory ratio to settle before
+        # the next allocation. Bounded so cleanup failures do not hang the suite.
+        wait_for_xpu_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
+
     def __exit__(self, exc_type, exc_value, traceback):
         # Explicitly shutdown the engine core to release GPU resources
         # This is needed because when executing consecutive tests, the GC
@@ -1360,6 +1384,7 @@ def __exit__(self, exc_type, exc_value, traceback):
         torch._dynamo.reset()
         cleanup_dist_env_and_memory()
         self._wait_for_rocm_memory_release(gpu_memory_utilization)
+        self._wait_for_xpu_memory_release(gpu_memory_utilization)
 
 
 @pytest.fixture(scope="session")

tests/models/language/generation/test_hybrid.py

@@ -9,6 +9,7 @@
 from tests.models.registry import HF_EXAMPLE_MODELS
 from tests.utils import multi_gpu_test
 from vllm import LLM
+from vllm.config import CUDAGraphMode
 from vllm.engine.arg_utils import EngineArgs
 from vllm.platforms import current_platform
 from vllm.sampling_params import SamplingParams
@@ -210,6 +211,8 @@ def test_mamba_cache_cg_padding(
     cudagraph_dispatcher.initialize_cudagraph_keys(
         vllm_config.compilation_config.cudagraph_mode
     )
+    if cudagraph_dispatcher.cudagraph_mode == CUDAGraphMode.NONE:
+        pytest.skip("CUDA/XPU graph is disabled.Please enable it to run this test. ")
     while (
         len(example_prompts)
         == cudagraph_dispatcher.dispatch(len(example_prompts))[1].num_tokens

tests/utils.py

@@ -1540,6 +1540,13 @@ def record_gpu_memory_usage_stats(
             mem_info = amdsmi_get_gpu_vram_usage(dev_handle)
             gb_used = mem_info["vram_used"] / 2**10
             gb_total = mem_info["vram_total"] / 2**10
+        elif current_platform.is_xpu():
+            # nvml/amdsmi are unavailable on XPU. Query device memory through
+            # torch.accelerator.get_memory_info, which the XPU platform patches
+            # to return (free, total) bytes via Level Zero.
+            free_b, total_b = torch.accelerator.get_memory_info(device)
+            gb_used = (total_b - free_b) / 2**30
+            gb_total = total_b / 2**30
         else:
             dev_handle = get_nvml_device_handle(device)
             mem_info = nvmlDeviceGetMemoryInfo(dev_handle)
@@ -1710,6 +1717,37 @@ def wait_for_rocm_memory_to_settle(
     )
 
 
+def wait_for_xpu_memory_to_settle(
+    *,
+    threshold_ratio: float | dict[int, float] | None = 0.1,
+    timeout_s: float = 240,
+) -> None:
+    """Block until XPU device memory usage drops below ``threshold_ratio``.
+
+    Like ROCm, XPU (Level Zero) can release device memory lazily after an
+    engine shuts down, so back-to-back model loads in a single test process
+    can OOM the *next* engine/model startup even after
+    ``cleanup_dist_env_and_memory``. This gives the driver time to actually
+    release device memory before the next allocation. No-op off XPU.
+    """
+    if not current_platform.is_xpu():
+        return
+
+    num_gpus = current_platform.device_count()
+    if num_gpus == 0:
+        return
+    if threshold_ratio is None:
+        threshold_ratio = 0.1
+
+    wait_for_gpu_memory_to_clear(
+        devices=list(range(num_gpus)),
+        threshold_ratio=threshold_ratio,
+        timeout_s=timeout_s,
+        stable_duration_s=2.0,
+        poll_interval_s=1.0,
+    )
+
+
 _P = ParamSpec("_P")