Merge remote-tracking branch 'origin/main' into patchy/async_ngram_v2_pr

Author: njhill

.buildkite/hardware_tests/cpu.yaml

@@ -11,6 +11,7 @@ steps:
   - CMakeLists.txt
   - vllm/_custom_ops.py
   - tests/kernels/attention/test_cpu_attn.py
+  - tests/v1/attention/test_group_head_counts.py
   - tests/kernels/moe/test_cpu_fused_moe.py
   - tests/kernels/moe/test_cpu_quant_fused_moe.py
   - tests/kernels/test_onednn.py
@@ -27,6 +28,7 @@ steps:
     - |
       bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 30m "
       pytest -x -v -s tests/kernels/attention/test_cpu_attn.py
+      pytest -x -v -s tests/v1/attention/test_group_head_counts.py
       pytest -x -v -s tests/kernels/moe/test_cpu_fused_moe.py
       pytest -x -v -s tests/kernels/moe/test_cpu_quant_fused_moe.py
       pytest -x -v -s tests/kernels/mamba/test_cpu_short_conv.py

.buildkite/scripts/hardware_ci/run-amd-test.sh

@@ -261,6 +261,8 @@ validate_native_workspace() {
 }
 
 prepare_native_workspace() {
+  local test_commands="${1:-}"
+
   if [[ "${VLLM_CI_USE_ARTIFACTS:-0}" != "1" ]]; then
     echo "Native CI requires VLLM_CI_USE_ARTIFACTS=1"
     return 1
@@ -281,6 +283,8 @@ prepare_native_workspace() {
   local recorded_base=""
   local recorded_commit=""
   local recorded_wheel=""
+  local checkout=""
+  local checkout_commit=""
   local workspace_dir="${VLLM_CI_WORKSPACE:-/vllm-workspace}"
   local wheel_dir=""
   local attempt=0
@@ -400,6 +404,53 @@ prepare_native_workspace() {
     return 1
   fi
 
+  # The ROCm artifact intentionally contains only the installed wheel and the
+  # test workspace. The Python-only compilation job also needs setup.py and the
+  # vllm source tree, so overlay the matching Buildkite checkout for that job.
+  if [[ "${test_commands}" == *python_only_compile.sh* ]]; then
+    checkout="${BUILDKITE_BUILD_CHECKOUT_PATH:-}"
+    if [[ -z "${checkout}" || ! -d "${checkout}" ]]; then
+      echo "Python-only native CI requires BUILDKITE_BUILD_CHECKOUT_PATH" >&2
+      return 1
+    fi
+    if ! git -c "safe.directory=${checkout}" -C "${checkout}" \
+      rev-parse --is-inside-work-tree >/dev/null 2>&1; then
+      echo "Buildkite checkout is not a Git worktree: ${checkout}" >&2
+      return 1
+    fi
+    checkout_commit=$(
+      git -c "safe.directory=${checkout}" -C "${checkout}" rev-parse HEAD
+    ) || return 1
+    if [[ "${checkout_commit}" != "${recorded_commit}" ]]; then
+      echo "Buildkite checkout ${checkout_commit} does not match ROCm artifact ${recorded_commit}" >&2
+      return 1
+    fi
+
+    # setup.py normally derives this from .git via setuptools-scm. The native
+    # source overlay deliberately excludes Git metadata, so preserve the exact
+    # version from the already installed, artifact-matched wheel.
+    VLLM_VERSION_OVERRIDE=$(
+      python3 -c 'import importlib.metadata as m; print(m.version("vllm"))'
+    ) || return 1
+    export VLLM_VERSION_OVERRIDE
+    VLLM_PRECOMPILED_WHEEL_LOCATION="${wheels[0]}"
+    export VLLM_PRECOMPILED_WHEEL_LOCATION
+    echo "INFO: native Python-only wheel=${VLLM_PRECOMPILED_WHEEL_LOCATION}"
+
+    echo "--- Overlaying full source checkout for Python-only compilation"
+    # Archive the verified commit instead of copying the worktree so dirty or
+    # untracked agent files cannot contaminate the artifact-matched workspace.
+    git -c "safe.directory=${checkout}" -C "${checkout}" \
+      archive --format=tar "${recorded_commit}" \
+      | tar --no-same-owner -C "${workspace_dir}" -xf - || return 1
+    for required_source in setup.py pyproject.toml vllm; do
+      if [[ ! -e "${workspace_dir}/${required_source}" ]]; then
+        echo "Full source checkout is missing ${required_source}" >&2
+        return 1
+      fi
+    done
+  fi
+
   return 0
 }
 
@@ -434,6 +485,12 @@ initialize_native_environment() {
   TIKTOKEN_RS_CACHE_DIR="${HF_HOME}/tiktoken-rs-cache"
   : "${HF_HUB_DOWNLOAD_TIMEOUT:=300}"
   : "${HF_HUB_ETAG_TIMEOUT:=60}"
+  if [[ "${VLLM_CI_EXPECTED_GPU_COUNT:-1}" == "0" ]]; then
+    # CPU-only native jobs intentionally reuse the ROCm wheel. Make that target
+    # explicit so platform selection does not depend on wheel metadata.
+    VLLM_TARGET_DEVICE=cpu
+    export VLLM_TARGET_DEVICE
+  fi
   export TMPDIR VLLM_RPC_BASE_PATH
   export TORCHINDUCTOR_CACHE_DIR TRITON_CACHE_DIR VLLM_CACHE_ROOT XDG_CACHE_HOME
   export HF_HOME HF_DATASETS_CACHE HF_HUB_DOWNLOAD_TIMEOUT HF_HUB_ETAG_TIMEOUT
@@ -948,7 +1005,13 @@ if is_native_runtime; then
     echo "Failed to initialize the native test environment"
     exit 1
   fi
-  if ! prepare_native_workspace; then
+  if [[ "${commands}" == *python_only_compile.sh* ]]; then
+    # This no-GPU job validates the ROCm precompiled/editable install path,
+    # rather than CPU runtime platform selection.
+    VLLM_TARGET_DEVICE=rocm
+    export VLLM_TARGET_DEVICE
+  fi
+  if ! prepare_native_workspace "${commands}"; then
     echo "Failed to prepare native test workspace"
     exit 1
   fi

.buildkite/test-amd.yaml

@@ -411,8 +411,9 @@ steps:
 
 - label: Python-only Installation # TBD
   timeout_in_minutes: 180
-  mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
-  agent_pool: mi250_1
+  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
+  agent_pool: mi300_1
   optional: true
   working_dir: "/vllm-workspace/tests"
   source_file_dependencies:
@@ -2092,7 +2093,7 @@ steps:
   - tests/models/multimodal/generation
   - tests/models/multimodal/test_mapping.py
   commands:
-  - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py  --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/processing
+  - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py  --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/generation/test_vit_cudagraph.py --ignore models/multimodal/processing
   - pytest -v -s models/multimodal/generation/test_memory_leak.py -m core_model
   - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model
 
@@ -3671,7 +3672,8 @@ steps:
   - vllm/
   - tests/models/multimodal/generation
   commands:
-  - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py  --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/processing
+  - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py  --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/generation/test_vit_cudagraph.py --ignore models/multimodal/processing
+  - pytest -v -s models/multimodal/generation/test_vit_cudagraph.py -m core_model
   - pytest -v -s models/multimodal/generation/test_memory_leak.py -m core_model
   - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model
 
@@ -3771,6 +3773,7 @@ steps:
   - tests/v1/executor
   - tests/v1/kv_offload
   - tests/v1/worker
+  - tests/v1/cudagraph
   - tests/v1/kv_connector/unit
   - tests/v1/metrics
   - tests/entrypoints/openai/correctness/test_lmeval.py
@@ -3780,6 +3783,7 @@ steps:
   - pytest -v -s v1/executor
   - pytest -v -s v1/kv_offload
   - pytest -v -s v1/worker
+  - pytest -v -s v1/cudagraph/test_encoder_cudagraph.py
   - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit
   - pytest -v -s -m 'not cpu_test' v1/metrics
   - pip install -U git+https://github.com/vllm-project/lm-evaluation-harness.git@streaming-api

.buildkite/test_areas/kernels.yaml

@@ -339,6 +339,28 @@ steps:
     # e2e
     - pytest -v -s tests/models/quantization/test_nvfp4.py
 
+- label: B12X Linear Kernels (DGX Spark) Nightly
+  key: b12x-linear-kernels-dgx-spark-nightly
+  timeout_in_minutes: 30
+  device: dgx-spark
+  optional: true
+  num_devices: 1
+  depends_on:
+  - arm64-image-build
+  source_file_dependencies:
+  - setup.py
+  - vllm/model_executor/kernels/linear/
+  - vllm/model_executor/warmup/b12x_warmup.py
+  - vllm/utils/b12x.py
+  - tests/model_executor/kernels/test_b12x_linear.py
+  - tests/model_executor/test_b12x_warmup.py
+  - tests/kernels/quantization/test_block_fp8.py
+  commands:
+  - uv pip install --system b12x==1.2.4
+  - pytest -v -s model_executor/kernels/test_b12x_linear.py
+      model_executor/test_b12x_warmup.py
+      kernels/quantization/test_block_fp8.py -k b12x
+
 - label: Kernels Helion Test
   key: kernels-helion-test
   timeout_in_minutes: 115

.buildkite/test_areas/lm_eval.yaml

@@ -195,20 +195,21 @@ steps:
   commands:
     - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor-dp-ep/config-b200.txt
 
-- label: LM Eval Humming f16 (A100 - TEMPORARY)
+- label: LM Eval Humming f16 (A100 - TEMPORARY) %N
   key: lm-eval-humming-f16-a100
   timeout_in_minutes: 75
   device: a100
   optional: true
   num_devices: 1
+  parallelism: 3
   source_file_dependencies:
   - vllm/model_executor/layers/quantization/humming.py
   - vllm/model_executor/layers/quantization/utils/humming_utils.py
   - vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
   - vllm/model_executor/layers/fused_moe/oracle/
   - vllm/model_executor/kernels/linear/
   commands:
-  - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config.txt
+  - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-a100-shard-$$BUILDKITE_PARALLEL_JOB.txt
 
 - label: LM Eval Humming Act int8 (A100 - TEMPORARY)
   key: lm-eval-humming-act-a100

.buildkite/test_areas/misc.yaml

@@ -127,6 +127,7 @@ steps:
     - pytest -v -s v1/test_kv_cache_spec_registry.py
     - pytest -v -s v1/cudagraph/test_cudagraph_manager.py
     - pytest -v -s -m 'cpu_test' v1/kv_connector/unit
+    - pytest -v -s -m 'cpu_test' v1/ec_connector/unit
     - pytest -v -s -m 'cpu_test' v1/metrics
 
 - label: Extract Hidden States Integration
@@ -274,7 +275,8 @@ steps:
   - bash standalone_tests/python_only_compile.sh
   mirror:
     amd:
-      device: mi250_1
+      dind: false
+      device: mi300_1
       timeout_in_minutes: 55
       soft_fail: true
       depends_on:

.buildkite/test_areas/plugins.yaml

@@ -75,6 +75,9 @@ steps:
   - vllm/model_executor/layers/quantization
   - tests/plugins_tests/bitsandbytes
   commands:
+    # bitsandbytes' int8 quant syncs internally, and it is out-of-tree, so
+    # there is nothing to wrap on the vLLM side.
+    - unset VLLM_GPU_SYNC_CHECK
     - pip install "vllm-bnb-plugin >= 0.0.1"
     - pytest -v -s plugins_tests/bitsandbytes -m 'not distributed'
 
@@ -89,5 +92,8 @@ steps:
   - vllm/model_executor/layers/quantization
   - tests/plugins_tests/bitsandbytes
   commands:
+    # bitsandbytes' int8 quant syncs internally, and it is out-of-tree, so
+    # there is nothing to wrap on the vLLM side.
+    - unset VLLM_GPU_SYNC_CHECK
     - pip install "vllm-bnb-plugin >= 0.0.1"
     - pytest -v -s plugins_tests/bitsandbytes -m 'distributed(num_gpus=2)'

CMakeLists.txt

@@ -222,7 +222,8 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
   # `+PTX` in TORCH_CUDA_ARCH_LIST is not preserved here. If a kernel really
   # needs PTX, add `+PTX` to that kernel's component-specific arch list below.
   #
-  clear_cuda_arches(CUDA_ARCH_FLAGS)
+  clear_cuda_gencode_flags(CUDA_ARCH_FLAGS)
+  warn_if_ptx_arch_requested("${CUDA_ARCH_FLAGS}")
   extract_unique_cuda_archs_ascending(CUDA_ARCHS "${CUDA_ARCH_FLAGS}")
   message(STATUS "CUDA target architectures: ${CUDA_ARCHS}")
   # Filter the target architectures by the supported supported archs

cmake/external_projects/flashkda.cmake

@@ -13,7 +13,7 @@ else()
   FetchContent_Declare(
     flashkda
     GIT_REPOSITORY https://github.com/vllm-project/FlashKDA.git
-    GIT_TAG b5d11010ff01c1d4a683c0dde42e76cbeaa8107f
+    GIT_TAG 053de1b716ef3255873e02d2d28f4adf09951978
     GIT_PROGRESS TRUE
     GIT_SUBMODULES cutlass
   )

cmake/utils.cmake

@@ -220,11 +220,11 @@ endmacro()
 #
 # Example:
 #   CMAKE_CUDA_FLAGS="-Wall -gencode arch=compute_70,code=sm_70 -gencode arch=compute_75,code=sm_75"
-#   clear_cuda_arches(CUDA_ARCH_FLAGS)
+#   clear_cuda_gencode_flags(CUDA_ARCH_FLAGS)
 #   CUDA_ARCH_FLAGS="-gencode arch=compute_70,code=sm_70;-gencode arch=compute_75,code=sm_75"
 #   CMAKE_CUDA_FLAGS="-Wall"
 #
-macro(clear_cuda_arches CUDA_ARCH_FLAGS)
+macro(clear_cuda_gencode_flags CUDA_ARCH_FLAGS)
     # Extract all `-gencode` flags from `CMAKE_CUDA_FLAGS`
     string(REGEX MATCHALL "-gencode arch=[^ ]+" CUDA_ARCH_FLAGS
       ${CMAKE_CUDA_FLAGS})
@@ -235,6 +235,26 @@ macro(clear_cuda_arches CUDA_ARCH_FLAGS)
       ${CMAKE_CUDA_FLAGS})
 endmacro()
 
+#
+# Warn when a caller requested PTX code generation through global CUDA arch
+# flags. vLLM removes those flags and reapplies per-source gencode flags, so the
+# user's global PTX request will not be preserved.
+#
+function(warn_if_ptx_arch_requested CUDA_ARCH_FLAGS)
+  foreach(_ARCH_FLAG ${CUDA_ARCH_FLAGS})
+    if(_ARCH_FLAG MATCHES "code=.*compute_[0-9]+[af]?")
+      message(WARNING
+        "PTX code generation requested in CUDA architecture flags "
+        "(${_ARCH_FLAG}), but vLLM does not preserve global PTX requests "
+        "when normalizing per-source CUDA architectures. Remove '+PTX' from "
+        "TORCH_CUDA_ARCH_LIST or rely on vLLM's built-in per-kernel PTX "
+        "selection.")
+      return()
+    endif()
+  endforeach()
+endfunction()
+
+
 #
 # Extract unique CUDA architectures from a list of compute capabilities codes in 
 # the form `<major><minor>[<letter>]`, convert them to the form sort