[Bugfix][Model] Pad SigLIP text prompts to the trained sequence length

SigLIP is trained with padding="max_length" and without an attention mask, so
padding tokens are part of the input and the pooled embedding is read from the
last position. Text embeddings computed without padding are not aligned with the
image embeddings, which makes image-text similarity unusable.

Callers that cannot pass tokenization kwargs — notably /v1/embeddings, which
builds TokenizeParams from scratch in entrypoints/pooling/base/protocol.py —
silently get wrong values: the request still returns 200 with a correctly shaped
vector.

Gate the default on the tokenizer's declared inputs, so a finetune trained with
masking (attention_mask in model_input_names) opts out on its own.

Test: each case in _run_test now carries separate HF and vLLM tokenization
kwargs. With explicit_padding=False vLLM is told nothing about padding and still
has to reach the same embedding as HF, which is the regression being guarded.

Co-authored-by: Claude <noreply@anthropic.com>
Signed-off-by: tmduc3 <tmduc3@rd.misa.com.vn>

Author: tmduc3

tests/models/multimodal/pooling/test_siglip.py

@@ -33,7 +33,9 @@
 def _run_test(
     hf_runner: type[HfRunner],
     vllm_runner: type[VllmRunner],
-    input_cases: list[tuple[list[str], PromptImageInput, dict[str, Any]]],
+    input_cases: list[
+        tuple[list[str], PromptImageInput, dict[str, Any], dict[str, Any]]
+    ],
     model: str,
     *,
     dtype: str,
@@ -50,9 +52,9 @@ def _run_test(
             vllm_model.embed(
                 input_texts,
                 images=input_images,
-                tokenization_kwargs=tokenization_kwargs,
+                tokenization_kwargs=vllm_tokenization_kwargs,
             )
-            for input_texts, input_images, tokenization_kwargs in input_cases
+            for input_texts, input_images, _, vllm_tokenization_kwargs in input_cases
         ]
 
         texts = [HF_TEXT_PROMPTS[0]]
@@ -65,7 +67,7 @@ def _run_test(
 
     with hf_runner(model, dtype=dtype, auto_cls=SiglipModel) as hf_model:
         hf_outputs_per_case = []
-        for input_texts, input_images, tokenization_kwargs in input_cases:
+        for input_texts, input_images, tokenization_kwargs, _ in input_cases:
             all_inputs = hf_model.get_inputs(
                 input_texts,
                 images=input_images,
@@ -103,25 +105,34 @@ def _run_test(
 
 @pytest.mark.parametrize("model", MODELS)
 @pytest.mark.parametrize("dtype", ["float"])
+@pytest.mark.parametrize("explicit_padding", [True, False])
 def test_models(
     hf_runner,
     vllm_runner,
     image_assets,
     model: str,
     dtype: str,
+    explicit_padding: bool,
 ) -> None:
+    """Text embeddings must match HF whether or not the caller asks for padding.
+
+    SigLIP is trained with ``padding="max_length"`` and vLLM now applies it by
+    default, so ``explicit_padding=False`` sends vLLM no padding kwargs at all
+    and still has to reach the same embedding. Callers that cannot pass
+    tokenization kwargs (the OpenAI-compatible server) would otherwise get
+    embeddings that are not aligned with the image embeddings.
+    """
     text_images = [None] * len(HF_TEXT_PROMPTS)
     images = [asset.pil_image for asset in image_assets]
+    padding_kwargs = {"padding": "max_length", "max_length": 64}
     input_cases = [
         (
             HF_TEXT_PROMPTS,
             text_images,
-            {
-                "padding": "max_length",
-                "max_length": 64,
-            },
+            padding_kwargs,
+            padding_kwargs if explicit_padding else {},
         ),
-        (HF_IMAGE_PROMPTS, images, {}),
+        (HF_IMAGE_PROMPTS, images, {}, {}),
     ]
 
     _run_test(

tests/renderers/test_completions.py

@@ -34,6 +34,7 @@ class MockModelConfig:
     tokenizer_revision = None
     tokenizer_mode = "auto"
     hf_config = MockHFConfig()
+    max_model_len: int = 1024
     encoder_config: dict[str, Any] | None = None
     enable_prompt_embeds: bool = True
     skip_tokenizer_init: bool = False

tests/renderers/test_default_padding.py

@@ -0,0 +1,102 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Model-level padding defaults must reach frontends that build their own
+`TokenizeParams`, such as the OpenAI-compatible pooling endpoints."""
+
+import pytest
+
+from vllm.engine.arg_utils import EngineArgs
+from vllm.inputs import TextPrompt
+from vllm.renderers import TokenizeParams
+from vllm.renderers.hf import HfRenderer
+from vllm.tokenizers import get_tokenizer
+
+MODEL = "google/siglip-base-patch16-224"
+TRAINED_LENGTH = 64
+
+
+@pytest.fixture(scope="module")
+def renderer() -> HfRenderer:
+    vllm_config = EngineArgs(
+        model=MODEL,
+        runner="pooling",
+        dtype="float32",
+        max_model_len=TRAINED_LENGTH,
+        enforce_eager=True,
+    ).create_engine_config()
+
+    return HfRenderer(vllm_config, get_tokenizer(MODEL))
+
+
+def _request_built_params() -> TokenizeParams:
+    """What the pooling endpoints build from a request: no padding field."""
+    return TokenizeParams(
+        max_total_tokens=TRAINED_LENGTH,
+        max_output_tokens=0,
+        add_special_tokens=True,
+    )
+
+
+def test_model_default_is_applied(renderer: HfRenderer):
+    assert renderer.default_cmpl_tok_params.pad_prompt_tokens == -1
+
+    prompt = TextPrompt(prompt="a photo of a stop sign")
+    tokenized = renderer._tokenize_singleton_prompt(prompt, _request_built_params())
+
+    assert len(tokenized["prompt_token_ids"]) == TRAINED_LENGTH
+
+
+def test_explicit_padding_is_not_overridden(renderer: HfRenderer):
+    params = TokenizeParams(
+        max_total_tokens=TRAINED_LENGTH,
+        max_output_tokens=0,
+        pad_prompt_tokens=16,
+    )
+    prompt = TextPrompt(prompt="a photo of a stop sign")
+
+    tokenized = renderer._tokenize_singleton_prompt(prompt, params)
+
+    assert len(tokenized["prompt_token_ids"]) == 16
+
+
+def test_multimodal_prompts_are_not_padded(renderer: HfRenderer):
+    """Image prompts carry placeholder text that processing replaces."""
+    prompt = TextPrompt(prompt="", multi_modal_data={"image": []})
+
+    tokenized = renderer._tokenize_singleton_prompt(prompt, _request_built_params())
+
+    assert len(tokenized["prompt_token_ids"]) < TRAINED_LENGTH
+
+
+def _declare_model_inputs(
+    renderer: HfRenderer, monkeypatch: pytest.MonkeyPatch, names: list[str] | None
+):
+    info = renderer.get_mm_processor().info
+    init_kwargs = dict(info.get_tokenizer().init_kwargs)
+
+    if names is None:
+        init_kwargs.pop("model_input_names", None)
+    else:
+        init_kwargs["model_input_names"] = names
+
+    monkeypatch.setattr(info.get_tokenizer(), "init_kwargs", init_kwargs)
+    return info
+
+
+def test_checkpoint_consuming_attention_mask_opts_out(
+    renderer: HfRenderer, monkeypatch: pytest.MonkeyPatch
+):
+    """A checkpoint that declares an attention mask is left unpadded."""
+    info = _declare_model_inputs(renderer, monkeypatch, ["input_ids", "attention_mask"])
+
+    assert info.get_default_tok_params().pad_prompt_tokens is None
+
+
+def test_undeclared_model_inputs_still_pad(
+    renderer: HfRenderer, monkeypatch: pytest.MonkeyPatch
+):
+    """`SiglipTokenizer.model_input_names` defaults to containing
+    `attention_mask`, so a checkpoint that declares nothing must still pad."""
+    info = _declare_model_inputs(renderer, monkeypatch, None)
+
+    assert info.get_default_tok_params().pad_prompt_tokens == -1

vllm/model_executor/models/siglip.py

@@ -53,6 +53,7 @@
     PromptUpdate,
     TimingContext,
 )
+from vllm.renderers import TokenizeParams
 from vllm.sequence import IntermediateTensors
 from vllm.utils.gpu_sync_debug import gpu_sync_allowed
 from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -146,6 +147,22 @@ def get_max_image_tokens(self) -> int:
             image_width=target_width, image_height=target_height
         )
 
+    def get_default_tok_params(self) -> TokenizeParams:
+        # SigLIP is trained with padding="max_length" and no attention mask, so
+        # the pooled embedding is read from the last position and unpadded text
+        # does not line up with the image embeddings. A checkpoint that declares
+        # attention_mask in tokenizer_config.json consumes a mask; leave it be.
+        # init_kwargs is read rather than tokenizer.model_input_names, which
+        # falls back to a class default that already contains attention_mask.
+        declared = getattr(self.get_tokenizer(), "init_kwargs", {}).get(
+            "model_input_names"
+        )
+        tok_params = super().get_default_tok_params()
+        if declared is not None and "attention_mask" in declared:
+            return tok_params
+
+        return tok_params.with_kwargs(pad_prompt_tokens=-1)
+
 
 class SiglipDummyInputsBuilder(BaseDummyInputsBuilder[SiglipProcessingInfo]):
     def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:

vllm/renderers/base.py

@@ -5,6 +5,7 @@
 from abc import ABC, abstractmethod
 from collections.abc import Mapping, Sequence
 from concurrent.futures import Executor, ThreadPoolExecutor
+from dataclasses import replace
 from functools import cached_property
 from typing import TYPE_CHECKING, Any, Generic, overload
 
@@ -513,11 +514,31 @@ def _tokenize_singleton_prompt(  # type: ignore[misc]
         params: TokenizeParams,
     ) -> EmbedsPrompt: ...
 
+    def _apply_default_padding(
+        self,
+        prompt: SingletonDictPrompt,
+        params: TokenizeParams,
+    ) -> TokenizeParams:
+        # Frontends build TokenizeParams from the request and do not inherit
+        # model-level defaults, so a model that is only correct with padded
+        # inputs (SigLIP) would be served unpadded. Multi-modal prompts are left
+        # alone: their text is replaced by placeholder tokens during processing.
+        if params.pad_prompt_tokens is not None or prompt.get("multi_modal_data"):
+            return params
+
+        default = self.default_cmpl_tok_params.pad_prompt_tokens
+        if default is None:
+            return params
+
+        return replace(params, pad_prompt_tokens=default)
+
     def _tokenize_singleton_prompt(
         self,
         prompt: SingletonDictPrompt,
         params: TokenizeParams,
     ) -> SingletonTokPrompt:
+        params = self._apply_default_padding(prompt, params)
+
         if "prompt_token_ids" not in prompt and "prompt_embeds" not in prompt:
             if not isinstance(prompt.get("prompt"), str):
                 raise TypeError(
@@ -554,6 +575,8 @@ async def _tokenize_singleton_prompt_async(
         prompt: SingletonDictPrompt,
         params: TokenizeParams,
     ) -> SingletonTokPrompt:
+        params = self._apply_default_padding(prompt, params)
+
         if "prompt_token_ids" not in prompt and "prompt_embeds" not in prompt:
             if not isinstance(prompt.get("prompt"), str):
                 raise TypeError(