[MM] Remove text components from ProcessorInputs

docs/contributing/model/multimodal.md

@@ -456,13 +456,11 @@ return a schema of the tensors outputted by the HF processor that are related to
             prompt: str,
             mm_data: Mapping[str, object],
             mm_kwargs: Mapping[str, object],
-            tok_kwargs: Mapping[str, object],
         ) -> BatchFeature:
             processed_outputs = super()._call_hf_processor(
                 prompt=prompt,
                 mm_data=mm_data,
                 mm_kwargs=mm_kwargs,
-                tok_kwargs=tok_kwargs,
             )
 
             pixel_values = processed_outputs.get("pixel_values")
@@ -479,11 +477,6 @@ return a schema of the tensors outputted by the HF processor that are related to
             return processed_outputs
         ```
 
-    !!! note
-        The `_call_hf_processor` method specifies both `mm_kwargs` and `tok_kwargs` for
-        processing. `mm_kwargs` is used to both initialize and call the huggingface
-        processor, whereas `tok_kwargs` is only used to call the huggingface processor.
-
     Since `pixel_values` is now a list with one tensor per image, we can override
     [_get_mm_fields_config][vllm.multimodal.processing.BaseMultiModalProcessor._get_mm_fields_config] as follows:
 

tests/models/multimodal/processing/test_audioflamingo3.py

@@ -107,7 +107,7 @@ def test_audio_chunk_counting(mock_ctx):
     mm_data = {"audio": [audio_1, audio_2]}
     prompt = "<|user|>Listen.<|end|>"
 
-    processed = processor._call_hf_processor(prompt, mm_data, {}, {})
+    processed = processor._call_hf_processor(prompt, mm_data, {})
 
     chunk_counts = processed["chunk_counts"]
 

tests/models/multimodal/processing/test_common.py

@@ -145,10 +145,10 @@ def get_transformers_backend_model_ids_to_test():
     )
 
 
-def get_text_token_prompts(
+def get_token_prompt(
     processor: BaseMultiModalProcessor,
     mm_data: MultiModalDataDict,
-):
+) -> list[int]:
     dummy_inputs = processor.dummy_inputs
     tokenizer: TokenizerLike = processor.info.get_tokenizer()
     model_config = processor.info.ctx.model_config
@@ -178,21 +178,10 @@ def get_text_token_prompts(
             mm_options={},
         )
 
-    text_prompt: str | None
-    token_prompt: list[int]
-    if isinstance(inputs.prompt, list):
-        text_prompt = None
-        token_prompt = inputs.prompt
-    elif isinstance(inputs.prompt, str):
-        text_prompt = inputs.prompt
-        token_prompt = tokenizer.encode(
-            text_prompt,
-            **processor.info.get_default_tok_params().get_encode_kwargs(),
-        )
-    else:
+    if not isinstance(inputs.prompt, list):
         raise TypeError(type(inputs.prompt))
 
-    return text_prompt, token_prompt
+    return inputs.prompt
 
 
 def random_vision_chunk(
@@ -358,7 +347,7 @@ def _test_processing_correctness_one(
 ):
     model_type = model_config.hf_config.model_type
 
-    text_prompt, token_prompt = get_text_token_prompts(baseline_processor, mm_data)
+    token_prompt = get_token_prompt(baseline_processor, mm_data)
     mm_items = baseline_processor.info.parse_mm_data(mm_data)
     ignore_mm_keys = _IGNORE_MM_KEYS.get(model_type, set[str]())
 
@@ -381,55 +370,10 @@ def _test_processing_correctness_one(
         msg=(
             f"Failed ({batch_idx=}, {hit_rate=}, "
             f"{num_batches=}, {simplify_rate=}, "
-            f"{text_prompt=}, {token_prompt=}, {mm_data=})"
+            f"{token_prompt=}, {mm_data=})"
         ),
     )
 
-    if text_prompt is not None:
-        baseline_text_result = baseline_processor(
-            text_prompt,
-            mm_items=mm_items,
-            hf_processor_mm_kwargs={},
-        )
-        cached_text_result = cached_processor(
-            text_prompt,
-            mm_items=mm_items,
-            hf_processor_mm_kwargs={},
-        )
-
-        _assert_inputs_equal(
-            baseline_text_result,
-            cached_text_result,
-            ignore_mm_keys=ignore_mm_keys,
-            msg=(
-                f"Failed ({batch_idx=}, {hit_rate=}, "
-                f"{num_batches=}, {simplify_rate=}, "
-                f"{text_prompt=}, {token_prompt=}, {mm_data=})"
-            ),
-        )
-
-        _assert_inputs_equal(
-            baseline_text_result,
-            baseline_tokenized_result,
-            ignore_mm_keys=ignore_mm_keys,
-            msg=(
-                f"Failed ({batch_idx=}, {hit_rate=}, "
-                f"{num_batches=}, {simplify_rate=}, "
-                f"{text_prompt=}, {token_prompt=}, {mm_data=})"
-            ),
-        )
-
-        _assert_inputs_equal(
-            cached_text_result,
-            cached_tokenized_result,
-            ignore_mm_keys=ignore_mm_keys,
-            msg=(
-                f"Failed ({batch_idx=}, {hit_rate=}, "
-                f"{num_batches=}, {simplify_rate=}, "
-                f"{text_prompt=}, {token_prompt=}, {mm_data=})"
-            ),
-        )
-
 
 @pytest.mark.parametrize("model_id", get_model_ids_to_test())
 @pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])

tests/models/multimodal/processing/test_llava_next.py

@@ -11,6 +11,7 @@
 from vllm.multimodal import MULTIMODAL_REGISTRY
 from vllm.multimodal.parse import ImageSize
 from vllm.multimodal.processing import BaseMultiModalProcessor
+from vllm.tokenizers.hf import maybe_make_thread_pool
 
 from ...utils import build_model_context
 
@@ -144,7 +145,14 @@ def test_processor_prompt_replacements_regression(model_id, num_imgs):
         mm_processor_kwargs=None,
         limit_mm_per_prompt={"image": num_imgs},
     )
-    processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
+
+    # Avoid tokenizer already borrowed error
+    maybe_make_thread_pool(ctx.tokenizer)
+
+    processor = MULTIMODAL_REGISTRY.create_processor(
+        ctx.model_config,
+        tokenizer=ctx.tokenizer,
+    )
 
     image_ratios = [
         (171, 152),
@@ -177,7 +185,14 @@ def test_processor_prompt_replacements_all(model_id, num_imgs):
         mm_processor_kwargs=None,
         limit_mm_per_prompt={"image": num_imgs},
     )
-    processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
+
+    # Avoid tokenizer already borrowed error
+    maybe_make_thread_pool(ctx.tokenizer)
+
+    processor = MULTIMODAL_REGISTRY.create_processor(
+        ctx.model_config,
+        tokenizer=ctx.tokenizer,
+    )
 
     seen_aspect_ratios = set[float]()
     image_sizes = list[ImageSize]()

tests/models/multimodal/processing/test_llava_onevision.py

@@ -11,6 +11,7 @@
 from vllm.multimodal import MULTIMODAL_REGISTRY
 from vllm.multimodal.parse import ImageSize
 from vllm.multimodal.processing import BaseMultiModalProcessor
+from vllm.tokenizers.hf import maybe_make_thread_pool
 
 from ...utils import build_model_context
 
@@ -42,7 +43,15 @@ def test_processor_max_tokens(model_id):
         mm_processor_kwargs=None,
         limit_mm_per_prompt={"image": 1},
     )
-    processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
+
+    # Avoid tokenizer already borrowed error
+    maybe_make_thread_pool(ctx.tokenizer)
+
+    processor = MULTIMODAL_REGISTRY.create_processor(
+        ctx.model_config,
+        tokenizer=ctx.tokenizer,
+    )
+
     info = processor.info
 
     seen_aspect_ratios = set[float]()
@@ -142,7 +151,14 @@ def test_processor_prompt_replacements_regression(model_id, num_imgs):
         mm_processor_kwargs=None,
         limit_mm_per_prompt={"image": num_imgs},
     )
-    processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
+
+    # Avoid tokenizer already borrowed error
+    maybe_make_thread_pool(ctx.tokenizer)
+
+    processor = MULTIMODAL_REGISTRY.create_processor(
+        ctx.model_config,
+        tokenizer=ctx.tokenizer,
+    )
 
     image_ratios = [
         (171, 152),

tests/models/multimodal/processing/test_moss_audio.py

@@ -35,12 +35,10 @@
 
 
 class _Tokenizer:
-    def encode(self, text, add_special_tokens=False):
-        del add_special_tokens
+    def encode(self, text, **kwargs):
         return [ord(char) for char in text]
 
     def decode(self, token_ids, **kwargs):
-        del kwargs
         return "".join(chr(token_id) for token_id in token_ids)
 
     def batch_decode(self, batch_token_ids, **kwargs):

tests/models/multimodal/processing/test_openvla.py

@@ -185,11 +185,6 @@ def test_openvla_prompt_update_inserts_image_tokens_after_bos() -> None:
     image = Image.new("RGB", (640, 480), color=(255, 255, 255))
     mm_items = MultiModalDataItems({"image": ImageProcessorItems([image])})
 
-    assert (
-        processor._hf_processor_applies_updates("In: test\nOut:", mm_items, {}, {})
-        is False
-    )
-
     prompt_update = processor._get_prompt_updates(mm_items, {}, {})[0]
     resolved = prompt_update.resolve(0)
     content = resolved.content

tests/models/multimodal/processing/test_tensor_schema.py

@@ -37,7 +37,7 @@
 from ....utils import create_new_process_for_each_test
 from ...registry import HF_EXAMPLE_MODELS
 from ...utils import dummy_hf_overrides
-from .test_common import get_model_ids_to_test, get_text_token_prompts
+from .test_common import get_model_ids_to_test, get_token_prompt
 
 ImageInput = list[Image.Image]
 VideoInput: TypeAlias = (
@@ -107,10 +107,10 @@ def create_batched_mm_kwargs(
     }
 
     # video metadata will be added back to the resized video data here.
-    text_prompt, token_prompt = get_text_token_prompts(processor, resized_mm_data)
+    token_prompt = get_token_prompt(processor, resized_mm_data)
 
     mm_kwargs = processor(
-        prompt=token_prompt if text_prompt is None else text_prompt,
+        prompt=token_prompt,
         mm_items=processor.info.parse_mm_data(resized_mm_data),
         hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
     )["mm_kwargs"].require_data()

vllm/model_executor/models/audioflamingo3.py

@@ -382,7 +382,6 @@ def _call_hf_processor(
         prompt: str,
         mm_data: Mapping[str, object],
         mm_kwargs: Mapping[str, Any],
-        tok_kwargs: Mapping[str, object],
     ) -> BatchFeature:
         processor_mm_data = dict(mm_data)
         audios = processor_mm_data.pop("audios", None)
@@ -393,7 +392,6 @@ def _call_hf_processor(
             prompt=prompt,
             mm_data=processor_mm_data,
             mm_kwargs=mm_kwargs,
-            tok_kwargs=tok_kwargs,
         )
 
         if "input_features_mask" in outputs:

vllm/model_executor/models/bagel.py

@@ -273,15 +273,6 @@ def get_dummy_mm_data(
 class BagelMultiModalProcessor(BaseMultiModalProcessor[BagelProcessingInfo]):
     """Multimodal processor for BAGEL model."""
 
-    def _hf_processor_applies_updates(
-        self,
-        prompt_text: str,
-        mm_items: MultiModalDataItems,
-        hf_processor_mm_kwargs: Mapping[str, object],
-        tokenization_kwargs: Mapping[str, object],
-    ) -> bool:
-        return False
-
     def _get_prompt_updates(
         self,
         mm_items: MultiModalDataItems,

vllm/model_executor/models/blip2.py

@@ -472,7 +472,6 @@ def _call_hf_processor(
         prompt: str,
         mm_data: Mapping[str, object],
         mm_kwargs: Mapping[str, object],
-        tok_kwargs: Mapping[str, object],
     ) -> BatchFeature:
         if not mm_data:
             # HF processor always adds placeholders even when there's no image
@@ -484,7 +483,6 @@ def _call_hf_processor(
             prompt=prompt,
             mm_data=mm_data,
             mm_kwargs=mm_kwargs,
-            tok_kwargs=tok_kwargs,
         )
 
     def _get_mm_fields_config(

vllm/model_executor/models/chameleon.py

@@ -140,7 +140,6 @@ def _call_hf_processor(
         prompt: str,
         mm_data: Mapping[str, object],
         mm_kwargs: Mapping[str, object],
-        tok_kwargs: Mapping[str, object],
     ) -> BatchFeature:
         if not mm_data:
             prompt_ids = self.info.get_tokenizer().encode(prompt)
@@ -151,7 +150,6 @@ def _call_hf_processor(
             prompt=prompt,
             mm_data=mm_data,
             mm_kwargs=mm_kwargs,
-            tok_kwargs=tok_kwargs,
         )
 
     def _apply_hf_processor_tokens_only(

vllm/model_executor/models/cheers.py

@@ -497,18 +497,8 @@ def _call_hf_processor(
         prompt: str,
         mm_data: Mapping[str, object],
         mm_kwargs: Mapping[str, object],
-        tok_kwargs: Mapping[str, object],
     ) -> BatchFeature:
-        return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
-
-    def _hf_processor_applies_updates(
-        self,
-        prompt_text: str,
-        mm_items: MultiModalDataItems,
-        hf_processor_mm_kwargs: Mapping[str, object],
-        tokenization_kwargs: Mapping[str, object],
-    ) -> bool:
-        return False
+        return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
 
     def _get_prompt_updates(
         self,

vllm/model_executor/models/clip.py

@@ -208,40 +208,17 @@ def apply(
         timing_ctx: TimingContext,
     ) -> MultiModalInput:
         if inputs.mm_data_items:
-            if isinstance(inputs.prompt, str):
-                if len(inputs.prompt) > 0:
-                    raise ValueError(
-                        "CLIP accepts text-only or image-only inputs, not both! "
-                        "You must pass an image with an empty text prompt."
-                    )
+            special_tokens = self.info.get_tokenizer().all_special_ids
+            if all(tok in special_tokens for tok in inputs.prompt):
+                inputs.prompt = []
             else:
-                special_tokens = self.info.get_tokenizer().all_special_ids
-                if all(tok in special_tokens for tok in inputs.prompt):
-                    inputs.prompt = []
-                else:
-                    raise ValueError(
-                        "CLIP accepts text-only or image-only inputs, not both! "
-                        "You must pass an image with an empty token prompt."
-                    )
-
-            # For multi-modal data, the prompt after processing should
-            # only contain the dummy image tokens
-            inputs.tokenization_kwargs = {
-                **inputs.tokenization_kwargs,
-                "add_special_tokens": False,
-            }
+                raise ValueError(
+                    "CLIP accepts text-only or image-only inputs, not both! "
+                    "You must pass an image with an empty token prompt."
+                )
 
         return super().apply(inputs, timing_ctx)
 
-    def _hf_processor_applies_updates(
-        self,
-        prompt_text: str,
-        mm_items: MultiModalDataItems,
-        hf_processor_mm_kwargs: Mapping[str, object],
-        tokenization_kwargs: Mapping[str, object],
-    ) -> bool:
-        return False
-
     def _get_mm_fields_config(
         self,
         hf_inputs: BatchFeature,