diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md
index 85efca1e9b22..f9c70adec763 100644
--- a/docs/models/supported_models.md
+++ b/docs/models/supported_models.md
@@ -409,6 +409,8 @@ th {
| `GraniteSWAForCausalLM` | Granite SWA | `ibm-granite/granite-swash-2b` | ✅︎ | ✅︎ |
| `GritLM` | GritLM | `parasail-ai/GritLM-7B-vllm`. | ✅︎ | ✅︎ |
| `HrmTextForCausalLM` | HRM-Text | `sapientinc/HRM-Text-1B`, etc. | | |
+| `HunYuanDenseV1ForCausalLM` | Hunyuan Dense | `tencent/Hunyuan-7B-Instruct` | ✅︎ | ✅︎ |
+| `HunYuanMoEV1ForCausalLM` | Hunyuan-A13B | `tencent/Hunyuan-A13B-Instruct`, `tencent/Hunyuan-A13B-Pretrain`, `tencent/Hunyuan-A13B-Instruct-FP8`, etc. | ✅︎ | ✅︎ |
| `HYV3ForCausalLM` | HY3 | `tencent/Hy3-preview-Base`, `tencent/Hy3-preview` | ✅︎ | ✅︎ |
| `HyperCLOVAXForCausalLM` | HyperCLOVAX-SEED-Think-14B | `naver-hyperclovax/HyperCLOVAX-SEED-Think-14B` | ✅︎ | ✅︎ |
| `InternLM2ForCausalLM` | InternLM2 | `internlm/internlm2-7b`, `internlm/internlm2-chat-7b`, etc. | ✅︎ | ✅︎ |
@@ -476,8 +478,6 @@ Some models are supported only via the [Transformers modeling backend](#transfor
| Architecture | Models | Example HF Models | [LoRA](../features/lora.md) | [PP](../serving/parallelism_scaling.md) |
| ------------ | ------ | ----------------- | -------------------- | ------------------------- |
| `GPTBigCodeForCausalLM` | StarCoder, SantaCoder, WizardCoder | `bigcode/starcoder`, `bigcode/gpt_bigcode-santacoder`, `WizardLM/WizardCoder-15B-V1.0`, etc. | ✅︎ | |
-| `HunYuanDenseV1ForCausalLM` | Hunyuan Dense | `tencent/Hunyuan-7B-Instruct` | ✅︎ | ✅︎ |
-| `HunYuanMoEV1ForCausalLM` | Hunyuan-A13B | `tencent/Hunyuan-A13B-Instruct`, `tencent/Hunyuan-A13B-Pretrain`, `tencent/Hunyuan-A13B-Instruct-FP8`, etc. | ✅︎ | ✅︎ |
| `OlmoForCausalLM` | OLMo | `allenai/OLMo-1B-hf`, `allenai/OLMo-7B-hf`, etc. | ✅︎ | ✅︎ |
| `Olmo2ForCausalLM` | OLMo2 | `allenai/OLMo-2-0425-1B`, etc. | ✅︎ | ✅︎ |
| `SmolLM3ForCausalLM` | SmolLM3 | `HuggingFaceTB/SmolLM3-3B` | ✅︎ | ✅︎ |
@@ -552,6 +552,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
| `HCXVisionForCausalLM` | HyperCLOVAX-SEED-Vision-Instruct-3B | T + I+ + V+ | `naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B` | | |
| `HCXVisionV2ForCausalLM` | HyperCLOVAX-SEED-Think-32B | T + I+ + V+ | `naver-hyperclovax/HyperCLOVAX-SEED-Think-32B` | | |
| `H2OVLChatModel` | H2OVL | T + IE+ | `h2oai/h2ovl-mississippi-800m`, `h2oai/h2ovl-mississippi-2b`, etc. | ✅︎ | ✅︎ |
+| `HunYuanVLForConditionalGeneration` | HunyuanOCR | T + IE+ | `tencent/HunyuanOCR`, etc. | ✅︎ | ✅︎ |
| `Idefics3ForConditionalGeneration` | Idefics3 | T + I | `HuggingFaceM4/Idefics3-8B-Llama3`, etc. | ✅︎ | |
| `IsaacForConditionalGeneration` | Isaac | T + I+ | `PerceptronAI/Isaac-0.1` | ✅︎ | ✅︎ |
| `InternS1ForConditionalGeneration` | Intern-S1 | T + IE+ + VE+ | `internlm/Intern-S1`, `internlm/Intern-S1-mini`, etc. | ✅︎ | ✅︎ |
@@ -628,7 +629,6 @@ Some models are supported only via the [Transformers modeling backend](#transfor
| Architecture | Models | Inputs | Example HF Models | [LoRA](../features/lora.md) | [PP](../serving/parallelism_scaling.md) |
| ------------ | ------ | ------ | ----------------- | --------------------------- | --------------------------------------- |
| `Emu3ForConditionalGeneration` | Emu3 | T + I+ | `BAAI/Emu3-Chat-hf` | ✅︎ | ✅︎ |
-| `HunYuanVLForConditionalGeneration` | HunyuanOCR | T + IE+ | `tencent/HunyuanOCR`, etc. | ✅︎ | ✅︎ |
| `VibeVoiceAsrForConditionalGeneration` | VibeVoice-ASR | T + A+ | `microsoft/VibeVoice-ASR-HF` | ✅︎ | ✅︎ |
^ You need to set the architecture name via `--hf-overrides` to match the one in vLLM.
diff --git a/vllm/model_executor/models/hunyuan_v1.py b/vllm/model_executor/models/hunyuan_v1.py
new file mode 100644
index 000000000000..cf1933bd7dd5
--- /dev/null
+++ b/vllm/model_executor/models/hunyuan_v1.py
@@ -0,0 +1,1038 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+# coding=utf-8
+# Copyright 2024 The HunYuan team.
+# Copyright 2023 The vLLM team.
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
+#
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
+# and OPT implementations in this library. It has been modified from its
+# original forms to accommodate minor architectural differences compared
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Inference-only HunYuan model compatible with HuggingFace weights."""
+
+import typing
+from collections.abc import Callable, Iterable
+from itertools import islice
+
+import regex as re
+import torch
+from torch import nn
+from transformers import PretrainedConfig
+
+from vllm.compilation.decorators import support_torch_compile
+from vllm.config import CacheConfig, VllmConfig, get_current_vllm_config
+from vllm.distributed import (
+ get_ep_group,
+ get_pp_group,
+ get_tensor_model_parallel_world_size,
+)
+from vllm.model_executor.layers.activation import SiluAndMul
+from vllm.model_executor.layers.attention import Attention
+from vllm.model_executor.layers.fused_moe import (
+ FusedMoEFactory,
+ fused_moe_make_expert_params_mapping,
+)
+from vllm.model_executor.layers.layernorm import RMSNorm
+from vllm.model_executor.layers.linear import (
+ ColumnParallelLinear,
+ MergedColumnParallelLinear,
+ QKVParallelLinear,
+ ReplicatedLinear,
+ RowParallelLinear,
+)
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
+from vllm.model_executor.layers.quantization import QuantizationConfig
+from vllm.model_executor.layers.rotary_embedding import get_rope
+from vllm.model_executor.layers.vocab_parallel_embedding import (
+ ParallelLMHead,
+ VocabParallelEmbedding,
+)
+from vllm.model_executor.model_loader.weight_utils import (
+ default_weight_loader,
+ maybe_remap_kv_scale_name,
+)
+from vllm.sequence import IntermediateTensors
+from vllm.v1.attention.backend import AttentionType
+
+from .interfaces import (
+ EagleModelMixin,
+ MixtureOfExperts,
+ SupportsEagle,
+ SupportsEagle3,
+ SupportsLoRA,
+ SupportsPP,
+)
+from .utils import (
+ AutoWeightsLoader,
+ PPMissingLayer,
+ is_pp_missing_parameter,
+ make_layers,
+ maybe_prefix,
+)
+
+
+def _is_moe(config: PretrainedConfig) -> bool:
+ num_experts = getattr(config, "num_experts", None)
+ if isinstance(num_experts, int):
+ return num_experts > 1
+ if isinstance(num_experts, list) and num_experts:
+ # Ensure all elements are integers before calling max.
+ if all(isinstance(e, int) for e in num_experts):
+ return max(num_experts) > 1
+ else:
+ return False
+ return False
+
+
+def _get_cla_factor(config: PretrainedConfig) -> int:
+ if not getattr(config, "use_cla", False):
+ return 1
+ return getattr(config, "cla_share_factor", 1)
+
+
+class HunYuanMLP(nn.Module):
+ def __init__(
+ self,
+ hidden_size: int,
+ intermediate_size: int,
+ hidden_act: str,
+ quant_config: QuantizationConfig | None = None,
+ bias: bool = False,
+ prefix: str = "",
+ reduce_results: bool = True,
+ ) -> None:
+ super().__init__()
+ self.gate_up_proj = MergedColumnParallelLinear(
+ input_size=hidden_size,
+ output_sizes=[intermediate_size] * 2,
+ bias=bias,
+ quant_config=quant_config,
+ prefix=f"{prefix}.gate_up_proj",
+ )
+ self.down_proj = RowParallelLinear(
+ input_size=intermediate_size,
+ output_size=hidden_size,
+ bias=bias,
+ quant_config=quant_config,
+ prefix=f"{prefix}.down_proj",
+ reduce_results=reduce_results,
+ )
+ if hidden_act != "silu":
+ raise ValueError(
+ f"Unsupported activation: {hidden_act}. Only silu is supported for now."
+ )
+ self.act_fn = SiluAndMul()
+
+ def forward(self, x):
+ gate_up, _ = self.gate_up_proj(x)
+ x = self.act_fn(gate_up)
+ x, _ = self.down_proj(x)
+ return x
+
+
+class HunYuanAttention(nn.Module):
+ def __init__(
+ self,
+ config: PretrainedConfig,
+ hidden_size: int,
+ num_heads: int,
+ num_kv_heads: int,
+ max_position_embeddings: int = 8192,
+ quant_config: QuantizationConfig | None = None,
+ bias: bool = False,
+ cache_config: CacheConfig | None = None,
+ prefix: str = "",
+ layer_id: int = -1,
+ ) -> None:
+ super().__init__()
+ self.hidden_size = hidden_size
+ tp_size = get_tensor_model_parallel_world_size()
+ self.total_num_heads = num_heads
+ assert self.total_num_heads % tp_size == 0
+ self.num_heads = self.total_num_heads // tp_size
+ self.total_num_kv_heads = num_kv_heads
+ if self.total_num_kv_heads >= tp_size:
+ # Number of KV heads is greater than TP size, so we partition
+ # the KV heads across multiple tensor parallel GPUs.
+ assert self.total_num_kv_heads % tp_size == 0
+ else:
+ # Number of KV heads is less than TP size, so we replicate
+ # the KV heads across multiple tensor parallel GPUs.
+ assert tp_size % self.total_num_kv_heads == 0
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
+
+ if hasattr(config, "head_dim") and config.head_dim:
+ self.head_dim = config.head_dim
+ elif hasattr(config, "attention_head_dim"):
+ self.head_dim = config.attention_head_dim
+ else:
+ self.head_dim = self.hidden_size // self.total_num_heads
+ self.q_size = self.num_heads * self.head_dim
+ self.kv_size = self.num_kv_heads * self.head_dim
+ self.scaling = self.head_dim**-0.5
+ self.max_position_embeddings = max_position_embeddings
+ self.use_qk_norm = getattr(config, "use_qk_norm", False)
+ self.layer_id = layer_id
+
+ self.qkv_proj = QKVParallelLinear(
+ hidden_size=hidden_size,
+ head_size=self.head_dim,
+ total_num_heads=self.total_num_heads,
+ total_num_kv_heads=self.total_num_kv_heads,
+ bias=bias,
+ quant_config=quant_config,
+ prefix=f"{prefix}.qkv_proj",
+ )
+
+ self.o_proj = RowParallelLinear(
+ input_size=self.total_num_heads * self.head_dim,
+ output_size=hidden_size,
+ bias=bias,
+ quant_config=quant_config,
+ prefix=f"{prefix}.o_proj",
+ )
+
+ self.rotary_emb = get_rope(
+ self.head_dim,
+ max_position=max_position_embeddings,
+ rope_parameters=config.rope_parameters,
+ is_neox_style=True,
+ )
+ self.attn = Attention(
+ self.num_heads,
+ self.head_dim,
+ self.scaling,
+ num_kv_heads=self.num_kv_heads,
+ cache_config=cache_config,
+ quant_config=quant_config,
+ prefix=f"{prefix}.attn",
+ )
+
+ if self.use_qk_norm:
+ self.query_layernorm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
+ self.key_layernorm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
+
+ def forward(
+ self,
+ positions: torch.Tensor,
+ hidden_states: torch.Tensor,
+ kv_states: tuple[torch.Tensor] | None = None,
+ ) -> torch.Tensor:
+ qkv, _ = self.qkv_proj(hidden_states)
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
+ q, k = self.rotary_emb(positions, q, k)
+ ori_k = k
+ if self.use_qk_norm:
+ q = self.query_layernorm(
+ q.view(-1, self.num_heads, self.head_dim),
+ )
+ k = self.key_layernorm(
+ k.view(-1, self.num_kv_heads, self.head_dim),
+ )
+
+ attn_output = self.attn(q, k, v)
+ # For o_proj
+ attn_output = attn_output.view(q.shape[0], -1)
+ output, _ = self.o_proj(attn_output)
+ return output, (ori_k, v)
+
+
+class HunYuanCrossAttention(nn.Module):
+ def __init__(
+ self,
+ config: PretrainedConfig,
+ hidden_size: int,
+ num_heads: int,
+ num_kv_heads: int,
+ max_position_embeddings: int = 8192,
+ quant_config: QuantizationConfig | None = None,
+ bias: bool = False,
+ cache_config: CacheConfig | None = None,
+ prefix: str = "",
+ layer_id: int = -1,
+ ) -> None:
+ super().__init__()
+ self.hidden_size = hidden_size
+ tp_size = get_tensor_model_parallel_world_size()
+ self.total_num_heads = num_heads
+ assert self.total_num_heads % tp_size == 0
+ self.num_heads = self.total_num_heads // tp_size
+ self.total_num_kv_heads = num_kv_heads
+ if self.total_num_kv_heads >= tp_size:
+ # Number of KV heads is greater than TP size, so we partition
+ # the KV heads across multiple tensor parallel GPUs.
+ assert self.total_num_kv_heads % tp_size == 0
+ else:
+ # Number of KV heads is less than TP size, so we replicate
+ # the KV heads across multiple tensor parallel GPUs.
+ assert tp_size % self.total_num_kv_heads == 0
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
+ # MistralConfig has an optional head_dim introduced by Mistral-Nemo
+ if hasattr(config, "head_dim"):
+ self.head_dim = config.head_dim
+ elif hasattr(config, "attention_head_dim"):
+ self.head_dim = config.attention_head_dim
+ else:
+ self.head_dim = self.hidden_size // self.total_num_heads
+ self.q_size = self.num_heads * self.head_dim
+ self.kv_size = self.num_kv_heads * self.head_dim
+ self.scaling = self.head_dim**-0.5
+ self.max_position_embeddings = max_position_embeddings
+ self.use_qk_norm = getattr(config, "use_qk_norm", False)
+ self.layer_id = layer_id
+
+ self.q_proj = ColumnParallelLinear(
+ hidden_size,
+ hidden_size,
+ bias=bias,
+ quant_config=quant_config,
+ prefix=f"{prefix}.q_proj",
+ )
+
+ self.o_proj = RowParallelLinear(
+ input_size=self.total_num_heads * self.head_dim,
+ output_size=hidden_size,
+ bias=bias,
+ quant_config=quant_config,
+ prefix=f"{prefix}.o_proj",
+ )
+
+ self.rotary_emb = get_rope(
+ self.head_dim,
+ max_position=max_position_embeddings,
+ rope_parameters=config.rope_parameters,
+ is_neox_style=True,
+ )
+ self.attn = Attention(
+ self.num_heads,
+ self.head_dim,
+ self.scaling,
+ num_kv_heads=self.num_kv_heads,
+ cache_config=cache_config,
+ quant_config=quant_config,
+ prefix=f"{prefix}.attn",
+ attn_type=AttentionType.ENCODER_DECODER,
+ )
+
+ if self.use_qk_norm:
+ self.query_layernorm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
+ self.key_layernorm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
+
+ def forward(
+ self,
+ positions: torch.Tensor,
+ hidden_states: torch.Tensor,
+ kv_states: tuple[torch.Tensor] | None = None,
+ ) -> torch.Tensor:
+ assert kv_states is not None
+ ori_k, v = kv_states # use last layer kv,
+ k = ori_k
+ q, _ = self.q_proj(hidden_states)
+ k_tmp = torch.empty_like(k) # Todo: reduant rotary embedding
+ q, _ = self.rotary_emb(positions, q, k_tmp)
+ if self.use_qk_norm:
+ q = self.query_layernorm(
+ q.view(-1, self.num_heads, self.head_dim),
+ )
+ k = self.key_layernorm(
+ k.view(-1, self.num_kv_heads, self.head_dim),
+ )
+
+ attn_output = self.attn(q, k, v)
+ # For o_proj
+ attn_output = attn_output.view(q.shape[0], -1)
+ output, _ = self.o_proj(attn_output)
+ return output, (ori_k, v)
+
+
+class HunYuanSparseMoeBlock(nn.Module):
+ def __init__(
+ self,
+ config: PretrainedConfig,
+ quant_config: QuantizationConfig | None = None,
+ layer_id: int = -1,
+ prefix: str = "",
+ enable_eplb: bool = False,
+ ):
+ super().__init__()
+ self.tp_size = get_tensor_model_parallel_world_size()
+
+ self.ep_group = get_ep_group().device_group
+ self.ep_size = self.ep_group.size()
+ self.n_routed_experts = config.num_experts
+
+ if self.tp_size > config.num_experts:
+ raise ValueError(
+ f"Tensor parallel size {self.tp_size} is greater than "
+ f"the number of experts {config.num_experts}."
+ )
+
+ # Get layer_id topk if config.moe_topk is a list
+ if isinstance(config.moe_topk, list):
+ assert layer_id >= 0
+ assert len(config.moe_topk) > layer_id
+ top_k = config.moe_topk[layer_id]
+ else:
+ top_k = config.moe_topk
+
+ # If it is moe, moe_intermediate_size is preferred
+ intermediate_size = config.intermediate_size
+ if config.moe_intermediate_size is not None:
+ intermediate_size = (
+ config.moe_intermediate_size
+ if isinstance(config.moe_intermediate_size, int)
+ else config.moe_intermediate_size[layer_id]
+ )
+
+ # Load balancing settings.
+ vllm_config = get_current_vllm_config()
+ eplb_config = vllm_config.parallel_config.eplb_config
+ self.enable_eplb = enable_eplb
+
+ self.n_logical_experts = self.n_routed_experts
+ self.n_redundant_experts = eplb_config.num_redundant_experts
+ self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
+ self.n_local_physical_experts = self.n_physical_experts // self.ep_size
+ self.gate = ReplicatedLinear(
+ config.hidden_size,
+ config.num_experts,
+ bias=False,
+ quant_config=None,
+ prefix=f"{prefix}.gate",
+ )
+ if config.use_mixed_mlp_moe > 0:
+ # Get layer_id num_shared_expert if config.num_shared_expert is
+ # a list.
+ if isinstance(config.num_shared_expert, list):
+ assert layer_id >= 0
+ assert len(config.num_shared_expert) > layer_id
+ num_shared_expert = config.num_shared_expert[layer_id]
+ else:
+ num_shared_expert = config.num_shared_expert
+
+ self.shared_mlp = HunYuanMLP(
+ hidden_size=config.hidden_size,
+ intermediate_size=config.intermediate_size * num_shared_expert,
+ hidden_act=config.hidden_act,
+ quant_config=quant_config,
+ reduce_results=False,
+ prefix=f"{prefix}.shared_mlp",
+ )
+ else:
+ self.shared_mlp = None
+
+ self.experts = FusedMoEFactory(
+ shared_experts=self.shared_mlp,
+ num_experts=self.n_routed_experts,
+ top_k=top_k,
+ hidden_size=config.hidden_size,
+ intermediate_size=intermediate_size,
+ renormalize=top_k > 1,
+ quant_config=quant_config,
+ prefix=f"{prefix}.experts",
+ enable_eplb=self.enable_eplb,
+ num_redundant_experts=self.n_redundant_experts,
+ )
+
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ # NOTE: hidden_states can have either 1D or 2D shape.
+ orig_shape = hidden_states.shape
+ hidden_dim = hidden_states.shape[-1]
+ hidden_states = hidden_states.view(-1, hidden_dim)
+
+ # router_logits: (num_tokens, n_experts)
+ router_logits, _ = self.gate(hidden_states)
+ final_hidden_states = self.experts(
+ hidden_states=hidden_states, router_logits=router_logits
+ )
+
+ return final_hidden_states.view(orig_shape)
+
+
+class HunYuanDecoderLayer(nn.Module):
+ def __init__(
+ self,
+ config: PretrainedConfig,
+ cache_config: CacheConfig | None = None,
+ quant_config: QuantizationConfig | None = None,
+ prefix: str = "",
+ layer_id: int = -1,
+ enable_eplb: bool = False,
+ ) -> None:
+ super().__init__()
+ assert layer_id >= 0
+ self.layer_id = layer_id
+ self.hidden_size = config.hidden_size
+ self.intermediate_size = (
+ config.intermediate_size
+ if isinstance(config.intermediate_size, int)
+ else config.intermediate_size[layer_id]
+ )
+ max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
+ attention_bias = getattr(config, "attention_bias", False) or getattr(
+ config, "bias", False
+ )
+ cla_factor = _get_cla_factor(config)
+ attention_type = (
+ AttentionType.ENCODER_DECODER
+ if layer_id >= 0 and layer_id % cla_factor != 0
+ else AttentionType.DECODER
+ )
+ if attention_type == AttentionType.DECODER:
+ self.self_attn = HunYuanAttention(
+ config=config,
+ hidden_size=self.hidden_size,
+ num_heads=config.num_attention_heads,
+ num_kv_heads=getattr(
+ config, "num_key_value_heads", config.num_attention_heads
+ ),
+ max_position_embeddings=max_position_embeddings,
+ quant_config=quant_config,
+ bias=attention_bias,
+ cache_config=cache_config,
+ prefix=f"{prefix}.self_attn",
+ layer_id=layer_id,
+ )
+ elif attention_type == AttentionType.ENCODER_DECODER:
+ self.self_attn = HunYuanCrossAttention(
+ config=config,
+ hidden_size=self.hidden_size,
+ num_heads=config.num_attention_heads,
+ num_kv_heads=getattr(
+ config, "num_key_value_heads", config.num_attention_heads
+ ),
+ max_position_embeddings=max_position_embeddings,
+ quant_config=quant_config,
+ bias=attention_bias,
+ cache_config=cache_config,
+ prefix=f"{prefix}.self_attn",
+ layer_id=layer_id,
+ )
+ else:
+ raise RuntimeError(f"Unsupported attention type: {attention_type}")
+
+ if _is_moe(config):
+ self.mlp = HunYuanSparseMoeBlock(
+ config=config,
+ quant_config=quant_config,
+ layer_id=layer_id,
+ prefix=f"{prefix}.mlp",
+ enable_eplb=enable_eplb,
+ )
+ else:
+ self.mlp = HunYuanMLP(
+ hidden_size=self.hidden_size,
+ intermediate_size=self.intermediate_size,
+ hidden_act=config.hidden_act,
+ quant_config=quant_config,
+ bias=getattr(config, "mlp_bias", False),
+ prefix=f"{prefix}.mlp",
+ )
+
+ self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+ self.post_attention_layernorm = RMSNorm(
+ config.hidden_size, eps=config.rms_norm_eps
+ )
+
+ def forward(
+ self,
+ positions: torch.Tensor,
+ hidden_states: torch.Tensor,
+ residual: torch.Tensor | None,
+ kv_states: tuple[torch.Tensor] | None = None,
+ ) -> tuple[torch.Tensor, torch.Tensor]:
+ # Self Attention
+ if residual is None:
+ residual = hidden_states
+ hidden_states = self.input_layernorm(hidden_states)
+ else:
+ hidden_states, residual = self.input_layernorm(hidden_states, residual)
+ hidden_states, ori_kv_states = self.self_attn(
+ positions=positions,
+ hidden_states=hidden_states,
+ kv_states=kv_states,
+ )
+
+ # Fully Connected
+ hidden_states, residual = self.post_attention_layernorm(hidden_states, residual)
+ hidden_states = self.mlp(hidden_states)
+ return hidden_states, residual, ori_kv_states
+
+
+@support_torch_compile(
+ dynamic_arg_dims={
+ "input_ids": 0,
+ # positions is of shape (xd, seq_len) if xdrope is enabled for hunyuan-vl,
+ # otherwise (seq_len, ).
+ "positions": -1,
+ "intermediate_tensors": 0,
+ "inputs_embeds": 0,
+ }
+)
+class HunYuanModel(nn.Module, EagleModelMixin):
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
+ super().__init__()
+
+ config = vllm_config.model_config.hf_config
+ cache_config = vllm_config.cache_config
+ quant_config = vllm_config.quant_config
+
+ eplb_config = vllm_config.parallel_config.eplb_config
+ enable_eplb = vllm_config.parallel_config.enable_eplb
+ self.num_redundant_experts = eplb_config.num_redundant_experts
+
+ self.config = config
+ self.quant_config = quant_config
+
+ self.vocab_size = config.vocab_size
+
+ if get_pp_group().is_first_rank or (
+ config.tie_word_embeddings and get_pp_group().is_last_rank
+ ):
+ self.embed_tokens = VocabParallelEmbedding(
+ self.vocab_size,
+ config.hidden_size,
+ quant_config=quant_config,
+ )
+ else:
+ self.embed_tokens = PPMissingLayer()
+ self.start_layer, self.end_layer, self.layers = make_layers(
+ config.num_hidden_layers,
+ lambda prefix: HunYuanDecoderLayer(
+ config=config,
+ layer_id=int(prefix.split(".")[-1]),
+ cache_config=cache_config,
+ quant_config=quant_config,
+ prefix=prefix,
+ enable_eplb=enable_eplb,
+ ),
+ prefix=f"{prefix}.layers",
+ )
+ if get_pp_group().is_last_rank:
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+ else:
+ self.norm = PPMissingLayer()
+
+ def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
+ return self.embed_tokens(input_ids)
+
+ def forward(
+ self,
+ input_ids: torch.Tensor | None,
+ positions: torch.Tensor,
+ intermediate_tensors: IntermediateTensors | None,
+ inputs_embeds: torch.Tensor | None = None,
+ ) -> torch.Tensor | IntermediateTensors:
+ if get_pp_group().is_first_rank:
+ if inputs_embeds is not None:
+ hidden_states = inputs_embeds
+ else:
+ hidden_states = self.embed_input_ids(input_ids)
+ residual = None
+ else:
+ assert intermediate_tensors is not None
+ hidden_states = intermediate_tensors["hidden_states"]
+ residual = intermediate_tensors["residual"]
+
+ cla_factor = _get_cla_factor(self.config)
+ prev_kv_states = None
+ aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, residual)
+ for i, layer in enumerate(
+ islice(self.layers, self.start_layer, self.end_layer)
+ ):
+ hidden_states, residual, kv_states = layer(
+ positions,
+ hidden_states,
+ residual,
+ prev_kv_states,
+ )
+
+ if getattr(self.config, "use_cla", False) and i % cla_factor == 0:
+ prev_kv_states = kv_states
+ else:
+ prev_kv_states = None
+
+ self._maybe_add_hidden_state(
+ aux_hidden_states, i + 1, hidden_states, residual
+ )
+
+ if not get_pp_group().is_last_rank:
+ return IntermediateTensors(
+ {"hidden_states": hidden_states, "residual": residual}
+ )
+
+ hidden_states, _ = self.norm(hidden_states, residual)
+
+ if len(aux_hidden_states) > 0:
+ return hidden_states, aux_hidden_states
+ return hidden_states
+
+ def _split_qkv_weight(self, qkv: torch.Tensor):
+ num_attention_heads = self.config.num_attention_heads
+ num_kv_heads = getattr(
+ self.config, "num_key_value_heads", self.config.num_attention_heads
+ )
+ num_key_value_groups = num_attention_heads // num_kv_heads
+ hidden_size = self.config.hidden_size
+
+ if hasattr(self.config, "head_dim"):
+ attention_head_dim = self.config.head_dim
+ elif hasattr(self.config, "attention_head_dim"):
+ attention_head_dim = self.config.attention_head_dim
+ else:
+ attention_head_dim = self.config.hidden_size // num_attention_heads
+
+ qkv = qkv.reshape(
+ num_kv_heads, num_key_value_groups + 2, attention_head_dim, hidden_size
+ )
+ q, k, v = torch.split(qkv, (num_key_value_groups, 1, 1), dim=1)
+ q = q.reshape(-1, hidden_size)
+ k = k.reshape(-1, hidden_size)
+ v = v.reshape(-1, hidden_size)
+ return torch.concat((q, k, v))
+
+ def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
+ if _is_moe(self.config):
+ # Params for weights, fp8 weight scales, fp8 activation scales
+ # (param_name, weight_name, expert_id, shard_id)
+ return fused_moe_make_expert_params_mapping(
+ self,
+ ckpt_gate_proj_name="gate_proj",
+ ckpt_down_proj_name="down_proj",
+ ckpt_up_proj_name="up_proj",
+ num_experts=self.config.num_experts,
+ num_redundant_experts=self.num_redundant_experts,
+ )
+ else:
+ return []
+
+ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
+ cla_factor = _get_cla_factor(self.config)
+ stacked_params_mapping = [
+ # (param_name, shard_name, shard_id)
+ (".qkv_proj", ".q_proj", "q"),
+ (".qkv_proj", ".k_proj", "k"),
+ (".qkv_proj", ".v_proj", "v"),
+ (".gate_up_proj", ".gate_proj", 0),
+ (".gate_up_proj", ".up_proj", 1),
+ ]
+
+ num_attention_heads = self.config.num_attention_heads
+ num_kv_heads = getattr(
+ self.config, "num_key_value_heads", self.config.num_attention_heads
+ )
+ split_params_mapping = [
+ (".gate_up_proj", ".gate_and_up_proj", 2, [(1, 1), (0, 1)], None),
+ (
+ ".qkv_proj",
+ ".qkv_proj",
+ num_attention_heads + num_kv_heads * 2,
+ [("q", num_attention_heads), ("k", num_kv_heads), ("v", num_kv_heads)],
+ self._split_qkv_weight,
+ ),
+ ]
+
+ params_dict = dict(self.named_parameters())
+ loaded_params: set[str] = set()
+ expert_params_mapping = self.get_expert_mapping()
+ for name, loaded_weight in weights:
+ if "rotary_emb.inv_freq" in name:
+ continue
+ if "gate_proj_bias" in name:
+ name = name.replace("gate_proj_bias", "gate_proj.bias")
+ if "up_proj_bias" in name:
+ name = name.replace("up_proj_bias", "up_proj.bias")
+ if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name:
+ # Models trained using ColossalAI may include these tensors in
+ # the checkpoint. Skip them.
+ continue
+ # With tie_word_embeddings, we can skip lm_head.weight
+ # The weight might appear unnecessarily in the files if the model is
+ # processed with quantization, LoRA, fine-tuning, etc.
+ if self.config.tie_word_embeddings and "lm_head.weight" in name:
+ continue
+
+ is_found = False
+ for param_name, weight_name, shard_id in stacked_params_mapping:
+ if weight_name not in name:
+ continue
+ if "mlp.experts" in name:
+ continue
+ # cross layer only have q_proj, skip qkv pack
+ if weight_name == ".q_proj":
+ match = re.search(r"layers\.\d+", name)
+ if match:
+ layer_id = int(match.group(0).split(".")[-1])
+ if cla_factor > 1 and layer_id % cla_factor != 0:
+ continue
+ name = name.replace(weight_name, param_name)
+ # Skip loading extra bias for GPTQ models.
+ if name.endswith(".bias") and name not in params_dict:
+ continue
+
+ if is_pp_missing_parameter(name, self):
+ continue
+
+ param = params_dict[name]
+ weight_loader = param.weight_loader
+ weight_loader(param, loaded_weight, shard_id)
+ loaded_params.add(name)
+ is_found = True
+ break
+ if is_found:
+ continue
+
+ for (
+ param_name,
+ weight_name,
+ den,
+ split_param,
+ func,
+ ) in split_params_mapping:
+ if weight_name not in name:
+ continue
+ name = name.replace(weight_name, param_name)
+ # Skip loading extra bias for GPTQ models.
+ if name.endswith(".bias") and name not in params_dict:
+ continue
+
+ if is_pp_missing_parameter(name, self):
+ continue
+
+ assert loaded_weight.shape[0] % den == 0
+ units = loaded_weight.shape[0] // den
+
+ param = params_dict[name]
+ weight_loader = param.weight_loader
+ offset = 0
+ for shard_id, num in split_param:
+ new_offset = offset + num * units
+ if func:
+ weight_loader(
+ param, func(loaded_weight)[offset:new_offset], shard_id
+ )
+ else:
+ weight_loader(param, loaded_weight[offset:new_offset], shard_id)
+ offset = new_offset
+
+ break
+ else:
+ # Skip loading extra bias for GPTQ models.
+ if name.endswith(".bias") and name not in params_dict:
+ continue
+ is_expert_weight = False
+ for mapping in expert_params_mapping:
+ param_name, weight_name, expert_id, shard_id = mapping
+ if weight_name not in name:
+ continue
+ # this is an expert weight and should not be
+ # attempted to load as other weights later
+ is_expert_weight = True
+
+ # Do not modify `name` since the loop may continue here
+ # Instead, create a new variable
+ name_mapped = name.replace(weight_name, param_name)
+ if is_pp_missing_parameter(name_mapped, self):
+ continue
+ param = params_dict[name_mapped]
+ # We should ask the weight loader to return success or not
+ # here since otherwise we may skip experts with other
+ # available replicas.
+ weight_loader = typing.cast(
+ Callable[..., bool], param.weight_loader
+ )
+ success = weight_loader(
+ param,
+ loaded_weight,
+ name_mapped,
+ shard_id=shard_id,
+ expert_id=expert_id,
+ return_success=True,
+ )
+ if success:
+ name = name_mapped
+ break
+ else:
+ if is_expert_weight:
+ # We've checked that this is an expert weight
+ # However it's not mapped locally to this rank
+ # So we simply skip it
+ continue
+ # Remapping the name of FP8 kv-scale.
+ name = maybe_remap_kv_scale_name(name, params_dict)
+ if name is None:
+ continue
+
+ if is_pp_missing_parameter(name, self):
+ continue
+
+ if "mlp.gate.wg." in name:
+ name = name.replace("wg.", "")
+
+ param = params_dict[name]
+ weight_loader = getattr(
+ param, "weight_loader", default_weight_loader
+ )
+ weight_loader(param, loaded_weight)
+ loaded_params.add(name)
+ return loaded_params
+
+
+class HunyuanV1ModelBase(
+ nn.Module, SupportsLoRA, SupportsPP, SupportsEagle, SupportsEagle3
+):
+ packed_modules_mapping = {
+ "qkv_proj": [
+ "q_proj",
+ "k_proj",
+ "v_proj",
+ ],
+ "gate_up_proj": [
+ "gate_proj",
+ "up_proj",
+ ],
+ }
+
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
+ super().__init__()
+
+ config = vllm_config.model_config.hf_config
+ quant_config = vllm_config.quant_config
+ self.config = config
+ self.quant_config = quant_config
+
+ self.model = HunYuanModel(
+ vllm_config=vllm_config,
+ prefix=maybe_prefix(prefix, "model"),
+ )
+ if get_pp_group().is_last_rank:
+ self.lm_head = ParallelLMHead(
+ config.vocab_size,
+ config.hidden_size,
+ quant_config=quant_config,
+ prefix=maybe_prefix(prefix, "lm_head"),
+ )
+ if config.tie_word_embeddings:
+ self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens)
+
+ logit_scale = getattr(config, "logit_scale", 1.0)
+ self.logits_processor = LogitsProcessor(
+ config.vocab_size, scale=logit_scale
+ )
+ else:
+ self.lm_head = PPMissingLayer()
+
+ def forward(
+ self,
+ input_ids: torch.Tensor | None,
+ positions: torch.Tensor,
+ intermediate_tensors: IntermediateTensors | None = None,
+ inputs_embeds: torch.Tensor | None = None,
+ ) -> torch.Tensor | IntermediateTensors:
+ model_output = self.model(
+ input_ids, positions, intermediate_tensors, inputs_embeds
+ )
+ return model_output
+
+ def compute_logits(
+ self,
+ hidden_states: torch.Tensor,
+ ) -> torch.Tensor | None:
+ logits = self.logits_processor(self.lm_head, hidden_states)
+ return logits
+
+ def make_empty_intermediate_tensors(
+ self, batch_size: int, dtype: torch.dtype, device: torch.device
+ ) -> IntermediateTensors:
+ return IntermediateTensors(
+ {
+ "hidden_states": torch.zeros(
+ (batch_size, self.config.hidden_size), dtype=dtype, device=device
+ ),
+ "residual": torch.zeros(
+ (batch_size, self.config.hidden_size), dtype=dtype, device=device
+ ),
+ }
+ )
+
+ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+ loader = AutoWeightsLoader(self)
+ return loader.load_weights(weights)
+
+ def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
+ return self.model.embed_input_ids(input_ids)
+
+
+class HunYuanMoEV1Base(HunyuanV1ModelBase, MixtureOfExperts):
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
+ super().__init__(vllm_config=vllm_config, prefix=prefix)
+
+ # Set MoE hyperparameters
+ self.num_expert_groups = 1
+ self.moe_layers = []
+ example_layer = None
+ for layer in self.model.layers:
+ if isinstance(layer, PPMissingLayer):
+ continue
+
+ assert isinstance(layer, HunYuanDecoderLayer)
+ if isinstance(layer.mlp, HunYuanSparseMoeBlock):
+ example_layer = layer.mlp
+ self.moe_layers.append(layer.mlp.experts)
+
+ if example_layer is None:
+ raise RuntimeError("No HunYuanMoE layer found in model.layers.")
+
+ self.num_moe_layers = len(self.moe_layers)
+ self.num_logical_experts = example_layer.n_logical_experts
+ self.num_physical_experts = example_layer.n_physical_experts
+ self.num_local_physical_experts = example_layer.n_local_physical_experts
+ self.num_routed_experts = example_layer.n_routed_experts
+ self.num_redundant_experts = example_layer.n_redundant_experts
+
+ def update_physical_experts_metadata(
+ self,
+ num_physical_experts: int,
+ num_local_physical_experts: int,
+ ) -> None:
+ assert self.num_local_physical_experts == num_local_physical_experts
+ self.num_physical_experts = num_physical_experts
+ self.num_local_physical_experts = num_local_physical_experts
+ self.num_redundant_experts = num_physical_experts - self.num_logical_experts
+ for layer in self.model.layers:
+ if isinstance(layer.mlp, HunYuanSparseMoeBlock):
+ moe = layer.mlp
+ moe.n_local_physical_experts = num_local_physical_experts
+ moe.n_physical_experts = num_physical_experts
+ moe.n_redundant_experts = self.num_redundant_experts
+ moe.experts.update_expert_map()
+
+ def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
+ return self.model.get_expert_mapping()
+
+
+class HunYuanDenseV1Base(HunyuanV1ModelBase):
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
+ super().__init__(vllm_config=vllm_config, prefix=prefix)
+
+
+class HunYuanDenseV1ForCausalLM(HunYuanDenseV1Base):
+ pass
+
+
+class HunYuanMoEV1ForCausalLM(HunYuanMoEV1Base):
+ pass
diff --git a/vllm/model_executor/models/hunyuan_vision.py b/vllm/model_executor/models/hunyuan_vision.py
new file mode 100644
index 000000000000..4454ce47c1a6
--- /dev/null
+++ b/vllm/model_executor/models/hunyuan_vision.py
@@ -0,0 +1,1047 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+# coding=utf-8
+# Copyright 2025 The HunYuan team.
+# Copyright 2025 The vLLM team.
+# Copyright 2025 EleutherAI and the HuggingFace Inc. team. All rights reserved.
+#
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
+# and OPT implementations in this library. It has been modified from its
+# original forms to accommodate minor architectural differences compared
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Inference-only HunYuan-VL model compatible with HuggingFace weights."""
+
+from collections.abc import Callable, Iterable, Mapping, Sequence
+from functools import partial
+from typing import Annotated, Any, Literal, TypeAlias
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from transformers import BatchFeature, HunYuanVLProcessor
+from transformers.models.hunyuan_vl.image_processing_hunyuan_vl import (
+ HunYuanVLImageProcessor,
+ smart_resize,
+)
+
+from vllm.config import VllmConfig
+from vllm.config.multimodal import BaseDummyOptions
+from vllm.distributed import parallel_state
+from vllm.distributed import utils as dist_utils
+from vllm.inputs import ModalityData, MultiModalDataDict
+from vllm.logger import init_logger
+from vllm.model_executor.layers.activation import get_act_fn
+from vllm.model_executor.layers.attention import MMEncoderAttention
+from vllm.model_executor.layers.layernorm import RMSNorm
+from vllm.model_executor.layers.linear import (
+ ColumnParallelLinear,
+ QKVParallelLinear,
+ RowParallelLinear,
+)
+from vllm.model_executor.layers.quantization import QuantizationConfig
+from vllm.model_executor.models.module_mapping import MultiModelKeys
+from vllm.multimodal import MULTIMODAL_REGISTRY
+from vllm.multimodal.inputs import (
+ ImageItem,
+ MultiModalFeatureSpec,
+ MultiModalFieldConfig,
+ MultiModalKwargsItems,
+)
+from vllm.multimodal.parse import (
+ DictEmbeddingItems,
+ ImageSize,
+ ModalityDataItems,
+ MultiModalDataItems,
+ MultiModalDataParser,
+)
+from vllm.multimodal.processing import (
+ BaseDummyInputsBuilder,
+ BaseMultiModalProcessor,
+ BaseProcessingInfo,
+ PromptReplacement,
+ PromptUpdate,
+ PromptUpdateDetails,
+)
+from vllm.sequence import IntermediateTensors
+from vllm.transformers_utils.configs.hunyuan_vl import (
+ HunYuanVLConfig,
+ HunYuanVLVisionConfig,
+)
+from vllm.utils.tensor_schema import TensorSchema, TensorShape
+
+from .interfaces import (
+ MultiModalEmbeddings,
+ SupportsEagle,
+ SupportsEagle3,
+ SupportsLoRA,
+ SupportsMultiModal,
+ SupportsPP,
+ SupportsQuant,
+ SupportsXDRoPE,
+)
+from .utils import (
+ AutoWeightsLoader,
+ WeightsMapper,
+ init_vllm_registered_model,
+ maybe_prefix,
+)
+from .vision import is_vit_use_data_parallel
+
+logger = init_logger(__name__)
+
+# === Vision Inputs === #
+
+
+class HunYuanVLImagePixelInputs(TensorSchema):
+ """
+ Dimensions:
+ - np: Number of patches
+ - ni: Number of images
+ - cps: Number of channels * patch_size * patch_size
+ """
+
+ type: Literal["pixel_values"]
+
+ pixel_values: Annotated[
+ torch.Tensor,
+ TensorShape("np", "cps"),
+ ]
+
+ image_grid_thw: Annotated[
+ torch.Tensor,
+ TensorShape("ni", 3),
+ ]
+
+
+class HunYuanVLImageEmbeddingInputs(TensorSchema):
+ """
+ Dimensions:
+ - nf: Number of image features
+ - hs: Hidden size
+ - ni: Number of images
+ """
+
+ type: Literal["image_embeds"]
+
+ image_embeds: Annotated[
+ torch.Tensor,
+ TensorShape("nf", "hs"),
+ ]
+
+ image_grid_thw: Annotated[
+ torch.Tensor,
+ TensorShape("ni", 3),
+ ]
+
+
+HunYuanVLImageInputs: TypeAlias = (
+ HunYuanVLImagePixelInputs | HunYuanVLImageEmbeddingInputs
+)
+
+# === Vision Encoder === #
+
+
+class HunYuanVisionMLP(nn.Module):
+ def __init__(
+ self,
+ in_features: int,
+ hidden_features: int,
+ bias: bool = True,
+ act_fn: Callable[[torch.Tensor], torch.Tensor] = F.gelu,
+ quant_config: QuantizationConfig | None = None,
+ prefix: str = "",
+ ):
+ super().__init__()
+ use_data_parallel = is_vit_use_data_parallel()
+ self.dense_h_to_4h = ColumnParallelLinear(
+ in_features,
+ hidden_features,
+ bias=bias,
+ quant_config=quant_config,
+ prefix=f"{prefix}.dense_h_to_4h",
+ disable_tp=use_data_parallel,
+ )
+ self.dense_4h_to_h = RowParallelLinear(
+ hidden_features,
+ in_features,
+ bias=bias,
+ quant_config=quant_config,
+ prefix=f"{prefix}.dense_4h_to_h",
+ disable_tp=use_data_parallel,
+ )
+ self.act_fn = act_fn
+
+ def forward(self, x: torch.Tensor):
+ x_up, _ = self.dense_h_to_4h(x)
+ x_down, _ = self.dense_4h_to_h(self.act_fn(x_up))
+ return x_down
+
+
+class HunYuanVisionAttention(nn.Module):
+ def __init__(
+ self,
+ embed_dim: int,
+ num_heads: int,
+ projection_size: int,
+ quant_config: QuantizationConfig | None = None,
+ prefix: str = "",
+ ) -> None:
+ super().__init__()
+ # Per attention head and per partition values.
+ use_data_parallel = is_vit_use_data_parallel()
+ self.tp_size = (
+ 1
+ if use_data_parallel
+ else parallel_state.get_tensor_model_parallel_world_size()
+ )
+ self.hidden_size_per_attention_head = dist_utils.divide(
+ projection_size, num_heads
+ )
+ self.num_attention_heads_per_partition = dist_utils.divide(
+ num_heads, self.tp_size
+ )
+
+ self.qkv = QKVParallelLinear(
+ hidden_size=embed_dim,
+ head_size=self.hidden_size_per_attention_head,
+ total_num_heads=num_heads,
+ total_num_kv_heads=num_heads,
+ bias=True,
+ quant_config=quant_config,
+ prefix=f"{prefix}.qkv",
+ disable_tp=use_data_parallel,
+ )
+
+ self.o_proj = RowParallelLinear(
+ input_size=projection_size,
+ output_size=embed_dim,
+ quant_config=quant_config,
+ prefix=f"{prefix}.o_proj",
+ disable_tp=use_data_parallel,
+ )
+
+ self.scale = self.hidden_size_per_attention_head**-0.5
+ self.attn = MMEncoderAttention(
+ self.num_attention_heads_per_partition,
+ self.hidden_size_per_attention_head,
+ self.scale,
+ prefix=f"{prefix}.attn",
+ )
+
+ def forward(
+ self,
+ x: torch.Tensor,
+ ) -> torch.Tensor:
+ qkv, _ = self.qkv(x)
+ q, k, v = qkv.chunk(3, dim=-1)
+ out = self.attn(q, k, v)
+ output, _ = self.o_proj(out)
+ return output
+
+
+class HunYuanVisionBlock(nn.Module):
+ def __init__(
+ self,
+ dim: int,
+ num_heads: int,
+ mlp_hidden_dim: int,
+ act_fn: Callable[[torch.Tensor], torch.Tensor] = F.gelu,
+ norm_layer: Callable[[int], nn.Module] | None = None,
+ quant_config: QuantizationConfig | None = None,
+ prefix: str = "",
+ ) -> None:
+ super().__init__()
+ if norm_layer is None:
+ norm_layer = partial(nn.LayerNorm, eps=1e-6)
+ self.input_layernorm = norm_layer(dim)
+ self.post_attention_layernorm = norm_layer(dim)
+ self.self_attn = HunYuanVisionAttention(
+ embed_dim=dim,
+ num_heads=num_heads,
+ projection_size=dim,
+ quant_config=quant_config,
+ prefix=f"{prefix}.self_attn",
+ )
+ self.mlp = HunYuanVisionMLP(
+ dim,
+ mlp_hidden_dim,
+ act_fn=act_fn,
+ bias=True,
+ quant_config=quant_config,
+ prefix=f"{prefix}.mlp",
+ )
+
+ def forward(
+ self,
+ x: torch.Tensor,
+ ) -> torch.Tensor:
+ x = x + self.self_attn(self.input_layernorm(x))
+ x = x + self.mlp(self.post_attention_layernorm(x))
+ return x
+
+
+class HunYuanVisionPatchEmbed(nn.Module):
+ def __init__(self, config: HunYuanVLVisionConfig):
+ super().__init__()
+
+ self.config = config
+ self.embed_dim = config.hidden_size
+ self.patch_size = config.patch_size
+ self.num_channels = config.num_channels
+ self.spatial_merge_size = config.spatial_merge_size
+ self.interpolate_mode = config.interpolate_mode
+
+ self.patch_embedding = nn.Conv2d(
+ in_channels=config.num_channels,
+ out_channels=self.embed_dim,
+ kernel_size=self.patch_size,
+ stride=self.patch_size,
+ bias=True,
+ )
+
+ self.max_num_patches = (config.max_image_size // self.patch_size) ** 2
+
+ self.num_positions = self.max_num_patches + 1
+ self.position_edge = int(self.num_positions**0.5)
+ # first token is cls token, skip it
+ self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim)
+
+ self.patch_pos_embed = None
+
+ def forward(
+ self, pixel_values: torch.Tensor, grid_thw: list[list[int]]
+ ) -> torch.Tensor:
+ num_patches = pixel_values.size(0)
+ pixel_values = pixel_values.reshape(
+ num_patches, self.num_channels, self.patch_size, self.patch_size
+ )
+
+ patch_embeds = self.patch_embedding(pixel_values)
+ patch_embeds = patch_embeds.squeeze(-1).squeeze(-1).unsqueeze(0)
+
+ if self.patch_pos_embed is None:
+ patch_pos_shape = (
+ 1,
+ self.position_edge,
+ self.position_edge,
+ self.embed_dim,
+ )
+ self.patch_pos_embed = (
+ self.position_embedding.weight[1:, :]
+ .reshape(patch_pos_shape)
+ .permute(0, 3, 1, 2)
+ .float()
+ )
+
+ patch_pos_embed_list = []
+ for grid in grid_thw:
+ _, h0, w0 = grid
+ # we add a small number to avoid floating point error in the interpolation
+ # see discussion at https://github.com/facebookresearch/dino/issues/8
+ h0, w0 = h0 + 0.1, w0 + 0.1
+ patch_pos_embed = nn.functional.interpolate(
+ self.patch_pos_embed,
+ scale_factor=(h0 / self.position_edge, w0 / self.position_edge),
+ mode=self.interpolate_mode,
+ align_corners=False,
+ )
+
+ patch_pos_embed = (
+ patch_pos_embed.reshape(self.embed_dim, -1)
+ .transpose(0, 1)
+ .unsqueeze(0)
+ .to(patch_embeds.dtype)
+ )
+ patch_pos_embed_list.append(patch_pos_embed)
+
+ patch_pos_embed = torch.cat(patch_pos_embed_list, dim=1)
+ embeddings = patch_embeds + patch_pos_embed
+
+ return embeddings
+
+
+class HunYuanVisionPatchMerger(nn.Module):
+ def __init__(
+ self,
+ in_channels,
+ out_channels,
+ spatial_merge_size=2,
+ rms_norm_eps=1e-5,
+ prefix="",
+ ):
+ super().__init__()
+ self.spatial_merge_size = spatial_merge_size
+ embed_std = out_channels**-0.5
+
+ self.proj = nn.Sequential(
+ nn.Conv2d(
+ in_channels,
+ in_channels * 2,
+ kernel_size=spatial_merge_size,
+ stride=spatial_merge_size,
+ ),
+ nn.GELU(),
+ nn.Conv2d(in_channels * 2, in_channels * 4, kernel_size=1),
+ )
+ self.mlp = nn.Linear(in_channels * 4, out_channels)
+
+ self.image_newline = nn.Parameter(torch.randn(in_channels * 4) * embed_std)
+ self.image_begin = nn.Parameter(torch.randn(out_channels) * embed_std)
+ self.image_end = nn.Parameter(torch.randn(out_channels) * embed_std)
+ self.image_sep = nn.Parameter(torch.randn(out_channels) * embed_std)
+
+ self.before_rms = RMSNorm(in_channels, eps=rms_norm_eps)
+ self.after_rms = RMSNorm(out_channels, eps=rms_norm_eps)
+
+ def forward(self, x, size=(16, 16)):
+ x = self.before_rms(x)
+
+ h, w = size
+ dtype = x.dtype
+ x = x.permute(0, 2, 1).reshape(x.shape[0], -1, h, w)
+
+ x = self.proj(x) # b,c,h,w
+ b, c, h, w = x.shape
+ x = torch.cat(
+ [x, self.image_newline.reshape(1, c, 1, 1).expand(b, c, h, 1).to(dtype)],
+ dim=-1,
+ )
+ x = x.reshape(b, c, -1).permute(0, 2, 1)
+ x = self.mlp(x)
+
+ begin = self.image_begin.reshape(1, 1, -1).expand(b, 1, x.shape[-1]).to(dtype)
+ end = self.image_end.reshape(1, 1, -1).expand(b, 1, x.shape[-1]).to(dtype)
+ x = torch.cat([begin, x, end], dim=1)
+
+ return self.after_rms(x)
+
+
+class HunYuanVisionTransformer(nn.Module):
+ hf_to_vllm_mapper = WeightsMapper(
+ orig_to_new_stacked={
+ ".q_proj": (".qkv", "q"),
+ ".k_proj": (".qkv", "k"),
+ ".v_proj": (".qkv", "v"),
+ }
+ )
+
+ def __init__(
+ self,
+ vision_config: HunYuanVLVisionConfig,
+ quant_config: QuantizationConfig | None = None,
+ prefix: str = "",
+ ) -> None:
+ super().__init__()
+
+ num_hidden_layers = vision_config.num_hidden_layers
+ self.hidden_size = vision_config.hidden_size
+ self.num_heads = vision_config.num_attention_heads
+ self.spatial_merge_size = vision_config.spatial_merge_size
+
+ from vllm.compilation.backends import set_model_tag
+
+ with set_model_tag("HunYuanVisionPatchEmbed"):
+ self.embeddings = HunYuanVisionPatchEmbed(vision_config)
+
+ norm_layer = partial(nn.LayerNorm, eps=vision_config.rms_norm_eps)
+
+ with set_model_tag("HunYuanVisionBlock"):
+ self.layers = nn.ModuleList(
+ [
+ HunYuanVisionBlock(
+ dim=vision_config.hidden_size,
+ num_heads=vision_config.num_attention_heads,
+ mlp_hidden_dim=vision_config.intermediate_size,
+ act_fn=get_act_fn(vision_config.hidden_act),
+ norm_layer=norm_layer,
+ quant_config=quant_config,
+ prefix=f"{prefix}.layers.{layer_idx}",
+ )
+ for layer_idx in range(num_hidden_layers)
+ ]
+ )
+
+ with set_model_tag("HunYuanVisionPatchMerger"):
+ self.perceive = HunYuanVisionPatchMerger(
+ vision_config.hidden_size,
+ vision_config.out_hidden_size,
+ spatial_merge_size=vision_config.spatial_merge_size,
+ rms_norm_eps=vision_config.rms_norm_eps,
+ prefix=f"{prefix}.perceive",
+ )
+
+ @property
+ def dtype(self) -> torch.dtype:
+ return self.embeddings.patch_embedding.weight.dtype
+
+ @property
+ def device(self) -> torch.device:
+ return self.embeddings.patch_embedding.weight.device
+
+ def forward(
+ self,
+ x: torch.Tensor,
+ grid_thw: list[list[int]],
+ ) -> torch.Tensor:
+ # patchify
+ seq_len = x.size(0)
+ cu_seqlens: list = [0]
+
+ hidden_states = x.to(device=self.device, dtype=self.dtype)
+ # embeddings = patch_embeds + patch_pos_embed
+ hidden_states = self.embeddings(hidden_states, grid_thw)
+
+ for t, h, w in grid_thw:
+ t, h, w = int(t), int(h), int(w)
+ cu_seqlens.append(h * w)
+
+ cu_seqlens = torch.tensor(cu_seqlens, dtype=torch.int32)
+ cu_seqlens = torch.cumsum(cu_seqlens, dim=0, dtype=torch.int32)
+
+ cu_seqlens = cu_seqlens.to(device=self.device, non_blocking=True)
+
+ hidden_states = hidden_states.reshape(seq_len, -1)
+ hidden_states = hidden_states.unsqueeze(0)
+
+ # build per-image lengths once
+ split_lengths = [int(h) * int(w) for (_, h, w) in grid_thw]
+ for layer in self.layers:
+ # hidden_states: (1, T_total, D)
+ parts = hidden_states.split(split_lengths, dim=1) # list of (1, L_i, D)
+ parts = [layer(p) for p in parts]
+ hidden_states = torch.cat(parts, dim=1)
+
+ # adapter
+ split_lengths = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist()
+ split_items = hidden_states.split(split_lengths, dim=1)
+ image_embeds_list = []
+ for grid, split_item in zip(grid_thw, split_items):
+ image_embeds_list.append(
+ self.perceive(split_item.contiguous(), size=grid[1:]).squeeze(0)
+ )
+
+ return image_embeds_list
+
+ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+ loader = AutoWeightsLoader(self)
+ return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+
+
+def _hunyuan_vl_field_config(hf_inputs: Mapping[str, torch.Tensor]):
+ image_grid_thw = hf_inputs.get("image_grid_thw", torch.empty((0, 3)))
+ image_grid_sizes = image_grid_thw.prod(-1)
+ return dict(
+ pixel_values=MultiModalFieldConfig.flat_from_sizes("image", image_grid_sizes),
+ image_embeds=MultiModalFieldConfig.flat_from_sizes("image", image_grid_sizes),
+ image_grid_thw=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
+ )
+
+
+class HunYuanVLMultiModalDataParser(MultiModalDataParser):
+ # The patch grid is what sizes the placeholder range.
+ embedding_fields = {
+ "image": {"image_embeds": "values", "image_grid_thw": "metadata"},
+ }
+
+ def _parse_image_data(
+ self,
+ data: dict[str, torch.Tensor] | ModalityData[ImageItem],
+ ) -> ModalityDataItems[Any, Any] | None:
+ if isinstance(data, dict):
+ required, optional = self.embedding_field_sets("image")
+ return DictEmbeddingItems(
+ data,
+ modality="image",
+ required_fields=required,
+ optional_fields=optional,
+ fields_factory=_hunyuan_vl_field_config,
+ )
+
+ return super()._parse_image_data(data)
+
+
+class HunYuanVLProcessingInfo(BaseProcessingInfo):
+ def get_hf_config(self):
+ return self.ctx.get_hf_config(HunYuanVLConfig)
+
+ def get_hf_processor(
+ self,
+ **kwargs: object,
+ ) -> HunYuanVLProcessor:
+ # transformers>=5.13 replaced `use_fast` with `backend`; pin the
+ # PIL backend to match the released HunyuanOCR checkpoint packing.
+ kwargs.pop("use_fast", None)
+ kwargs.setdefault("backend", "pil")
+ return self.ctx.get_hf_processor(
+ HunYuanVLProcessor,
+ **kwargs,
+ )
+
+ def get_image_processor(
+ self,
+ **kwargs: object,
+ ) -> HunYuanVLImageProcessor:
+ return self.get_hf_processor(**kwargs).image_processor
+
+ def get_data_parser(self):
+ return HunYuanVLMultiModalDataParser(
+ expected_hidden_size=self._get_expected_hidden_size(),
+ allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
+ )
+
+ def get_supported_mm_limits(self) -> Mapping[str, int | None]:
+ return {"image": None}
+
+ def get_mm_max_tokens_per_item(
+ self,
+ seq_len: int,
+ mm_counts: Mapping[str, int],
+ ) -> Mapping[str, int]:
+ max_image_tokens = self.get_max_image_tokens()
+ # TODO: support video
+ max_video_tokens = 0
+ return {"image": max_image_tokens, "video": max_video_tokens}
+
+ def _get_vision_info(
+ self,
+ *,
+ image_width: int,
+ image_height: int,
+ num_frames: int = 1,
+ do_resize: bool = True,
+ image_processor: HunYuanVLImageProcessor,
+ mm_kwargs: Mapping[str, object],
+ ) -> tuple[ImageSize, int]:
+ hf_config = self.get_hf_config()
+ vision_config = hf_config.vision_config
+ patch_size = vision_config.patch_size
+ spatial_merge_size = vision_config.spatial_merge_size
+
+ mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs)
+ size = image_processor.size
+ if override_size := mm_kwargs.get("size"):
+ size = size | override_size
+ if (override_min_pixels := mm_kwargs.get("min_pixels")) is not None:
+ size = size | {"shortest_edge": override_min_pixels}
+ if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None:
+ size = size | {"longest_edge": override_max_pixels}
+
+ if do_resize:
+ resized_height, resized_width = smart_resize(
+ height=image_height,
+ width=image_width,
+ factor=patch_size * spatial_merge_size,
+ min_pixels=size["shortest_edge"],
+ max_pixels=size["longest_edge"],
+ )
+ preprocessed_size = ImageSize(width=resized_width, height=resized_height)
+ else:
+ preprocessed_size = ImageSize(width=image_width, height=image_height)
+
+ grid_t = 1
+ grid_h = preprocessed_size.height // patch_size
+ grid_w = preprocessed_size.width // patch_size
+
+ num_vision_tokens = (
+ grid_t * grid_h // spatial_merge_size * (grid_w // spatial_merge_size + 1)
+ + 2
+ )
+
+ return preprocessed_size, num_vision_tokens
+
+ def get_num_image_tokens(
+ self,
+ *,
+ image_width: int,
+ image_height: int,
+ image_processor: HunYuanVLImageProcessor,
+ mm_kwargs: Mapping[str, object],
+ ) -> int:
+ _, num_image_tokens = self._get_vision_info(
+ image_width=image_width,
+ image_height=image_height,
+ image_processor=image_processor,
+ mm_kwargs=mm_kwargs,
+ )
+ return num_image_tokens
+
+ def get_image_size_with_most_features(self) -> ImageSize:
+ image_processor = self.get_image_processor()
+
+ max_image_size, _ = self._get_vision_info(
+ image_width=512,
+ image_height=8192,
+ image_processor=image_processor,
+ mm_kwargs={},
+ )
+ return max_image_size
+
+ def get_max_image_tokens(self) -> int:
+ image_processor = self.get_image_processor()
+ target_width, target_height = self.get_image_size_with_most_features()
+
+ return self.get_num_image_tokens(
+ image_width=target_width,
+ image_height=target_height,
+ image_processor=image_processor,
+ mm_kwargs={},
+ )
+
+
+class HunYuanVLDummyInputsBuilder(BaseDummyInputsBuilder[HunYuanVLProcessingInfo]):
+ def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
+ num_images = mm_counts.get("image", 0)
+
+ hf_processor = self.info.get_hf_processor(typ=HunYuanVLProcessor)
+ image_placeholder = (
+ f"{hf_processor.image_start_token}{hf_processor.image_token}"
+ f"{hf_processor.image_end_token}"
+ )
+
+ return image_placeholder * num_images
+
+ def get_dummy_mm_data(
+ self,
+ seq_len: int,
+ mm_counts: Mapping[str, int],
+ mm_options: Mapping[str, BaseDummyOptions],
+ ) -> MultiModalDataDict:
+ num_images = mm_counts.get("image", 1)
+
+ target_width, target_height = self.info.get_image_size_with_most_features()
+
+ return {
+ "image": self._get_dummy_images(
+ width=target_width, height=target_height, num_images=num_images
+ ),
+ }
+
+
+class HunYuanVLMultiModalProcessor(BaseMultiModalProcessor[HunYuanVLProcessingInfo]):
+ def _call_hf_processor(
+ self,
+ prompt: str,
+ mm_data: Mapping[str, object],
+ mm_kwargs: Mapping[str, object],
+ ) -> BatchFeature:
+ hf_processor = self.info.get_hf_processor(**mm_kwargs)
+ # HunYuanVLProcessor requires image placeholders wrapped with start/end tokens.
+ if mm_data.get("images") is not None and prompt:
+ img_tok = hf_processor.image_token
+ wrapped = (
+ f"{hf_processor.image_start_token}{img_tok}"
+ f"{hf_processor.image_end_token}"
+ )
+ if img_tok in prompt and wrapped not in prompt:
+ prompt = prompt.replace(img_tok, wrapped)
+ return self.info.ctx.call_hf_processor(
+ hf_processor,
+ dict(text=prompt, **mm_data),
+ mm_kwargs,
+ )
+
+ def _get_prompt_updates(
+ self,
+ mm_items: MultiModalDataItems,
+ hf_processor_mm_kwargs: Mapping[str, Any],
+ out_mm_kwargs: MultiModalKwargsItems,
+ ) -> Sequence[PromptUpdate]:
+ hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
+ image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs)
+
+ token_ids = {
+ "image": hf_processor.image_token_id,
+ "image_start": hf_processor.image_start_token_id,
+ "image_end": hf_processor.image_end_token_id,
+ }
+
+ merge_size = image_processor.merge_size
+
+ def get_replacement_hunyuan_vl(item_idx: int, modality: str):
+ out_item = out_mm_kwargs[modality][item_idx]
+ grid_thw = out_item[f"{modality}_grid_thw"].data
+ assert isinstance(grid_thw, torch.Tensor)
+
+ _, grid_h, grid_w = grid_thw
+ num_tokens = (int(grid_h) // merge_size) * (
+ int(grid_w) // merge_size + 1
+ ) + 2
+ tokens = (
+ [token_ids[f"{modality}_start"]]
+ + [token_ids[modality]] * num_tokens
+ + [token_ids[f"{modality}_end"]]
+ )
+ return PromptUpdateDetails.select_token_id(tokens, token_ids[modality])
+
+ return [
+ PromptReplacement(
+ modality=modality,
+ target=[
+ token_ids[f"{modality}_start"],
+ token_ids[modality],
+ token_ids[f"{modality}_end"],
+ ],
+ replacement=partial(get_replacement_hunyuan_vl, modality=modality),
+ )
+ for modality in ("image",)
+ ]
+
+ def _get_mm_fields_config(
+ self,
+ hf_inputs: BatchFeature,
+ hf_processor_mm_kwargs: Mapping[str, object],
+ ) -> Mapping[str, MultiModalFieldConfig]:
+ return _hunyuan_vl_field_config(hf_inputs)
+
+
+@MULTIMODAL_REGISTRY.register_processor(
+ HunYuanVLMultiModalProcessor,
+ info=HunYuanVLProcessingInfo,
+ dummy_inputs=HunYuanVLDummyInputsBuilder,
+)
+class HunYuanVLForConditionalGeneration(
+ nn.Module,
+ SupportsMultiModal,
+ SupportsLoRA,
+ SupportsPP,
+ SupportsQuant,
+ SupportsXDRoPE,
+ SupportsEagle,
+ SupportsEagle3,
+):
+ # To ensure correct weight loading and mapping.
+ hf_to_vllm_mapper = WeightsMapper(
+ orig_to_new_prefix={
+ # mapping for new names in checkpoint saved after transformers v4.52
+ "vit.vit.": "visual.",
+ "vit.": "visual.",
+ "model.": "language_model.model.",
+ }
+ )
+
+ supports_encoder_tp_data = True
+
+ def get_xdrope_input_positions(
+ self,
+ input_tokens: list[int],
+ mm_features: list[MultiModalFeatureSpec],
+ ) -> torch.Tensor:
+ kwargs = MultiModalFeatureSpec.gather_kwargs(
+ mm_features,
+ {"image_grid_thw"},
+ )
+ image_grid_thw = [item.tolist() for item in kwargs.get("image_grid_thw", [])]
+
+ hf_config = self.config
+ image_start_token_id = hf_config.image_start_token_id
+ spatial_merge_size = hf_config.vision_config.spatial_merge_size
+ xd_num = len(hf_config.rope_scaling["xdrope_section"])
+
+ input_tokens_tensor = torch.tensor(input_tokens)
+ image_start_indices = torch.argwhere(
+ input_tokens_tensor == image_start_token_id
+ ).squeeze(1)
+
+ p_index = torch.arange(len(input_tokens_tensor))
+ w_index = torch.arange(len(input_tokens_tensor))
+ h_index = torch.arange(len(input_tokens_tensor))
+ t_index = torch.arange(len(input_tokens_tensor))
+ for image_index in range(len(image_start_indices)):
+ # +1 : first image_token, +2: for xdrope positions
+ pos = image_start_indices[image_index] + 2
+ t, h, w = image_grid_thw[image_index]
+ _, llm_grid_h, llm_grid_w = (
+ t,
+ h // spatial_merge_size,
+ w // spatial_merge_size,
+ )
+
+ token_num = (llm_grid_w + 1) * llm_grid_h
+ w_index[pos : pos + token_num].copy_(
+ torch.arange(0, llm_grid_w + 1)
+ .reshape(1, -1)
+ .expand(llm_grid_h, -1)
+ .reshape(-1)
+ )
+ h_index[pos : pos + token_num].copy_(
+ torch.arange(0, llm_grid_h)
+ .reshape(-1, 1)
+ .expand(-1, llm_grid_w + 1)
+ .reshape(-1)
+ )
+ t_index[pos : pos + token_num] = image_index
+
+ if xd_num == 4:
+ llm_positions = torch.stack([p_index, w_index, h_index, t_index])
+ elif xd_num == 3:
+ llm_positions = torch.stack([w_index, h_index, t_index])
+
+ return llm_positions
+
+ @classmethod
+ def get_placeholder_str(cls, modality: str, i: int) -> str | None:
+ if modality.startswith("image"):
+ return "<｜hy_place▁holder▁no▁100｜><｜hy_place▁holder▁no▁102｜><｜hy_place▁holder▁no▁101｜>" # noqa: E501
+
+ raise ValueError("Only image modality is supported")
+
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
+ super().__init__()
+ config: HunYuanVLConfig = vllm_config.model_config.hf_config
+
+ self.config = config
+
+ with self._mark_tower_model(vllm_config, {"image"}):
+ self.visual = HunYuanVisionTransformer(
+ config.vision_config,
+ quant_config=vllm_config.quant_config,
+ prefix=maybe_prefix(prefix, "visual"),
+ )
+
+ with self._mark_language_model(vllm_config):
+ self.language_model = init_vllm_registered_model(
+ vllm_config=vllm_config,
+ prefix=maybe_prefix(prefix, "language_model.model"),
+ architectures=[
+ "HunYuanDenseV1ForCausalLM",
+ "HunYuanMoEV1ForCausalLM",
+ ],
+ )
+
+ self.make_empty_intermediate_tensors = (
+ self.language_model.make_empty_intermediate_tensors
+ )
+
+ def _parse_and_validate_image_input(
+ self, **kwargs: object
+ ) -> HunYuanVLImageInputs | None:
+ pixel_values = kwargs.pop("pixel_values", None)
+ image_embeds = kwargs.pop("image_embeds", None)
+ image_grid_thw = kwargs.pop("image_grid_thw", None)
+
+ if pixel_values is None and image_embeds is None:
+ return None
+
+ # TODO: refine
+ if isinstance(pixel_values, list):
+ pixel_values = torch.cat(pixel_values, dim=0)
+ if len(pixel_values.shape) == 3:
+ last_dim = pixel_values.shape[-1]
+ pixel_values = pixel_values.reshape(-1, last_dim)
+ image_grid_thw = image_grid_thw.reshape(-1, 3)
+
+ if pixel_values is not None:
+ return HunYuanVLImagePixelInputs(
+ type="pixel_values",
+ pixel_values=pixel_values,
+ image_grid_thw=image_grid_thw,
+ )
+
+ if image_embeds is not None:
+ return HunYuanVLImageEmbeddingInputs(
+ type="image_embeds",
+ image_embeds=image_embeds,
+ image_grid_thw=image_grid_thw,
+ )
+
+ def _process_image_input(
+ self, image_input: HunYuanVLImageInputs
+ ) -> tuple[torch.Tensor, ...]:
+ grid_thw = image_input["image_grid_thw"]
+ assert grid_thw.ndim == 2
+ grid_thw_list = grid_thw.tolist()
+
+ if image_input["type"] == "image_embeds":
+ image_embeds = image_input["image_embeds"].type(self.visual.dtype)
+ else:
+ pixel_values = image_input["pixel_values"]
+
+ # TODO: use_data_parallel (split image_embeds in visual)
+ image_embeds = self.visual(pixel_values, grid_thw=grid_thw_list)
+
+ return image_embeds
+
+ def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
+ mm_input_by_modality = {}
+
+ # Preserve the order of modalities if there are multiple of them
+ # from the order of kwargs.
+ for input_key in kwargs:
+ if (
+ input_key in ("pixel_values", "image_embeds")
+ and "image" not in mm_input_by_modality
+ ):
+ mm_input_by_modality["image"] = self._parse_and_validate_image_input(
+ **kwargs
+ )
+ return mm_input_by_modality
+
+ def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
+ mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs)
+ if not mm_input_by_modality:
+ return []
+
+ # The result multimodal_embeddings is tuple of tensors, with each
+ # tensor correspoending to a multimodal data item (image or video).
+ multimodal_embeddings: tuple[torch.Tensor, ...] = ()
+
+ # NOTE: It is important to iterate over the keys in this dictionary
+ # to preserve the order of the modalities.
+ for modality in mm_input_by_modality:
+ multimodal_input = mm_input_by_modality[modality]
+ if modality == "image":
+ image_embeddings = self._process_image_input(multimodal_input)
+ multimodal_embeddings += tuple(image_embeddings)
+ return multimodal_embeddings
+
+ def forward(
+ self,
+ input_ids: torch.Tensor | None,
+ positions: torch.Tensor,
+ intermediate_tensors: IntermediateTensors | None,
+ inputs_embeds: torch.Tensor | None,
+ **kwargs: object,
+ ) -> torch.Tensor | IntermediateTensors:
+ if intermediate_tensors is not None:
+ inputs_embeds = None
+
+ hidden_states = self.language_model(
+ input_ids=input_ids,
+ positions=positions,
+ intermediate_tensors=intermediate_tensors,
+ inputs_embeds=inputs_embeds,
+ )
+ return hidden_states
+
+ def compute_logits(
+ self,
+ hidden_states: torch.Tensor,
+ ) -> torch.Tensor | None:
+ return self.language_model.compute_logits(hidden_states)
+
+ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+ loader = AutoWeightsLoader(self)
+ return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+
+ def get_mm_mapping(self) -> MultiModelKeys:
+ """
+ Get the module prefix in multimodal models
+ """
+ return MultiModelKeys.from_string_field(
+ language_model="language_model.model",
+ connector="visual.perceive",
+ tower_model="visual",
+ )
diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py
index d8e10ba75f40..24fc66a08999 100644
--- a/vllm/model_executor/models/registry.py
+++ b/vllm/model_executor/models/registry.py
@@ -131,6 +131,8 @@
"GraniteSWAForCausalLM": ("granite", "GraniteForCausalLM"),
"GritLM": ("gritlm", "GritLM"),
"HrmTextForCausalLM": ("hrm_text", "HrmTextForCausalLM"),
+ "HunYuanMoEV1ForCausalLM": ("hunyuan_v1", "HunYuanMoEV1ForCausalLM"),
+ "HunYuanDenseV1ForCausalLM": ("hunyuan_v1", "HunYuanDenseV1ForCausalLM"),
"HYV3ForCausalLM": ("hy_v3", "HYV3ForCausalLM"),
"HCXVisionForCausalLM": ("hyperclovax_vision", "HCXVisionForCausalLM"),
"HCXVisionV2ForCausalLM": ("hyperclovax_vision_v2", "HCXVisionV2ForCausalLM"),
@@ -439,6 +441,10 @@
"Granite4VisionForConditionalGeneration",
),
"H2OVLChatModel": ("h2ovl", "H2OVLChatModel"),
+ "HunYuanVLForConditionalGeneration": (
+ "hunyuan_vision",
+ "HunYuanVLForConditionalGeneration",
+ ),
"InternVLChatModel": ("internvl", "InternVLChatModel"),
"InternS1ForConditionalGeneration": (
"interns1",
@@ -694,8 +700,6 @@
_TRANSFORMERS_SUPPORTED_MODELS = {
# Text generation models
"GPTBigCodeForCausalLM": ("transformers", "TransformersForCausalLM"),
- "HunYuanDenseV1ForCausalLM": ("transformers", "TransformersForCausalLM"),
- "HunYuanMoEV1ForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
"OlmoForCausalLM": ("transformers", "TransformersForCausalLM"),
"Olmo2ForCausalLM": ("transformers", "TransformersForCausalLM"),
"SmolLM3ForCausalLM": ("transformers", "TransformersForCausalLM"),
@@ -706,10 +710,6 @@
"transformers",
"TransformersMultiModalForCausalLM",
),
- "HunYuanVLForConditionalGeneration": (
- "transformers",
- "TransformersMultiModalForCausalLM",
- ),
"VibeVoiceAsrForConditionalGeneration": (
"transformers",
"TransformersMultiModalForCausalLM",
diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py
index d58a386f583f..26d4bd6e1b36 100644
--- a/vllm/transformers_utils/config.py
+++ b/vllm/transformers_utils/config.py
@@ -95,6 +95,7 @@ def __getitem__(self, key):
granite4_vision="Granite4VisionConfig",
hyperclovax="HyperCLOVAXConfig",
hyperclovax_vlm="HCXVisionConfig",
+ hunyuan_vl="HunYuanVLConfig",
hy_v3="HYV3Config",
isaac="IsaacConfig",
kimi_k2="DeepseekV3Config", # Kimi K2 uses same architecture as DeepSeek V3
diff --git a/vllm/transformers_utils/configs/__init__.py b/vllm/transformers_utils/configs/__init__.py
index d0b3b962e4c1..c38cc4b4b8cc 100644
--- a/vllm/transformers_utils/configs/__init__.py
+++ b/vllm/transformers_utils/configs/__init__.py
@@ -44,6 +44,9 @@
"FunAudioChatConfig": "vllm.transformers_utils.configs.funaudiochat",
"FunAudioChatAudioEncoderConfig": "vllm.transformers_utils.configs.funaudiochat",
"Granite4VisionConfig": "vllm.transformers_utils.configs.granite4_vision",
+ "HunYuanVLConfig": "vllm.transformers_utils.configs.hunyuan_vl",
+ "HunYuanVLTextConfig": "vllm.transformers_utils.configs.hunyuan_vl",
+ "HunYuanVLVisionConfig": "vllm.transformers_utils.configs.hunyuan_vl",
"HCXVisionConfig": "vllm.transformers_utils.configs.hyperclovax",
"HYV3Config": "vllm.transformers_utils.configs.hy_v3",
"HyperCLOVAXConfig": "vllm.transformers_utils.configs.hyperclovax",
@@ -138,6 +141,9 @@
"FunAudioChatConfig",
"FunAudioChatAudioEncoderConfig",
"Granite4VisionConfig",
+ "HunYuanVLConfig",
+ "HunYuanVLTextConfig",
+ "HunYuanVLVisionConfig",
"HCXVisionConfig",
"HYV3Config",
"HyperCLOVAXConfig",
diff --git a/vllm/transformers_utils/configs/hunyuan_vl.py b/vllm/transformers_utils/configs/hunyuan_vl.py
new file mode 100644
index 000000000000..548dcfefcaf6
--- /dev/null
+++ b/vllm/transformers_utils/configs/hunyuan_vl.py
@@ -0,0 +1,281 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+# adapted from https://github.com/ManaEstras/transformers/blob/v4.57.1.hyvl/src/transformers/models/hunyuan_vl/configuration_hunyuan_vl.py
+
+from transformers import PretrainedConfig
+
+
+class HunYuanVLVisionConfig(PretrainedConfig):
+ model_type = "hunyuan_vl"
+ base_config_key = "vision_config"
+
+ def __init__(
+ self,
+ hidden_act="gelu",
+ hidden_size=1152,
+ intermediate_size=4304,
+ interpolate_mode="bilinear",
+ rms_norm_eps=1e-05,
+ learnable_mlp_pooling_size=0,
+ num_attention_heads=16,
+ num_key_value_heads=None,
+ num_channels=3,
+ num_hidden_layers=27,
+ out_hidden_size=4096,
+ patch_size=16,
+ remove_prenorm=True,
+ spatial_merge_size=2,
+ temporal_patch_size=1,
+ resize_resolution=2048,
+ img_max_token_num=4096,
+ max_image_size=2048,
+ video_max_image_size=768,
+ video_min_image_size=256,
+ min_image_size=512,
+ anyres_vit_max_image_size=2048,
+ max_vit_seq_len=16384,
+ text_hidden_size=3072,
+ **kwargs,
+ ):
+ super().__init__(**kwargs)
+
+ self.hidden_act = hidden_act
+ self.hidden_size = hidden_size
+ self.intermediate_size = intermediate_size
+ self.interpolate_mode = interpolate_mode
+ self.learnable_mlp_pooling_size = learnable_mlp_pooling_size
+ self.num_attention_heads = num_attention_heads
+ if not num_key_value_heads:
+ self.num_key_value_heads = num_attention_heads
+ else:
+ self.num_key_value_heads = num_key_value_heads
+ self.num_channels = num_channels
+ self.num_hidden_layers = num_hidden_layers
+ self.out_hidden_size = out_hidden_size
+ self.patch_size = patch_size
+ self.remove_prenorm = remove_prenorm
+ self.spatial_merge_size = spatial_merge_size
+ self.temporal_patch_size = temporal_patch_size
+ self.rms_norm_eps = rms_norm_eps
+
+ self.resize_resolution = resize_resolution
+ self.img_max_token_num = img_max_token_num
+ self.max_image_size = max_image_size
+ self.min_image_size = min_image_size
+ self.video_max_image_size = video_max_image_size
+ self.video_min_image_size = video_min_image_size
+ self.anyres_vit_max_image_size = anyres_vit_max_image_size
+ self.max_vit_seq_len = max_vit_seq_len
+ self.text_hidden_size = text_hidden_size
+
+
+class HunYuanVLTextConfig(PretrainedConfig):
+ r"""
+ This is the configuration class to store the configuration of a [`HunYuanVLTextConfig`]. It is used to instantiate an
+ HunYuan model according to the specified arguments, defining the model architecture. Instantiating a configuration
+ with the defaults will yield a similar configuration to that of the HunYuan-7B.
+ Hunyuan-7B-Instruct [tencent/Hunyuan-7B-Instruct](https://huggingface.co/tencent/Hunyuan-7B-Instruct).
+
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
+
+
+ Args:
+ vocab_size (`int`, *optional*, defaults to 290943):
+ Vocabulary size of the HunYuan model. Defines the number of different tokens that can be represented by the
+ `inputs_ids` passed when calling [`HunYuanVLTextConfig`]
+ hidden_size (`int`, *optional*, defaults to 4096):
+ Dimension of the hidden representations.
+ intermediate_size (`int`, *optional*, defaults to 11008):
+ Dimension of the MLP representations or shared MLP representations.
+ num_hidden_layers (`int`, *optional*, defaults to 32):
+ Number of hidden layers in the Transformer decoder.
+ num_attention_heads (`int`, *optional*, defaults to 32):
+ Number of attention heads for each attention layer in the Transformer decoder.
+ num_key_value_heads (`int`, *optional*):
+ This is the number of key_value heads that should be used to implement Grouped Query Attention. If
+ `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
+ `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
+ converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
+ by meanpooling all the original heads within that group. For more details checkout [this
+ paper](https://huggingface.co/papers/2305.13245). If it is not specified, will default to
+ `num_attention_heads`.
+ hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
+ The non-linear activation function (function or string) in the decoder.
+ max_position_embeddings (`int`, *optional*, defaults to 2048):
+ The maximum sequence length that this model might ever be used with.
+ initializer_range (`float`, *optional*, defaults to 0.02):
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+ rms_norm_eps (`float`, *optional*, defaults to 1e-05):
+ The epsilon used by the rms normalization layers.
+ use_cache (`bool`, *optional*, defaults to `True`):
+ Whether or not the model should return the last key/values attentions (not used by all models). Only
+ relevant if `config.is_decoder=True`.
+ pad_token_id (`int`, *optional*, defaults to 0):
+ Padding token id.
+ bos_token_id (`int`, *optional*, defaults to 1):
+ Beginning of stream token id.
+ eos_token_id (`int`, *optional*, defaults to 2):
+ End of stream token id.
+ eod_token_id (int, *optional*, defaults to 3):
+ Token ID representing the end-of-document marker. Used to indicate the termination of a text sequence.
+ Example: In multi-document processing, this token helps the model distinguish between separate documents.
+ pretraining_tp (`int`, *optional*, defaults to 1):
+ Experimental feature. Tensor parallelism rank used during pretraining. Please refer to [this
+ document](https://huggingface.co/docs/transformers/parallelism) to understand more about it. This value is
+ necessary to ensure exact reproducibility of the pretraining results. Please refer to [this
+ issue](https://github.com/pytorch/pytorch/issues/76232).
+ tie_word_embeddings (`bool`, *optional*, defaults to `False`):
+ Whether to tie weight embeddings
+ rope_theta (`float`, *optional*, defaults to 10000.0):
+ The base period of the RoPE embeddings.
+ rope_scaling (`Dict`, *optional*):
+ Dictionary containing the scaling configuration for the RoPE embeddings. Currently supports two scaling
+ strategies: linear and dynamic. Their scaling factor must be a float greater than 1. The expected format is
+ `{"type": strategy name, "factor": scaling factor}`. When using this flag, don't update
+ `max_position_embeddings` to the expected new maximum. See the following thread for more information on how
+ these scaling strategies behave:
+ https://www.reddit.com/r/LocalLLaMA/comments/14mrgpr/dynamically_scaled_rope_further_increases/. This is an
+ experimental feature, subject to breaking API changes in future versions.
+ attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
+ Whether to use a bias in the query, key, value and output projection layers during self-attention.
+ attention_dropout (`float`, *optional*, defaults to 0.0):
+ The dropout ratio for the attention probabilities.
+ head_dim (`int`, *optional*, defaults to 128):
+ The attention head dimension.
+ """ # noqa: E501
+
+ model_type = "hunyuan_vl_text"
+ keys_to_ignore_at_inference = ["past_key_values"]
+
+ def __init__(
+ self,
+ vocab_size=290943,
+ hidden_size=4096,
+ intermediate_size: int = 11008,
+ num_hidden_layers=32,
+ num_attention_heads=32,
+ num_key_value_heads=None,
+ hidden_act="silu",
+ max_position_embeddings=2048,
+ initializer_range=0.02,
+ rms_norm_eps=1e-5,
+ use_cache=True,
+ pad_token_id=0,
+ bos_token_id=1,
+ eos_token_id=2,
+ eod_token_id=3,
+ pretraining_tp=1,
+ tie_word_embeddings=False,
+ rope_theta=10000.0,
+ rope_scaling=None,
+ attention_bias=False,
+ attention_dropout=0.0,
+ head_dim=None,
+ **kwargs,
+ ):
+ self.vocab_size = vocab_size
+ self.max_position_embeddings = max_position_embeddings
+ self.hidden_size = hidden_size
+ self.intermediate_size = intermediate_size
+ self.num_hidden_layers = num_hidden_layers
+ self.num_attention_heads = num_attention_heads
+ self.head_dim = head_dim
+ # for backward compatibility
+ if num_key_value_heads is None:
+ num_key_value_heads = num_attention_heads
+
+ self.num_key_value_heads = num_key_value_heads
+ self.hidden_act = hidden_act
+ self.initializer_range = initializer_range
+ self.rms_norm_eps = rms_norm_eps
+ self.pretraining_tp = pretraining_tp
+ self.use_cache = use_cache
+ self.rope_theta = rope_theta
+ self.rope_scaling = rope_scaling
+ self.attention_bias = attention_bias
+ self.attention_dropout = attention_dropout
+
+ super().__init__(
+ pad_token_id=pad_token_id,
+ bos_token_id=bos_token_id,
+ eos_token_id=eos_token_id,
+ tie_word_embeddings=tie_word_embeddings,
+ **kwargs,
+ )
+
+
+class HunYuanVLConfig(PretrainedConfig):
+ model_type = "hunyuan_vl"
+ sub_configs = {
+ "vision_config": HunYuanVLVisionConfig,
+ "text_config": HunYuanVLTextConfig,
+ }
+ keys_to_ignore_at_inference = ["past_key_values"]
+
+ def __init__(
+ self,
+ text_config=None,
+ vision_config=None,
+ im_start_id=120118,
+ im_end_id=120119,
+ image_token_id=120120,
+ im_newline_id=120121,
+ video_start_id=120122,
+ video_end_id=120123,
+ **kwargs,
+ ):
+ # We need to init super() here so that it does not reset values
+ # that are in text config to the BaseClass defaults. The Base
+ # config has many text related defaults and not all defaults are
+ # same as for `HunYuanVLTextConfig`.
+ super().__init__(**kwargs)
+
+ if isinstance(vision_config, dict):
+ self.vision_config = self.sub_configs["vision_config"](**vision_config)
+ elif vision_config is None:
+ self.vision_config = self.sub_configs["vision_config"]()
+
+ if isinstance(text_config, dict):
+ self.text_config = self.sub_configs["text_config"](**text_config)
+ elif text_config is None:
+ # For BC use all kwargs to init `TextConfig`
+ self.text_config = self.sub_configs["text_config"](**kwargs)
+
+ self.image_token_id = image_token_id
+ self.im_start_id = im_start_id
+ self.im_end_id = im_end_id
+ self.im_newline_id = im_newline_id
+ self.video_start_id = video_start_id
+ self.video_end_id = video_end_id
+
+ self.vision_config.text_hidden_size = self.text_config.hidden_size
+
+ # Attention implementation to use. It sets it recursively on sub-configs
+ # so we call it again in the end.
+ self._attn_implementation = kwargs.pop("attn_implementation", None)
+
+ def __setattr__(self, key, value):
+ if (
+ (text_config := super().__getattribute__("__dict__").get("text_config"))
+ is not None
+ and key not in ["dtype", "_attn_implementation_internal"]
+ and key in text_config.__dict__
+ ):
+ setattr(text_config, key, value)
+ else:
+ super().__setattr__(key, value)
+
+ def __getattribute__(self, key):
+ if "text_config" in super().__getattribute__("__dict__") and key not in [
+ "_name_or_path",
+ "model_type",
+ "dtype",
+ "_attn_implementation_internal",
+ ]:
+ text_config = super().__getattribute__("text_config")
+ if key in text_config.__dict__:
+ return getattr(text_config, key)
+
+ return super().__getattribute__(key)