vllm.models.qwen3_5.nvidia.model

Inference-only Qwen3.5 Series compatible with HuggingFace weights.

Classes:

• Qwen3_5ForConditionalGeneration –
• Qwen3_5RMSNorm –

  Gemma-style RMS normalization backed by flashinfer kernels.

Qwen3_5ForConditionalGeneration

Bases: Qwen3VLForConditionalGeneration, IsHybrid

Methods:

• forward –

  Run forward pass for Qwen3.5.

Source code in vllm/models/qwen3_5/nvidia/model.py

@MULTIMODAL_REGISTRY.register_processor(
    Qwen3VLMultiModalProcessor,
    info=Qwen3_5ProcessingInfo,
    dummy_inputs=Qwen3VLDummyInputsBuilder,
)
class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration, IsHybrid):
    supports_multimodal_pruning = True

    hf_to_vllm_mapper = (
        Qwen3VLForConditionalGeneration.hf_to_vllm_mapper
        | WeightsMapper(orig_to_new_prefix={"mtp.": None})
    )

    packed_modules_mapping = Qwen3VLForConditionalGeneration.packed_modules_mapping | {
        "in_proj_qkvz": ["in_proj_qkv", "in_proj_z"],
        "in_proj_ba": ["in_proj_b", "in_proj_a"],
    }

    def __init__(self, *, vllm_config: VllmConfig, prefix: str = "model"):
        # protocols have not __init__ method, so we need to use nn.Module.__init__
        nn.Module.__init__(self)
        config: Qwen3_5Config = vllm_config.model_config.hf_config
        quant_config = vllm_config.quant_config
        multimodal_config = cast(
            MultiModalConfig, vllm_config.model_config.multimodal_config
        )

        self.config = config
        self.model_config = vllm_config.model_config
        self.multimodal_config = multimodal_config
        self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
        self.is_multimodal_pruning_enabled = (
            multimodal_config.is_multimodal_pruning_enabled()
        )
        self.video_pruning_rate = self.multimodal_config.video_pruning_rate
        self._tokenizer = cached_tokenizer_from_config(vllm_config.model_config)

        # attributes needed by EVS-related functions inherited from Qwen3-VL
        self.use_deepstack = hasattr(config.vision_config, "deepstack_visual_indexes")
        self.deepstack_num_level = (
            len(config.vision_config.deepstack_visual_indexes)
            if self.use_deepstack
            else 0
        )
        self.visual_dim = config.vision_config.out_hidden_size
        self.multiscale_dim = self.visual_dim * self.deepstack_num_level

        with self._mark_tower_model(vllm_config, {"image", "video"}):
            self.visual = Qwen3_VisionTransformer(
                config.vision_config,
                norm_eps=getattr(config, "rms_norm_eps", 1e-6),
                quant_config=quant_config,
                input_norm=build_mm_input_norm(self.model_config),
                prefix=maybe_prefix(prefix, "visual"),
            )

        with self._mark_language_model(vllm_config):
            self.language_model = Qwen3_5ForCausalLM(
                vllm_config=vllm_config, prefix=maybe_prefix(prefix, "language_model")
            )

        self.make_empty_intermediate_tensors = (  # type: ignore[method-assign]
            self.language_model.make_empty_intermediate_tensors
        )

    def embed_input_ids(
        self,
        input_ids: torch.Tensor,
        multimodal_embeddings: MultiModalEmbeddings | None = None,
        *,
        is_multimodal: torch.Tensor | None = None,
    ) -> torch.Tensor:
        inputs_embeds = self._embed_text_input_ids(
            input_ids,
            self.language_model.embed_input_ids,
            is_multimodal=is_multimodal,
        )

        if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
            return inputs_embeds

        is_multimodal = _require_is_multimodal(is_multimodal)

        inputs_embeds = _merge_multimodal_embeddings(
            inputs_embeds=inputs_embeds,
            multimodal_embeddings=multimodal_embeddings,
            is_multimodal=is_multimodal,
        )

        return inputs_embeds

    def forward(
        self,
        input_ids: torch.Tensor,
        positions: torch.Tensor,
        intermediate_tensors: IntermediateTensors | None = None,
        inputs_embeds: torch.Tensor | None = None,
        **kwargs: object,
    ) -> torch.Tensor | IntermediateTensors:
        """Run forward pass for Qwen3.5.

        With mrope enabled (default for Qwen3VL-class models), ``positions``
        has shape `(3, seq_len)`, otherwise `(seq_len,)`.
        """
        if intermediate_tensors is not None:
            inputs_embeds = None

        hidden_states = self.language_model.model(
            input_ids=input_ids,
            positions=positions,
            intermediate_tensors=intermediate_tensors,
            inputs_embeds=inputs_embeds,
        )

        return hidden_states

    def compute_logits_local(self, hidden_states: torch.Tensor) -> torch.Tensor:
        return self.language_model.compute_logits_local(hidden_states)

    def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
        loader = AutoWeightsLoader(self)
        return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)

    @classmethod
    def get_mamba_state_dtype_from_config(
        cls,
        vllm_config: "VllmConfig",
    ) -> tuple[torch.dtype, torch.dtype]:
        return MambaStateDtypeCalculator.gated_delta_net_state_dtype(
            vllm_config.model_config.dtype,
            vllm_config.cache_config.mamba_cache_dtype,
            vllm_config.cache_config.mamba_ssm_cache_dtype,
        )

    @classmethod
    def get_mamba_state_shape_from_config(
        cls, vllm_config: "VllmConfig"
    ) -> tuple[tuple[int, int], tuple[int, int, int]]:
        parallel_config = vllm_config.parallel_config
        hf_config = vllm_config.model_config.hf_text_config
        tp_size = parallel_config.tensor_parallel_size
        num_spec = (
            vllm_config.speculative_config.num_speculative_tokens
            if vllm_config.speculative_config
            else 0
        )
        return MambaStateShapeCalculator.gated_delta_net_state_shape(
            tp_size,
            hf_config.linear_num_key_heads,
            hf_config.linear_num_value_heads,
            hf_config.linear_key_head_dim,
            hf_config.linear_value_head_dim,
            hf_config.linear_conv_kernel_dim,
            num_spec,
        )

    @classmethod
    def get_mamba_state_copy_func(cls) -> tuple[MambaStateCopyFunc, MambaStateCopyFunc]:
        return MambaStateCopyFuncCalculator.gated_delta_net_state_copy_func()

forward(input_ids, positions, intermediate_tensors=None, inputs_embeds=None, **kwargs)

Run forward pass for Qwen3.5.

With mrope enabled (default for Qwen3VL-class models), positions has shape (3, seq_len), otherwise (seq_len,).

Source code in vllm/models/qwen3_5/nvidia/model.py

def forward(
    self,
    input_ids: torch.Tensor,
    positions: torch.Tensor,
    intermediate_tensors: IntermediateTensors | None = None,
    inputs_embeds: torch.Tensor | None = None,
    **kwargs: object,
) -> torch.Tensor | IntermediateTensors:
    """Run forward pass for Qwen3.5.

    With mrope enabled (default for Qwen3VL-class models), ``positions``
    has shape `(3, seq_len)`, otherwise `(seq_len,)`.
    """
    if intermediate_tensors is not None:
        inputs_embeds = None

    hidden_states = self.language_model.model(
        input_ids=input_ids,
        positions=positions,
        intermediate_tensors=intermediate_tensors,
        inputs_embeds=inputs_embeds,
    )

    return hidden_states

Qwen3_5RMSNorm

Bases: Module

Gemma-style RMS normalization backed by flashinfer kernels.

When residual is given, the fused add + norm runs in place and the updated (x, residual) pair is returned.

reduce_inputs: in forward_fused, an x that comes with a residual is the un-reduced TP partial of a row-parallel linear, and the norm absorbs its all-reduce. An x without a residual (first decoder layer, MTP layer) is always complete.

Methods:

• forward_fused –

  forward with the preceding TP all-reduce (reduce_inputs)

Source code in vllm/models/qwen3_5/nvidia/model.py

class Qwen3_5RMSNorm(nn.Module):
    """Gemma-style RMS normalization backed by flashinfer kernels.

    When ``residual`` is given, the fused add + norm runs in place and the
    updated ``(x, residual)`` pair is returned.

    ``reduce_inputs``: in ``forward_fused``, an ``x`` that comes with a
    residual is the un-reduced TP partial of a row-parallel linear, and the
    norm absorbs its all-reduce. An ``x`` without a residual (first decoder
    layer, MTP layer) is always complete.
    """

    def __init__(
        self, hidden_size: int, eps: float = 1e-6, reduce_inputs: bool = False
    ) -> None:
        super().__init__()
        self.weight = nn.Parameter(torch.zeros(hidden_size))
        set_weight_attrs(self.weight, {"dummy_weight_value": 0.0})
        self.variance_epsilon = eps
        self.reduce_inputs = reduce_inputs

    def forward(
        self,
        x: torch.Tensor,
        residual: torch.Tensor | None = None,
    ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
        from flashinfer.norm import gemma_fused_add_rmsnorm, gemma_rmsnorm

        if residual is None:
            return gemma_rmsnorm(x, self.weight, self.variance_epsilon)

        # gemma_fused_add_rmsnorm mutates x and residual in place.
        gemma_fused_add_rmsnorm(x, residual, self.weight, self.variance_epsilon)
        return x, residual

    def forward_fused(
        self,
        x: torch.Tensor,
        residual: torch.Tensor | None,
        linear: LinearBase | None,
        *,
        write_y: bool = False,
    ) -> tuple[QuantizedActivation | None, torch.Tensor | None, torch.Tensor]:
        """``forward`` with the preceding TP all-reduce (``reduce_inputs``)
        and the consumer ``linear``'s input quantization fused in.

        When ``linear``'s input quant key is supported, the normed activation
        is quantized for it; x is then not modified.

        Returns the quantized normed activation (or None), the normed
        activation in x's dtype (when the quantized one is None, or when
        ``write_y``; else None) and the residual (x when ``residual`` is None).
        """
        allreduce = self.reduce_inputs and residual is not None
        key = None if linear is None else _norm_quant_key(linear)
        if key is None:
            if allreduce:
                assert residual is not None
                return None, *fused_allreduce_gemma_rms_norm(x, residual, self)
            if residual is None:
                return None, self(x), x
            return None, *self(x, residual)

        assert linear is not None
        nvfp4 = key == kNvfp4Dynamic
        scale = linear.input_global_scale_inv if nvfp4 else linear.input_scale
        if allreduce:
            assert residual is not None
            out = fused_allreduce_gemma_rms_norm_quant(
                x, residual, self.weight, self.variance_epsilon, scale, key
            )
            if out is not None:
                return out
            # Above the FlashInfer workspace budget: explicit all-reduce, then
            # the Triton norm-quant.
            x = tensor_model_parallel_all_reduce(x)
        q, block_scale, y = gemma_add_rms_norm_quant(
            x, residual, self.weight, self.variance_epsilon, scale, nvfp4, write_y
        )
        qa = QuantizedActivation(
            data=q,
            scale=block_scale,
            orig_dtype=x.dtype,
            orig_shape=x.shape,
            quant_key=key,
        )
        return qa, y, x if residual is None else residual

forward_fused(x, residual, linear, *, write_y=False)

forward with the preceding TP all-reduce (reduce_inputs) and the consumer linear's input quantization fused in.

When linear's input quant key is supported, the normed activation is quantized for it; x is then not modified.

Returns the quantized normed activation (or None), the normed activation in x's dtype (when the quantized one is None, or when write_y; else None) and the residual (x when residual is None).

Source code in vllm/models/qwen3_5/nvidia/model.py

def forward_fused(
    self,
    x: torch.Tensor,
    residual: torch.Tensor | None,
    linear: LinearBase | None,
    *,
    write_y: bool = False,
) -> tuple[QuantizedActivation | None, torch.Tensor | None, torch.Tensor]:
    """``forward`` with the preceding TP all-reduce (``reduce_inputs``)
    and the consumer ``linear``'s input quantization fused in.

    When ``linear``'s input quant key is supported, the normed activation
    is quantized for it; x is then not modified.

    Returns the quantized normed activation (or None), the normed
    activation in x's dtype (when the quantized one is None, or when
    ``write_y``; else None) and the residual (x when ``residual`` is None).
    """
    allreduce = self.reduce_inputs and residual is not None
    key = None if linear is None else _norm_quant_key(linear)
    if key is None:
        if allreduce:
            assert residual is not None
            return None, *fused_allreduce_gemma_rms_norm(x, residual, self)
        if residual is None:
            return None, self(x), x
        return None, *self(x, residual)

    assert linear is not None
    nvfp4 = key == kNvfp4Dynamic
    scale = linear.input_global_scale_inv if nvfp4 else linear.input_scale
    if allreduce:
        assert residual is not None
        out = fused_allreduce_gemma_rms_norm_quant(
            x, residual, self.weight, self.variance_epsilon, scale, key
        )
        if out is not None:
            return out
        # Above the FlashInfer workspace budget: explicit all-reduce, then
        # the Triton norm-quant.
        x = tensor_model_parallel_all_reduce(x)
    q, block_scale, y = gemma_add_rms_norm_quant(
        x, residual, self.weight, self.variance_epsilon, scale, nvfp4, write_y
    )
    qa = QuantizedActivation(
        data=q,
        scale=block_scale,
        orig_dtype=x.dtype,
        orig_shape=x.shape,
        quant_key=key,
    )
    return qa, y, x if residual is None else residual

_fused_gdn_gated_rms_norm_quant(*, x, out, out_proj, **kwargs)

GDN gated norm, quantized for out_proj when it takes static FP8.

Source code in vllm/models/qwen3_5/nvidia/model.py

def _fused_gdn_gated_rms_norm_quant(
    *,
    x: torch.Tensor,
    out: torch.Tensor,
    out_proj: LinearBase,
    **kwargs,
) -> torch.Tensor | QuantizedActivation:
    """GDN gated norm, quantized for out_proj when it takes static FP8."""
    if get_input_quant_key(out_proj) != kFp8StaticTensorSym:
        fused_gdn_gated_rms_norm(x=x, out=out, **kwargs)
        return out
    scale = out_proj.input_scale
    q = torch.empty(x.shape, dtype=FP8_DTYPE, device=x.device)
    fused_gdn_gated_rms_norm(x=x, out=q, scale=scale, **kwargs)
    return QuantizedActivation(
        data=q,
        scale=scale,
        orig_dtype=x.dtype,
        orig_shape=x.shape,
        quant_key=kFp8StaticTensorSym,
    )

_norm_quant_key(linear)

The input quant key of linear that the fused norms can produce.

Checked per call: keys are exposed after weight loading, and LoRA hides them (requires_unquantized_input).

Source code in vllm/models/qwen3_5/nvidia/model.py

def _norm_quant_key(linear: nn.Module) -> QuantKey | None:
    """The input quant key of ``linear`` that the fused norms can produce.

    Checked per call: keys are exposed after weight loading, and LoRA hides
    them (requires_unquantized_input).
    """
    key = get_input_quant_key(linear)
    if key == kFp8StaticTensorSym:
        return key
    # The norm writes block scales for exactly K = hidden_size, in whole
    # 4-column scale tiles (weights only require K % 32 == 0).
    if (
        key == kNvfp4Dynamic
        and nvfp4_weight_padding_bytes(linear) == 0
        and linear.input_size_per_partition % 64 == 0
    ):
        return key
    return None