vllm.model_executor.models.config

Classes:

• ColQwen3_5Config –

  Apply the attention contract declared by a ColQwen3.5 checkpoint.

• D3ForSequenceClassificationConfig –

  Apply the contract declared by a Decision 3.0 decision_config.json.

• DiffusionGemmaModelForBlockDiffusionConfig –
• Gemma4Config –
• HybridAttentionMambaModelConfig –
• JinaEmbeddingsV5ModelConfig –

  Config handler for Jina Embeddings V5 embedding models.

• KimiK3ForConditionalGenerationConfig –

  Route MXFP4-checkpointed Kimi-K3 MoE experts to the MXFP4 interface.

• LlamaNemotronVLConfig –

  Config handler for LlamaNemotronVL embedding models.

• MambaModelConfig –
• MiniMaxM3SparseForConditionalGenerationConfig –
• NemotronHForCausalLMConfig –
• Qwen3_5ForConditionalGenerationConfig –
• Qwen4ExpForConditionalGenerationConfig –

  Apply the Qwen3.5 hybrid-cache contract to Qwen4Exp.

• Qwen4ExpMTPConfig –

  Preserve MRoPE for a VL target and use 1D RoPE for a text target.

• UnlimitedOCRForCausalLMConfig –

ColQwen3_5Config

Bases: Qwen3_5ForConditionalGenerationConfig

Apply the attention contract declared by a ColQwen3.5 checkpoint.

Source code in vllm/model_executor/models/config.py

class ColQwen3_5Config(Qwen3_5ForConditionalGenerationConfig):
    """Apply the attention contract declared by a ColQwen3.5 checkpoint."""

    @staticmethod
    def verify_and_update_model_config(model_config: "ModelConfig") -> None:
        configs = {
            id(config): config
            for config in (
                model_config.hf_config,
                model_config.hf_text_config,
            )
        }
        declarations = [
            contract
            for config in configs.values()
            if (contract := getattr(config, "retrieval_attention_contract", None))
            is not None
        ]
        supported = {"causal", "bidirectional"}
        if not declarations:
            raise ValueError(
                "ColQwen3.5 checkpoints must declare "
                "retrieval_attention_contract as 'causal' or 'bidirectional'"
            )
        if (
            any(not isinstance(contract, str) for contract in declarations)
            or any(contract not in supported for contract in declarations)
            or len(set(declarations)) != 1
        ):
            raise ValueError(
                "unsupported or conflicting ColQwen3.5 "
                f"retrieval_attention_contract declarations: {declarations!r}"
            )
        is_causal = declarations[0] == "causal"
        for config in configs.values():
            config.is_causal = is_causal

D3ForSequenceClassificationConfig

Bases: Qwen3_5ForConditionalGenerationConfig

Apply the contract declared by a Decision 3.0 decision_config.json.

Source code in vllm/model_executor/models/config.py

class D3ForSequenceClassificationConfig(Qwen3_5ForConditionalGenerationConfig):
    """Apply the contract declared by a Decision 3.0 `decision_config.json`."""

    @staticmethod
    def verify_and_update_model_config(model_config: "ModelConfig") -> None:
        hf_config = model_config.hf_config
        decision = getattr(hf_config, "decision_config", None)
        if not isinstance(decision, dict):
            raise ValueError("Decision 3.0 checkpoints need a decision_config.json")
        if decision.get("format_version") != 1:
            raise ValueError(
                "unsupported decision_config.json format_version "
                f"{decision.get('format_version')!r}; vLLM reads version 1"
            )
        if decision.get("pooling", "last") != "last":
            raise ValueError(
                f"unsupported decision_config.json pooling {decision['pooling']!r}"
            )
        attention_mode = decision.get("attention_mode", "causal")
        if attention_mode not in ("causal", "noncausal_full_attention"):
            raise ValueError(
                f"unsupported decision_config.json attention_mode {attention_mode!r}"
            )
        codes = decision.get("codes")
        token_ids = decision.get("token_ids")
        if not codes or not isinstance(token_ids, list) or len(codes) != len(token_ids):
            raise ValueError(
                "decision_config.json must list one token id per answer code"
            )

        # The full attention layers read the whole prompt in both directions;
        # the Gated DeltaNet layers stay causal.
        is_causal = attention_mode == "causal"
        for config in {
            id(config): config for config in (hf_config, model_config.hf_text_config)
        }.values():
            config.is_causal = is_causal
        # One readout row per answer code.
        hf_config.id2label = dict(enumerate(codes))

        # Probabilities only exist over a question's own options, so the head
        # returns raw code logits unless activation is requested.
        pooler_config = model_config.pooler_config
        assert pooler_config is not None
        if pooler_config.use_activation is None:
            pooler_config.use_activation = False

DiffusionGemmaModelForBlockDiffusionConfig

Bases: VerifyAndUpdateConfig

Methods:

• verify_and_update_config –

  Set up the diffusion config and defaults for DiffusionGemma.

Source code in vllm/model_executor/models/config.py

class DiffusionGemmaModelForBlockDiffusionConfig(VerifyAndUpdateConfig):
    @classmethod
    def verify_and_update_config(cls, vllm_config: "VllmConfig") -> None:
        """Set up the diffusion config and defaults for DiffusionGemma.

        Auto-creates DiffusionConfig from the HF config when the user
        didn't pass ``--diffusion-config``. Diffusion sampling params are
        read straight from generation_config.json at sampler-build time
        (see DiffusionGemma's custom_sampler), not injected here.
        """
        # Inherit Gemma4's attention backend selection (FA4 on Hopper,
        # TRITON_ATTN fallback for heterogeneous head dims).
        Gemma4Config.verify_and_update_config(vllm_config)

        from vllm.v1.attention.backends.registry import AttentionBackendEnum

        attention_config = vllm_config.attention_config
        if attention_config.backend == AttentionBackendEnum.FLASHINFER:
            raise ValueError(
                "FlashInfer does not support DiffusionGemma's mixed "
                "causal/bidirectional attention. Use --attention-backend "
                "FLASH_ATTN or TRITON_ATTN instead."
            )
        if attention_config.backend is None and not attention_config.use_non_causal:
            # Only the sm_100 default backend order reads this flag (it puts
            # FlashInfer first for causal models). Elsewhere Gemma4Config above
            # keeps FlashInfer out: FA4 on every layer, else TRITON_ATTN.
            attention_config.use_non_causal = True
            logger.info(
                "DiffusionGemma mixes causal and bidirectional attention in "
                "one batch; setting use_non_causal=True so the default backend "
                "order prefers FlashAttention over FlashInfer."
            )

        # Auto-create DiffusionConfig from HF config if not provided.
        if vllm_config.diffusion_config is None:
            from vllm.config.diffusion import DiffusionConfig

            hf_config = vllm_config.model_config.hf_config
            canvas_length = getattr(hf_config, "canvas_length", 256)
            vllm_config.diffusion_config = DiffusionConfig(
                canvas_length=canvas_length,
            )

        # The diffusion sampler materializes [num_seqs, canvas_length, vocab]
        # fp32 transients, so concurrency is memory-bound (>8 OOMs a single H200).
        # Default to 8 when the user didn't pass --max-num-seqs.
        # We can't see the original None here (the engine already filled a generic
        # default), so use >= DEFAULT_MAX_NUM_SEQS as a proxy, (the default is much
        # larger than any deliberate value for this model)
        from vllm.config.scheduler import SchedulerConfig

        sc = vllm_config.scheduler_config
        if sc is not None and sc.max_num_seqs >= SchedulerConfig.DEFAULT_MAX_NUM_SEQS:
            sc.max_num_seqs = 8

        # Remove the model's generation_config.json cap on max_new_tokens
        # (256) so DiffusionGemma behaves like every other model: no
        # server-wide limit, each request controls its own output length
        # via max_tokens.  Setting to None causes get_diff_sampling_param
        # to skip this key entirely.
        model_config = vllm_config.model_config
        if "max_new_tokens" not in model_config.override_generation_config:
            model_config.override_generation_config["max_new_tokens"] = None
            logger.info(
                "DiffusionGemma: removing server-wide max_new_tokens cap "
                "from generation_config.json (use "
                "--override-generation-config to set a custom limit).",
            )

verify_and_update_config(vllm_config) classmethod

Set up the diffusion config and defaults for DiffusionGemma.

Auto-creates DiffusionConfig from the HF config when the user didn't pass --diffusion-config. Diffusion sampling params are read straight from generation_config.json at sampler-build time (see DiffusionGemma's custom_sampler), not injected here.

Source code in vllm/model_executor/models/config.py

@classmethod
def verify_and_update_config(cls, vllm_config: "VllmConfig") -> None:
    """Set up the diffusion config and defaults for DiffusionGemma.

    Auto-creates DiffusionConfig from the HF config when the user
    didn't pass ``--diffusion-config``. Diffusion sampling params are
    read straight from generation_config.json at sampler-build time
    (see DiffusionGemma's custom_sampler), not injected here.
    """
    # Inherit Gemma4's attention backend selection (FA4 on Hopper,
    # TRITON_ATTN fallback for heterogeneous head dims).
    Gemma4Config.verify_and_update_config(vllm_config)

    from vllm.v1.attention.backends.registry import AttentionBackendEnum

    attention_config = vllm_config.attention_config
    if attention_config.backend == AttentionBackendEnum.FLASHINFER:
        raise ValueError(
            "FlashInfer does not support DiffusionGemma's mixed "
            "causal/bidirectional attention. Use --attention-backend "
            "FLASH_ATTN or TRITON_ATTN instead."
        )
    if attention_config.backend is None and not attention_config.use_non_causal:
        # Only the sm_100 default backend order reads this flag (it puts
        # FlashInfer first for causal models). Elsewhere Gemma4Config above
        # keeps FlashInfer out: FA4 on every layer, else TRITON_ATTN.
        attention_config.use_non_causal = True
        logger.info(
            "DiffusionGemma mixes causal and bidirectional attention in "
            "one batch; setting use_non_causal=True so the default backend "
            "order prefers FlashAttention over FlashInfer."
        )

    # Auto-create DiffusionConfig from HF config if not provided.
    if vllm_config.diffusion_config is None:
        from vllm.config.diffusion import DiffusionConfig

        hf_config = vllm_config.model_config.hf_config
        canvas_length = getattr(hf_config, "canvas_length", 256)
        vllm_config.diffusion_config = DiffusionConfig(
            canvas_length=canvas_length,
        )

    # The diffusion sampler materializes [num_seqs, canvas_length, vocab]
    # fp32 transients, so concurrency is memory-bound (>8 OOMs a single H200).
    # Default to 8 when the user didn't pass --max-num-seqs.
    # We can't see the original None here (the engine already filled a generic
    # default), so use >= DEFAULT_MAX_NUM_SEQS as a proxy, (the default is much
    # larger than any deliberate value for this model)
    from vllm.config.scheduler import SchedulerConfig

    sc = vllm_config.scheduler_config
    if sc is not None and sc.max_num_seqs >= SchedulerConfig.DEFAULT_MAX_NUM_SEQS:
        sc.max_num_seqs = 8

    # Remove the model's generation_config.json cap on max_new_tokens
    # (256) so DiffusionGemma behaves like every other model: no
    # server-wide limit, each request controls its own output length
    # via max_tokens.  Setting to None causes get_diff_sampling_param
    # to skip this key entirely.
    model_config = vllm_config.model_config
    if "max_new_tokens" not in model_config.override_generation_config:
        model_config.override_generation_config["max_new_tokens"] = None
        logger.info(
            "DiffusionGemma: removing server-wide max_new_tokens cap "
            "from generation_config.json (use "
            "--override-generation-config to set a custom limit).",
        )

Gemma4Config

Bases: VerifyAndUpdateConfig

Methods:

• verify_and_update_config –

  Configure attention for heterogeneous head dimensions.

Source code in vllm/model_executor/models/config.py

class Gemma4Config(VerifyAndUpdateConfig):
    @staticmethod
    def verify_and_update_config(vllm_config: "VllmConfig") -> None:
        """Configure attention for heterogeneous head dimensions.

        Gemma4 uses different head dimensions for sliding window vs full attention
        layers. The default FA3 on Hopper cannot handle head_dim > 256.

        On SM90 with FP8 KV cache, use FA3 for supported layers and let the generic
        FlashAttention selector upgrade larger head dimensions to FA4.
        The multimodal-prefix composite routes image masks to Triton and causal
        requests to this per-layer FA3/FA4 selection. For other configurations,
        force FA4 for all layers to avoid the mixed
        FA3+FA4 penalty.
        When FA4 is not available, fall back to Triton.
        """
        model_config = vllm_config.model_config
        arch_config = model_config.model_arch_config
        layer_types = getattr(model_config.hf_text_config, "layer_types", None) or []
        head_dims = {
            layer_types[i]: arch_config[i].head_size
            for i in range(min(arch_config.total_num_hidden_layers, len(layer_types)))
        }

        if len(set(head_dims.values())) <= 1:
            return

        from vllm.platforms import current_platform
        from vllm.v1.attention.backends.fa_utils import is_fa_version_supported
        from vllm.v1.attention.backends.registry import AttentionBackendEnum

        max_head_dim = max(head_dims.values())

        if is_fa_version_supported(4) and max_head_dim <= 512:
            if (
                vllm_config.attention_config.flash_attn_version is None
                and vllm_config.attention_config.backend
                in (None, AttentionBackendEnum.FLASH_ATTN)
            ):
                use_per_layer_fa = current_platform.is_device_capability_family(
                    90
                ) and vllm_config.cache_config.cache_dtype.startswith("fp8")
                if use_per_layer_fa:
                    logger.info(
                        "Gemma4 model has heterogeneous head dimensions %s. Using "
                        "per-layer FA3/FA4 selection for FP8 KV cache on SM90.",
                        head_dims,
                    )
                else:
                    vllm_config.attention_config.flash_attn_version = 4
                    logger.info(
                        "Gemma4 model has heterogeneous head dimensions %s. Using FA4 "
                        "for all layers to avoid mixed FA3/FA4 penalty.",
                        head_dims,
                    )
        elif current_platform.is_xpu():
            logger.info(
                "Gemma4 model has heterogeneous head dimensions %s. Leaving the "
                "attention backend choice to the XPU platform.",
                head_dims,
            )
        elif vllm_config.attention_config.backend is None:
            vllm_config.attention_config.backend = AttentionBackendEnum.TRITON_ATTN
            logger.info(
                "Gemma4 model has heterogeneous head dimensions "
                "%s. FA4 not available, forcing TRITON_ATTN backend.",
                head_dims,
            )

verify_and_update_config(vllm_config) staticmethod

Configure attention for heterogeneous head dimensions.

Gemma4 uses different head dimensions for sliding window vs full attention layers. The default FA3 on Hopper cannot handle head_dim > 256.

On SM90 with FP8 KV cache, use FA3 for supported layers and let the generic FlashAttention selector upgrade larger head dimensions to FA4. The multimodal-prefix composite routes image masks to Triton and causal requests to this per-layer FA3/FA4 selection. For other configurations, force FA4 for all layers to avoid the mixed FA3+FA4 penalty. When FA4 is not available, fall back to Triton.

Source code in vllm/model_executor/models/config.py

@staticmethod
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
    """Configure attention for heterogeneous head dimensions.

    Gemma4 uses different head dimensions for sliding window vs full attention
    layers. The default FA3 on Hopper cannot handle head_dim > 256.

    On SM90 with FP8 KV cache, use FA3 for supported layers and let the generic
    FlashAttention selector upgrade larger head dimensions to FA4.
    The multimodal-prefix composite routes image masks to Triton and causal
    requests to this per-layer FA3/FA4 selection. For other configurations,
    force FA4 for all layers to avoid the mixed
    FA3+FA4 penalty.
    When FA4 is not available, fall back to Triton.
    """
    model_config = vllm_config.model_config
    arch_config = model_config.model_arch_config
    layer_types = getattr(model_config.hf_text_config, "layer_types", None) or []
    head_dims = {
        layer_types[i]: arch_config[i].head_size
        for i in range(min(arch_config.total_num_hidden_layers, len(layer_types)))
    }

    if len(set(head_dims.values())) <= 1:
        return

    from vllm.platforms import current_platform
    from vllm.v1.attention.backends.fa_utils import is_fa_version_supported
    from vllm.v1.attention.backends.registry import AttentionBackendEnum

    max_head_dim = max(head_dims.values())

    if is_fa_version_supported(4) and max_head_dim <= 512:
        if (
            vllm_config.attention_config.flash_attn_version is None
            and vllm_config.attention_config.backend
            in (None, AttentionBackendEnum.FLASH_ATTN)
        ):
            use_per_layer_fa = current_platform.is_device_capability_family(
                90
            ) and vllm_config.cache_config.cache_dtype.startswith("fp8")
            if use_per_layer_fa:
                logger.info(
                    "Gemma4 model has heterogeneous head dimensions %s. Using "
                    "per-layer FA3/FA4 selection for FP8 KV cache on SM90.",
                    head_dims,
                )
            else:
                vllm_config.attention_config.flash_attn_version = 4
                logger.info(
                    "Gemma4 model has heterogeneous head dimensions %s. Using FA4 "
                    "for all layers to avoid mixed FA3/FA4 penalty.",
                    head_dims,
                )
    elif current_platform.is_xpu():
        logger.info(
            "Gemma4 model has heterogeneous head dimensions %s. Leaving the "
            "attention backend choice to the XPU platform.",
            head_dims,
        )
    elif vllm_config.attention_config.backend is None:
        vllm_config.attention_config.backend = AttentionBackendEnum.TRITON_ATTN
        logger.info(
            "Gemma4 model has heterogeneous head dimensions "
            "%s. FA4 not available, forcing TRITON_ATTN backend.",
            head_dims,
        )

HybridAttentionMambaModelConfig

Bases: VerifyAndUpdateConfig

Methods:

• verify_and_update_config –

  Perform early validation and setup for hybrid attention/mamba models.

Source code in vllm/model_executor/models/config.py

class HybridAttentionMambaModelConfig(VerifyAndUpdateConfig):
    @classmethod
    def verify_and_update_config(cls, vllm_config: "VllmConfig") -> None:
        """Perform early validation and setup for hybrid attention/mamba models.

        Block size alignment with mamba page sizes is handled later by
        Platform.update_block_size_for_backend(), which runs after model
        layers are constructed and the attention backend is known.

        Args:
            vllm_config: vLLM Config

        """
        # Enable FULL_AND_PIECEWISE by default
        MambaModelConfig.verify_and_update_config(vllm_config)

verify_and_update_config(vllm_config) classmethod

Perform early validation and setup for hybrid attention/mamba models.

Block size alignment with mamba page sizes is handled later by Platform.update_block_size_for_backend(), which runs after model layers are constructed and the attention backend is known.

Parameters:

• vllm_config

  (VllmConfig) –

  vLLM Config

Source code in vllm/model_executor/models/config.py

@classmethod
def verify_and_update_config(cls, vllm_config: "VllmConfig") -> None:
    """Perform early validation and setup for hybrid attention/mamba models.

    Block size alignment with mamba page sizes is handled later by
    Platform.update_block_size_for_backend(), which runs after model
    layers are constructed and the attention backend is known.

    Args:
        vllm_config: vLLM Config

    """
    # Enable FULL_AND_PIECEWISE by default
    MambaModelConfig.verify_and_update_config(vllm_config)

JinaEmbeddingsV5ModelConfig

Bases: VerifyAndUpdateConfig

Config handler for Jina Embeddings V5 embedding models.

Methods:

• verify_and_update_model_config –

  Enable the bidirectional encoder backbone for -nano checkpoints.

Source code in vllm/model_executor/models/config.py

class JinaEmbeddingsV5ModelConfig(VerifyAndUpdateConfig):
    """Config handler for Jina Embeddings V5 embedding models."""

    @staticmethod
    def verify_and_update_model_config(model_config: "ModelConfig") -> None:
        """Enable the bidirectional encoder backbone for -nano checkpoints.

        The V5 family ships more than one backbone under a single
        `architectures` entry: the `-small` variants are Qwen3 decoders, while
        `-nano` is a bidirectional EuroBERT encoder. Upstream ships a separate
        `configuration_*.py` per repository, so the config carries no backbone
        field and the encoder variants are only identifiable by
        `is_decoder=False`. For encoder checkpoints, set `is_causal=False` so the
        Llama backbone uses EncoderOnlyAttention; `JinaEmbeddingsV5Model` then
        dispatches to its encoder implementation.
        """
        if getattr(model_config.hf_config, "is_decoder", True):
            return

        model_config.hf_config.is_causal = False

verify_and_update_model_config(model_config) staticmethod

Enable the bidirectional encoder backbone for -nano checkpoints.

The V5 family ships more than one backbone under a single architectures entry: the -small variants are Qwen3 decoders, while -nano is a bidirectional EuroBERT encoder. Upstream ships a separate configuration_*.py per repository, so the config carries no backbone field and the encoder variants are only identifiable by is_decoder=False. For encoder checkpoints, set is_causal=False so the Llama backbone uses EncoderOnlyAttention; JinaEmbeddingsV5Model then dispatches to its encoder implementation.

Source code in vllm/model_executor/models/config.py

@staticmethod
def verify_and_update_model_config(model_config: "ModelConfig") -> None:
    """Enable the bidirectional encoder backbone for -nano checkpoints.

    The V5 family ships more than one backbone under a single
    `architectures` entry: the `-small` variants are Qwen3 decoders, while
    `-nano` is a bidirectional EuroBERT encoder. Upstream ships a separate
    `configuration_*.py` per repository, so the config carries no backbone
    field and the encoder variants are only identifiable by
    `is_decoder=False`. For encoder checkpoints, set `is_causal=False` so the
    Llama backbone uses EncoderOnlyAttention; `JinaEmbeddingsV5Model` then
    dispatches to its encoder implementation.
    """
    if getattr(model_config.hf_config, "is_decoder", True):
        return

    model_config.hf_config.is_causal = False

KimiK3ForConditionalGenerationConfig

Bases: VerifyAndUpdateConfig

Route MXFP4-checkpointed Kimi-K3 MoE experts to the MXFP4 interface.

Kimi-K3 ships its routed experts as compressed-tensors mxfp4-pack-quantized (quant_method="compressed-tensors"), which lands them on CompressedTensorsW4A4Mxfp4MoEMethod and its narrow kernel selection. Rewriting quant_method to "mxfp4" selects Mxfp4Config (hence Mxfp4MoEMethod) with its full backend set, while any non-MXFP4 checkpoint is left untouched. Covers both the main model and the MTP draft.

model_arch_config.quantization_config is a separate dict, snapshotted in ModelConfig.__init__ before this hook runs, and it is what _verify_quantization reads when resolving the quant method. Patch it alongside the hf configs so the rewrite lands before resolution; otherwise the main model still resolves to compressed-tensors.

Source code in vllm/model_executor/models/config.py

class KimiK3ForConditionalGenerationConfig(VerifyAndUpdateConfig):
    """Route MXFP4-checkpointed Kimi-K3 MoE experts to the MXFP4 interface.

    Kimi-K3 ships its routed experts as compressed-tensors
    ``mxfp4-pack-quantized`` (``quant_method="compressed-tensors"``), which
    lands them on ``CompressedTensorsW4A4Mxfp4MoEMethod`` and its narrow kernel
    selection. Rewriting ``quant_method`` to ``"mxfp4"`` selects ``Mxfp4Config``
    (hence ``Mxfp4MoEMethod``) with its full backend set, while any non-MXFP4
    checkpoint is left untouched. Covers both the main model and the MTP draft.

    ``model_arch_config.quantization_config`` is a separate dict, snapshotted in
    ``ModelConfig.__init__`` before this hook runs, and it is what
    ``_verify_quantization`` reads when resolving the quant method. Patch it
    alongside the hf configs so the rewrite lands before resolution; otherwise
    the main model still resolves to compressed-tensors.
    """

    @staticmethod
    def verify_and_update_model_config(model_config: "ModelConfig") -> None:
        for cfg in (
            model_config.hf_config,
            model_config.hf_text_config,
            model_config.model_arch_config,
        ):
            quant_config = getattr(cfg, "quantization_config", None)
            if (
                isinstance(quant_config, dict)
                and quant_config.get("quant_method") == "compressed-tensors"
                and quant_config.get("format") == "mxfp4-pack-quantized"
            ):
                quant_config["quant_method"] = "mxfp4"

LlamaNemotronVLConfig

Bases: VerifyAndUpdateConfig

Config handler for LlamaNemotronVL embedding models.

Source code in vllm/model_executor/models/config.py

class LlamaNemotronVLConfig(VerifyAndUpdateConfig):
    """Config handler for LlamaNemotronVL embedding models."""

    @staticmethod
    def verify_and_update_model_config(model_config: "ModelConfig") -> None:
        from vllm.config.pooler import SequencePoolingType

        hf_config = model_config.hf_config

        # Set bidirectional attention on the language model config
        hf_config.is_causal = False
        if hasattr(hf_config, "llm_config"):
            hf_config.llm_config.is_causal = False

        if hasattr(hf_config, "vision_config"):
            hf_config.patch_size = hf_config.vision_config.patch_size

        # Set up pooling type
        pooling_type_map: dict[str, SequencePoolingType] = {
            "avg": "MEAN",
            "cls": "CLS",
            "last": "LAST",
        }

        # Get pooling type from config (check both top-level and llm_config)
        pooling = getattr(hf_config, "pooling", None)
        if pooling is None and hasattr(hf_config, "llm_config"):
            pooling = getattr(hf_config.llm_config, "pooling", "avg")

        assert isinstance(pooling, str)

        pooling_type = pooling_type_map.get(pooling)
        if pooling_type is None:
            raise ValueError(f"pool_type {pooling!r} not supported")

        assert model_config.pooler_config is not None
        model_config.pooler_config.seq_pooling_type = pooling_type

MambaModelConfig

Bases: VerifyAndUpdateConfig

Methods:

• verify_and_update_config –

  Enable FULL_AND_PIECEWISE cuda graph mode by default (required

Source code in vllm/model_executor/models/config.py

class MambaModelConfig(VerifyAndUpdateConfig):
    @classmethod
    def verify_and_update_config(cls, vllm_config: "VllmConfig") -> None:
        """Enable FULL_AND_PIECEWISE cuda graph mode by default (required
        to get good performance for mamba layers in V1).

        Args:
            vllm_config: vLLM Config

        """
        model_config = vllm_config.model_config
        cache_config = vllm_config.cache_config

        if cache_config.enable_prefix_caching:
            if cache_config.mamba_cache_mode == "none":
                cache_config.mamba_cache_mode = "align"
                logger.info(
                    "Mamba cache mode is set to '%s' for %s by default "
                    "when prefix caching is enabled",
                    cache_config.mamba_cache_mode,
                    model_config.architecture,
                )
            assert vllm_config.scheduler_config.enable_chunked_prefill, (
                "Chunked prefill is required for Mamba prefix caching."
            )
            # By default, mamba block size will be set to max_model_len (see
            # below). When enabling prefix caching, we align mamba block size
            # to the block size as the basic granularity for prefix caching.
            if cache_config.mamba_block_size is None:
                cache_config.mamba_block_size = cache_config.block_size
        else:
            if cache_config.mamba_cache_mode != "none":
                cache_config.mamba_cache_mode = "none"
                logger.warning(
                    "Mamba cache mode is set to 'none' when prefix caching is disabled"
                )
            if cache_config.mamba_block_size is None:
                cache_config.mamba_block_size = model_config.max_model_len

verify_and_update_config(vllm_config) classmethod

Enable FULL_AND_PIECEWISE cuda graph mode by default (required to get good performance for mamba layers in V1).

Parameters:

• vllm_config

  (VllmConfig) –

  vLLM Config

Source code in vllm/model_executor/models/config.py

@classmethod
def verify_and_update_config(cls, vllm_config: "VllmConfig") -> None:
    """Enable FULL_AND_PIECEWISE cuda graph mode by default (required
    to get good performance for mamba layers in V1).

    Args:
        vllm_config: vLLM Config

    """
    model_config = vllm_config.model_config
    cache_config = vllm_config.cache_config

    if cache_config.enable_prefix_caching:
        if cache_config.mamba_cache_mode == "none":
            cache_config.mamba_cache_mode = "align"
            logger.info(
                "Mamba cache mode is set to '%s' for %s by default "
                "when prefix caching is enabled",
                cache_config.mamba_cache_mode,
                model_config.architecture,
            )
        assert vllm_config.scheduler_config.enable_chunked_prefill, (
            "Chunked prefill is required for Mamba prefix caching."
        )
        # By default, mamba block size will be set to max_model_len (see
        # below). When enabling prefix caching, we align mamba block size
        # to the block size as the basic granularity for prefix caching.
        if cache_config.mamba_block_size is None:
            cache_config.mamba_block_size = cache_config.block_size
    else:
        if cache_config.mamba_cache_mode != "none":
            cache_config.mamba_cache_mode = "none"
            logger.warning(
                "Mamba cache mode is set to 'none' when prefix caching is disabled"
            )
        if cache_config.mamba_block_size is None:
            cache_config.mamba_block_size = model_config.max_model_len

MiniMaxM3SparseForConditionalGenerationConfig

Bases: MiniMaxM3SparseConfig

Methods:

• verify_and_update_model_config –

  Reject IndexCache overrides set on the outer VL config, where the

Source code in vllm/model_executor/models/config.py

class MiniMaxM3SparseForConditionalGenerationConfig(MiniMaxM3SparseConfig):
    INDEX_CACHE_KEYS = ("use_index_cache", "index_topk_freq", "index_skip_topk_offset")

    @classmethod
    def verify_and_update_model_config(cls, model_config: "ModelConfig") -> None:
        """Reject IndexCache overrides set on the outer VL config, where the
        ROCm text model never reads them."""
        from vllm.platforms import current_platform

        overrides = model_config.hf_overrides
        if not current_platform.is_rocm() or not isinstance(overrides, dict):
            return
        misplaced = [key for key in cls.INDEX_CACHE_KEYS if key in overrides]
        if misplaced:
            suggested = {"text_config": {key: overrides[key] for key in misplaced}}
            raise ValueError(
                "MiniMax-M3 reads IndexCache settings from text_config, so "
                f"top-level {', '.join(misplaced)} would be ignored. Pass them as "
                f"--hf-overrides '{json.dumps(suggested)}'."
            )

verify_and_update_model_config(model_config) classmethod

Reject IndexCache overrides set on the outer VL config, where the ROCm text model never reads them.

Source code in vllm/model_executor/models/config.py

@classmethod
def verify_and_update_model_config(cls, model_config: "ModelConfig") -> None:
    """Reject IndexCache overrides set on the outer VL config, where the
    ROCm text model never reads them."""
    from vllm.platforms import current_platform

    overrides = model_config.hf_overrides
    if not current_platform.is_rocm() or not isinstance(overrides, dict):
        return
    misplaced = [key for key in cls.INDEX_CACHE_KEYS if key in overrides]
    if misplaced:
        suggested = {"text_config": {key: overrides[key] for key in misplaced}}
        raise ValueError(
            "MiniMax-M3 reads IndexCache settings from text_config, so "
            f"top-level {', '.join(misplaced)} would be ignored. Pass them as "
            f"--hf-overrides '{json.dumps(suggested)}'."
        )

NemotronHForCausalLMConfig

Bases: VerifyAndUpdateConfig

Methods:

• update_mamba_ssm_cache_dtype –

  Update mamba_ssm_cache_dtype for NemotronH models when set to 'auto'

Attributes:

• DEFAULT_MAMBA_SSM_CACHE_DTYPE (MambaDType) –

  Only float32 is known to have no accuracy issues by default.

Source code in vllm/model_executor/models/config.py

class NemotronHForCausalLMConfig(VerifyAndUpdateConfig):
    DEFAULT_MAMBA_SSM_CACHE_DTYPE: "MambaDType" = "float32"
    """Only `float32` is known to have no accuracy issues by default."""

    @classmethod
    def update_mamba_ssm_cache_dtype(
        cls, *, cache_config: "CacheConfig", hf_config: "PreTrainedConfig"
    ) -> None:
        """Update mamba_ssm_cache_dtype for NemotronH models when set to 'auto'
        (or not explicitly set), to the value specified in the HF config, or to
        `float32` if not specified.
        """
        if cache_config.mamba_ssm_cache_dtype == "auto":
            mamba_ssm_cache_dtype: MambaDType = getattr(
                hf_config, "mamba_ssm_cache_dtype", cls.DEFAULT_MAMBA_SSM_CACHE_DTYPE
            )
            logger.info(
                "Updating mamba_ssm_cache_dtype to '%s' for NemotronH model",
                mamba_ssm_cache_dtype,
            )
            cache_config.mamba_ssm_cache_dtype = mamba_ssm_cache_dtype

    @classmethod
    def verify_and_update_config(cls, vllm_config: "VllmConfig") -> None:
        cls.update_mamba_ssm_cache_dtype(
            cache_config=vllm_config.cache_config,
            hf_config=vllm_config.model_config.hf_config,
        )

DEFAULT_MAMBA_SSM_CACHE_DTYPE = 'float32' class-attribute instance-attribute

Only float32 is known to have no accuracy issues by default.

update_mamba_ssm_cache_dtype(*, cache_config, hf_config) classmethod

Update mamba_ssm_cache_dtype for NemotronH models when set to 'auto' (or not explicitly set), to the value specified in the HF config, or to float32 if not specified.

Source code in vllm/model_executor/models/config.py

@classmethod
def update_mamba_ssm_cache_dtype(
    cls, *, cache_config: "CacheConfig", hf_config: "PreTrainedConfig"
) -> None:
    """Update mamba_ssm_cache_dtype for NemotronH models when set to 'auto'
    (or not explicitly set), to the value specified in the HF config, or to
    `float32` if not specified.
    """
    if cache_config.mamba_ssm_cache_dtype == "auto":
        mamba_ssm_cache_dtype: MambaDType = getattr(
            hf_config, "mamba_ssm_cache_dtype", cls.DEFAULT_MAMBA_SSM_CACHE_DTYPE
        )
        logger.info(
            "Updating mamba_ssm_cache_dtype to '%s' for NemotronH model",
            mamba_ssm_cache_dtype,
        )
        cache_config.mamba_ssm_cache_dtype = mamba_ssm_cache_dtype

Qwen3_5ForConditionalGenerationConfig

Bases: VerifyAndUpdateConfig

Methods:

• verify_and_update_config –

  Update mamba_ssm_cache_dtype for Qwen3.5 models when set to 'auto'

Source code in vllm/model_executor/models/config.py

class Qwen3_5ForConditionalGenerationConfig(VerifyAndUpdateConfig):
    @staticmethod
    def verify_and_update_config(vllm_config: "VllmConfig") -> None:
        """Update mamba_ssm_cache_dtype for Qwen3.5 models when set to 'auto'
        (or not explicitly set), to the value specified in the HF config's
        mamba_ssm_dtype field. Warn if the user explicitly overrides it to a
        different value.
        """
        cache_config = vllm_config.cache_config
        hf_text_config = vllm_config.model_config.hf_text_config
        mamba_ssm_dtype = getattr(hf_text_config, "mamba_ssm_dtype", None)
        if cache_config.mamba_ssm_cache_dtype == "auto":
            if mamba_ssm_dtype is not None:
                cache_config.mamba_ssm_cache_dtype = mamba_ssm_dtype
        elif (
            mamba_ssm_dtype is not None
            and cache_config.mamba_ssm_cache_dtype != mamba_ssm_dtype
        ):
            logger.warning(
                "Qwen3.5 model specifies mamba_ssm_dtype='%s' in its config, "
                "but --mamba-ssm-cache-dtype='%s' was passed. "
                "Using the user-specified value.",
                mamba_ssm_dtype,
                cache_config.mamba_ssm_cache_dtype,
            )

verify_and_update_config(vllm_config) staticmethod

Update mamba_ssm_cache_dtype for Qwen3.5 models when set to 'auto' (or not explicitly set), to the value specified in the HF config's mamba_ssm_dtype field. Warn if the user explicitly overrides it to a different value.

Source code in vllm/model_executor/models/config.py

@staticmethod
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
    """Update mamba_ssm_cache_dtype for Qwen3.5 models when set to 'auto'
    (or not explicitly set), to the value specified in the HF config's
    mamba_ssm_dtype field. Warn if the user explicitly overrides it to a
    different value.
    """
    cache_config = vllm_config.cache_config
    hf_text_config = vllm_config.model_config.hf_text_config
    mamba_ssm_dtype = getattr(hf_text_config, "mamba_ssm_dtype", None)
    if cache_config.mamba_ssm_cache_dtype == "auto":
        if mamba_ssm_dtype is not None:
            cache_config.mamba_ssm_cache_dtype = mamba_ssm_dtype
    elif (
        mamba_ssm_dtype is not None
        and cache_config.mamba_ssm_cache_dtype != mamba_ssm_dtype
    ):
        logger.warning(
            "Qwen3.5 model specifies mamba_ssm_dtype='%s' in its config, "
            "but --mamba-ssm-cache-dtype='%s' was passed. "
            "Using the user-specified value.",
            mamba_ssm_dtype,
            cache_config.mamba_ssm_cache_dtype,
        )

Qwen4ExpForConditionalGenerationConfig

Bases: Qwen3_5ForConditionalGenerationConfig

Apply the Qwen3.5 hybrid-cache contract to Qwen4Exp.

Source code in vllm/model_executor/models/config.py

class Qwen4ExpForConditionalGenerationConfig(Qwen3_5ForConditionalGenerationConfig):
    """Apply the Qwen3.5 hybrid-cache contract to Qwen4Exp."""

    @staticmethod
    def verify_and_update_config(vllm_config: "VllmConfig") -> None:
        from vllm.platforms import current_platform

        Qwen3_5ForConditionalGenerationConfig.verify_and_update_config(vllm_config)
        text_config = vllm_config.model_config.hf_text_config
        if text_config.hc_count <= 1:
            raise ValueError("Qwen4Exp requires hc_count > 1")
        parallel_config = vllm_config.parallel_config
        # MoE SP also enables HC SP without the explicit HC flag.
        if (
            current_platform.is_cuda_alike()
            and parallel_config.use_sequence_parallel_moe
            and parallel_config.pipeline_parallel_size != 1
        ):
            raise ValueError("Qwen4Exp MoE SP requires PP=1")

        uses_ple_or_qsa = bool(text_config.ple_layer_ids) or (
            getattr(text_config, "indexer_n_heads", None) is not None
        )
        if uses_ple_or_qsa and (
            parallel_config.enable_dbo or parallel_config.ubatch_size > 1
        ):
            raise NotImplementedError(
                "Qwen4Exp PLE/QSA does not support dual-batch overlap or microbatching"
            )
        multimodal_config = vllm_config.model_config.multimodal_config
        if multimodal_config is not None and multimodal_config.language_model_only:
            _strip_qwen4_exp_mrope(vllm_config.model_config)
        spec_config = vllm_config.speculative_config
        if spec_config is not None and spec_config.method not in {
            "mtp",
            "ngram",
            "ngram_gpu",
        }:
            raise NotImplementedError(
                "Qwen4Exp speculative decoding supports only its native MTP "
                "checkpoint and linear n-gram proposers"
            )

Qwen4ExpMTPConfig

Bases: Qwen4ExpForConditionalGenerationConfig

Preserve MRoPE for a VL target and use 1D RoPE for a text target.

Source code in vllm/model_executor/models/config.py

class Qwen4ExpMTPConfig(Qwen4ExpForConditionalGenerationConfig):
    """Preserve MRoPE for a VL target and use 1D RoPE for a text target."""

    @staticmethod
    def verify_and_update_config(vllm_config: "VllmConfig") -> None:
        Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(vllm_config)
        if hasattr(vllm_config.model_config.hf_config, "vision_config"):
            return
        _strip_qwen4_exp_mrope(vllm_config.model_config)

UnlimitedOCRForCausalLMConfig

Bases: VerifyAndUpdateConfig

Methods:

• verify_and_update_config –

  Configure Unlimited-OCR attention backends for R-SWA and vision.

Source code in vllm/model_executor/models/config.py

class UnlimitedOCRForCausalLMConfig(VerifyAndUpdateConfig):
    @staticmethod
    def verify_and_update_config(vllm_config: "VllmConfig") -> None:
        """Configure Unlimited-OCR attention backends for R-SWA and vision.

        Backend selection — controlled by the standard ``--attention-config``
        CLI argument (priority order):

          1. ``--attention-config '{"backend": "FLASH_ATTN"}'``
             → FA4 + rswa_mask_mod.  Exact token-level R-SWA.
               ``flash_attn_version`` is forced to 4 if not already set (R-SWA
               mask_mod requires FA4; FA3 cannot express it).  Raises if FA4 is
               not available on this device.

          2. ``--attention-config '{"backend": "FLEX_ATTENTION"}'``
             → FlexAttention R-SWA via Triton block mask.

          3. ``--attention-config '{"backend": "TRITON_ATTN"}'``
             → Triton unified attention with an R-SWA decode mask.

          4. ``--attention-config '{"backend": "auto"}'`` (or omitted)
             → Auto-detect: FA4 if available (H20/H100 SM90), else TritonAttention.

        Regardless of backend, prefix caching is disabled for this model: R-SWA
        decode-phase KV is not a pure causal function of the prefix (so decode
        blocks are not reusable), and single-turn image-led OCR prompts rarely
        hit the prefix cache.

        Example — force FlexAttention even on a machine with FA4::

            vllm serve baidu/Unlimited-OCR \\
                --attention-config '{"backend": "FLEX_ATTENTION"}'
        """
        from vllm.v1.attention.backends.fa_utils import is_fa_version_supported
        from vllm.v1.attention.backends.registry import AttentionBackendEnum

        if getattr(vllm_config.model_config.hf_text_config, "unlimited_ocr2", False):
            UnlimitedOCRForCausalLMConfig._configure_v2(vllm_config)
            return

        attn_config = vllm_config.attention_config
        fa4_available = is_fa_version_supported(4)

        # ── step 1: resolve backend ─────────────────────────────────────────
        # None means the user did not explicitly specify a backend; auto-select.
        if attn_config.backend is None:
            attn_config.backend = (
                AttentionBackendEnum.FLASH_ATTN
                if fa4_available
                else AttentionBackendEnum.TRITON_ATTN
            )
            logger.info(
                "Unlimited-OCR: auto-selected attention backend=%s (fa4_available=%s).",
                attn_config.backend.value,
                fa4_available,
            )

        # ── step 2: configure the chosen backend ────────────────────────────
        if attn_config.backend == AttentionBackendEnum.FLASH_ATTN:
            if not fa4_available:
                raise RuntimeError(
                    "Unlimited-OCR: --attention-config backend=FLASH_ATTN "
                    "requires FA4 (rswa_mask_mod), but FA4 is not available on "
                    "this device/installation. Use backend=TRITON_ATTN or "
                    "FLEX_ATTENTION, or upgrade vllm-flash-attn."
                )
            # On SM90 (H20), the default FA version is FA3 regardless of FA4
            # availability (FA4 is only auto-upgraded when head_size > 256).
            # The R-SWA mask_mod requires FA4, so force the version globally.
            if attn_config.flash_attn_version is None:
                attn_config.flash_attn_version = 4
            elif attn_config.flash_attn_version < 4:
                logger.warning(
                    "Unlimited-OCR: flash_attn_version=%d cannot express the "
                    "R-SWA mask_mod; upgrading to 4.",
                    attn_config.flash_attn_version,
                )
                attn_config.flash_attn_version = 4
            logger.info(
                "Unlimited-OCR: FlashAttention FA%d + rswa_mask_mod — exact R-SWA.",
                attn_config.flash_attn_version,
            )

        elif attn_config.backend == AttentionBackendEnum.TRITON_ATTN:
            logger.info(
                "Unlimited-OCR: TritonAttention — R-SWA via unified attention mask."
            )

        elif attn_config.backend == AttentionBackendEnum.FLEX_ATTENTION:
            logger.info(
                "Unlimited-OCR: FlexAttention — R-SWA via Triton block mask%s.",
                ""
                if not fa4_available
                else (
                    " (FA4 available but not used; pass backend=FLASH_ATTN to upgrade)"
                ),
            )

        else:
            raise ValueError(
                f"Unlimited-OCR: unsupported attention backend "
                f"{attn_config.backend!r} for R-SWA. "
                "Use FLASH_ATTN (FA4), TRITON_ATTN or FLEX_ATTENTION."
            )

        # R-SWA windows the *generated* tokens, so a decode-token's KV is not a
        # pure causal function of the prefix and cannot be safely reused across
        # requests via prefix caching. Only the prompt/image prefix is cacheable,
        # but OCR is single-turn with image-led prompts that rarely share a
        # prefix, so prefix caching brings little benefit while complicating the
        # KV cache manager. Disable it for this model.
        cache_config = vllm_config.cache_config
        if cache_config.enable_prefix_caching:
            cache_config.enable_prefix_caching = False
            logger.info(
                "Unlimited-OCR: disabling prefix caching (R-SWA decode KV is not "
                "cacheable, and single-turn image-led prompts rarely hit the "
                "prefix cache)."
            )

        UnlimitedOCRForCausalLMConfig._configure_vision(vllm_config)

    @staticmethod
    def _configure_vision(vllm_config: "VllmConfig") -> None:
        from vllm.v1.attention.backends.registry import AttentionBackendEnum

        mm_config = getattr(vllm_config.model_config, "multimodal_config", None)
        if mm_config is not None:
            if mm_config.mm_encoder_attn_backend is None:
                mm_config.mm_encoder_attn_backend = AttentionBackendEnum.FLASH_ATTN
            elif mm_config.mm_encoder_attn_backend == AttentionBackendEnum.FLASHINFER:
                logger.warning(
                    "Unlimited-OCR: FlashInfer is not supported for the vision "
                    "encoder (the CLIP stage runs full attention without "
                    "cu_seqlens); falling back to FlashAttention."
                )
                mm_config.mm_encoder_attn_backend = AttentionBackendEnum.FLASH_ATTN

    @staticmethod
    def verify_and_update_model_config(model_config: "ModelConfig") -> None:
        text_config = model_config.hf_config.text_config
        text_config.architectures = ["DeepseekV2ForCausalLM"]
        sparse = getattr(model_config.hf_config, "sparse_attention_config", None)
        if sparse is not None:
            required = {
                "training_stage": "dash",
                "dash_stage0": "head_cls",
                "sparse_main_rope_mode": "page2d",
                "sparse_rope_mode": "page2d",
                "sparse_rope_page_span": 0,
                "use_sparse_attention": True,
                "image_bucket_constraint": True,
                "dash_image_only": True,
            }
            for key, value in required.items():
                if sparse.get(key) != value:
                    raise ValueError(f"Unlimited-OCR-2 requires {key}={value!r}")
            if (
                sparse.get("sparse_attention_freq")
                != [1] * text_config.num_hidden_layers
            ):
                raise ValueError("Unlimited-OCR-2 requires DASH in every decoder layer")
            if (
                sparse["sparse_index_dim"]
                != text_config.hidden_size // text_config.num_attention_heads
            ):
                raise ValueError("Unlimited-OCR-2 requires index_dim == head_dim")
            if sparse["dash_sigma"] <= 0 or sparse["dash_topk_blocks"] <= 0:
                raise ValueError("Unlimited-OCR-2 requires positive sigma and top-K")
            # Token/page RoPE plus two DASH cache metadata channels.
            pairs = sparse.get("sparse_rope_page_pairs", 1)
            text_config.mrope_section = [
                sparse["sparse_index_dim"] // 2 - pairs,
                pairs,
                0,
                0,
            ]
            text_config.architectures = ["UnlimitedOCR2TextForCausalLM"]
            text_config.unlimited_ocr2 = True
            text_config.sparse_attention_config = sparse
            window = sparse["sparse_local_block"] * sparse["sparse_block_size"]
            requested = getattr(model_config.hf_config, "rswa_window", None)
            if requested not in (None, 0, window):
                raise ValueError(
                    f"Unlimited-OCR-2 requires rswa_window={window} or 0 (full KV)"
                )
            model_config.hf_config.rswa_window = None if requested == 0 else window
            text_config.rswa_window = model_config.hf_config.rswa_window
            model_config.model_arch_config.rswa_window = text_config.rswa_window
            return
        if getattr(model_config.hf_config, "rswa_window", None) is None:
            model_config.hf_config.rswa_window = 128
        # Propagate rswa_window to text_config so that DeepseekAttention (which
        # receives text_config as its vllm_config.model_config.hf_config via
        # init_vllm_registered_model) can read it and create RSWAAttention.
        rswa_window = model_config.hf_config.rswa_window
        text_config.rswa_window = rswa_window

    @staticmethod
    def _configure_v2(vllm_config: "VllmConfig") -> None:
        from vllm.compilation.breakable_cudagraph import is_breakable_cudagraph_enabled
        from vllm.config.compilation import CompilationMode, CUDAGraphMode
        from vllm.v1.attention.backends.registry import AttentionBackendEnum

        if not vllm_config.use_v2_model_runner:
            raise ValueError(
                "Unlimited-OCR-2 requires Model Runner V2; set "
                "VLLM_USE_V2_MODEL_RUNNER=1"
            )
        # Generic JIT warmup submits text-only requests, which DASH cannot use.
        vllm_config.kernel_config.enable_jit_warmup = False
        compilation = vllm_config.compilation_config
        breakable = is_breakable_cudagraph_enabled()
        if not vllm_config.model_config.enforce_eager:
            if compilation.mode not in (None, CompilationMode.NONE):
                raise ValueError("Unlimited-OCR-2 currently requires compile mode NONE")
            compilation.mode = CompilationMode.NONE
            if compilation.cudagraph_mode is None:
                compilation.cudagraph_mode = (
                    CUDAGraphMode.FULL_AND_PIECEWISE
                    if breakable
                    else CUDAGraphMode.FULL_DECODE_ONLY
                )
            allowed_modes = {CUDAGraphMode.NONE, CUDAGraphMode.FULL_DECODE_ONLY}
            if breakable:
                allowed_modes.update(
                    {CUDAGraphMode.PIECEWISE, CUDAGraphMode.FULL_AND_PIECEWISE}
                )
            if compilation.cudagraph_mode not in allowed_modes:
                raise ValueError(
                    "Unlimited-OCR-2 prefill graphs require "
                    "VLLM_USE_BREAKABLE_CUDAGRAPH=1 and mode "
                    "PIECEWISE or FULL_AND_PIECEWISE; full prefill graphs "
                    "are not supported"
                )
        parallel = vllm_config.parallel_config
        unsupported = {
            "pipeline parallelism": parallel.pipeline_parallel_size != 1,
            "context parallelism": (
                parallel.decode_context_parallel_size != 1
                or parallel.prefill_context_parallel_size != 1
            ),
            "dual batch overlap": parallel.enable_dbo,
            "LoRA": vllm_config.lora_config is not None,
            "speculative decoding": vllm_config.speculative_config is not None,
            "quantized weights": vllm_config.model_config.quantization is not None,
            "non-default KV dtype": vllm_config.cache_config.cache_dtype != "auto",
            "KV transfer": vllm_config.kv_transfer_config is not None,
        }
        for feature, enabled in unsupported.items():
            if enabled:
                raise ValueError(f"Unlimited-OCR-2 does not support {feature}")
        vllm_config.cache_config.enable_prefix_caching = False
        backend = (
            vllm_config.attention_config.backend or AttentionBackendEnum.FLASH_ATTN
        )
        if backend not in (
            AttentionBackendEnum.FLASH_ATTN,
            AttentionBackendEnum.FLASHINFER,
        ):
            raise ValueError("Unlimited-OCR-2 supports FLASH_ATTN or FLASHINFER")
        text_config = vllm_config.model_config.hf_config.text_config
        text_config.unlimited_ocr2_attention_backend = backend.name
        if backend == AttentionBackendEnum.FLASHINFER:
            compilation.cudagraph_num_of_warmups = max(
                1, compilation.cudagraph_num_of_warmups
            )
        # DASH owns sparse selection and compute. Reuse Triton's paged-cache
        # metadata, including R-SWA reclamation, for both compute backends.
        vllm_config.attention_config.backend = AttentionBackendEnum.TRITON_ATTN
        logger.info("Unlimited-OCR-2 DASH compute backend: %s", backend.name)
        UnlimitedOCRForCausalLMConfig._configure_vision(vllm_config)
        logger.info(
            "Unlimited-OCR-2: output KV window=%s; input KV is retained for DASH.",
            vllm_config.model_config.rswa_window,
        )

verify_and_update_config(vllm_config) staticmethod

Configure Unlimited-OCR attention backends for R-SWA and vision.

Backend selection — controlled by the standard --attention-config CLI argument (priority order):

1. --attention-config '{"backend": "FLASH_ATTN"}' → FA4 + rswa_mask_mod. Exact token-level R-SWA. flash_attn_version is forced to 4 if not already set (R-SWA mask_mod requires FA4; FA3 cannot express it). Raises if FA4 is not available on this device.

2. --attention-config '{"backend": "FLEX_ATTENTION"}' → FlexAttention R-SWA via Triton block mask.

3. --attention-config '{"backend": "TRITON_ATTN"}' → Triton unified attention with an R-SWA decode mask.

4. --attention-config '{"backend": "auto"}' (or omitted) → Auto-detect: FA4 if available (H20/H100 SM90), else TritonAttention.

Regardless of backend, prefix caching is disabled for this model: R-SWA decode-phase KV is not a pure causal function of the prefix (so decode blocks are not reusable), and single-turn image-led OCR prompts rarely hit the prefix cache.

Example — force FlexAttention even on a machine with FA4::

vllm serve baidu/Unlimited-OCR \
    --attention-config '{"backend": "FLEX_ATTENTION"}'

Source code in vllm/model_executor/models/config.py

@staticmethod
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
    """Configure Unlimited-OCR attention backends for R-SWA and vision.

    Backend selection — controlled by the standard ``--attention-config``
    CLI argument (priority order):

      1. ``--attention-config '{"backend": "FLASH_ATTN"}'``
         → FA4 + rswa_mask_mod.  Exact token-level R-SWA.
           ``flash_attn_version`` is forced to 4 if not already set (R-SWA
           mask_mod requires FA4; FA3 cannot express it).  Raises if FA4 is
           not available on this device.

      2. ``--attention-config '{"backend": "FLEX_ATTENTION"}'``
         → FlexAttention R-SWA via Triton block mask.

      3. ``--attention-config '{"backend": "TRITON_ATTN"}'``
         → Triton unified attention with an R-SWA decode mask.

      4. ``--attention-config '{"backend": "auto"}'`` (or omitted)
         → Auto-detect: FA4 if available (H20/H100 SM90), else TritonAttention.

    Regardless of backend, prefix caching is disabled for this model: R-SWA
    decode-phase KV is not a pure causal function of the prefix (so decode
    blocks are not reusable), and single-turn image-led OCR prompts rarely
    hit the prefix cache.

    Example — force FlexAttention even on a machine with FA4::

        vllm serve baidu/Unlimited-OCR \\
            --attention-config '{"backend": "FLEX_ATTENTION"}'
    """
    from vllm.v1.attention.backends.fa_utils import is_fa_version_supported
    from vllm.v1.attention.backends.registry import AttentionBackendEnum

    if getattr(vllm_config.model_config.hf_text_config, "unlimited_ocr2", False):
        UnlimitedOCRForCausalLMConfig._configure_v2(vllm_config)
        return

    attn_config = vllm_config.attention_config
    fa4_available = is_fa_version_supported(4)

    # ── step 1: resolve backend ─────────────────────────────────────────
    # None means the user did not explicitly specify a backend; auto-select.
    if attn_config.backend is None:
        attn_config.backend = (
            AttentionBackendEnum.FLASH_ATTN
            if fa4_available
            else AttentionBackendEnum.TRITON_ATTN
        )
        logger.info(
            "Unlimited-OCR: auto-selected attention backend=%s (fa4_available=%s).",
            attn_config.backend.value,
            fa4_available,
        )

    # ── step 2: configure the chosen backend ────────────────────────────
    if attn_config.backend == AttentionBackendEnum.FLASH_ATTN:
        if not fa4_available:
            raise RuntimeError(
                "Unlimited-OCR: --attention-config backend=FLASH_ATTN "
                "requires FA4 (rswa_mask_mod), but FA4 is not available on "
                "this device/installation. Use backend=TRITON_ATTN or "
                "FLEX_ATTENTION, or upgrade vllm-flash-attn."
            )
        # On SM90 (H20), the default FA version is FA3 regardless of FA4
        # availability (FA4 is only auto-upgraded when head_size > 256).
        # The R-SWA mask_mod requires FA4, so force the version globally.
        if attn_config.flash_attn_version is None:
            attn_config.flash_attn_version = 4
        elif attn_config.flash_attn_version < 4:
            logger.warning(
                "Unlimited-OCR: flash_attn_version=%d cannot express the "
                "R-SWA mask_mod; upgrading to 4.",
                attn_config.flash_attn_version,
            )
            attn_config.flash_attn_version = 4
        logger.info(
            "Unlimited-OCR: FlashAttention FA%d + rswa_mask_mod — exact R-SWA.",
            attn_config.flash_attn_version,
        )

    elif attn_config.backend == AttentionBackendEnum.TRITON_ATTN:
        logger.info(
            "Unlimited-OCR: TritonAttention — R-SWA via unified attention mask."
        )

    elif attn_config.backend == AttentionBackendEnum.FLEX_ATTENTION:
        logger.info(
            "Unlimited-OCR: FlexAttention — R-SWA via Triton block mask%s.",
            ""
            if not fa4_available
            else (
                " (FA4 available but not used; pass backend=FLASH_ATTN to upgrade)"
            ),
        )

    else:
        raise ValueError(
            f"Unlimited-OCR: unsupported attention backend "
            f"{attn_config.backend!r} for R-SWA. "
            "Use FLASH_ATTN (FA4), TRITON_ATTN or FLEX_ATTENTION."
        )

    # R-SWA windows the *generated* tokens, so a decode-token's KV is not a
    # pure causal function of the prefix and cannot be safely reused across
    # requests via prefix caching. Only the prompt/image prefix is cacheable,
    # but OCR is single-turn with image-led prompts that rarely share a
    # prefix, so prefix caching brings little benefit while complicating the
    # KV cache manager. Disable it for this model.
    cache_config = vllm_config.cache_config
    if cache_config.enable_prefix_caching:
        cache_config.enable_prefix_caching = False
        logger.info(
            "Unlimited-OCR: disabling prefix caching (R-SWA decode KV is not "
            "cacheable, and single-turn image-led prompts rarely hit the "
            "prefix cache)."
        )

    UnlimitedOCRForCausalLMConfig._configure_vision(vllm_config)

_indexer_sparse_logits_supported(vllm_config)

Whether the DeepSeek V4.1 sparse-logits indexer can run: a two-level (candidate block) model on SM100 with DeepGEMM's sparse MQA kernels and DeepSelect, no decode/prefill context parallelism, and a DeepSelect top-k backend. Mirrors DeepseekV41SparseIndexerMetadataBuilder's checks.

Source code in vllm/model_executor/models/config.py

def _indexer_sparse_logits_supported(vllm_config: "VllmConfig") -> bool:
    """Whether the DeepSeek V4.1 sparse-logits indexer can run: a two-level
    (candidate block) model on SM100 with DeepGEMM's sparse MQA kernels and
    DeepSelect, no decode/prefill context parallelism, and a DeepSelect top-k
    backend. Mirrors DeepseekV41SparseIndexerMetadataBuilder's checks."""
    from vllm.platforms import current_platform

    hf_config = vllm_config.model_config.hf_text_config
    block = getattr(hf_config, "candidate_block_size", 0)
    topk_blocks = getattr(hf_config, "candidate_topk_blocks", 0)
    parallel = vllm_config.parallel_config
    if (
        block <= 0
        or topk_blocks <= 0
        or not current_platform.is_cuda()
        or not current_platform.is_device_capability_family(100)
        or parallel.decode_context_parallel_size > 1
        or parallel.prefill_context_parallel_size > 1
        or vllm_config.kernel_config.sparse_indexer_topk_backend
        not in ("auto", "deep_select")
    ):
        return False
    from vllm.model_executor.kernels.attention.dsa.sparse_mqa_logits import (
        check_deep_select_layout,
        has_deep_select,
        pick_sparse_block_kv,
    )
    from vllm.utils.deep_gemm import has_deep_gemm_sparse_mqa

    if not (has_deep_gemm_sparse_mqa() and has_deep_select()):
        return False
    try:
        pick_sparse_block_kv(block)
        check_deep_select_layout(
            topk_blocks * block, getattr(hf_config, "index_topk", 0)
        )
    except ValueError:
        return False
    return True

_set_deepseek_indexer_logits_dtype(vllm_config)

Default DeepSeek indexer logits to BF16 where the MQA logits kernels also reduce in BF16 (SM100) and to FP32 elsewhere.

Source code in vllm/model_executor/models/config.py

def _set_deepseek_indexer_logits_dtype(vllm_config: "VllmConfig") -> None:
    """Default DeepSeek indexer logits to BF16 where the MQA logits kernels
    also reduce in BF16 (SM100) and to FP32 elsewhere."""
    from vllm.utils.deep_gemm import mqa_logits_use_mx_scales

    attention_config = vllm_config.attention_config
    attention_config.indexer_logits_dtype = (
        attention_config.resolve_indexer_logits_dtype(
            "bfloat16" if mqa_logits_use_mx_scales() else "float32"
        )
    )