vllm.model_executor.layers.fused_moe.oracle.int8

Classes:

• Int8MoEKernelOracle –

  INT8 MoE backend selection for one deployment configuration.

Functions:

• convert_to_int8_moe_kernel_format –

  Convert INT8 MoE weights to backend-specific kernel format.

• map_int8_backend –

  Map user's MoEBackend to Int8MoeBackend.

• select_int8_moe_backend –

  Select the primary Int8 MoE backend.

Int8MoEKernelOracle

Bases: MoEKernelOracle[Int8MoeBackend]

INT8 MoE backend selection for one deployment configuration.

Source code in vllm/model_executor/layers/fused_moe/oracle/int8.py

class Int8MoEKernelOracle(MoEKernelOracle[Int8MoeBackend]):
    """INT8 MoE backend selection for one deployment configuration."""

    name = "Int8"

    def backend_to_kernel_cls(
        self, backend: Int8MoeBackend
    ) -> list[type[mk.FusedMoEExperts]]:
        return backend_to_kernel_cls(backend)

_get_priority_backends(moe_config)

Get available backends in priority order based on platform and config.

Source code in vllm/model_executor/layers/fused_moe/oracle/int8.py

def _get_priority_backends(
    moe_config: FusedMoEConfig,
) -> list[Int8MoeBackend]:
    """Get available backends in priority order based on platform and config."""
    return [
        Int8MoeBackend.TRITON,
        Int8MoeBackend.HUMMING,
        Int8MoeBackend.CPU,
    ]

_humming_int8_weight_schema(weight, weight_scale)

Build the humming compressed-tensors int8 schema from the canonical on-device tensors; humming does the signed-int8 -> native conversion.

Source code in vllm/model_executor/layers/fused_moe/oracle/int8.py

def _humming_int8_weight_schema(
    weight: torch.Tensor, weight_scale: torch.Tensor
) -> dict[str, Any]:
    """Build the humming compressed-tensors int8 schema from the canonical
    on-device tensors; humming does the signed-int8 -> native conversion."""
    config: dict[str, Any] = {
        "quant_method": "compressed-tensors",
        "format": "int-quantized",
        "type": "int",
        "num_bits": 8,
        "symmetric": True,
        "strategy": "channel",
    }
    num_experts, num_output = weight.shape[0], weight.shape[-2]
    if weight_scale.numel() < num_experts * num_output:
        config["strategy"] = "tensor"
    return config

convert_to_int8_moe_kernel_format(int8_backend, w13, w2, layer=None, w13_scale=None)

Convert INT8 MoE weights to backend-specific kernel format.

Source code in vllm/model_executor/layers/fused_moe/oracle/int8.py

def convert_to_int8_moe_kernel_format(
    int8_backend: Int8MoeBackend,
    w13: torch.Tensor,
    w2: torch.Tensor,
    layer: torch.nn.Module | None = None,
    w13_scale: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
    """Convert INT8 MoE weights to backend-specific kernel format."""
    if int8_backend == Int8MoeBackend.HUMMING:
        from vllm.model_executor.layers.quantization.utils.humming import (
            convert_to_humming_moe_kernel_format,
        )

        assert layer is not None
        # The Humming conversion expects canonical CT scale parameter names.
        # Online int8 produces per-channel (E, N) w*_scale; expose them as
        # (E, N, 1) w*_weight_scale tensors before conversion.
        for sub in ("w13", "w2"):
            if hasattr(layer, f"{sub}_weight_scale"):
                continue
            scale = getattr(layer, f"{sub}_scale").data
            if scale.dim() < 3:
                scale = scale.unsqueeze(-1)
            replace_parameter(layer, f"{sub}_weight_scale", scale)
            delattr(layer, f"{sub}_scale")
        convert_to_humming_moe_kernel_format(
            layer,
            quant_config=_humming_int8_weight_schema(w13, layer.w13_weight_scale),
        )
        return layer.w13_weight, layer.w2_weight
    elif int8_backend not in (Int8MoeBackend.TRITON, Int8MoeBackend.CPU):
        raise ValueError(f"Unsupported Int8 MoE backend: {int8_backend.value}")

    return w13, w2

map_int8_backend(runner_backend)

Map user's MoEBackend to Int8MoeBackend.

Source code in vllm/model_executor/layers/fused_moe/oracle/int8.py

def map_int8_backend(runner_backend: MoEBackend) -> Int8MoeBackend:
    """Map user's MoEBackend to Int8MoeBackend."""
    mapping = {
        "triton": Int8MoeBackend.TRITON,
        "humming": Int8MoeBackend.HUMMING,
    }
    if backend := mapping.get(runner_backend):
        return backend
    raise ValueError(
        f"moe_backend='{runner_backend}' is not supported for Int8 MoE. "
        f"Expected one of {list(mapping.keys())}."
    )

select_int8_moe_backend(config, weight_key=kInt8StaticChannelSym, activation_key=kInt8DynamicTokenSym)

Select the primary Int8 MoE backend. Note: Shape-specific fallbacks may still occur at runtime.

Source code in vllm/model_executor/layers/fused_moe/oracle/int8.py

def select_int8_moe_backend(
    config: FusedMoEConfig,
    weight_key: QuantKey | None = kInt8StaticChannelSym,
    activation_key: QuantKey | None = kInt8DynamicTokenSym,
) -> tuple[Int8MoeBackend, type[mk.FusedMoEExperts]]:
    """Select the primary Int8 MoE backend.
    Note: Shape-specific fallbacks may still occur at runtime.
    """
    AVAILABLE_BACKENDS = _get_priority_backends(config)

    oracle = Int8MoEKernelOracle(config, weight_key, activation_key)

    # Handle explicit moe_backend from user.
    runner_backend = config.moe_backend
    if runner_backend != "auto":
        requested_backend = map_int8_backend(runner_backend)
        return oracle.return_or_raise(requested_backend, AVAILABLE_BACKENDS)

    # Select kernels in order of backend.
    for backend in AVAILABLE_BACKENDS:
        kernel_cls, _ = oracle.try_backend(backend)
        if kernel_cls is not None:
            logger.info_once(oracle.make_log_backend(backend, AVAILABLE_BACKENDS))
            return backend, kernel_cls

    raise NotImplementedError(
        "No Int8 MoE backend supports the deployment configuration "
        f"(weight_key={weight_key}, activation_key={activation_key}). "
        "Set `VLLM_LOGGING_LEVEL=DEBUG` to see per-backend unsupported reasons."
    )