Skip to content

vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata

Metadata dataclasses and helpers for the NIXL connector.

Classes:

Functions:

HeartbeatInfo dataclass

Heartbeat data for a single remote engine, sent from D worker to P.

Source code in vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py 
204
205
206
207
208
209
210
211
212
@dataclass
class HeartbeatInfo:
    """Heartbeat data for a single remote engine, sent from D worker to P."""

    req_ids: set[ReqId]
    host: str
    port: int
    tp_size: int
    pp_size: int = 1

NixlHandshakePayload dataclass

Bases: KVConnectorHandshakeMetadata

Wrapper for NIXL handshake sent over the wire.

Enables two-phase decoding for graceful compatibility checking: 1. Decode NixlHandshakePayload to get compatibility_hash 2. Compute local hash and compare 3. Only if hashes match, decode agent_metadata_bytes

This prevents decoder errors when NixlAgentMetadata schema is incompatible, allowing graceful failure with clear error message.

Source code in vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py 
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
@dataclass
class NixlHandshakePayload(KVConnectorHandshakeMetadata):
    """
    Wrapper for NIXL handshake sent over the wire.

    Enables two-phase decoding for graceful compatibility checking:
    1. Decode NixlHandshakePayload to get compatibility_hash
    2. Compute local hash and compare
    3. Only if hashes match, decode agent_metadata_bytes

    This prevents decoder errors when NixlAgentMetadata schema is
    incompatible, allowing graceful failure with clear error message.
    """

    compatibility_hash: str
    agent_metadata_bytes: bytes  # NixlAgentMetadata encoded

_get_speculative_compatibility_factors(vllm_config)

Return NIXL compatibility factors for hidden-state-based speculators.

Source code in vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py 
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
def _get_speculative_compatibility_factors(
    vllm_config: VllmConfig,
) -> dict[str, Any] | None:
    """Return NIXL compatibility factors for hidden-state-based speculators."""
    speculative_config = vllm_config.speculative_config
    if speculative_config is None or not speculative_config.use_eagle():
        return None

    draft_model_config = speculative_config.draft_model_config
    assert draft_model_config is not None
    auxiliary_layer_ids = getattr(
        draft_model_config.hf_config,
        "eagle_aux_hidden_state_layer_ids",
        None,
    )

    # kv_cache_dtype is a user override that defaults to None, meaning "inherit
    # the target's --kv-cache-dtype". Resolve it to the effective value so an
    # explicit setting on one side and inheritance on the other (same effective
    # dtype) don't spuriously mismatch.
    kv_cache_dtype = (
        speculative_config.kv_cache_dtype or vllm_config.cache_config.cache_dtype
    )

    # Note: the draft attention_backend is intentionally not hashed. Its only
    # transfer-relevant effect is the KV block layout/size, which is validated
    # per region at runtime in _validate_remote_agent_handshake. The connector
    # only sees the raw override here (usually None = auto-select), never the
    # resolved backend, so hashing it would cause false mismatches without
    # catching anything the runtime layout check misses.
    return {
        "method": speculative_config.method,
        "model": draft_model_config.model,
        "revision": draft_model_config.revision,
        "code_revision": draft_model_config.code_revision,
        "parallel_drafting": speculative_config.parallel_drafting,
        "kv_cache_dtype": str(kv_cache_dtype),
        "auxiliary_layer_ids": (
            tuple(auxiliary_layer_ids) if auxiliary_layer_ids is not None else None
        ),
    }

compute_nixl_compatibility_hash(vllm_config, attn_backend_name, transfer_mode='pull')

Compute compatibility hash for NIXL KV transfer.

Hash only the factors that affect whether two NIXL instances can successfully transfer KV cache data.

Factors included: - vLLM version and NIXL connector version - Model architecture (name, dtype, KV heads, layers) - KV cache format (dtype, sliding window) - Attention backend - EAGLE/MTP configuration that affects transferred state - Transfer mode (push vs pull)

The transfer mode is included because the push (WRITE) and pull (READ) connectors use incompatible transfer protocols; a push connector and a pull connector must never complete a handshake with each other.

Note: Factors like tensor_parallel_size, block_size, and kv_cache_layout are validated at runtime in _validate_remote_agent_handshake and are not included in this hash to support heterogeneous deployments.

Note - the set of factors are likely to evolve significantly over time to be more or less permissive.

Returns:

  • str –

    SHA-256 hex digest

Source code in vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py 
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
def compute_nixl_compatibility_hash(
    vllm_config: VllmConfig,
    attn_backend_name: str,
    transfer_mode: str = "pull",
) -> str:
    """
    Compute compatibility hash for NIXL KV transfer.

    Hash only the factors that affect whether two NIXL instances can
    successfully transfer KV cache data.

    Factors included:
    - vLLM version and NIXL connector version
    - Model architecture (name, dtype, KV heads, layers)
    - KV cache format (dtype, sliding window)
    - Attention backend
    - EAGLE/MTP configuration that affects transferred state
    - Transfer mode (push vs pull)

    The transfer mode is included because the push (WRITE) and pull (READ)
    connectors use incompatible transfer protocols; a push connector and a
    pull connector must never complete a handshake with each other.

    Note: Factors like tensor_parallel_size, block_size, and kv_cache_layout
    are validated at runtime in _validate_remote_agent_handshake and are not
    included in this hash to support heterogeneous deployments.

    Note - the set of factors are likely to evolve significantly over
    time to be more or less permissive.

    Returns:
        SHA-256 hex digest
    """
    from vllm import __version__ as vllm_version
    from vllm.config.utils import hash_factors

    model_config = vllm_config.model_config
    cache_config = vllm_config.cache_config
    is_hma_enabled = not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager

    factors = {
        # Version compatibility
        "vllm_version": vllm_version,
        "nixl_connector_version": NIXL_CONNECTOR_VERSION,
        # Model architecture - affects KV cache shape
        "model": model_config.model,
        "dtype": str(model_config.dtype),
        "num_kv_heads": model_config.get_total_num_kv_heads(),
        "head_size": model_config.get_head_size(),
        "num_hidden_layers": model_config.get_total_num_hidden_layers(),
        # Attention backend and KV cache dtype affect memory layout
        "attn_backend_name": attn_backend_name,
        "cache_dtype": str(cache_config.cache_dtype),
        "is_hma_enabled": is_hma_enabled,
        "speculative_config": _get_speculative_compatibility_factors(vllm_config),
        # push (WRITE) and pull (READ) connectors are protocol-incompatible
        "transfer_mode": transfer_mode,
    }

    compat_hash = hash_factors(factors)
    logger.debug(
        "NIXL compatibility hash: %s (model=%s, dtype=%s, num_kv_heads=%d, "
        "cache_dtype=%s, attn_backend=%s)",
        compat_hash,
        factors["model"],
        factors["dtype"],
        factors["num_kv_heads"],
        factors["cache_dtype"],
        attn_backend_name,
    )
    return compat_hash