Skip to content

vllm.v1.worker.gpu.attn_utils

Classes:

Functions:

AttentionCGSupportInfo dataclass

Methods:

  • narrow –

    Return an info tightened by support if it is more restrictive.

Source code in vllm/v1/worker/gpu/attn_utils.py 
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
@dataclass(frozen=True)
class AttentionCGSupportInfo:
    min_cg_support: AttentionCGSupport = AttentionCGSupport.ALWAYS
    min_cg_attn_backend: str | None = None

    def narrow(
        self, support: AttentionCGSupport, backend: str | None
    ) -> "AttentionCGSupportInfo":
        """Return an info tightened by ``support`` if it is more restrictive.

        Lets attention groups built outside ``init_attn_backend`` (e.g.
        encoder-only layers) contribute to the runner's cudagraph decision.
        """
        if support.value < self.min_cg_support.value:
            return AttentionCGSupportInfo(support, backend)
        return self

narrow(support, backend)

Return an info tightened by support if it is more restrictive.

Lets attention groups built outside init_attn_backend (e.g. encoder-only layers) contribute to the runner's cudagraph decision.

Source code in vllm/v1/worker/gpu/attn_utils.py 
38
39
40
41
42
43
44
45
46
47
48
def narrow(
    self, support: AttentionCGSupport, backend: str | None
) -> "AttentionCGSupportInfo":
    """Return an info tightened by ``support`` if it is more restrictive.

    Lets attention groups built outside ``init_attn_backend`` (e.g.
    encoder-only layers) contribute to the runner's cudagraph decision.
    """
    if support.value < self.min_cg_support.value:
        return AttentionCGSupportInfo(support, backend)
    return self

compute_mm_prefix_ranges(req_ids, mm_features, sliding_window=None)

Compute PrefixLM bidirectional ranges for multimodal tokens.

Ranges exceeding sliding_window are skipped to prevent early tokens from attending across the entire image span.

Source code in vllm/v1/worker/gpu/attn_utils.py 
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
def compute_mm_prefix_ranges(
    req_ids: list[str],
    mm_features: dict[str, list[MultiModalFeatureSpec]],
    sliding_window: int | None = None,
) -> dict[int, list[tuple[int, int]]]:
    """Compute PrefixLM bidirectional ranges for multimodal tokens.

    Ranges exceeding sliding_window are skipped to prevent early tokens
    from attending across the entire image span.
    """
    req_doc_ranges: dict[int, list[tuple[int, int]]] = {}
    for req_idx, req_id in enumerate(req_ids):
        image_doc_ranges = []
        for mm_feature in mm_features.get(req_id, ()):
            if mm_feature.modality not in ("image", "video"):
                continue
            for r in mm_feature.mm_position.extract_embeds_range():
                if sliding_window is not None and (r[1] - r[0] + 1) > sliding_window:
                    continue
                image_doc_ranges.append(r)
        req_doc_ranges[req_idx] = image_doc_ranges
    return req_doc_ranges

get_query_lens_mismatch_unsupported_backend(attn_groups)

Name the first backend needing the CPU query lengths to be exact, if any.

The attention selector already excludes these when adaptive verification is enabled, but models that hard-wire their backend never consult it. See AttentionBackend.supports_device_cpu_query_lens_mismatch().

Source code in vllm/v1/worker/gpu/attn_utils.py 
168
169
170
171
172
173
174
175
176
177
178
179
180
181
def get_query_lens_mismatch_unsupported_backend(
    attn_groups: list[list[AttentionGroup]],
) -> str | None:
    """Name the first backend needing the CPU query lengths to be exact, if any.

    The attention selector already excludes these when adaptive verification is
    enabled, but models that hard-wire their backend never consult it. See
    AttentionBackend.supports_device_cpu_query_lens_mismatch().
    """
    for groups in attn_groups:
        for group in groups:
            if not group.backend.supports_device_cpu_query_lens_mismatch():
                return group.backend.__name__
    return None