Skip to content

vllm.entrypoints.openai.engine.protocol

Classes:

Functions:

GenerationError

Bases: VLLMServerError

raised when finish_reason indicates internal server error (500)

Source code in vllm/entrypoints/openai/engine/protocol.py 
442
443
444
445
446
447
class GenerationError(VLLMServerError):
    """raised when finish_reason indicates internal server error (500)"""

    def __init__(self, message: str = "Internal server error"):
        super().__init__(message)
        self.status_code = HTTPStatus.INTERNAL_SERVER_ERROR

PromptTokenUsageInfo

Bases: OpenAIBaseModel

Attributes:

Source code in vllm/entrypoints/openai/engine/protocol.py 
110
111
112
113
114
115
116
117
class PromptTokenUsageInfo(OpenAIBaseModel):
    cached_tokens: int | None = None
    created_cache_tokens: int | None = None
    multimodal_tokens: dict[str, int] | None = None
    """Prompt tokens contributed by each input modality, keyed by modality name
    (e.g. `image`, `audio`, `video`). A breakdown of the multimodal
    placeholder tokens already counted in `prompt_tokens`; `None` when the
    request has no multimodal input."""

multimodal_tokens = None class-attribute instance-attribute

Prompt tokens contributed by each input modality, keyed by modality name (e.g. image, audio, video). A breakdown of the multimodal placeholder tokens already counted in prompt_tokens; None when the request has no multimodal input.

SpeculativeDecodingMetrics

Bases: OpenAIBaseModel

Per-request speculative-decoding acceptance metrics.

Experimental, subject to change. Only populated for single-sequence requests (n == 1); null for n > 1, mirroring the timing metrics.

Source code in vllm/entrypoints/openai/engine/protocol.py 
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
class SpeculativeDecodingMetrics(OpenAIBaseModel):
    """Per-request speculative-decoding acceptance metrics.

    Experimental, subject to change. Only populated for single-sequence requests
    (`n == 1`); `null` for `n > 1`, mirroring the timing metrics.
    """

    mean_acceptance_length: float
    draft_acceptance_rate: float
    # Dense histogram: index j holds the number of verify steps that accepted
    # exactly j draft tokens (length num_spec_tokens + 1). Excludes the
    # always-accepted bonus token.
    acceptance_histogram: list[int]
    num_spec_steps: int
    num_accepted_draft_tokens: int
    num_draft_tokens: int
    num_spec_tokens: int
    # Ordered per-verify-step arrays; populated only at the `detailed` level.
    per_step_accepted: list[int] | None = None
    per_step_drafted: list[int] | None = None

structured_outputs_from_response_format(structured_outputs, response_format)

Apply response_format overrides to structured_outputs.

Source code in vllm/entrypoints/openai/engine/protocol.py 
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
def structured_outputs_from_response_format(
    structured_outputs: StructuredOutputsParams | None,
    response_format: AnyResponseFormat | None,
) -> StructuredOutputsParams | None:
    """Apply ``response_format`` overrides to ``structured_outputs``."""
    if response_format is None or response_format.type == "text":
        return structured_outputs

    overrides: dict[str, Any]
    if response_format.type == "json_object":
        overrides = {"json_object": True}
    elif response_format.type == "json_schema":
        json_schema = response_format.json_schema
        assert json_schema is not None
        overrides = {"json": json_schema.json_schema}
    else:
        assert isinstance(
            response_format,
            (
                LegacyStructuralTagResponseFormat,
                StructuralTagResponseFormat,
            ),
        )
        overrides = {
            "structural_tag": json.dumps(response_format.model_dump(by_alias=True))
        }

    if structured_outputs is None:
        return StructuredOutputsParams(**overrides)

    return replace(structured_outputs, **overrides)

validate_structural_tag_response_format(response_format)

Validate structural tags before they are sent to the engine.

Engine-side validation reports malformed structural tags as generation failures. OpenAI request parsing should classify them as bad requests.

Source code in vllm/entrypoints/openai/engine/protocol.py 
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
def validate_structural_tag_response_format(
    response_format: AnyStructuralTagResponseFormat | dict[str, Any],
) -> None:
    """Validate structural tags before they are sent to the engine.

    Engine-side validation reports malformed structural tags as generation
    failures. OpenAI request parsing should classify them as bad requests.
    """
    from pydantic import TypeAdapter, ValidationError

    if isinstance(response_format, dict):
        try:
            response_format = TypeAdapter(
                AnyStructuralTagResponseFormat
            ).validate_python(response_format)
        except ValidationError as exc:
            raise VLLMValidationError(
                "Invalid response_format structural_tag specification.",
                parameter="response_format",
            ) from exc

    try:
        payload = json.dumps(response_format.model_dump(by_alias=True))
        validate_structural_tag_payload(payload, parameter="response_format")
    except (TypeError, ValueError) as exc:
        raise VLLMValidationError(
            "Invalid response_format structural_tag specification.",
            parameter="response_format",
        ) from exc