Skip to content

[Pooling] Report input throughput for batched requests #53213

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
taneem-ibrahim wants to merge 4 commits into
base: main
Choose a base branch
from
Open

[Pooling] Report input throughput for batched requests #53213

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3d6013f
[Benchmark] Report batched pooling input throughput
taneem-ibrahim Aug 21, 2026
fc2928a
[Benchmark] Complete pooling throughput reporting
taneem-ibrahim Aug 21, 2026
fcefe0c
[Benchmark] Count batched token ID inputs
taneem-ibrahim Aug 21, 2026
80e3fef
Merge branch 'main' into report-batched-embedding-throughput
mergify[bot] Aug 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions rust/src/bench/src/backends/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ pub struct RequestFuncOutput {
pub prompt_len: usize,
pub error: String,
pub start_time: f64,
pub num_input_sequences: usize,
}

impl Default for RequestFuncOutput {
Expand All @@ -116,6 +117,7 @@ impl Default for RequestFuncOutput {
prompt_len: 0,
error: String::new(),
start_time: 0.0,
num_input_sequences: 1,
}
}
}
Expand Down
15 changes: 13 additions & 2 deletions rust/src/bench/src/backends/pooling.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,22 @@ impl PoolingBackend {
input: &RequestFuncInput,
client: &reqwest::Client,
) -> Result<RequestFuncOutput> {
let payload = self.build_payload(input);

// Preserve client-side prompt_len as fallback if server doesn't report usage.
let mut output = RequestFuncOutput {
prompt_len: input.prompt_len,
num_input_sequences: match self.kind {
BackendKind::OpenaiEmbeddings | BackendKind::VllmPooling => {
input.prompt_list.as_ref().map_or(1, |list| list.len())
}
BackendKind::OpenaiEmbeddingsChat => 1,
BackendKind::VllmRerank => payload
.get("documents")
.and_then(|documents| documents.as_array())
.map_or(0, |documents| documents.len()),
_ => unreachable!("PoolingBackend with non-pooling kind"),
},
Comment on lines +47 to +57

@yewentao256 yewentao256 Aug 21, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When prompt_list is None, documents can still come from extra_body, but this always reports one input even if multiple documents are sent?

..Default::default()
};

Expand All @@ -51,8 +64,6 @@ impl PoolingBackend {
&input.request_id,
);

let payload = self.build_payload(input);

let mut request = client.post(&input.api_url);
for (k, v) in &headers {
request = request.header(k, v);
Expand Down
5 changes: 5 additions & 0 deletions rust/src/bench/src/compare.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,11 @@ const METRICS: &[MetricDef] = &[
key: "request_throughput",
lower_is_better: false,
},
MetricDef {
label: "Input throughput (inputs/s)",
key: "input_sequence_throughput",
lower_is_better: false,
},
MetricDef {
label: "Output throughput (tok/s)",
key: "output_throughput",
Expand Down
6 changes: 6 additions & 0 deletions rust/src/bench/src/metrics/calculator.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,8 +184,10 @@ pub fn calculate_metrics(
completed,
failed,
total_input,
total_input_sequences: completed,
total_output,
request_throughput: completed as f64 / dur_s,
input_sequence_throughput: completed as f64 / dur_s,
request_goodput,
input_throughput: total_input as f64 / dur_s,
output_throughput: total_output as f64 / dur_s,
Expand Down Expand Up @@ -291,6 +293,7 @@ pub fn calculate_embedding_metrics(
selected_percentiles: &[f64],
) -> BenchmarkMetrics {
let mut total_input: usize = 0;
let mut total_input_sequences: usize = 0;
let mut completed: usize = 0;
let mut e2els: Vec<f64> = Vec::new();

Expand All @@ -299,6 +302,7 @@ pub fn calculate_embedding_metrics(
e2els.push(output.latency);
completed += 1;
total_input += output.prompt_len;
total_input_sequences += output.num_input_sequences;
}
}

Expand Down Expand Up @@ -344,8 +348,10 @@ pub fn calculate_embedding_metrics(
completed,
failed,
total_input,
total_input_sequences,
total_output: 0,
request_throughput: completed as f64 / dur_s,
input_sequence_throughput: total_input_sequences as f64 / dur_s,
request_goodput: 0.0,
input_throughput: total_input as f64 / dur_s,
output_throughput: 0.0,
Expand Down
2 changes: 2 additions & 0 deletions rust/src/bench/src/metrics/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,10 @@ pub struct BenchmarkMetrics {
pub completed: usize,
pub failed: usize,
pub total_input: usize,
pub total_input_sequences: usize,
pub total_output: usize,
pub request_throughput: f64,
pub input_sequence_throughput: f64,
pub request_goodput: f64,
pub input_throughput: f64,
pub output_throughput: f64,
Expand Down
20 changes: 17 additions & 3 deletions rust/src/bench/src/multi_run.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ use crate::error::Result;
/// Key metrics extracted from a single run's JSON result.
struct RunMetrics {
request_throughput: f64,
input_sequence_throughput: Option<f64>,
output_throughput: f64,
total_token_throughput: f64,
mean_ttft_ms: f64,
Expand Down Expand Up @@ -38,6 +39,7 @@ impl RunMetrics {
fn from_json(json: &serde_json::Value) -> Self {
Self {
request_throughput: get(json, "request_throughput"),
input_sequence_throughput: get_opt(json, "input_sequence_throughput"),
output_throughput: get(json, "output_throughput"),
total_token_throughput: get(json, "total_token_throughput"),
mean_ttft_ms: get(json, "mean_ttft_ms"),
Expand Down Expand Up @@ -71,6 +73,10 @@ fn get(json: &serde_json::Value, key: &str) -> f64 {
json.get(key).and_then(|v| v.as_f64()).unwrap_or(0.0)
}

fn get_opt(json: &serde_json::Value, key: &str) -> Option<f64> {
json.get(key).and_then(|v| v.as_f64())
}

fn get_ss_opt(json: &serde_json::Value, key: &str) -> Option<f64> {
json.get("steady_state")
.and_then(|ss| if ss.is_null() { None } else { Some(ss) })
Expand Down Expand Up @@ -119,8 +125,15 @@ fn print_multi_run_summary(runs: &[RunMetrics]) {
let n = runs.len();

// Collect each metric into a series, compute stats
let stats = vec![
compute_stats("Request throughput (req/s)", runs, |r| r.request_throughput),
let mut stats = vec![compute_stats("Request throughput (req/s)", runs, |r| {
r.request_throughput
})];
if runs.iter().all(|r| r.input_sequence_throughput.is_some()) {
stats.push(compute_stats("Input throughput (inputs/s)", runs, |r| {
r.input_sequence_throughput.unwrap_or_default()
}));
}
stats.extend([
compute_stats("Output throughput (tok/s)", runs, |r| r.output_throughput),
compute_stats("Total token throughput (tok/s)", runs, |r| {
r.total_token_throughput
Expand All @@ -138,7 +151,7 @@ fn print_multi_run_summary(runs: &[RunMetrics]) {
compute_stats("Completed requests", runs, |r| r.completed),
compute_stats("Failed requests", runs, |r| r.failed),
compute_stats("Duration (s)", runs, |r| r.duration),
];
]);

println!("{:=^80}", format!(" Multi-Run Summary ({n} runs) "));
println!(
Expand Down Expand Up @@ -307,6 +320,7 @@ mod tests {
fn mk_run(ss: Option<f64>) -> RunMetrics {
RunMetrics {
request_throughput: 0.0,
input_sequence_throughput: None,
output_throughput: 0.0,
total_token_throughput: 0.0,
mean_ttft_ms: 0.0,
Expand Down
6 changes: 6 additions & 0 deletions rust/src/bench/src/output/console.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,12 @@ pub fn print_results(
"{:<40} {:<10.2}",
"Request throughput (req/s):", metrics.request_throughput
);
if is_pooling {
println!(
"{:<40} {:<10.2}",
"Input throughput (inputs/s):", metrics.input_sequence_throughput
);
}
if metrics.request_goodput > 0.0 {
println!(
"{:<40} {:<10.2}",
Expand Down
10 changes: 10 additions & 0 deletions rust/src/bench/src/output/json.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,16 @@ pub fn build_result_json(
"request_throughput".into(),
serde_json::json!(metrics.request_throughput),
);
if is_pooling {
result.insert(
"total_input_sequences".into(),
serde_json::json!(metrics.total_input_sequences),
);
result.insert(
"input_sequence_throughput".into(),
serde_json::json!(metrics.input_sequence_throughput),
);
Comment on lines +106 to +109

@yewentao256 yewentao256 Aug 21, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we also expose this metric in the Rust compare, multi-run, and sweep summaries?

}
if config.goodput.is_empty() {
result.insert("request_goodput".into(), Value::Null);
} else {
Expand Down
3 changes: 3 additions & 0 deletions rust/src/bench/src/sweep.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -295,6 +295,7 @@ fn print_sweep_summary(param_name: &str, points: &[SweepPoint], summary_percenti
fn build_summary_columns(summary_percentiles: &[f64]) -> Vec<SummaryColumn> {
let mut columns = vec![
SummaryColumn::new("Req/s", "request_throughput", 10),
SummaryColumn::new("Inputs/s", "input_sequence_throughput", 10),
SummaryColumn::new("Tok/s", "output_throughput", 10),
SummaryColumn::new("Total tok/s", "total_token_throughput", 12),
SummaryColumn::new("SS req/s", SS_REQUEST_THROUGHPUT_KEY, 10),
Expand Down Expand Up @@ -452,6 +453,7 @@ mod tests {
headers,
vec![
"Req/s",
"Inputs/s",
"Tok/s",
"Total tok/s",
"SS req/s",
Expand Down Expand Up @@ -495,6 +497,7 @@ mod tests {
headers,
vec![
"Req/s",
"Inputs/s",
"Tok/s",
"Total tok/s",
"SS req/s",
Expand Down
2 changes: 1 addition & 1 deletion vllm/benchmarks/datasets/datasets.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ class SampleRequest:
Represents a single inference request for benchmarking.
"""

prompt: str | list[str] | list[int] | list[dict]
prompt: str | list[str] | list[int] | list[list[int]] | list[dict]
prompt_len: int
expected_output_len: int = 0
multi_modal_data: MultiModalDataDict | dict | list[dict] | None = None
Expand Down
16 changes: 14 additions & 2 deletions vllm/benchmarks/lib/endpoint_request_func.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ def add_chunk(self, chunk_bytes: bytes) -> list[str]:
class RequestFuncInput:
"""The input for the request function."""

prompt: str | list[str] | list[int] | list[dict[str, Any]]
prompt: str | list[str] | list[int] | list[list[int]] | list[dict[str, Any]]
api_url: str
prompt_len: int
output_len: int
Expand Down Expand Up @@ -100,6 +100,7 @@ class RequestFuncOutput:
error: str = ""
start_time: float = 0.0
input_audio_duration: float = 0.0 # in seconds
num_input_sequences: int = 1


class RequestFunc(Protocol):
Expand Down Expand Up @@ -586,8 +587,9 @@ async def _run_pooling_request(
payload: dict[str, Any],
headers: dict[str, Any],
pbar: tqdm | None = None,
num_input_sequences: int = 1,
) -> RequestFuncOutput:
output = RequestFuncOutput()
output = RequestFuncOutput(num_input_sequences=num_input_sequences)
st = time.perf_counter()
output.start_time = st
try:
Expand Down Expand Up @@ -617,6 +619,12 @@ async def _run_pooling_request(
return output


def _get_num_input_sequences(prompt: Any) -> int:
if prompt and isinstance(prompt, list) and isinstance(prompt[0], (str, list)):
return len(prompt)
return 1
Comment on lines +622 to +625

@yewentao256 yewentao256 Aug 21, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Will list[list[int]] be supported?

@taneem-ibrahim taneem-ibrahim Aug 21, 2026 •
edited
Loading

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Very good point. The current helper treats list[list[int]] as one input. I’ll count both list[str] and list[list[int]] as batches while preserving list[int] as a single input, and update the type annotations accordingly.

Reproducer to test:

.vent/bin/python - <<'PY'
from vllm.benchmarks.lib.endpoint_request_func import _get_num_input_sequences

cases = [
    ("text batch", ["a", "b"], 2),
    ("token-ID batch", [[101, 102], [103, 104]], 2),
    ("single token-ID input", [101, 102], 1),
    ("chat content", [{"type": "text", "text": "hello"}], 1),
]

for name, value, expected in cases:
    actual = _get_num_input_sequences(value)
    print(f"{name}: expected={expected}, actual={actual}")
    assert actual == expected

print("All input-shape checks passed")
PY

Prior to the latest commit, we get an AssertionError:

text batch: expected=2, actual=2
token-ID batch: expected=2, actual=1
Traceback (most recent call last):
  File "<stdin>", line 13, in <module>
AssertionError

After making the changes, it passes:

text batch: expected=2, actual=2
token-ID batch: expected=2, actual=2
single token-ID input: expected=1, actual=1
chat content: expected=1, actual=1
All input-shape checks passed

Thanks for the review!



async def async_request_openai_embeddings(
request_func_input: RequestFuncInput,
session: aiohttp.ClientSession,
Expand Down Expand Up @@ -645,6 +653,7 @@ async def async_request_openai_embeddings(
payload=payload,
headers=headers,
pbar=pbar,
num_input_sequences=_get_num_input_sequences(request_func_input.prompt),
)


Expand Down Expand Up @@ -681,6 +690,7 @@ async def async_request_vllm_rerank(
payload=payload,
headers=headers,
pbar=pbar,
num_input_sequences=len(request_func_input.prompt) - 1,
)


Expand Down Expand Up @@ -818,6 +828,7 @@ async def async_request_infinity_embeddings(
payload=payload,
headers=headers,
pbar=pbar,
num_input_sequences=_get_num_input_sequences(request_func_input.prompt),
)


Expand Down Expand Up @@ -866,6 +877,7 @@ async def async_request_vllm_pooling(
payload=payload,
headers=headers,
pbar=pbar,
num_input_sequences=_get_num_input_sequences(request_func_input.prompt),
)


Expand Down
14 changes: 14 additions & 0 deletions vllm/benchmarks/serve.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -357,7 +357,9 @@ class EmbedBenchmarkMetrics:
completed: int
failed: int
total_input: int
total_input_sequences: int
request_throughput: float
input_sequence_throughput: float
total_token_throughput: float
mean_e2el_ms: float
std_e2el_ms: float
Expand Down Expand Up @@ -520,6 +522,7 @@ def calculate_metrics_for_embeddings(
The calculated benchmark metrics.
"""
total_input = 0
total_input_sequences = 0
completed = 0
failed = 0
e2els: list[float] = []
Expand All @@ -528,6 +531,7 @@ def calculate_metrics_for_embeddings(
e2els.append(outputs[i].latency)
completed += 1
total_input += outputs[i].prompt_len
total_input_sequences += outputs[i].num_input_sequences
else:
failed += 1

Expand All @@ -541,7 +545,9 @@ def calculate_metrics_for_embeddings(
completed=completed,
failed=failed,
total_input=total_input,
total_input_sequences=total_input_sequences,
request_throughput=completed / dur_s,
input_sequence_throughput=total_input_sequences / dur_s,
total_token_throughput=total_input / dur_s,
mean_e2el_ms=np.mean(e2els or 0) * 1000,
std_e2el_ms=np.std(e2els or 0) * 1000,
Expand Down Expand Up @@ -1181,6 +1187,12 @@ async def probe_loop():
"Request throughput (req/s):", metrics.request_throughput
)
)
if isinstance(metrics, EmbedBenchmarkMetrics):
print(
"{:<40} {:<10.2f}".format(
"Input throughput (inputs/s):", metrics.input_sequence_throughput
)
)
if goodput_config_dict and isinstance(metrics, BenchmarkMetrics):
print(
"{:<40} {:<10.2f}".format(
Expand Down Expand Up @@ -1284,7 +1296,9 @@ async def probe_loop():
"duration": benchmark_duration,
"completed": metrics.completed,
"total_input_tokens": metrics.total_input,
"total_input_sequences": metrics.total_input_sequences,
"request_throughput": metrics.request_throughput,
"input_sequence_throughput": metrics.input_sequence_throughput,
"total_token_throughput": metrics.total_token_throughput,
"input_lens": [output.prompt_len for output in outputs],
"errors": [output.error for output in outputs],
Expand Down
Loading