[Benchmark] Complete pooling throughput reporting

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>

Author: taneem-ibrahim

rust/src/bench/src/backends/pooling.rs

@@ -39,6 +39,8 @@ impl PoolingBackend {
         input: &RequestFuncInput,
         client: &reqwest::Client,
     ) -> Result<RequestFuncOutput> {
+        let payload = self.build_payload(input);
+
         // Preserve client-side prompt_len as fallback if server doesn't report usage.
         let mut output = RequestFuncOutput {
             prompt_len: input.prompt_len,
@@ -47,9 +49,10 @@ impl PoolingBackend {
                     input.prompt_list.as_ref().map_or(1, |list| list.len())
                 }
                 BackendKind::OpenaiEmbeddingsChat => 1,
-                BackendKind::VllmRerank => {
-                    input.prompt_list.as_ref().map_or(1, |list| list.len().saturating_sub(1))
-                }
+                BackendKind::VllmRerank => payload
+                    .get("documents")
+                    .and_then(|documents| documents.as_array())
+                    .map_or(0, |documents| documents.len()),
                 _ => unreachable!("PoolingBackend with non-pooling kind"),
             },
             ..Default::default()
@@ -61,8 +64,6 @@ impl PoolingBackend {
             &input.request_id,
         );
 
-        let payload = self.build_payload(input);
-
         let mut request = client.post(&input.api_url);
         for (k, v) in &headers {
             request = request.header(k, v);

rust/src/bench/src/compare.rs

@@ -16,6 +16,11 @@ const METRICS: &[MetricDef] = &[
         key: "request_throughput",
         lower_is_better: false,
     },
+    MetricDef {
+        label: "Input throughput (inputs/s)",
+        key: "input_sequence_throughput",
+        lower_is_better: false,
+    },
     MetricDef {
         label: "Output throughput (tok/s)",
         key: "output_throughput",

rust/src/bench/src/multi_run.rs

@@ -7,6 +7,7 @@ use crate::error::Result;
 /// Key metrics extracted from a single run's JSON result.
 struct RunMetrics {
     request_throughput: f64,
+    input_sequence_throughput: Option<f64>,
     output_throughput: f64,
     total_token_throughput: f64,
     mean_ttft_ms: f64,
@@ -38,6 +39,7 @@ impl RunMetrics {
     fn from_json(json: &serde_json::Value) -> Self {
         Self {
             request_throughput: get(json, "request_throughput"),
+            input_sequence_throughput: get_opt(json, "input_sequence_throughput"),
             output_throughput: get(json, "output_throughput"),
             total_token_throughput: get(json, "total_token_throughput"),
             mean_ttft_ms: get(json, "mean_ttft_ms"),
@@ -71,6 +73,10 @@ fn get(json: &serde_json::Value, key: &str) -> f64 {
     json.get(key).and_then(|v| v.as_f64()).unwrap_or(0.0)
 }
 
+fn get_opt(json: &serde_json::Value, key: &str) -> Option<f64> {
+    json.get(key).and_then(|v| v.as_f64())
+}
+
 fn get_ss_opt(json: &serde_json::Value, key: &str) -> Option<f64> {
     json.get("steady_state")
         .and_then(|ss| if ss.is_null() { None } else { Some(ss) })
@@ -119,8 +125,15 @@ fn print_multi_run_summary(runs: &[RunMetrics]) {
     let n = runs.len();
 
     // Collect each metric into a series, compute stats
-    let stats = vec![
-        compute_stats("Request throughput (req/s)", runs, |r| r.request_throughput),
+    let mut stats = vec![compute_stats("Request throughput (req/s)", runs, |r| {
+        r.request_throughput
+    })];
+    if runs.iter().all(|r| r.input_sequence_throughput.is_some()) {
+        stats.push(compute_stats("Input throughput (inputs/s)", runs, |r| {
+            r.input_sequence_throughput.unwrap_or_default()
+        }));
+    }
+    stats.extend([
         compute_stats("Output throughput (tok/s)", runs, |r| r.output_throughput),
         compute_stats("Total token throughput (tok/s)", runs, |r| {
             r.total_token_throughput
@@ -138,7 +151,7 @@ fn print_multi_run_summary(runs: &[RunMetrics]) {
         compute_stats("Completed requests", runs, |r| r.completed),
         compute_stats("Failed requests", runs, |r| r.failed),
         compute_stats("Duration (s)", runs, |r| r.duration),
-    ];
+    ]);
 
     println!("{:=^80}", format!(" Multi-Run Summary ({n} runs) "));
     println!(
@@ -307,6 +320,7 @@ mod tests {
     fn mk_run(ss: Option<f64>) -> RunMetrics {
         RunMetrics {
             request_throughput: 0.0,
+            input_sequence_throughput: None,
             output_throughput: 0.0,
             total_token_throughput: 0.0,
             mean_ttft_ms: 0.0,

rust/src/bench/src/sweep.rs

@@ -295,6 +295,7 @@ fn print_sweep_summary(param_name: &str, points: &[SweepPoint], summary_percenti
 fn build_summary_columns(summary_percentiles: &[f64]) -> Vec<SummaryColumn> {
     let mut columns = vec![
         SummaryColumn::new("Req/s", "request_throughput", 10),
+        SummaryColumn::new("Inputs/s", "input_sequence_throughput", 10),
         SummaryColumn::new("Tok/s", "output_throughput", 10),
         SummaryColumn::new("Total tok/s", "total_token_throughput", 12),
         SummaryColumn::new("SS req/s", SS_REQUEST_THROUGHPUT_KEY, 10),
@@ -452,6 +453,7 @@ mod tests {
             headers,
             vec![
                 "Req/s",
+                "Inputs/s",
                 "Tok/s",
                 "Total tok/s",
                 "SS req/s",
@@ -495,6 +497,7 @@ mod tests {
             headers,
             vec![
                 "Req/s",
+                "Inputs/s",
                 "Tok/s",
                 "Total tok/s",
                 "SS req/s",