Improve release and benchmark tooling

src/bin/dosh-bench.rs

@@ -2,6 +2,8 @@ use anyhow::{Context, Result, anyhow};
 use clap::Parser;
 use portable_pty::{CommandBuilder, NativePtySystem, PtySize, PtySystem};
 use std::ffi::OsStr;
+use std::fmt::Write as _;
+use std::fs;
 use std::io::Read;
 use std::path::PathBuf;
 use std::process::Command;
@@ -63,6 +65,12 @@ struct Args {
     /// Emit machine-readable JSON (one object per metric, with raw samples).
     #[arg(long)]
     json: bool,
+    /// Emit a publishable Markdown benchmark report.
+    #[arg(long)]
+    markdown: bool,
+    /// Write the report to a file instead of stdout.
+    #[arg(long)]
+    output: Option<PathBuf>,
     /// Optional label printed in summary/JSON output (e.g. machine/OS identifier).
     #[arg(long)]
     label: Option<String>,
@@ -179,11 +187,14 @@ fn main() -> Result<()> {
         results.push(MetricSamples::new("mosh_start_true_ms", mosh_times));
     }
 
-    if args.json {
-        print_json(&args, &results);
+    let report = if args.json {
+        render_json(&args, &results)
+    } else if args.markdown {
+        render_markdown(&args, &results)
     } else {
-        print_table(&args, &results);
-    }
+        render_table(&args, &results)
+    };
+    write_report(&args, &report)?;
 
     run_assertions(&args, &results)?;
     Ok(())
@@ -560,17 +571,20 @@ fn percentile(sorted: &[f64], pct: f64) -> f64 {
     }
 }
 
-fn print_table(args: &Args, results: &[MetricSamples]) {
+fn render_table(args: &Args, results: &[MetricSamples]) -> String {
+    let mut out = String::new();
     if let Some(label) = &args.label {
-        println!("# label: {label}");
+        let _ = writeln!(out, "# label: {label}");
     }
-    println!(
+    let _ = writeln!(
+        out,
         "{:<24} {:>6} {:>9} {:>9} {:>9} {:>9} {:>9}",
         "metric", "n", "min", "median", "p95", "mean", "max"
     );
     for metric in results {
         let s = metric.stats();
-        println!(
+        let _ = writeln!(
+            out,
             "{:<24} {:>6} {:>9.2} {:>9.2} {:>9.2} {:>9.2} {:>9.2}",
             metric.name, s.count, s.min, s.median, s.p95, s.mean, s.max
         );
@@ -578,11 +592,12 @@ fn print_table(args: &Args, results: &[MetricSamples]) {
     // Raw per-iteration samples, so published numbers can include raw data.
     for metric in results {
         let ms: Vec<String> = metric.ms().iter().map(|v| format!("{v:.2}")).collect();
-        println!("{} samples_ms=[{}]", metric.name, ms.join(", "));
+        let _ = writeln!(out, "{} samples_ms=[{}]", metric.name, ms.join(", "));
     }
+    out
 }
 
-fn print_json(args: &Args, results: &[MetricSamples]) {
+fn render_json(args: &Args, results: &[MetricSamples]) -> String {
     let mut entries = Vec::new();
     for metric in results {
         let s = metric.stats();
@@ -603,11 +618,58 @@ fn print_json(args: &Args, results: &[MetricSamples]) {
         Some(label) => format!("\"{}\"", label.replace('"', "\\\"")),
         None => "null".to_string(),
     };
-    println!(
+    format!(
         "{{\"label\":{label},\"iterations\":{},\"metrics\":[{}]}}",
         args.iterations.max(1),
         entries.join(",")
+    )
+}
+
+fn render_markdown(args: &Args, results: &[MetricSamples]) -> String {
+    let mut out = String::new();
+    let label = args.label.as_deref().unwrap_or("Dosh benchmark");
+    let _ = writeln!(out, "# {label}\n");
+    let _ = writeln!(out, "- server: `{}`", args.server);
+    let _ = writeln!(out, "- iterations: `{}`", args.iterations.max(1));
+    let _ = writeln!(
+        out,
+        "- generated_by: `dosh-bench {}`\n",
+        env!("CARGO_PKG_VERSION")
     );
+    let _ = writeln!(
+        out,
+        "| metric | n | min ms | median ms | p95 ms | mean ms | max ms |"
+    );
+    let _ = writeln!(out, "|---|---:|---:|---:|---:|---:|---:|");
+    for metric in results {
+        let s = metric.stats();
+        let _ = writeln!(
+            out,
+            "| `{}` | {} | {:.2} | {:.2} | {:.2} | {:.2} | {:.2} |",
+            metric.name, s.count, s.min, s.median, s.p95, s.mean, s.max
+        );
+    }
+    let _ = writeln!(out, "\n## Raw Samples\n");
+    for metric in results {
+        let ms: Vec<String> = metric.ms().iter().map(|v| format!("{v:.2}")).collect();
+        let _ = writeln!(out, "- `{}`: [{}]", metric.name, ms.join(", "));
+    }
+    out
+}
+
+fn write_report(args: &Args, report: &str) -> Result<()> {
+    if let Some(path) = &args.output {
+        if let Some(parent) = path.parent()
+            && !parent.as_os_str().is_empty()
+        {
+            fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?;
+        }
+        fs::write(path, report).with_context(|| format!("write {}", path.display()))?;
+        println!("wrote {}", path.display());
+    } else {
+        println!("{report}");
+    }
+    Ok(())
 }
 
 fn metric_mean(results: &[MetricSamples], name: &str) -> Option<f64> {
@@ -677,3 +739,69 @@ fn default_client_path() -> PathBuf {
         .and_then(|path| path.parent().map(|parent| parent.join("dosh-client")))
         .unwrap_or_else(|| PathBuf::from("dosh-client"))
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn args_for_render() -> Args {
+        Args {
+            server: "local".to_string(),
+            session: "default".to_string(),
+            ssh_port: 22,
+            dosh_port: 50000,
+            dosh_host: None,
+            iterations: 2,
+            local_auth: true,
+            cold_native: false,
+            cached_ticket: false,
+            resume: false,
+            client: None,
+            ssh_auth_command: "~/.local/bin/dosh-auth".to_string(),
+            ssh_key: None,
+            ssh_known_hosts: None,
+            ssh_control_path: None,
+            controlmaster: false,
+            no_cache: false,
+            warm_cache: false,
+            skip_ssh_baseline: true,
+            include_mosh: false,
+            mosh: PathBuf::from("mosh"),
+            mosh_server_command: "mosh-server".to_string(),
+            mosh_port: None,
+            json: false,
+            markdown: true,
+            output: None,
+            label: Some("test host".to_string()),
+            assert_ssh_plus_ms: None,
+            assert_mosh_minus_ms: None,
+            assert_dosh_max_ms: None,
+        }
+    }
+
+    #[test]
+    fn markdown_report_contains_summary_and_samples() {
+        let args = args_for_render();
+        let metrics = vec![MetricSamples::new(
+            "dosh_cached_attach_ms",
+            vec![Duration::from_millis(3), Duration::from_millis(5)],
+        )];
+        let report = render_markdown(&args, &metrics);
+        assert!(report.contains("# test host"));
+        assert!(report.contains("| `dosh_cached_attach_ms` | 2 | 3.00"));
+        assert!(report.contains("- `dosh_cached_attach_ms`: [3.00, 5.00]"));
+    }
+
+    #[test]
+    fn json_report_contains_metrics() {
+        let args = args_for_render();
+        let metrics = vec![MetricSamples::new(
+            "dosh_local_attach_ms",
+            vec![Duration::from_millis(7)],
+        )];
+        let report = render_json(&args, &metrics);
+        assert!(report.contains("\"label\":\"test host\""));
+        assert!(report.contains("\"metric\":\"dosh_local_attach_ms\""));
+        assert!(report.contains("\"samples_ms\":[7.0000]"));
+    }
+}