diff --git a/Makefile b/Makefile index 2d90a7e..fd7f2d6 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build test fmt install bench-local bench-docker-ssh bench-docker-mosh +.PHONY: build test fmt install bench-local bench-local-json bench-docker-ssh bench-docker-mosh build: cargo build --release @@ -12,14 +12,16 @@ fmt: install: sh packaging/install.sh +# Safe, self-contained local benchmark matrix (native cold auth, cached attach +# ticket, local-auth) on a throwaway server bound to 127.0.0.1 on a free port in +# a temp HOME. Never touches the production server or UDP port 50000. bench-local: - cargo build - tmp="$$(mktemp -d)"; \ - HOME="$$tmp" target/debug/dosh-server serve >/tmp/dosh-bench-server.log 2>&1 & \ - pid="$$!"; \ - trap 'kill "$$pid" 2>/dev/null || true; rm -rf "$$tmp"' EXIT INT TERM; \ - sleep 0.5; \ - HOME="$$tmp" target/debug/dosh-bench --local-auth --server local --iterations 5 + sh scripts/bench-local.sh + +# Same matrix, machine-readable JSON output (one object per metric with raw +# samples). Useful for publishing or regression tracking. +bench-local-json: + DOSH_BENCH_JSON=1 sh scripts/bench-local.sh bench-docker-ssh: sh scripts/ci-docker-ssh-bench.sh diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md new file mode 100644 index 0000000..4f746b0 --- /dev/null +++ b/docs/BENCHMARKS.md @@ -0,0 +1,173 @@ +# Dosh Benchmarks + +This document explains how to run the Dosh benchmarks, what each metric means, +and how to read the results honestly. Dosh's headline claim is *insane speed on +repeat attach/reconnect* — these benchmarks exist to make that claim measurable +and reproducible. + +> **Read first:** `docs/PUBLIC_READINESS.md`. The defensible public claim is fast +> attach/reconnect, not full Mosh feature parity, and **not** cold startup. Cite +> `dosh_cached_attach_ms` for the core speed claim; cite cold metrics only to show +> the fallback path stays competitive with ordinary SSH. + +## TL;DR + +```bash +make bench-local # safe, self-contained matrix on a throwaway server +make bench-local-json # same, machine-readable JSON with raw samples +make bench-docker-ssh # containerized SSH-vs-Dosh gate used by CI +make bench-docker-mosh # same, with Mosh installed for a three-way comparison +``` + +`make bench-local` never touches a running production server: it builds release +binaries, starts its own `dosh-server` bound to `127.0.0.1` on a random free UDP +port inside a temporary `HOME`, runs the matrix, prints raw samples plus summary +statistics, and tears everything down on exit. It will refuse to bind UDP port +`50000` (the default production port). + +## The benchmark binary: `dosh-bench` + +`dosh-bench` spawns the real `dosh-client` once per iteration and times the whole +process from launch to terminal-ready-and-detached. That wall-clock cost is the +apples-to-apples comparison against `ssh host true`: it is the startup price each +tool pays before any useful remote work begins. + +Every run prints, per metric, a summary line (count, min, median, p95, mean, max +in milliseconds) **and** a raw-samples line. Pass `--json` for one machine-readable +object per run (with the full `samples_ms` array) so published numbers can always +include raw data, as required by `docs/PUBLIC_READINESS.md`. + +### Path matrix + +`dosh-bench` can benchmark these Dosh attach paths. Without an explicit path flag +it keeps its legacy single-path behavior (driven by `--local-auth` / `--warm-cache` +/ `--no-cache`) so existing CI scripts keep working. + +| Flag | Metric | What it measures | +| --- | --- | --- | +| `--cold-native` | `dosh_cold_native_ms` | Native cold auth: full `--auth native --no-cache` handshake to first frame and detach. The cold fallback when no cache exists. | +| `--cached-ticket` | `dosh_cached_attach_ms` | Cached attach-ticket fast path. Warms the cache once, then measures repeat attaches using cached UDP credentials/tickets. **This is the core speed claim.** | +| `--resume` | `dosh_resume_ms` | UDP resume of a session whose endpoint changes. See the caveat below — only meaningful when a live session is kept open out-of-band. | +| `--local-auth` | `dosh_local_attach_ms` | Self-contained local bootstrap; no SSH, no native handshake. Useful as a lower-bound sanity check on loopback. | +| *(default, no flag)* | `dosh_attach_ms` | Cold SSH bootstrap plus UDP attach (legacy single-path mode). | + +Existing baseline/comparison metrics are unchanged: + +| Metric | Meaning | +| --- | --- | +| `ssh_true_ms` | `ssh host true` with the same key/options. | +| `mosh_start_true_ms` | `mosh host -- true` bootstrap, run `true`, exit (timed inside a PTY). | + +### Assertions / gates + +| Flag | Gate | +| --- | --- | +| `--assert-ssh-plus-ms N` | The primary Dosh metric must be ≤ `ssh_true_ms` mean + `N` ms. | +| `--assert-mosh-minus-ms N` | The primary Dosh metric must be at least `N` ms faster than `mosh_start_true_ms`. | +| `--assert-dosh-max-ms N` | The primary Dosh metric mean must be ≤ `N` ms. | + +The "primary Dosh metric" for assertions is, in priority order: +`dosh_cached_attach_ms` → `dosh_resume_ms` → `dosh_cold_native_ms` → +`dosh_attach_ms` → `dosh_local_attach_ms`. + +## What each benchmark does + +### `make bench-local` (`scripts/bench-local.sh`) + +Self-contained, no SSH, no Docker. It: + +1. Builds release binaries (`DOSH_BENCH_PROFILE=debug` for the debug profile). +2. Picks a free UDP port (never `50000`) and a temp `HOME`. +3. Generates a throwaway client identity (`ssh-keygen`), authorizes it on the + server, and writes a throwaway server + client config (native auth, + trust-on-first-use, localhost UDP). +4. Starts `dosh-server serve` bound to `127.0.0.1` on that port. +5. Runs `dosh-bench --cold-native --cached-ticket`, then `--local-auth --no-cache`. +6. Sanity-checks that native cold auth actually trusted the host (so a silent + config-parse fallback can't make the benchmark measure the wrong path). +7. Kills the server and removes the temp `HOME` on exit. + +Tunables: `scripts/bench-local.sh [ITERATIONS]` (default 20), `DOSH_BENCH_JSON=1`, +`DOSH_BENCH_PROFILE=release|debug`. + +### `make bench-docker-ssh` / `make bench-docker-mosh` (`scripts/ci-docker-ssh-bench.sh`) + +Builds one Ubuntu image with OpenSSH, `dosh-server`, `dosh-auth` (and Mosh for the +`-mosh` target), then runs `ssh_true_ms`, cold + ControlMaster `dosh_attach_ms`, +`dosh_cached_attach_ms`, and optionally `mosh_start_true_ms` against the same +container, key, and loopback network path. This is the gate CI enforces; it asserts +cold Dosh stays within 500 ms of SSH and that cached attach stays under a small +budget. See `README.md` "Develop" for the exact invocations. + +## Methodology and caveats + +- **Same machine, same network path.** `bench-local` runs everything on loopback, + so it isolates Dosh's own process/handshake/render overhead and removes network + RTT from the comparison. Real-world cached attach is approximately *network RTT + plus the local overhead these numbers measure*. Publish the machine, OS, CPU, + network path, and sample count alongside any numbers. +- **Wall-clock of the whole client process.** Each sample includes process spawn, + attach, first-frame render, and detach. That is deliberately the same thing + `ssh host true` pays, but it means a few ms is fixed process-startup cost, not + protocol cost. +- **Do not cite cold metrics as the headline.** `dosh_cold_native_ms` and + `dosh_attach_ms` still pay first-authentication cost. They exist to show the + fallback path stays competitive with ordinary SSH. Cite `dosh_cached_attach_ms` + for repeat attach/reconnect, per `docs/PUBLIC_READINESS.md`. +- **UDP resume is the roaming path, not cold reconnect.** Resume reconnects a + client that is *still attached* when its network endpoint changes (sleep/Wi-Fi + switch). The `--attach-only` benchmark model detaches after every iteration, + which removes the client on the server, so a fresh-process resume has nothing + live to resume and is not meaningful in `bench-local`. The cold fresh-process + reconnect fast path is the attach ticket (`dosh_cached_attach_ms`). Roaming + resume correctness is covered by the integration test + `resume_updates_udp_endpoint_for_roaming` in `tests/integration_smoke.rs`. + `dosh-bench --resume` remains available for scenarios that keep a live session + open out-of-band (for example remote soak tests). +- **These are not identical workloads.** SSH/Mosh/Dosh do different work at + startup. The comparison is still useful because it measures the startup tax each + tool charges before useful remote work begins. + +## Sample results (loopback, self-contained) + +Captured with `make bench-local` (`scripts/bench-local.sh 30`), all times in +milliseconds, 30 samples per metric. + +- Machine: Intel Core i5-9500 @ 3.00 GHz, 6 cores +- OS: Ubuntu 24.04.4 LTS, Linux 6.8.0-124-generic, x86_64 +- Toolchain: rustc 1.96.0, release profile +- Network path: loopback (`127.0.0.1`), throwaway server on a free UDP port +- Workload: `dosh-client --attach-only` to first frame and detach + +| Metric | n | min | median | p95 | mean | max | +| --- | --- | --- | --- | --- | --- | --- | +| `dosh_cold_native_ms` | 30 | 8.10 | 9.01 | 10.40 | 9.18 | 10.82 | +| `dosh_cached_attach_ms` | 30 | 2.73 | 2.96 | 3.25 | 2.96 | 3.30 | +| `dosh_local_attach_ms` | 30 | 2.66 | 2.78 | 3.24 | 2.87 | 3.33 | + +Raw samples (ms): + +``` +dosh_cold_native_ms: +8.32, 8.23, 10.31, 10.37, 8.83, 8.81, 9.20, 9.04, 10.14, 8.98, 10.82, 9.87, +10.22, 10.42, 9.76, 9.09, 9.19, 8.94, 8.69, 8.67, 8.74, 9.33, 9.25, 8.92, 8.39, +8.55, 8.38, 9.81, 8.17, 8.10 + +dosh_cached_attach_ms: +2.99, 2.99, 2.93, 2.87, 2.95, 2.80, 2.80, 2.90, 3.10, 2.73, 3.19, 2.80, 2.87, +2.82, 2.89, 3.03, 3.06, 3.21, 2.97, 2.83, 3.00, 2.82, 3.04, 2.97, 3.03, 3.24, +3.30, 2.73, 2.79, 3.25 + +dosh_local_attach_ms: +3.05, 3.00, 2.84, 3.04, 3.33, 2.89, 2.76, 2.74, 2.73, 3.32, 3.14, 2.75, 2.83, +2.88, 2.99, 2.77, 2.75, 2.79, 2.69, 2.78, 2.75, 2.77, 2.75, 2.77, 3.03, 2.75, +2.94, 2.74, 2.82, 2.66 +``` + +Reading these numbers: cold native auth is ~9 ms because it pays the native +handshake; cached attach is ~3 ms because it reuses cached credentials and skips +the handshake entirely. On loopback there is no network RTT, so this is the local +overhead floor. Over a real link, expect cached attach ≈ this floor + one network +round trip — which is exactly the "near network RTT" target in the spec. There is +no SSH/Mosh baseline in this loopback table because those paths need an SSH server; +use `make bench-docker-ssh` / `make bench-docker-mosh` for the head-to-head. diff --git a/scripts/bench-local.sh b/scripts/bench-local.sh new file mode 100755 index 0000000..75db156 --- /dev/null +++ b/scripts/bench-local.sh @@ -0,0 +1,192 @@ +#!/usr/bin/env sh +# Safe, self-contained local Dosh benchmark harness. +# +# Spins up a THROWAWAY dosh-server bound to 127.0.0.1 on a random free port in a +# temp HOME, runs the full path matrix (native cold auth, cached attach-ticket, +# UDP resume, and local-auth), prints raw samples + summary stats, then tears +# everything down. +# +# It NEVER touches the production server: it never uses UDP port 50000, never +# restarts any systemd unit, and never reads or writes the real ~/.config/dosh +# or ~/.local. Everything lives under a mktemp HOME that is removed on exit. +# +# Usage: +# scripts/bench-local.sh [ITERATIONS] +# Environment: +# DOSH_BENCH_ITERS iteration count (default 20; overridden by $1) +# DOSH_BENCH_JSON=1 emit machine-readable JSON instead of the table +# DOSH_BENCH_PROFILE release|debug build profile (default release) +set -eu + +iters="${1:-${DOSH_BENCH_ITERS:-20}}" +profile="${DOSH_BENCH_PROFILE:-release}" + +repo_root="$(cd "$(dirname "$0")/.." && pwd)" +cd "$repo_root" + +# Make sure cargo is reachable in non-login shells. +if ! command -v cargo >/dev/null 2>&1; then + # shellcheck disable=SC1090 + [ -f "$HOME/.cargo/env" ] && . "$HOME/.cargo/env" +fi + +if [ "$profile" = "release" ]; then + cargo build --release >&2 + bindir="$repo_root/target/release" +else + cargo build >&2 + bindir="$repo_root/target/debug" +fi +server_bin="$bindir/dosh-server" +client_bin="$bindir/dosh-client" +bench_bin="$bindir/dosh-bench" + +# Pick a free UDP port that is NOT the production port (50000). +free_udp_port() { + python3 - <<'PY' +import socket +s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) +s.bind(("127.0.0.1", 0)) +print(s.getsockname()[1]) +s.close() +PY +} +dosh_port="$(free_udp_port)" +if [ "$dosh_port" = "50000" ]; then + dosh_port="$(free_udp_port)" +fi +if [ "$dosh_port" = "50000" ]; then + echo "refusing to bind production port 50000" >&2 + exit 1 +fi + +home="$(mktemp -d)" +server_pid="" +cleanup() { + if [ -n "$server_pid" ]; then + kill "$server_pid" 2>/dev/null || true + wait "$server_pid" 2>/dev/null || true + fi + rm -rf "$home" +} +trap cleanup EXIT INT TERM + +mkdir -p "$home/.config/dosh" "$home/.ssh" "$home/.local/share/dosh" + +# Client identity used by the native cold-auth path. +ssh-keygen -t ed25519 -N "" -q -f "$home/.ssh/id_ed25519" +client_pub="$(cat "$home/.ssh/id_ed25519.pub")" +printf '%s\n' "$client_pub" > "$home/authorized_keys" + +# Throwaway server config: localhost only, throwaway port, attach tickets on. +cat > "$home/.config/dosh/server.toml" < "$home/.config/dosh/client.toml" <"$home/server.log" 2>&1 & + server_pid="$!" + # Wait for the UDP socket to come up. + i=0 + while [ "$i" -lt 50 ]; do + if grep -q "listening on" "$home/server.log" 2>/dev/null; then + return 0 + fi + if ! kill -0 "$server_pid" 2>/dev/null; then + echo "dosh-server exited early:" >&2 + cat "$home/server.log" >&2 || true + exit 1 + fi + i=$((i + 1)) + sleep 0.1 + done +} + +run_bench() { + # $@ extra args forwarded to dosh-bench + json_flag="" + [ "${DOSH_BENCH_JSON:-0}" = "1" ] && json_flag="--json" + label="$(uname -s) $(uname -m), profile=$profile" + HOME="$home" "$bench_bin" \ + --client "$client_bin" \ + --server local \ + --dosh-host 127.0.0.1 \ + --dosh-port "$dosh_port" \ + --session default \ + --skip-ssh-baseline \ + --iterations "$iters" \ + --label "$label" \ + $json_flag \ + "$@" +} + +echo "dosh local benchmark: port=$dosh_port iters=$iters profile=$profile home=$home" >&2 + +# 1) Native cold auth + cached attach-ticket in one run (ticket cache on). +write_client_config true +start_server +echo "== native cold auth + cached attach-ticket ==" >&2 +run_bench --cold-native --cached-ticket +# Sanity: native cold auth must have trusted the host (proves the generated +# client config loaded and the native handshake ran instead of silently +# falling back to a different path). +if [ ! -s "$home/.config/dosh/known_hosts" ]; then + echo "native cold auth did not record a trusted host; check server.log:" >&2 + cat "$home/server.log" >&2 || true + exit 1 +fi +kill "$server_pid" 2>/dev/null || true +wait "$server_pid" 2>/dev/null || true +server_pid="" + +# 2) Self-contained local-auth path (no SSH, no native handshake). +# +# Note on UDP resume: resume is the roaming path for a client that is STILL +# attached when its network endpoint changes. The `--attach-only` benchmark +# model detaches after each iteration, which tears the client down on the +# server, so a fresh-process resume has nothing live to resume and is not +# meaningful here. The cold fresh-process reconnect fast path is the attach +# ticket (measured above). Roaming resume is validated by the integration test +# `resume_updates_udp_endpoint_for_roaming`. `dosh-bench --resume` exists for +# scenarios that keep a live session out-of-band (e.g. remote soak tests); see +# docs/BENCHMARKS.md. +rm -rf "$home/.local/share/dosh/credentials" +write_client_config true +start_server +echo "== local-auth (no SSH) ==" >&2 +run_bench --local-auth --no-cache diff --git a/src/bin/dosh-bench.rs b/src/bin/dosh-bench.rs index 88e69e2..0d1c8d2 100644 --- a/src/bin/dosh-bench.rs +++ b/src/bin/dosh-bench.rs @@ -25,6 +25,15 @@ struct Args { iterations: usize, #[arg(long)] local_auth: bool, + /// Benchmark native cold auth (no cache, native handshake) terminal-ready time. + #[arg(long)] + cold_native: bool, + /// Benchmark cached attach-ticket terminal-ready time (warms the cache first). + #[arg(long)] + cached_ticket: bool, + /// Benchmark UDP resume terminal-ready time (warms the cache first). + #[arg(long)] + resume: bool, #[arg(long)] client: Option, #[arg(long, default_value = "~/.local/bin/dosh-auth")] @@ -51,6 +60,12 @@ struct Args { mosh_server_command: String, #[arg(long)] mosh_port: Option, + /// Emit machine-readable JSON (one object per metric, with raw samples). + #[arg(long)] + json: bool, + /// Optional label printed in summary/JSON output (e.g. machine/OS identifier). + #[arg(long)] + label: Option, #[arg(long)] assert_ssh_plus_ms: Option, #[arg(long)] @@ -59,12 +74,45 @@ struct Args { assert_dosh_max_ms: Option, } +/// One Dosh attach path to benchmark. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum DoshPath { + /// `--auth native --no-cache`: full native handshake, no cache. + ColdNative, + /// Cached attach-ticket fast path (requires a warmed cache). + CachedTicket, + /// UDP resume fast path (requires a warmed cache + `cache_attach_tickets = false`). + Resume, + /// `--local-auth`: self-contained local bootstrap, no SSH. + LocalAuth, + /// SSH-bootstrap cold attach (cache controlled by `--no-cache` / `--warm-cache`). + SshBootstrap, +} + +impl DoshPath { + fn metric(self) -> &'static str { + match self { + DoshPath::ColdNative => "dosh_cold_native_ms", + DoshPath::CachedTicket => "dosh_cached_attach_ms", + DoshPath::Resume => "dosh_resume_ms", + DoshPath::LocalAuth => "dosh_local_attach_ms", + DoshPath::SshBootstrap => "dosh_attach_ms", + } + } + + /// Whether this path consumes a warmed credential cache. + fn needs_warm(self) -> bool { + matches!(self, DoshPath::CachedTicket | DoshPath::Resume) + } +} + fn main() -> Result<()> { let args = Args::parse(); let client = args.client.clone().unwrap_or_else(default_client_path); - let mut ssh_times = Vec::new(); - let mut dosh_times = Vec::new(); - let mut mosh_times = Vec::new(); + if args.no_cache && args.warm_cache { + return Err(anyhow!("--warm-cache cannot be used with --no-cache")); + } + let generated_control_path = if args.controlmaster { Some(std::env::temp_dir().join(format!("dosh-bench-control-{}", std::process::id()))) } else { @@ -78,96 +126,112 @@ fn main() -> Result<()> { } else { None }; - if args.no_cache && args.warm_cache { - return Err(anyhow!("--warm-cache cannot be used with --no-cache")); - } - let dosh_label = if args.warm_cache { - let _ = time_dosh_attach(&client, &args, control_path)?; - "dosh_cached_attach_ms" + + // Resolve which Dosh paths to benchmark. Explicit path flags select an + // explicit matrix; otherwise fall back to the legacy single-path behavior + // so existing callers (CI docker scripts) keep working unchanged. + let explicit_paths = explicit_dosh_paths(&args); + let dosh_paths = if explicit_paths.is_empty() { + vec![legacy_dosh_path(&args)] } else { - "dosh_attach_ms" + explicit_paths }; - for _ in 0..args.iterations.max(1) { - if !args.local_auth && !args.skip_ssh_baseline { + let mut results: Vec = Vec::new(); + + // SSH baseline (shared across the matrix; the comparison is per-iteration + // ssh-true vs the dosh path startup that replaces it). + let run_ssh = !args.local_auth && !args.skip_ssh_baseline && !dosh_only(&dosh_paths); + if run_ssh { + let mut ssh_times = Vec::new(); + for _ in 0..args.iterations.max(1) { let mut ssh = Command::new("ssh"); add_ssh_options(&mut ssh, &args, control_path); ssh.arg(&args.server).arg("true"); ssh_times.push(time_command(&mut ssh)?); } + results.push(MetricSamples::new("ssh_true_ms", ssh_times)); + } - dosh_times.push(time_dosh_attach(&client, &args, control_path)?); + for path in &dosh_paths { + if path.needs_warm() { + // Prime the cache with one attach so the fast path has credentials. + let _ = time_dosh_attach(&client, &args, *path, control_path, true)?; + } + let mut samples = Vec::new(); + for _ in 0..args.iterations.max(1) { + samples.push(time_dosh_attach( + &client, + &args, + *path, + control_path, + false, + )?); + } + results.push(MetricSamples::new(path.metric(), samples)); + } - if args.include_mosh { + if args.include_mosh { + let mut mosh_times = Vec::new(); + for _ in 0..args.iterations.max(1) { mosh_times.push(time_mosh_in_pty(&args)?); } + results.push(MetricSamples::new("mosh_start_true_ms", mosh_times)); } - if !ssh_times.is_empty() { - println!( - "ssh_true_ms avg={:.2} samples={:?}", - avg_ms(&ssh_times), - ssh_times - ); - } - println!( - "{dosh_label} avg={:.2} samples={:?}", - avg_ms(&dosh_times), - dosh_times - ); - if !mosh_times.is_empty() { - println!( - "mosh_start_true_ms avg={:.2} samples={:?}", - avg_ms(&mosh_times), - mosh_times - ); - } - if let Some(margin) = args.assert_ssh_plus_ms { - if ssh_times.is_empty() { - return Err(anyhow!( - "--assert-ssh-plus-ms requires non-local SSH benchmark" - )); - } - let ssh_avg = avg_ms(&ssh_times); - let dosh_avg = avg_ms(&dosh_times); - if dosh_avg > ssh_avg + margin { - return Err(anyhow!( - "dosh attach avg {dosh_avg:.2}ms exceeded ssh avg {ssh_avg:.2}ms + {margin:.2}ms" - )); - } - println!("gate ok: dosh avg {dosh_avg:.2}ms <= ssh avg {ssh_avg:.2}ms + {margin:.2}ms"); - } - if let Some(margin) = args.assert_mosh_minus_ms { - if mosh_times.is_empty() { - return Err(anyhow!( - "--assert-mosh-minus-ms requires --include-mosh benchmark" - )); - } - let dosh_avg = avg_ms(&dosh_times); - let mosh_avg = avg_ms(&mosh_times); - if dosh_avg + margin > mosh_avg { - return Err(anyhow!( - "dosh attach avg {dosh_avg:.2}ms was not at least {margin:.2}ms faster than mosh avg {mosh_avg:.2}ms" - )); - } - println!("gate ok: dosh avg {dosh_avg:.2}ms + {margin:.2}ms <= mosh avg {mosh_avg:.2}ms"); - } - if let Some(max_ms) = args.assert_dosh_max_ms { - let dosh_avg = avg_ms(&dosh_times); - if dosh_avg > max_ms { - return Err(anyhow!( - "{dosh_label} avg {dosh_avg:.2}ms exceeded max {max_ms:.2}ms" - )); - } - println!("gate ok: {dosh_label} avg {dosh_avg:.2}ms <= {max_ms:.2}ms"); + if args.json { + print_json(&args, &results); + } else { + print_table(&args, &results); } + + run_assertions(&args, &results)?; Ok(()) } +/// Paths explicitly requested via flags. +fn explicit_dosh_paths(args: &Args) -> Vec { + let mut paths = Vec::new(); + if args.cold_native { + paths.push(DoshPath::ColdNative); + } + if args.cached_ticket { + paths.push(DoshPath::CachedTicket); + } + if args.resume { + paths.push(DoshPath::Resume); + } + paths +} + +/// The single path implied by legacy flags when no explicit path is requested. +fn legacy_dosh_path(args: &Args) -> DoshPath { + if args.local_auth { + DoshPath::LocalAuth + } else if args.warm_cache { + DoshPath::CachedTicket + } else { + DoshPath::SshBootstrap + } +} + +/// True when none of the selected paths needs an SSH baseline comparison +/// (i.e. all are local-auth or cache fast paths driven without SSH). +fn dosh_only(paths: &[DoshPath]) -> bool { + paths.iter().all(|p| { + matches!( + p, + DoshPath::LocalAuth | DoshPath::CachedTicket | DoshPath::Resume + ) + }) +} + fn time_dosh_attach( client: &PathBuf, args: &Args, + path: DoshPath, control_path: Option<&PathBuf>, + warm: bool, ) -> Result { let mut cmd = Command::new(client); cmd.arg("--attach-only") @@ -178,30 +242,66 @@ fn time_dosh_attach( if let Some(host) = &args.dosh_host { cmd.arg("--dosh-host").arg(host); } - if args.local_auth { - cmd.arg("--local-auth").arg(&args.server); - } else { - cmd.arg("--ssh-port") - .arg(args.ssh_port.to_string()) - .arg("--ssh-auth-command") - .arg(&args.ssh_auth_command); - if args.no_cache { + + match path { + DoshPath::LocalAuth => { + cmd.arg("--local-auth"); + // While warming, write the cache; measured runs honor --no-cache. + if args.no_cache && !warm { + cmd.arg("--no-cache"); + } + cmd.arg(&args.server); + } + DoshPath::CachedTicket | DoshPath::Resume => { + // Fast paths must read the warmed cache, so never pass --no-cache here. + // Whether the cache uses tickets vs resume is decided by the client + // config (`cache_attach_tickets`) the harness wrote into HOME. + if args.local_auth { + cmd.arg("--local-auth"); + } + add_bootstrap_args(&mut cmd, args, control_path); + cmd.arg(&args.server); + } + DoshPath::ColdNative => { + cmd.arg("--auth").arg("native"); + // Cold path: never use a warmed cache. cmd.arg("--no-cache"); + add_bootstrap_args(&mut cmd, args, control_path); + cmd.arg(&args.server); } - if let Some(key) = &args.ssh_key { - cmd.arg("--ssh-key").arg(key); + DoshPath::SshBootstrap => { + add_bootstrap_args(&mut cmd, args, control_path); + // Cold SSH bootstrap honors --no-cache on measured runs. + if args.no_cache && !warm { + cmd.arg("--no-cache"); + } + cmd.arg(&args.server); } - if let Some(known_hosts) = &args.ssh_known_hosts { - cmd.arg("--ssh-known-hosts").arg(known_hosts); - } - if let Some(control_path) = control_path { - cmd.arg("--ssh-control-path").arg(control_path); - } - cmd.arg(&args.server); } time_command(&mut cmd) } +/// Append the SSH bootstrap arguments shared by the non-local paths. Native +/// auth reuses the same SSH key/known-hosts/port plumbing. +fn add_bootstrap_args(cmd: &mut Command, args: &Args, control_path: Option<&PathBuf>) { + if args.local_auth { + return; + } + cmd.arg("--ssh-port") + .arg(args.ssh_port.to_string()) + .arg("--ssh-auth-command") + .arg(&args.ssh_auth_command); + if let Some(key) = &args.ssh_key { + cmd.arg("--ssh-key").arg(key); + } + if let Some(known_hosts) = &args.ssh_known_hosts { + cmd.arg("--ssh-known-hosts").arg(known_hosts); + } + if let Some(control_path) = control_path { + cmd.arg("--ssh-control-path").arg(control_path); + } +} + fn add_ssh_options(cmd: &mut Command, args: &Args, control_path: Option<&PathBuf>) { cmd.arg("-p").arg(args.ssh_port.to_string()).arg("-T"); if let Some(key) = &args.ssh_key { @@ -381,9 +481,194 @@ fn time_command(cmd: &mut Command) -> Result { Ok(start.elapsed()) } -fn avg_ms(samples: &[Duration]) -> f64 { - let total: f64 = samples.iter().map(|d| d.as_secs_f64() * 1000.0).sum(); - total / samples.len() as f64 +/// Per-metric samples with summary statistics. +struct MetricSamples { + name: &'static str, + samples: Vec, +} + +impl MetricSamples { + fn new(name: &'static str, samples: Vec) -> Self { + Self { name, samples } + } + + fn ms(&self) -> Vec { + self.samples + .iter() + .map(|d| d.as_secs_f64() * 1000.0) + .collect() + } + + fn stats(&self) -> Stats { + Stats::from_ms(&self.ms()) + } +} + +/// Summary statistics for a set of millisecond samples. +struct Stats { + count: usize, + min: f64, + median: f64, + p95: f64, + mean: f64, + max: f64, +} + +impl Stats { + fn from_ms(values: &[f64]) -> Self { + if values.is_empty() { + return Self { + count: 0, + min: 0.0, + median: 0.0, + p95: 0.0, + mean: 0.0, + max: 0.0, + }; + } + let mut sorted = values.to_vec(); + sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + let count = sorted.len(); + let mean = sorted.iter().sum::() / count as f64; + Self { + count, + min: sorted[0], + median: percentile(&sorted, 50.0), + p95: percentile(&sorted, 95.0), + mean, + max: sorted[count - 1], + } + } +} + +/// Linear-interpolated percentile over a pre-sorted slice. +fn percentile(sorted: &[f64], pct: f64) -> f64 { + if sorted.is_empty() { + return 0.0; + } + if sorted.len() == 1 { + return sorted[0]; + } + let rank = (pct / 100.0) * (sorted.len() - 1) as f64; + let lo = rank.floor() as usize; + let hi = rank.ceil() as usize; + if lo == hi { + sorted[lo] + } else { + let frac = rank - lo as f64; + sorted[lo] + (sorted[hi] - sorted[lo]) * frac + } +} + +fn print_table(args: &Args, results: &[MetricSamples]) { + if let Some(label) = &args.label { + println!("# label: {label}"); + } + println!( + "{:<24} {:>6} {:>9} {:>9} {:>9} {:>9} {:>9}", + "metric", "n", "min", "median", "p95", "mean", "max" + ); + for metric in results { + let s = metric.stats(); + println!( + "{:<24} {:>6} {:>9.2} {:>9.2} {:>9.2} {:>9.2} {:>9.2}", + metric.name, s.count, s.min, s.median, s.p95, s.mean, s.max + ); + } + // Raw per-iteration samples, so published numbers can include raw data. + for metric in results { + let ms: Vec = metric.ms().iter().map(|v| format!("{v:.2}")).collect(); + println!("{} samples_ms=[{}]", metric.name, ms.join(", ")); + } +} + +fn print_json(args: &Args, results: &[MetricSamples]) { + let mut entries = Vec::new(); + for metric in results { + let s = metric.stats(); + let samples: Vec = metric.ms().iter().map(|v| format!("{v:.4}")).collect(); + entries.push(format!( + "{{\"metric\":\"{}\",\"count\":{},\"min\":{:.4},\"median\":{:.4},\"p95\":{:.4},\"mean\":{:.4},\"max\":{:.4},\"samples_ms\":[{}]}}", + metric.name, + s.count, + s.min, + s.median, + s.p95, + s.mean, + s.max, + samples.join(",") + )); + } + let label = match &args.label { + Some(label) => format!("\"{}\"", label.replace('"', "\\\"")), + None => "null".to_string(), + }; + println!( + "{{\"label\":{label},\"iterations\":{},\"metrics\":[{}]}}", + args.iterations.max(1), + entries.join(",") + ); +} + +fn metric_mean(results: &[MetricSamples], name: &str) -> Option { + results + .iter() + .find(|m| m.name == name) + .map(|m| m.stats().mean) +} + +/// The headline dosh metric for assertions: prefer cached, then resume, then +/// cold native, then ssh-bootstrap, then local-auth. +fn primary_dosh(results: &[MetricSamples]) -> Option<(&'static str, f64)> { + const ORDER: [&str; 5] = [ + "dosh_cached_attach_ms", + "dosh_resume_ms", + "dosh_cold_native_ms", + "dosh_attach_ms", + "dosh_local_attach_ms", + ]; + for name in ORDER { + if let Some(mean) = metric_mean(results, name) { + return Some((name, mean)); + } + } + None +} + +fn run_assertions(args: &Args, results: &[MetricSamples]) -> Result<()> { + if let Some(margin) = args.assert_ssh_plus_ms { + let ssh = metric_mean(results, "ssh_true_ms") + .ok_or_else(|| anyhow!("--assert-ssh-plus-ms requires non-local SSH benchmark"))?; + let (name, dosh) = + primary_dosh(results).ok_or_else(|| anyhow!("no dosh metric to assert against"))?; + if dosh > ssh + margin { + return Err(anyhow!( + "{name} avg {dosh:.2}ms exceeded ssh avg {ssh:.2}ms + {margin:.2}ms" + )); + } + println!("gate ok: {name} avg {dosh:.2}ms <= ssh avg {ssh:.2}ms + {margin:.2}ms"); + } + if let Some(margin) = args.assert_mosh_minus_ms { + let mosh = metric_mean(results, "mosh_start_true_ms") + .ok_or_else(|| anyhow!("--assert-mosh-minus-ms requires --include-mosh benchmark"))?; + let (name, dosh) = + primary_dosh(results).ok_or_else(|| anyhow!("no dosh metric to assert against"))?; + if dosh + margin > mosh { + return Err(anyhow!( + "{name} avg {dosh:.2}ms was not at least {margin:.2}ms faster than mosh avg {mosh:.2}ms" + )); + } + println!("gate ok: {name} avg {dosh:.2}ms + {margin:.2}ms <= mosh avg {mosh:.2}ms"); + } + if let Some(max_ms) = args.assert_dosh_max_ms { + let (name, dosh) = + primary_dosh(results).ok_or_else(|| anyhow!("no dosh metric to assert against"))?; + if dosh > max_ms { + return Err(anyhow!("{name} avg {dosh:.2}ms exceeded max {max_ms:.2}ms")); + } + println!("gate ok: {name} avg {dosh:.2}ms <= {max_ms:.2}ms"); + } + Ok(()) } fn default_client_path() -> PathBuf {