Palav/dosh
1
0
Fork 0
Code Issues Pull Requests Actions 58 Packages Projects Releases 51 Wiki Activity

Add benchmark path matrix and safe local harness

Browse Source 
Extend dosh-bench to benchmark the full attach path matrix and emit both raw
per-iteration samples and summary statistics (count/min/median/p95/mean/max):

- --cold-native:   native cold auth terminal-ready (dosh_cold_native_ms)
- --cached-ticket: cached attach-ticket fast path (dosh_cached_attach_ms)
- --resume:        UDP resume path (dosh_resume_ms; roaming-only, see docs)
- --local-auth:    self-contained local bootstrap, no SSH (dosh_local_attach_ms)
- default:         legacy SSH-bootstrap cold attach (dosh_attach_ms)

Existing ssh-bootstrap/local-auth/mosh modes and all assert gates are preserved;
legacy flags (--local-auth/--warm-cache/--no-cache) keep their prior behavior so
the CI docker scripts run unchanged. Add --json (machine-readable raw samples)
and --label. Add a table/JSON renderer and percentile-based summary stats.

Add scripts/bench-local.sh: a safe, self-contained harness that builds release,
spins up a throwaway dosh-server bound to 127.0.0.1 on a random free UDP port in
a temp HOME (never port 50000, never a systemd unit), runs the matrix, prints
results, sanity-checks native auth actually trusted the host, and cleans up.

Point `make bench-local` at the new harness and add `make bench-local-json`;
keep bench-docker-* targets intact.

Add docs/BENCHMARKS.md: how to run each benchmark, metric meanings, methodology
and caveats (resume is the roaming path, not cold reconnect; cite cached attach
as the core claim per PUBLIC_READINESS.md), plus a real loopback sample table
with machine/OS/sample-count and raw samples.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
DuProcess
2026-06-14 10:33:37 -04:00
parent 6c14d669b8
commit c48f2280a0
4 changed files with 751 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/bin/dosh-bench.rs
+376 -91
View File
@@ -25,6 +25,15 @@ struct Args {
iterations: usize,
#[arg(long)]
local_auth: bool,
/// Benchmark native cold auth (no cache, native handshake) terminal-ready time.
#[arg(long)]
cold_native: bool,
/// Benchmark cached attach-ticket terminal-ready time (warms the cache first).
#[arg(long)]
cached_ticket: bool,
/// Benchmark UDP resume terminal-ready time (warms the cache first).
#[arg(long)]
resume: bool,
#[arg(long)]
client: Option<PathBuf>,
#[arg(long, default_value = "~/.local/bin/dosh-auth")]
@@ -51,6 +60,12 @@ struct Args {
mosh_server_command: String,
#[arg(long)]
mosh_port: Option<String>,
/// Emit machine-readable JSON (one object per metric, with raw samples).
#[arg(long)]
json: bool,
/// Optional label printed in summary/JSON output (e.g. machine/OS identifier).
#[arg(long)]
label: Option<String>,
#[arg(long)]
assert_ssh_plus_ms: Option<f64>,
#[arg(long)]
@@ -59,12 +74,45 @@ struct Args {
assert_dosh_max_ms: Option<f64>,
}
/// One Dosh attach path to benchmark.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum DoshPath {
/// `--auth native --no-cache`: full native handshake, no cache.
ColdNative,
/// Cached attach-ticket fast path (requires a warmed cache).
CachedTicket,
/// UDP resume fast path (requires a warmed cache + `cache_attach_tickets = false`).
Resume,
/// `--local-auth`: self-contained local bootstrap, no SSH.
LocalAuth,
/// SSH-bootstrap cold attach (cache controlled by `--no-cache` / `--warm-cache`).
SshBootstrap,
}
impl DoshPath {
fn metric(self) -> &'static str {
match self {
DoshPath::ColdNative => "dosh_cold_native_ms",
DoshPath::CachedTicket => "dosh_cached_attach_ms",
DoshPath::Resume => "dosh_resume_ms",
DoshPath::LocalAuth => "dosh_local_attach_ms",
DoshPath::SshBootstrap => "dosh_attach_ms",
}
}
/// Whether this path consumes a warmed credential cache.
fn needs_warm(self) -> bool {
matches!(self, DoshPath::CachedTicket | DoshPath::Resume)
}
}
fn main() -> Result<()> {
let args = Args::parse();
let client = args.client.clone().unwrap_or_else(default_client_path);
let mut ssh_times = Vec::new();
let mut dosh_times = Vec::new();
let mut mosh_times = Vec::new();
if args.no_cache && args.warm_cache {
return Err(anyhow!("--warm-cache cannot be used with --no-cache"));
}
let generated_control_path = if args.controlmaster {
Some(std::env::temp_dir().join(format!("dosh-bench-control-{}", std::process::id())))
} else {
@@ -78,96 +126,112 @@ fn main() -> Result<()> {
} else {
None
};
if args.no_cache && args.warm_cache {
return Err(anyhow!("--warm-cache cannot be used with --no-cache"));
}
let dosh_label = if args.warm_cache {
let _ = time_dosh_attach(&client, &args, control_path)?;
"dosh_cached_attach_ms"
// Resolve which Dosh paths to benchmark. Explicit path flags select an
// explicit matrix; otherwise fall back to the legacy single-path behavior
// so existing callers (CI docker scripts) keep working unchanged.
let explicit_paths = explicit_dosh_paths(&args);
let dosh_paths = if explicit_paths.is_empty() {
vec![legacy_dosh_path(&args)]
} else {
"dosh_attach_ms"
explicit_paths
};
for _ in 0..args.iterations.max(1) {
if !args.local_auth && !args.skip_ssh_baseline {
let mut results: Vec<MetricSamples> = Vec::new();
// SSH baseline (shared across the matrix; the comparison is per-iteration
// ssh-true vs the dosh path startup that replaces it).
let run_ssh = !args.local_auth && !args.skip_ssh_baseline && !dosh_only(&dosh_paths);
if run_ssh {
let mut ssh_times = Vec::new();
for _ in 0..args.iterations.max(1) {
let mut ssh = Command::new("ssh");
add_ssh_options(&mut ssh, &args, control_path);
ssh.arg(&args.server).arg("true");
ssh_times.push(time_command(&mut ssh)?);
}
results.push(MetricSamples::new("ssh_true_ms", ssh_times));
}
dosh_times.push(time_dosh_attach(&client, &args, control_path)?);
for path in &dosh_paths {
if path.needs_warm() {
// Prime the cache with one attach so the fast path has credentials.
let _ = time_dosh_attach(&client, &args, *path, control_path, true)?;
}
let mut samples = Vec::new();
for _ in 0..args.iterations.max(1) {
samples.push(time_dosh_attach(
&client,
&args,
*path,
control_path,
false,
)?);
}
results.push(MetricSamples::new(path.metric(), samples));
}
if args.include_mosh {
if args.include_mosh {
let mut mosh_times = Vec::new();
for _ in 0..args.iterations.max(1) {
mosh_times.push(time_mosh_in_pty(&args)?);
}
results.push(MetricSamples::new("mosh_start_true_ms", mosh_times));
}
if !ssh_times.is_empty() {
println!(
"ssh_true_ms avg={:.2} samples={:?}",
avg_ms(&ssh_times),
ssh_times
);
}
println!(
"{dosh_label} avg={:.2} samples={:?}",
avg_ms(&dosh_times),
dosh_times
);
if !mosh_times.is_empty() {
println!(
"mosh_start_true_ms avg={:.2} samples={:?}",
avg_ms(&mosh_times),
mosh_times
);
}
if let Some(margin) = args.assert_ssh_plus_ms {
if ssh_times.is_empty() {
return Err(anyhow!(
"--assert-ssh-plus-ms requires non-local SSH benchmark"
));
}
let ssh_avg = avg_ms(&ssh_times);
let dosh_avg = avg_ms(&dosh_times);
if dosh_avg > ssh_avg + margin {
return Err(anyhow!(
"dosh attach avg {dosh_avg:.2}ms exceeded ssh avg {ssh_avg:.2}ms + {margin:.2}ms"
));
}
println!("gate ok: dosh avg {dosh_avg:.2}ms <= ssh avg {ssh_avg:.2}ms + {margin:.2}ms");
}
if let Some(margin) = args.assert_mosh_minus_ms {
if mosh_times.is_empty() {
return Err(anyhow!(
"--assert-mosh-minus-ms requires --include-mosh benchmark"
));
}
let dosh_avg = avg_ms(&dosh_times);
let mosh_avg = avg_ms(&mosh_times);
if dosh_avg + margin > mosh_avg {
return Err(anyhow!(
"dosh attach avg {dosh_avg:.2}ms was not at least {margin:.2}ms faster than mosh avg {mosh_avg:.2}ms"
));
}
println!("gate ok: dosh avg {dosh_avg:.2}ms + {margin:.2}ms <= mosh avg {mosh_avg:.2}ms");
}
if let Some(max_ms) = args.assert_dosh_max_ms {
let dosh_avg = avg_ms(&dosh_times);
if dosh_avg > max_ms {
return Err(anyhow!(
"{dosh_label} avg {dosh_avg:.2}ms exceeded max {max_ms:.2}ms"
));
}
println!("gate ok: {dosh_label} avg {dosh_avg:.2}ms <= {max_ms:.2}ms");
if args.json {
print_json(&args, &results);
} else {
print_table(&args, &results);
}
run_assertions(&args, &results)?;
Ok(())
}
/// Paths explicitly requested via flags.
fn explicit_dosh_paths(args: &Args) -> Vec<DoshPath> {
let mut paths = Vec::new();
if args.cold_native {
paths.push(DoshPath::ColdNative);
}
if args.cached_ticket {
paths.push(DoshPath::CachedTicket);
}
if args.resume {
paths.push(DoshPath::Resume);
}
paths
}
/// The single path implied by legacy flags when no explicit path is requested.
fn legacy_dosh_path(args: &Args) -> DoshPath {
if args.local_auth {
DoshPath::LocalAuth
} else if args.warm_cache {
DoshPath::CachedTicket
} else {
DoshPath::SshBootstrap
}
}
/// True when none of the selected paths needs an SSH baseline comparison
/// (i.e. all are local-auth or cache fast paths driven without SSH).
fn dosh_only(paths: &[DoshPath]) -> bool {
paths.iter().all(|p| {
matches!(
p,
DoshPath::LocalAuth | DoshPath::CachedTicket | DoshPath::Resume
)
})
}
fn time_dosh_attach(
client: &PathBuf,
args: &Args,
path: DoshPath,
control_path: Option<&PathBuf>,
warm: bool,
) -> Result<Duration> {
let mut cmd = Command::new(client);
cmd.arg("--attach-only")
@@ -178,30 +242,66 @@ fn time_dosh_attach(
if let Some(host) = &args.dosh_host {
cmd.arg("--dosh-host").arg(host);
}
if args.local_auth {
cmd.arg("--local-auth").arg(&args.server);
} else {
cmd.arg("--ssh-port")
.arg(args.ssh_port.to_string())
.arg("--ssh-auth-command")
.arg(&args.ssh_auth_command);
if args.no_cache {
match path {
DoshPath::LocalAuth => {
cmd.arg("--local-auth");
// While warming, write the cache; measured runs honor --no-cache.
if args.no_cache && !warm {
cmd.arg("--no-cache");
}
cmd.arg(&args.server);
}
DoshPath::CachedTicket | DoshPath::Resume => {
// Fast paths must read the warmed cache, so never pass --no-cache here.
// Whether the cache uses tickets vs resume is decided by the client
// config (`cache_attach_tickets`) the harness wrote into HOME.
if args.local_auth {
cmd.arg("--local-auth");
}
add_bootstrap_args(&mut cmd, args, control_path);
cmd.arg(&args.server);
}
DoshPath::ColdNative => {
cmd.arg("--auth").arg("native");
// Cold path: never use a warmed cache.
cmd.arg("--no-cache");
add_bootstrap_args(&mut cmd, args, control_path);
cmd.arg(&args.server);
}
if let Some(key) = &args.ssh_key {
cmd.arg("--ssh-key").arg(key);
DoshPath::SshBootstrap => {
add_bootstrap_args(&mut cmd, args, control_path);
// Cold SSH bootstrap honors --no-cache on measured runs.
if args.no_cache && !warm {
cmd.arg("--no-cache");
}
cmd.arg(&args.server);
}
if let Some(known_hosts) = &args.ssh_known_hosts {
cmd.arg("--ssh-known-hosts").arg(known_hosts);
}
if let Some(control_path) = control_path {
cmd.arg("--ssh-control-path").arg(control_path);
}
cmd.arg(&args.server);
}
time_command(&mut cmd)
}
/// Append the SSH bootstrap arguments shared by the non-local paths. Native
/// auth reuses the same SSH key/known-hosts/port plumbing.
fn add_bootstrap_args(cmd: &mut Command, args: &Args, control_path: Option<&PathBuf>) {
if args.local_auth {
return;
}
cmd.arg("--ssh-port")
.arg(args.ssh_port.to_string())
.arg("--ssh-auth-command")
.arg(&args.ssh_auth_command);
if let Some(key) = &args.ssh_key {
cmd.arg("--ssh-key").arg(key);
}
if let Some(known_hosts) = &args.ssh_known_hosts {
cmd.arg("--ssh-known-hosts").arg(known_hosts);
}
if let Some(control_path) = control_path {
cmd.arg("--ssh-control-path").arg(control_path);
}
}
fn add_ssh_options(cmd: &mut Command, args: &Args, control_path: Option<&PathBuf>) {
cmd.arg("-p").arg(args.ssh_port.to_string()).arg("-T");
if let Some(key) = &args.ssh_key {
@@ -381,9 +481,194 @@ fn time_command(cmd: &mut Command) -> Result<Duration> {
Ok(start.elapsed())
}
fn avg_ms(samples: &[Duration]) -> f64 {
let total: f64 = samples.iter().map(|d| d.as_secs_f64() * 1000.0).sum();
total / samples.len() as f64
/// Per-metric samples with summary statistics.
struct MetricSamples {
name: &'static str,
samples: Vec<Duration>,
}
impl MetricSamples {
fn new(name: &'static str, samples: Vec<Duration>) -> Self {
Self { name, samples }
}
fn ms(&self) -> Vec<f64> {
self.samples
.iter()
.map(|d| d.as_secs_f64() * 1000.0)
.collect()
}
fn stats(&self) -> Stats {
Stats::from_ms(&self.ms())
}
}
/// Summary statistics for a set of millisecond samples.
struct Stats {
count: usize,
min: f64,
median: f64,
p95: f64,
mean: f64,
max: f64,
}
impl Stats {
fn from_ms(values: &[f64]) -> Self {
if values.is_empty() {
return Self {
count: 0,
min: 0.0,
median: 0.0,
p95: 0.0,
mean: 0.0,
max: 0.0,
};
}
let mut sorted = values.to_vec();
sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
let count = sorted.len();
let mean = sorted.iter().sum::<f64>() / count as f64;
Self {
count,
min: sorted[0],
median: percentile(&sorted, 50.0),
p95: percentile(&sorted, 95.0),
mean,
max: sorted[count - 1],
}
}
}
/// Linear-interpolated percentile over a pre-sorted slice.
fn percentile(sorted: &[f64], pct: f64) -> f64 {
if sorted.is_empty() {
return 0.0;
}
if sorted.len() == 1 {
return sorted[0];
}
let rank = (pct / 100.0) * (sorted.len() - 1) as f64;
let lo = rank.floor() as usize;
let hi = rank.ceil() as usize;
if lo == hi {
sorted[lo]
} else {
let frac = rank - lo as f64;
sorted[lo] + (sorted[hi] - sorted[lo]) * frac
}
}
fn print_table(args: &Args, results: &[MetricSamples]) {
if let Some(label) = &args.label {
println!("# label: {label}");
}
println!(
"{:<24} {:>6} {:>9} {:>9} {:>9} {:>9} {:>9}",
"metric", "n", "min", "median", "p95", "mean", "max"
);
for metric in results {
let s = metric.stats();
println!(
"{:<24} {:>6} {:>9.2} {:>9.2} {:>9.2} {:>9.2} {:>9.2}",
metric.name, s.count, s.min, s.median, s.p95, s.mean, s.max
);
}
// Raw per-iteration samples, so published numbers can include raw data.
for metric in results {
let ms: Vec<String> = metric.ms().iter().map(|v| format!("{v:.2}")).collect();
println!("{} samples_ms=[{}]", metric.name, ms.join(", "));
}
}
fn print_json(args: &Args, results: &[MetricSamples]) {
let mut entries = Vec::new();
for metric in results {
let s = metric.stats();
let samples: Vec<String> = metric.ms().iter().map(|v| format!("{v:.4}")).collect();
entries.push(format!(
"{{\"metric\":\"{}\",\"count\":{},\"min\":{:.4},\"median\":{:.4},\"p95\":{:.4},\"mean\":{:.4},\"max\":{:.4},\"samples_ms\":[{}]}}",
metric.name,
s.count,
s.min,
s.median,
s.p95,
s.mean,
s.max,
samples.join(",")
));
}
let label = match &args.label {
Some(label) => format!("\"{}\"", label.replace('"', "\\\"")),
None => "null".to_string(),
};
println!(
"{{\"label\":{label},\"iterations\":{},\"metrics\":[{}]}}",
args.iterations.max(1),
entries.join(",")
);
}
fn metric_mean(results: &[MetricSamples], name: &str) -> Option<f64> {
results
.iter()
.find(|m| m.name == name)
.map(|m| m.stats().mean)
}
/// The headline dosh metric for assertions: prefer cached, then resume, then
/// cold native, then ssh-bootstrap, then local-auth.
fn primary_dosh(results: &[MetricSamples]) -> Option<(&'static str, f64)> {
const ORDER: [&str; 5] = [
"dosh_cached_attach_ms",
"dosh_resume_ms",
"dosh_cold_native_ms",
"dosh_attach_ms",
"dosh_local_attach_ms",
];
for name in ORDER {
if let Some(mean) = metric_mean(results, name) {
return Some((name, mean));
}
}
None
}
fn run_assertions(args: &Args, results: &[MetricSamples]) -> Result<()> {
if let Some(margin) = args.assert_ssh_plus_ms {
let ssh = metric_mean(results, "ssh_true_ms")
.ok_or_else(|| anyhow!("--assert-ssh-plus-ms requires non-local SSH benchmark"))?;
let (name, dosh) =
primary_dosh(results).ok_or_else(|| anyhow!("no dosh metric to assert against"))?;
if dosh > ssh + margin {
return Err(anyhow!(
"{name} avg {dosh:.2}ms exceeded ssh avg {ssh:.2}ms + {margin:.2}ms"
));
}
println!("gate ok: {name} avg {dosh:.2}ms <= ssh avg {ssh:.2}ms + {margin:.2}ms");
}
if let Some(margin) = args.assert_mosh_minus_ms {
let mosh = metric_mean(results, "mosh_start_true_ms")
.ok_or_else(|| anyhow!("--assert-mosh-minus-ms requires --include-mosh benchmark"))?;
let (name, dosh) =
primary_dosh(results).ok_or_else(|| anyhow!("no dosh metric to assert against"))?;
if dosh + margin > mosh {
return Err(anyhow!(
"{name} avg {dosh:.2}ms was not at least {margin:.2}ms faster than mosh avg {mosh:.2}ms"
));
}
println!("gate ok: {name} avg {dosh:.2}ms + {margin:.2}ms <= mosh avg {mosh:.2}ms");
}
if let Some(max_ms) = args.assert_dosh_max_ms {
let (name, dosh) =
primary_dosh(results).ok_or_else(|| anyhow!("no dosh metric to assert against"))?;
if dosh > max_ms {
return Err(anyhow!("{name} avg {dosh:.2}ms exceeded max {max_ms:.2}ms"));
}
println!("gate ok: {name} avg {dosh:.2}ms <= {max_ms:.2}ms");
}
Ok(())
}
fn default_client_path() -> PathBuf {