Add benchmark path matrix and safe local harness
Extend dosh-bench to benchmark the full attach path matrix and emit both raw per-iteration samples and summary statistics (count/min/median/p95/mean/max): - --cold-native: native cold auth terminal-ready (dosh_cold_native_ms) - --cached-ticket: cached attach-ticket fast path (dosh_cached_attach_ms) - --resume: UDP resume path (dosh_resume_ms; roaming-only, see docs) - --local-auth: self-contained local bootstrap, no SSH (dosh_local_attach_ms) - default: legacy SSH-bootstrap cold attach (dosh_attach_ms) Existing ssh-bootstrap/local-auth/mosh modes and all assert gates are preserved; legacy flags (--local-auth/--warm-cache/--no-cache) keep their prior behavior so the CI docker scripts run unchanged. Add --json (machine-readable raw samples) and --label. Add a table/JSON renderer and percentile-based summary stats. Add scripts/bench-local.sh: a safe, self-contained harness that builds release, spins up a throwaway dosh-server bound to 127.0.0.1 on a random free UDP port in a temp HOME (never port 50000, never a systemd unit), runs the matrix, prints results, sanity-checks native auth actually trusted the host, and cleans up. Point `make bench-local` at the new harness and add `make bench-local-json`; keep bench-docker-* targets intact. Add docs/BENCHMARKS.md: how to run each benchmark, metric meanings, methodology and caveats (resume is the roaming path, not cold reconnect; cite cached attach as the core claim per PUBLIC_READINESS.md), plus a real loopback sample table with machine/OS/sample-count and raw samples. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
DuProcess
Executable
+192
@@ -0,0 +1,192 @@
|
||||
#!/usr/bin/env sh
|
||||
# Safe, self-contained local Dosh benchmark harness.
|
||||
#
|
||||
# Spins up a THROWAWAY dosh-server bound to 127.0.0.1 on a random free port in a
|
||||
# temp HOME, runs the full path matrix (native cold auth, cached attach-ticket,
|
||||
# UDP resume, and local-auth), prints raw samples + summary stats, then tears
|
||||
# everything down.
|
||||
#
|
||||
# It NEVER touches the production server: it never uses UDP port 50000, never
|
||||
# restarts any systemd unit, and never reads or writes the real ~/.config/dosh
|
||||
# or ~/.local. Everything lives under a mktemp HOME that is removed on exit.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/bench-local.sh [ITERATIONS]
|
||||
# Environment:
|
||||
# DOSH_BENCH_ITERS iteration count (default 20; overridden by $1)
|
||||
# DOSH_BENCH_JSON=1 emit machine-readable JSON instead of the table
|
||||
# DOSH_BENCH_PROFILE release|debug build profile (default release)
|
||||
set -eu
|
||||
|
||||
iters="${1:-${DOSH_BENCH_ITERS:-20}}"
|
||||
profile="${DOSH_BENCH_PROFILE:-release}"
|
||||
|
||||
repo_root="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
cd "$repo_root"
|
||||
|
||||
# Make sure cargo is reachable in non-login shells.
|
||||
if ! command -v cargo >/dev/null 2>&1; then
|
||||
# shellcheck disable=SC1090
|
||||
[ -f "$HOME/.cargo/env" ] && . "$HOME/.cargo/env"
|
||||
fi
|
||||
|
||||
if [ "$profile" = "release" ]; then
|
||||
cargo build --release >&2
|
||||
bindir="$repo_root/target/release"
|
||||
else
|
||||
cargo build >&2
|
||||
bindir="$repo_root/target/debug"
|
||||
fi
|
||||
server_bin="$bindir/dosh-server"
|
||||
client_bin="$bindir/dosh-client"
|
||||
bench_bin="$bindir/dosh-bench"
|
||||
|
||||
# Pick a free UDP port that is NOT the production port (50000).
|
||||
free_udp_port() {
|
||||
python3 - <<'PY'
|
||||
import socket
|
||||
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||
s.bind(("127.0.0.1", 0))
|
||||
print(s.getsockname()[1])
|
||||
s.close()
|
||||
PY
|
||||
}
|
||||
dosh_port="$(free_udp_port)"
|
||||
if [ "$dosh_port" = "50000" ]; then
|
||||
dosh_port="$(free_udp_port)"
|
||||
fi
|
||||
if [ "$dosh_port" = "50000" ]; then
|
||||
echo "refusing to bind production port 50000" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
home="$(mktemp -d)"
|
||||
server_pid=""
|
||||
cleanup() {
|
||||
if [ -n "$server_pid" ]; then
|
||||
kill "$server_pid" 2>/dev/null || true
|
||||
wait "$server_pid" 2>/dev/null || true
|
||||
fi
|
||||
rm -rf "$home"
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
mkdir -p "$home/.config/dosh" "$home/.ssh" "$home/.local/share/dosh"
|
||||
|
||||
# Client identity used by the native cold-auth path.
|
||||
ssh-keygen -t ed25519 -N "" -q -f "$home/.ssh/id_ed25519"
|
||||
client_pub="$(cat "$home/.ssh/id_ed25519.pub")"
|
||||
printf '%s\n' "$client_pub" > "$home/authorized_keys"
|
||||
|
||||
# Throwaway server config: localhost only, throwaway port, attach tickets on.
|
||||
cat > "$home/.config/dosh/server.toml" <<EOF
|
||||
port = $dosh_port
|
||||
bind = "127.0.0.1"
|
||||
scrollback = 5000
|
||||
auth_ttl_secs = 30
|
||||
attach_ticket_ttl_secs = 3600
|
||||
allow_attach_tickets = true
|
||||
client_timeout_secs = 60
|
||||
retransmit_window = 256
|
||||
default_input_mode = "read-write"
|
||||
prewarm_sessions = ["default"]
|
||||
create_on_attach = true
|
||||
shell = "/bin/sh"
|
||||
sessions_dir = "$home/sessions"
|
||||
secret_path = "$home/secret"
|
||||
host_key = "$home/host_key"
|
||||
authorized_keys = ["$home/authorized_keys"]
|
||||
EOF
|
||||
|
||||
# Client config: native auth, trust-on-first-use (no SSH needed), localhost UDP.
|
||||
write_client_config() {
|
||||
# $1 = cache_attach_tickets (true|false)
|
||||
cat > "$home/.config/dosh/client.toml" <<EOF
|
||||
server = "local"
|
||||
dosh_host = "127.0.0.1"
|
||||
dosh_port = $dosh_port
|
||||
default_session = "default"
|
||||
reconnect_timeout_secs = 5
|
||||
view_only = false
|
||||
predict = false
|
||||
cache_attach_tickets = $1
|
||||
credential_cache = "$home/.local/share/dosh/credentials"
|
||||
known_hosts = "$home/.config/dosh/known_hosts"
|
||||
auth_preference = "native"
|
||||
trust_on_first_use = true
|
||||
EOF
|
||||
}
|
||||
|
||||
start_server() {
|
||||
HOME="$home" "$server_bin" serve --config "$home/.config/dosh/server.toml" \
|
||||
>"$home/server.log" 2>&1 &
|
||||
server_pid="$!"
|
||||
# Wait for the UDP socket to come up.
|
||||
i=0
|
||||
while [ "$i" -lt 50 ]; do
|
||||
if grep -q "listening on" "$home/server.log" 2>/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
if ! kill -0 "$server_pid" 2>/dev/null; then
|
||||
echo "dosh-server exited early:" >&2
|
||||
cat "$home/server.log" >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
i=$((i + 1))
|
||||
sleep 0.1
|
||||
done
|
||||
}
|
||||
|
||||
run_bench() {
|
||||
# $@ extra args forwarded to dosh-bench
|
||||
json_flag=""
|
||||
[ "${DOSH_BENCH_JSON:-0}" = "1" ] && json_flag="--json"
|
||||
label="$(uname -s) $(uname -m), profile=$profile"
|
||||
HOME="$home" "$bench_bin" \
|
||||
--client "$client_bin" \
|
||||
--server local \
|
||||
--dosh-host 127.0.0.1 \
|
||||
--dosh-port "$dosh_port" \
|
||||
--session default \
|
||||
--skip-ssh-baseline \
|
||||
--iterations "$iters" \
|
||||
--label "$label" \
|
||||
$json_flag \
|
||||
"$@"
|
||||
}
|
||||
|
||||
echo "dosh local benchmark: port=$dosh_port iters=$iters profile=$profile home=$home" >&2
|
||||
|
||||
# 1) Native cold auth + cached attach-ticket in one run (ticket cache on).
|
||||
write_client_config true
|
||||
start_server
|
||||
echo "== native cold auth + cached attach-ticket ==" >&2
|
||||
run_bench --cold-native --cached-ticket
|
||||
# Sanity: native cold auth must have trusted the host (proves the generated
|
||||
# client config loaded and the native handshake ran instead of silently
|
||||
# falling back to a different path).
|
||||
if [ ! -s "$home/.config/dosh/known_hosts" ]; then
|
||||
echo "native cold auth did not record a trusted host; check server.log:" >&2
|
||||
cat "$home/server.log" >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
kill "$server_pid" 2>/dev/null || true
|
||||
wait "$server_pid" 2>/dev/null || true
|
||||
server_pid=""
|
||||
|
||||
# 2) Self-contained local-auth path (no SSH, no native handshake).
|
||||
#
|
||||
# Note on UDP resume: resume is the roaming path for a client that is STILL
|
||||
# attached when its network endpoint changes. The `--attach-only` benchmark
|
||||
# model detaches after each iteration, which tears the client down on the
|
||||
# server, so a fresh-process resume has nothing live to resume and is not
|
||||
# meaningful here. The cold fresh-process reconnect fast path is the attach
|
||||
# ticket (measured above). Roaming resume is validated by the integration test
|
||||
# `resume_updates_udp_endpoint_for_roaming`. `dosh-bench --resume` exists for
|
||||
# scenarios that keep a live session out-of-band (e.g. remote soak tests); see
|
||||
# docs/BENCHMARKS.md.
|
||||
rm -rf "$home/.local/share/dosh/credentials"
|
||||
write_client_config true
|
||||
start_server
|
||||
echo "== local-auth (no SSH) ==" >&2
|
||||
run_bench --local-auth --no-cache
|
||||
Reference in New Issue
Block a user