Add hostile network TUI soak gate
ci / test (push) Has been cancelled
ci / fuzz-smoke (push) Has been cancelled
ci / windows-client (push) Has been cancelled
ci / package-release (linux-x86_64, ubuntu-latest) (push) Has been cancelled
ci / package-release (macos-aarch64, macos-14) (push) Has been cancelled
ci / package-release (macos-x86_64, macos-13) (push) Has been cancelled
ci / package-release (windows-x86_64, windows-latest) (push) Has been cancelled
ci / remote-bench (push) Has been cancelled

This commit is contained in:
DuProcess
2026-07-04 14:37:23 -04:00
parent 237ad52bef
commit c085137250
2 changed files with 142 additions and 9 deletions
+1
View File
@@ -27,6 +27,7 @@ reconnect-check:
hostile-network-check:
cargo test --test hostile_network -- --nocapture
DOSH_BADNET_SOAK_SECONDS=$${DOSH_BADNET_SOAK_SECONDS:-300} cargo test --test hostile_network bad_network_tui_work_soak_30m -- --ignored --nocapture
persistence-check:
cargo test --test integration_smoke session_survives_server_restart_same_shell_and_screen -- --nocapture
+141 -9
View File
@@ -258,7 +258,8 @@ fn relay_loop(
let mut upstream = new_upstream();
let mut client_addr: Option<SocketAddr> = None;
let mut rng = StdRng::seed_from_u64(seed);
let mut held: Option<(Vec<u8>, bool)> = None; // (packet, is_c2s) held for reorder
let mut held_c2s: Option<Vec<u8>> = None;
let mut held_s2c: Option<Vec<u8>> = None;
let mut buf = [0u8; 65535];
loop {
@@ -288,10 +289,9 @@ fn relay_loop(
&upstream,
server_addr,
packet,
true,
&controls,
&mut rng,
&mut held,
&mut held_c2s,
);
}
}
@@ -310,7 +310,12 @@ fn relay_loop(
let drop_pct = controls.drop_s2c_percent.load(Ordering::SeqCst);
if drop_pct == 0 || rng.gen_range(0..100) >= drop_pct {
forward_with_effects(
&front, dst, packet, false, &controls, &mut rng, &mut held,
&front,
dst,
packet,
&controls,
&mut rng,
&mut held_s2c,
);
}
}
@@ -336,23 +341,22 @@ fn forward_with_effects(
out: &UdpSocket,
dst: SocketAddr,
packet: Vec<u8>,
is_c2s: bool,
controls: &RelayControls,
rng: &mut StdRng,
held: &mut Option<(Vec<u8>, bool)>,
held: &mut Option<Vec<u8>>,
) {
// Reorder: if armed, hold this packet and release the previously held one
// afterward (so two consecutive packets swap order).
if controls.reorder_next.swap(false, Ordering::SeqCst) {
if let Some((prev, _)) = held.take() {
if let Some(prev) = held.take() {
let _ = out.send_to(&packet, dst);
let _ = out.send_to(&prev, dst);
return;
}
*held = Some((packet, is_c2s));
*held = Some(packet);
return;
}
if let Some((prev, _)) = held.take() {
if let Some(prev) = held.take() {
let _ = out.send_to(&prev, dst);
}
@@ -767,6 +771,46 @@ fn wait_for_text(socket: &UdpSocket, key: &[u8; 32], needle: &str, millis: u64)
acc.contains(needle)
}
fn wait_for_text_with_ack(
socket: &UdpSocket,
relay: &Relay,
client_id: [u8; 16],
seq: &mut u64,
key: &[u8; 32],
needle: &str,
millis: u64,
) -> bool {
let prev = socket.read_timeout().unwrap();
socket
.set_read_timeout(Some(Duration::from_millis(100)))
.unwrap();
let deadline = Instant::now() + Duration::from_millis(millis);
let mut acc = String::new();
while Instant::now() < deadline {
if let Some((_header, frame)) = recv_frame(socket, key) {
acc.push_str(&String::from_utf8_lossy(&frame.bytes));
let ack = protocol::encode_encrypted(
PacketKind::Ack,
client_id,
*seq,
frame.output_seq,
key,
CLIENT_TO_SERVER,
b"",
)
.unwrap();
*seq += 1;
socket.send_to(&ack, relay.front_addr()).unwrap();
if acc.contains(needle) {
socket.set_read_timeout(prev).unwrap();
return true;
}
}
}
socket.set_read_timeout(prev).unwrap();
acc.contains(needle)
}
#[test]
fn session_survives_packet_loss_and_reorder() {
let dir = tempfile::tempdir().unwrap();
@@ -820,6 +864,94 @@ fn session_survives_packet_loss_and_reorder() {
);
}
#[test]
#[ignore = "30-minute hostile-network TUI soak; run with `DOSH_BADNET_SOAK_SECONDS=1800 cargo test --test hostile_network bad_network_tui_work_soak_30m -- --ignored --nocapture`"]
fn bad_network_tui_work_soak_30m() {
let soak_secs = std::env::var("DOSH_BADNET_SOAK_SECONDS")
.ok()
.and_then(|value| value.parse::<u64>().ok())
.unwrap_or(30 * 60);
let dir = tempfile::tempdir().unwrap();
let port = free_udp_port();
let config = write_server_config(&dir, port);
let mut server = start_server(&dir, &config);
let relay = Relay::spawn(port, 0xBADC0DEu64);
let (socket, bootstrap, ok) = attach_through_relay(&config, &relay);
relay.set_drop_c2s(15);
relay.set_drop_s2c(20);
relay.set_dup(10);
let deadline = Instant::now() + Duration::from_secs(soak_secs);
let mut seq = 2u64;
let mut iteration = 0u64;
while Instant::now() < deadline {
if iteration % 2 == 0 {
relay.arm_reorder();
}
if iteration > 0 && iteration % 7 == 0 {
let _ = relay.rebind_upstream();
}
if iteration > 0 && iteration % 11 == 0 {
relay.set_drop_s2c(100);
thread::sleep(Duration::from_millis(750));
relay.set_drop_s2c(20);
}
let marker = format!("DOSH_BADNET_TUI_{iteration}");
let command = format!(
"stty -echo; \
printf '\\033[?1049h\\033[?2026h\\033[?25l'; \
for i in 1 2 3 4 5 6; do \
printf '\\033[%s;4H{} ⠀⠁⠃⠇⡇⣇⣧⣷⣿ █▇▆▅▄▃▂▁ %s\\033[0m' \"$i\" \"$i\"; \
done; \
printf '\\033[?25h\\033[?2026l\\033[?1049l'\n",
marker
);
let step_deadline = Instant::now() + Duration::from_secs(8);
let mut seen = false;
let mut attempts = 0;
while Instant::now() < step_deadline && attempts < 12 {
send_input(
&socket,
&relay,
ok.client_id,
seq,
0,
&bootstrap.session_key,
command.as_bytes(),
);
seq += 1;
attempts += 1;
if wait_for_text_with_ack(
&socket,
&relay,
ok.client_id,
&mut seq,
&bootstrap.session_key,
&marker,
500,
) {
seen = true;
break;
}
}
assert!(
seen,
"TUI marker {marker} did not arrive during hostile-network soak"
);
iteration += 1;
thread::sleep(Duration::from_millis(500));
}
relay.clear_impairments();
drop(relay);
let _ = server.kill();
let _ = server.wait();
}
#[test]
fn duplicated_and_replayed_input_is_applied_at_most_once() {
let dir = tempfile::tempdir().unwrap();