Add hostile network TUI soak gate
ci / test (push) Has been cancelled
ci / fuzz-smoke (push) Has been cancelled
ci / windows-client (push) Has been cancelled
ci / package-release (linux-x86_64, ubuntu-latest) (push) Has been cancelled
ci / package-release (macos-aarch64, macos-14) (push) Has been cancelled
ci / package-release (macos-x86_64, macos-13) (push) Has been cancelled
ci / package-release (windows-x86_64, windows-latest) (push) Has been cancelled
ci / remote-bench (push) Has been cancelled
ci / test (push) Has been cancelled
ci / fuzz-smoke (push) Has been cancelled
ci / windows-client (push) Has been cancelled
ci / package-release (linux-x86_64, ubuntu-latest) (push) Has been cancelled
ci / package-release (macos-aarch64, macos-14) (push) Has been cancelled
ci / package-release (macos-x86_64, macos-13) (push) Has been cancelled
ci / package-release (windows-x86_64, windows-latest) (push) Has been cancelled
ci / remote-bench (push) Has been cancelled
This commit is contained in:
@@ -27,6 +27,7 @@ reconnect-check:
|
||||
|
||||
hostile-network-check:
|
||||
cargo test --test hostile_network -- --nocapture
|
||||
DOSH_BADNET_SOAK_SECONDS=$${DOSH_BADNET_SOAK_SECONDS:-300} cargo test --test hostile_network bad_network_tui_work_soak_30m -- --ignored --nocapture
|
||||
|
||||
persistence-check:
|
||||
cargo test --test integration_smoke session_survives_server_restart_same_shell_and_screen -- --nocapture
|
||||
|
||||
+141
-9
@@ -258,7 +258,8 @@ fn relay_loop(
|
||||
let mut upstream = new_upstream();
|
||||
let mut client_addr: Option<SocketAddr> = None;
|
||||
let mut rng = StdRng::seed_from_u64(seed);
|
||||
let mut held: Option<(Vec<u8>, bool)> = None; // (packet, is_c2s) held for reorder
|
||||
let mut held_c2s: Option<Vec<u8>> = None;
|
||||
let mut held_s2c: Option<Vec<u8>> = None;
|
||||
let mut buf = [0u8; 65535];
|
||||
|
||||
loop {
|
||||
@@ -288,10 +289,9 @@ fn relay_loop(
|
||||
&upstream,
|
||||
server_addr,
|
||||
packet,
|
||||
true,
|
||||
&controls,
|
||||
&mut rng,
|
||||
&mut held,
|
||||
&mut held_c2s,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -310,7 +310,12 @@ fn relay_loop(
|
||||
let drop_pct = controls.drop_s2c_percent.load(Ordering::SeqCst);
|
||||
if drop_pct == 0 || rng.gen_range(0..100) >= drop_pct {
|
||||
forward_with_effects(
|
||||
&front, dst, packet, false, &controls, &mut rng, &mut held,
|
||||
&front,
|
||||
dst,
|
||||
packet,
|
||||
&controls,
|
||||
&mut rng,
|
||||
&mut held_s2c,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -336,23 +341,22 @@ fn forward_with_effects(
|
||||
out: &UdpSocket,
|
||||
dst: SocketAddr,
|
||||
packet: Vec<u8>,
|
||||
is_c2s: bool,
|
||||
controls: &RelayControls,
|
||||
rng: &mut StdRng,
|
||||
held: &mut Option<(Vec<u8>, bool)>,
|
||||
held: &mut Option<Vec<u8>>,
|
||||
) {
|
||||
// Reorder: if armed, hold this packet and release the previously held one
|
||||
// afterward (so two consecutive packets swap order).
|
||||
if controls.reorder_next.swap(false, Ordering::SeqCst) {
|
||||
if let Some((prev, _)) = held.take() {
|
||||
if let Some(prev) = held.take() {
|
||||
let _ = out.send_to(&packet, dst);
|
||||
let _ = out.send_to(&prev, dst);
|
||||
return;
|
||||
}
|
||||
*held = Some((packet, is_c2s));
|
||||
*held = Some(packet);
|
||||
return;
|
||||
}
|
||||
if let Some((prev, _)) = held.take() {
|
||||
if let Some(prev) = held.take() {
|
||||
let _ = out.send_to(&prev, dst);
|
||||
}
|
||||
|
||||
@@ -767,6 +771,46 @@ fn wait_for_text(socket: &UdpSocket, key: &[u8; 32], needle: &str, millis: u64)
|
||||
acc.contains(needle)
|
||||
}
|
||||
|
||||
fn wait_for_text_with_ack(
|
||||
socket: &UdpSocket,
|
||||
relay: &Relay,
|
||||
client_id: [u8; 16],
|
||||
seq: &mut u64,
|
||||
key: &[u8; 32],
|
||||
needle: &str,
|
||||
millis: u64,
|
||||
) -> bool {
|
||||
let prev = socket.read_timeout().unwrap();
|
||||
socket
|
||||
.set_read_timeout(Some(Duration::from_millis(100)))
|
||||
.unwrap();
|
||||
let deadline = Instant::now() + Duration::from_millis(millis);
|
||||
let mut acc = String::new();
|
||||
while Instant::now() < deadline {
|
||||
if let Some((_header, frame)) = recv_frame(socket, key) {
|
||||
acc.push_str(&String::from_utf8_lossy(&frame.bytes));
|
||||
let ack = protocol::encode_encrypted(
|
||||
PacketKind::Ack,
|
||||
client_id,
|
||||
*seq,
|
||||
frame.output_seq,
|
||||
key,
|
||||
CLIENT_TO_SERVER,
|
||||
b"",
|
||||
)
|
||||
.unwrap();
|
||||
*seq += 1;
|
||||
socket.send_to(&ack, relay.front_addr()).unwrap();
|
||||
if acc.contains(needle) {
|
||||
socket.set_read_timeout(prev).unwrap();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
socket.set_read_timeout(prev).unwrap();
|
||||
acc.contains(needle)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn session_survives_packet_loss_and_reorder() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
@@ -820,6 +864,94 @@ fn session_survives_packet_loss_and_reorder() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "30-minute hostile-network TUI soak; run with `DOSH_BADNET_SOAK_SECONDS=1800 cargo test --test hostile_network bad_network_tui_work_soak_30m -- --ignored --nocapture`"]
|
||||
fn bad_network_tui_work_soak_30m() {
|
||||
let soak_secs = std::env::var("DOSH_BADNET_SOAK_SECONDS")
|
||||
.ok()
|
||||
.and_then(|value| value.parse::<u64>().ok())
|
||||
.unwrap_or(30 * 60);
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let port = free_udp_port();
|
||||
let config = write_server_config(&dir, port);
|
||||
let mut server = start_server(&dir, &config);
|
||||
let relay = Relay::spawn(port, 0xBADC0DEu64);
|
||||
let (socket, bootstrap, ok) = attach_through_relay(&config, &relay);
|
||||
|
||||
relay.set_drop_c2s(15);
|
||||
relay.set_drop_s2c(20);
|
||||
relay.set_dup(10);
|
||||
|
||||
let deadline = Instant::now() + Duration::from_secs(soak_secs);
|
||||
let mut seq = 2u64;
|
||||
let mut iteration = 0u64;
|
||||
while Instant::now() < deadline {
|
||||
if iteration % 2 == 0 {
|
||||
relay.arm_reorder();
|
||||
}
|
||||
if iteration > 0 && iteration % 7 == 0 {
|
||||
let _ = relay.rebind_upstream();
|
||||
}
|
||||
if iteration > 0 && iteration % 11 == 0 {
|
||||
relay.set_drop_s2c(100);
|
||||
thread::sleep(Duration::from_millis(750));
|
||||
relay.set_drop_s2c(20);
|
||||
}
|
||||
|
||||
let marker = format!("DOSH_BADNET_TUI_{iteration}");
|
||||
let command = format!(
|
||||
"stty -echo; \
|
||||
printf '\\033[?1049h\\033[?2026h\\033[?25l'; \
|
||||
for i in 1 2 3 4 5 6; do \
|
||||
printf '\\033[%s;4H{} ⠀⠁⠃⠇⡇⣇⣧⣷⣿ █▇▆▅▄▃▂▁ %s\\033[0m' \"$i\" \"$i\"; \
|
||||
done; \
|
||||
printf '\\033[?25h\\033[?2026l\\033[?1049l'\n",
|
||||
marker
|
||||
);
|
||||
|
||||
let step_deadline = Instant::now() + Duration::from_secs(8);
|
||||
let mut seen = false;
|
||||
let mut attempts = 0;
|
||||
while Instant::now() < step_deadline && attempts < 12 {
|
||||
send_input(
|
||||
&socket,
|
||||
&relay,
|
||||
ok.client_id,
|
||||
seq,
|
||||
0,
|
||||
&bootstrap.session_key,
|
||||
command.as_bytes(),
|
||||
);
|
||||
seq += 1;
|
||||
attempts += 1;
|
||||
if wait_for_text_with_ack(
|
||||
&socket,
|
||||
&relay,
|
||||
ok.client_id,
|
||||
&mut seq,
|
||||
&bootstrap.session_key,
|
||||
&marker,
|
||||
500,
|
||||
) {
|
||||
seen = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
seen,
|
||||
"TUI marker {marker} did not arrive during hostile-network soak"
|
||||
);
|
||||
|
||||
iteration += 1;
|
||||
thread::sleep(Duration::from_millis(500));
|
||||
}
|
||||
|
||||
relay.clear_impairments();
|
||||
drop(relay);
|
||||
let _ = server.kill();
|
||||
let _ = server.wait();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn duplicated_and_replayed_input_is_applied_at_most_once() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
Reference in New Issue
Block a user