Add hostile network TUI soak gate
ci / test (push) Has been cancelled
ci / fuzz-smoke (push) Has been cancelled
ci / windows-client (push) Has been cancelled
ci / package-release (linux-x86_64, ubuntu-latest) (push) Has been cancelled
ci / package-release (macos-aarch64, macos-14) (push) Has been cancelled
ci / package-release (macos-x86_64, macos-13) (push) Has been cancelled
ci / package-release (windows-x86_64, windows-latest) (push) Has been cancelled
ci / remote-bench (push) Has been cancelled
ci / test (push) Has been cancelled
ci / fuzz-smoke (push) Has been cancelled
ci / windows-client (push) Has been cancelled
ci / package-release (linux-x86_64, ubuntu-latest) (push) Has been cancelled
ci / package-release (macos-aarch64, macos-14) (push) Has been cancelled
ci / package-release (macos-x86_64, macos-13) (push) Has been cancelled
ci / package-release (windows-x86_64, windows-latest) (push) Has been cancelled
ci / remote-bench (push) Has been cancelled
This commit is contained in:
@@ -27,6 +27,7 @@ reconnect-check:
|
|||||||
|
|
||||||
hostile-network-check:
|
hostile-network-check:
|
||||||
cargo test --test hostile_network -- --nocapture
|
cargo test --test hostile_network -- --nocapture
|
||||||
|
DOSH_BADNET_SOAK_SECONDS=$${DOSH_BADNET_SOAK_SECONDS:-300} cargo test --test hostile_network bad_network_tui_work_soak_30m -- --ignored --nocapture
|
||||||
|
|
||||||
persistence-check:
|
persistence-check:
|
||||||
cargo test --test integration_smoke session_survives_server_restart_same_shell_and_screen -- --nocapture
|
cargo test --test integration_smoke session_survives_server_restart_same_shell_and_screen -- --nocapture
|
||||||
|
|||||||
+141
-9
@@ -258,7 +258,8 @@ fn relay_loop(
|
|||||||
let mut upstream = new_upstream();
|
let mut upstream = new_upstream();
|
||||||
let mut client_addr: Option<SocketAddr> = None;
|
let mut client_addr: Option<SocketAddr> = None;
|
||||||
let mut rng = StdRng::seed_from_u64(seed);
|
let mut rng = StdRng::seed_from_u64(seed);
|
||||||
let mut held: Option<(Vec<u8>, bool)> = None; // (packet, is_c2s) held for reorder
|
let mut held_c2s: Option<Vec<u8>> = None;
|
||||||
|
let mut held_s2c: Option<Vec<u8>> = None;
|
||||||
let mut buf = [0u8; 65535];
|
let mut buf = [0u8; 65535];
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
@@ -288,10 +289,9 @@ fn relay_loop(
|
|||||||
&upstream,
|
&upstream,
|
||||||
server_addr,
|
server_addr,
|
||||||
packet,
|
packet,
|
||||||
true,
|
|
||||||
&controls,
|
&controls,
|
||||||
&mut rng,
|
&mut rng,
|
||||||
&mut held,
|
&mut held_c2s,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -310,7 +310,12 @@ fn relay_loop(
|
|||||||
let drop_pct = controls.drop_s2c_percent.load(Ordering::SeqCst);
|
let drop_pct = controls.drop_s2c_percent.load(Ordering::SeqCst);
|
||||||
if drop_pct == 0 || rng.gen_range(0..100) >= drop_pct {
|
if drop_pct == 0 || rng.gen_range(0..100) >= drop_pct {
|
||||||
forward_with_effects(
|
forward_with_effects(
|
||||||
&front, dst, packet, false, &controls, &mut rng, &mut held,
|
&front,
|
||||||
|
dst,
|
||||||
|
packet,
|
||||||
|
&controls,
|
||||||
|
&mut rng,
|
||||||
|
&mut held_s2c,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -336,23 +341,22 @@ fn forward_with_effects(
|
|||||||
out: &UdpSocket,
|
out: &UdpSocket,
|
||||||
dst: SocketAddr,
|
dst: SocketAddr,
|
||||||
packet: Vec<u8>,
|
packet: Vec<u8>,
|
||||||
is_c2s: bool,
|
|
||||||
controls: &RelayControls,
|
controls: &RelayControls,
|
||||||
rng: &mut StdRng,
|
rng: &mut StdRng,
|
||||||
held: &mut Option<(Vec<u8>, bool)>,
|
held: &mut Option<Vec<u8>>,
|
||||||
) {
|
) {
|
||||||
// Reorder: if armed, hold this packet and release the previously held one
|
// Reorder: if armed, hold this packet and release the previously held one
|
||||||
// afterward (so two consecutive packets swap order).
|
// afterward (so two consecutive packets swap order).
|
||||||
if controls.reorder_next.swap(false, Ordering::SeqCst) {
|
if controls.reorder_next.swap(false, Ordering::SeqCst) {
|
||||||
if let Some((prev, _)) = held.take() {
|
if let Some(prev) = held.take() {
|
||||||
let _ = out.send_to(&packet, dst);
|
let _ = out.send_to(&packet, dst);
|
||||||
let _ = out.send_to(&prev, dst);
|
let _ = out.send_to(&prev, dst);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
*held = Some((packet, is_c2s));
|
*held = Some(packet);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if let Some((prev, _)) = held.take() {
|
if let Some(prev) = held.take() {
|
||||||
let _ = out.send_to(&prev, dst);
|
let _ = out.send_to(&prev, dst);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -767,6 +771,46 @@ fn wait_for_text(socket: &UdpSocket, key: &[u8; 32], needle: &str, millis: u64)
|
|||||||
acc.contains(needle)
|
acc.contains(needle)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn wait_for_text_with_ack(
|
||||||
|
socket: &UdpSocket,
|
||||||
|
relay: &Relay,
|
||||||
|
client_id: [u8; 16],
|
||||||
|
seq: &mut u64,
|
||||||
|
key: &[u8; 32],
|
||||||
|
needle: &str,
|
||||||
|
millis: u64,
|
||||||
|
) -> bool {
|
||||||
|
let prev = socket.read_timeout().unwrap();
|
||||||
|
socket
|
||||||
|
.set_read_timeout(Some(Duration::from_millis(100)))
|
||||||
|
.unwrap();
|
||||||
|
let deadline = Instant::now() + Duration::from_millis(millis);
|
||||||
|
let mut acc = String::new();
|
||||||
|
while Instant::now() < deadline {
|
||||||
|
if let Some((_header, frame)) = recv_frame(socket, key) {
|
||||||
|
acc.push_str(&String::from_utf8_lossy(&frame.bytes));
|
||||||
|
let ack = protocol::encode_encrypted(
|
||||||
|
PacketKind::Ack,
|
||||||
|
client_id,
|
||||||
|
*seq,
|
||||||
|
frame.output_seq,
|
||||||
|
key,
|
||||||
|
CLIENT_TO_SERVER,
|
||||||
|
b"",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
*seq += 1;
|
||||||
|
socket.send_to(&ack, relay.front_addr()).unwrap();
|
||||||
|
if acc.contains(needle) {
|
||||||
|
socket.set_read_timeout(prev).unwrap();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
socket.set_read_timeout(prev).unwrap();
|
||||||
|
acc.contains(needle)
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn session_survives_packet_loss_and_reorder() {
|
fn session_survives_packet_loss_and_reorder() {
|
||||||
let dir = tempfile::tempdir().unwrap();
|
let dir = tempfile::tempdir().unwrap();
|
||||||
@@ -820,6 +864,94 @@ fn session_survives_packet_loss_and_reorder() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[ignore = "30-minute hostile-network TUI soak; run with `DOSH_BADNET_SOAK_SECONDS=1800 cargo test --test hostile_network bad_network_tui_work_soak_30m -- --ignored --nocapture`"]
|
||||||
|
fn bad_network_tui_work_soak_30m() {
|
||||||
|
let soak_secs = std::env::var("DOSH_BADNET_SOAK_SECONDS")
|
||||||
|
.ok()
|
||||||
|
.and_then(|value| value.parse::<u64>().ok())
|
||||||
|
.unwrap_or(30 * 60);
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let port = free_udp_port();
|
||||||
|
let config = write_server_config(&dir, port);
|
||||||
|
let mut server = start_server(&dir, &config);
|
||||||
|
let relay = Relay::spawn(port, 0xBADC0DEu64);
|
||||||
|
let (socket, bootstrap, ok) = attach_through_relay(&config, &relay);
|
||||||
|
|
||||||
|
relay.set_drop_c2s(15);
|
||||||
|
relay.set_drop_s2c(20);
|
||||||
|
relay.set_dup(10);
|
||||||
|
|
||||||
|
let deadline = Instant::now() + Duration::from_secs(soak_secs);
|
||||||
|
let mut seq = 2u64;
|
||||||
|
let mut iteration = 0u64;
|
||||||
|
while Instant::now() < deadline {
|
||||||
|
if iteration % 2 == 0 {
|
||||||
|
relay.arm_reorder();
|
||||||
|
}
|
||||||
|
if iteration > 0 && iteration % 7 == 0 {
|
||||||
|
let _ = relay.rebind_upstream();
|
||||||
|
}
|
||||||
|
if iteration > 0 && iteration % 11 == 0 {
|
||||||
|
relay.set_drop_s2c(100);
|
||||||
|
thread::sleep(Duration::from_millis(750));
|
||||||
|
relay.set_drop_s2c(20);
|
||||||
|
}
|
||||||
|
|
||||||
|
let marker = format!("DOSH_BADNET_TUI_{iteration}");
|
||||||
|
let command = format!(
|
||||||
|
"stty -echo; \
|
||||||
|
printf '\\033[?1049h\\033[?2026h\\033[?25l'; \
|
||||||
|
for i in 1 2 3 4 5 6; do \
|
||||||
|
printf '\\033[%s;4H{} ⠀⠁⠃⠇⡇⣇⣧⣷⣿ █▇▆▅▄▃▂▁ %s\\033[0m' \"$i\" \"$i\"; \
|
||||||
|
done; \
|
||||||
|
printf '\\033[?25h\\033[?2026l\\033[?1049l'\n",
|
||||||
|
marker
|
||||||
|
);
|
||||||
|
|
||||||
|
let step_deadline = Instant::now() + Duration::from_secs(8);
|
||||||
|
let mut seen = false;
|
||||||
|
let mut attempts = 0;
|
||||||
|
while Instant::now() < step_deadline && attempts < 12 {
|
||||||
|
send_input(
|
||||||
|
&socket,
|
||||||
|
&relay,
|
||||||
|
ok.client_id,
|
||||||
|
seq,
|
||||||
|
0,
|
||||||
|
&bootstrap.session_key,
|
||||||
|
command.as_bytes(),
|
||||||
|
);
|
||||||
|
seq += 1;
|
||||||
|
attempts += 1;
|
||||||
|
if wait_for_text_with_ack(
|
||||||
|
&socket,
|
||||||
|
&relay,
|
||||||
|
ok.client_id,
|
||||||
|
&mut seq,
|
||||||
|
&bootstrap.session_key,
|
||||||
|
&marker,
|
||||||
|
500,
|
||||||
|
) {
|
||||||
|
seen = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
seen,
|
||||||
|
"TUI marker {marker} did not arrive during hostile-network soak"
|
||||||
|
);
|
||||||
|
|
||||||
|
iteration += 1;
|
||||||
|
thread::sleep(Duration::from_millis(500));
|
||||||
|
}
|
||||||
|
|
||||||
|
relay.clear_impairments();
|
||||||
|
drop(relay);
|
||||||
|
let _ = server.kill();
|
||||||
|
let _ = server.wait();
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn duplicated_and_replayed_input_is_applied_at_most_once() {
|
fn duplicated_and_replayed_input_is_applied_at_most_once() {
|
||||||
let dir = tempfile::tempdir().unwrap();
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
|||||||
Reference in New Issue
Block a user