Recover from sustained terminal backpressure

tests/client_terminal_runtime.rs

@@ -51,6 +51,7 @@ enum ServerObservation {
     BulkSent,
     Input(Vec<u8>),
     Reconnected,
+    RenderResynced,
     Resize(u16, u16),
 }
 
@@ -67,7 +68,7 @@ fn native_client_terminal_round_trip_is_platform_complete() {
         .set_read_timeout(Some(Duration::from_secs(8)))
         .unwrap();
     let port = socket.local_addr().unwrap().port();
-    write_client_fixture(&home, &cache, port);
+    write_client_fixture(&home, &cache, port, 5);
 
     let (observation_tx, observation_rx) = mpsc::channel();
     let server = thread::spawn(move || run_fake_terminal_server(socket, observation_tx));
@@ -176,7 +177,7 @@ fn terminal_output_backpressure_does_not_block_input_transport() {
         .set_read_timeout(Some(Duration::from_secs(8)))
         .unwrap();
     let port = socket.local_addr().unwrap().port();
-    write_client_fixture(&home, &cache, port);
+    write_client_fixture(&home, &cache, port, 5);
     let (observation_tx, observation_rx) = mpsc::channel();
     let server = thread::spawn(move || run_backpressure_server(socket, observation_tx, PROBE));
 
@@ -240,7 +241,7 @@ fn idle_reconnect_restores_snapshot_and_orders_reordered_frames() {
         .set_read_timeout(Some(Duration::from_secs(8)))
         .unwrap();
     let port = socket.local_addr().unwrap().port();
-    write_client_fixture(&home, &cache, port);
+    write_client_fixture(&home, &cache, port, 1);
     let (observation_tx, observation_rx) = mpsc::channel();
     let server = thread::spawn(move || run_reconnect_server(socket, observation_tx, PROBE));
 
@@ -299,13 +300,92 @@ fn idle_reconnect_restores_snapshot_and_orders_reordered_frames() {
     assert!(status.success(), "Dosh client exited with {status:?}");
 }
 
-fn write_client_fixture(home: &Path, cache: &Path, port: u16) {
+#[test]
+fn renderer_overflow_resyncs_without_dropping_the_session() {
+    const PROBE: &[u8] = b"DOSH_OVERFLOW_INPUT\r";
+
+    let dir = tempfile::tempdir().unwrap();
+    let home = dir.path().join("home");
+    let cache = dir.path().join("credentials");
+    fs::create_dir_all(home.join(".config/dosh")).unwrap();
+    fs::create_dir_all(&cache).unwrap();
+
+    let socket = UdpSocket::bind("127.0.0.1:0").unwrap();
+    socket
+        .set_read_timeout(Some(Duration::from_secs(12)))
+        .unwrap();
+    let port = socket.local_addr().unwrap().port();
+    write_client_fixture(&home, &cache, port, 30);
+    let (observation_tx, observation_rx) = mpsc::channel();
+    let server = thread::spawn(move || run_overflow_server(socket, observation_tx, PROBE));
+
+    let pty = NativePtySystem::default();
+    let pair = pty
+        .openpty(PtySize {
+            rows: 24,
+            cols: 80,
+            pixel_width: 0,
+            pixel_height: 0,
+        })
+        .unwrap();
+    let mut reader = pair.master.try_clone_reader().unwrap();
+    let mut writer = pair.master.take_writer().unwrap();
+    let mut command = client_command(dir.path(), port);
+    command.env("HOME", home.to_string_lossy().to_string());
+    command.env("USERPROFILE", home.to_string_lossy().to_string());
+    command.env("APPDATA", home.to_string_lossy().to_string());
+    command.env("LOCALAPPDATA", home.to_string_lossy().to_string());
+    command.env("TERM", "xterm-256color");
+    let mut child = pair.slave.spawn_command(command).unwrap();
+    drop(pair.slave);
+
+    wait_for_observation(&observation_rx, Duration::from_secs(5), |observation| {
+        matches!(observation, ServerObservation::BulkSent)
+    });
+    writer.write_all(PROBE).unwrap();
+    writer.flush().unwrap();
+    wait_for_input(&observation_rx, PROBE, Duration::from_secs(2));
+
+    let output = Arc::new(Mutex::new(Vec::new()));
+    let reader_output = Arc::clone(&output);
+    let reader_thread = thread::spawn(move || {
+        let mut buf = [0u8; 16 * 1024];
+        loop {
+            match reader.read(&mut buf) {
+                Ok(0) | Err(_) => break,
+                Ok(n) => reader_output.lock().unwrap().extend_from_slice(&buf[..n]),
+            }
+        }
+    });
+    wait_for_observation(&observation_rx, Duration::from_secs(5), |observation| {
+        matches!(observation, ServerObservation::RenderResynced)
+    });
+    wait_for_output(
+        &output,
+        b"DOSH_OVERFLOW_RESYNC_DOSH_OVERFLOW_DONE",
+        Duration::from_secs(5),
+    );
+
+    let status = child.wait().unwrap();
+    drop(writer);
+    drop(pair.master);
+    reader_thread.join().unwrap();
+    server.join().unwrap();
+    assert!(status.success(), "Dosh client exited with {status:?}");
+}
+
+fn write_client_fixture(
+    home: &Path,
+    cache: &Path,
+    port: u16,
+    reconnect_timeout_secs: u64,
+) {
     let config = ClientConfig {
         server: "local".to_string(),
         dosh_host: Some("127.0.0.1".to_string()),
         dosh_port: port,
         cache_attach_tickets: false,
-        reconnect_timeout_secs: 1,
+        reconnect_timeout_secs,
         credential_cache: cache.to_string_lossy().to_string(),
         auth_preference: "ssh".to_string(),
         predict: false,
@@ -638,6 +718,101 @@ fn run_reconnect_server(
     }
 }
 
+fn run_overflow_server(
+    socket: UdpSocket,
+    observations: mpsc::Sender<ServerObservation>,
+    expected_input: &[u8],
+) {
+    const BULK_FRAMES: u64 = 640;
+
+    let mut resume_count = 0u64;
+    let mut bulk_sent = false;
+    let mut resynced = false;
+    let mut closed_sent = false;
+    let mut received_input = Vec::new();
+    let mut buf = [0u8; 65535];
+    loop {
+        let (n, source) = socket.recv_from(&mut buf).unwrap();
+        let packet = protocol::decode(&buf[..n]).unwrap();
+        match packet.header.kind {
+            PacketKind::ResumeRequest => {
+                resume_count += 1;
+                if resume_count == 1 {
+                    send_frame(
+                        &socket,
+                        source,
+                        PacketKind::ResumeOk,
+                        1,
+                        10,
+                        b"DOSH_OVERFLOW_READY",
+                        true,
+                        false,
+                    );
+                } else {
+                    send_frame(
+                        &socket,
+                        source,
+                        PacketKind::ResumeOk,
+                        700,
+                        700,
+                        b"DOSH_OVERFLOW_RESYNC_",
+                        true,
+                        false,
+                    );
+                    resynced = true;
+                    observations
+                        .send(ServerObservation::RenderResynced)
+                        .unwrap();
+                }
+            }
+            PacketKind::Ack if !bulk_sent => {
+                let bulk = [b'x'; 1024];
+                for index in 0..BULK_FRAMES {
+                    send_frame(
+                        &socket,
+                        source,
+                        PacketKind::Frame,
+                        2 + index,
+                        11 + index,
+                        &bulk,
+                        false,
+                        false,
+                    );
+                    thread::sleep(Duration::from_millis(1));
+                }
+                bulk_sent = true;
+                observations.send(ServerObservation::BulkSent).unwrap();
+            }
+            PacketKind::Ack if resynced && !closed_sent => {
+                send_frame(
+                    &socket,
+                    source,
+                    PacketKind::Frame,
+                    701,
+                    701,
+                    b"DOSH_OVERFLOW_DONE",
+                    false,
+                    true,
+                );
+                closed_sent = true;
+            }
+            PacketKind::Input => {
+                let plain =
+                    protocol::decrypt_body(&packet, &SESSION_KEY, CLIENT_TO_SERVER).unwrap();
+                let input: Input = protocol::from_body(&plain).unwrap();
+                received_input.extend_from_slice(&input.bytes);
+                observations
+                    .send(ServerObservation::Input(input.bytes))
+                    .unwrap();
+            }
+            _ => {}
+        }
+        if closed_sent && contains(&received_input, expected_input) {
+            break;
+        }
+    }
+}
+
 #[allow(clippy::too_many_arguments)]
 fn send_frame(
     socket: &UdpSocket,