peer.rs

  1use super::{
  2    proto::{self, AnyTypedEnvelope, EnvelopedMessage, MessageStream, RequestMessage},
  3    Connection,
  4};
  5use anyhow::{anyhow, Context, Result};
  6use collections::HashMap;
  7use futures::{
  8    channel::{mpsc, oneshot},
  9    stream::BoxStream,
 10    FutureExt, SinkExt, StreamExt,
 11};
 12use parking_lot::{Mutex, RwLock};
 13use serde::{ser::SerializeStruct, Serialize};
 14use std::sync::atomic::Ordering::SeqCst;
 15use std::{
 16    fmt,
 17    future::Future,
 18    marker::PhantomData,
 19    sync::{
 20        atomic::{self, AtomicU32},
 21        Arc,
 22    },
 23    time::Duration,
 24};
 25use tracing::instrument;
 26
 27#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, Serialize)]
 28pub struct ConnectionId(pub u32);
 29
 30impl fmt::Display for ConnectionId {
 31    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
 32        self.0.fmt(f)
 33    }
 34}
 35
 36#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
 37pub struct PeerId(pub u32);
 38
 39impl fmt::Display for PeerId {
 40    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
 41        self.0.fmt(f)
 42    }
 43}
 44
 45pub struct Receipt<T> {
 46    pub sender_id: ConnectionId,
 47    pub message_id: u32,
 48    payload_type: PhantomData<T>,
 49}
 50
 51impl<T> Clone for Receipt<T> {
 52    fn clone(&self) -> Self {
 53        Self {
 54            sender_id: self.sender_id,
 55            message_id: self.message_id,
 56            payload_type: PhantomData,
 57        }
 58    }
 59}
 60
 61impl<T> Copy for Receipt<T> {}
 62
 63pub struct TypedEnvelope<T> {
 64    pub sender_id: ConnectionId,
 65    pub original_sender_id: Option<PeerId>,
 66    pub message_id: u32,
 67    pub payload: T,
 68}
 69
 70impl<T> TypedEnvelope<T> {
 71    pub fn original_sender_id(&self) -> Result<PeerId> {
 72        self.original_sender_id
 73            .ok_or_else(|| anyhow!("missing original_sender_id"))
 74    }
 75}
 76
 77impl<T: RequestMessage> TypedEnvelope<T> {
 78    pub fn receipt(&self) -> Receipt<T> {
 79        Receipt {
 80            sender_id: self.sender_id,
 81            message_id: self.message_id,
 82            payload_type: PhantomData,
 83        }
 84    }
 85}
 86
 87pub struct Peer {
 88    pub connections: RwLock<HashMap<ConnectionId, ConnectionState>>,
 89    next_connection_id: AtomicU32,
 90}
 91
 92#[derive(Clone, Serialize)]
 93pub struct ConnectionState {
 94    #[serde(skip)]
 95    outgoing_tx: mpsc::UnboundedSender<proto::Message>,
 96    next_message_id: Arc<AtomicU32>,
 97    #[serde(skip)]
 98    response_channels:
 99        Arc<Mutex<Option<HashMap<u32, oneshot::Sender<(proto::Envelope, oneshot::Sender<()>)>>>>>,
100}
101
102const KEEPALIVE_INTERVAL: Duration = Duration::from_secs(1);
103const WRITE_TIMEOUT: Duration = Duration::from_secs(2);
104pub const RECEIVE_TIMEOUT: Duration = Duration::from_secs(5);
105
106impl Peer {
107    pub fn new() -> Arc<Self> {
108        Arc::new(Self {
109            connections: Default::default(),
110            next_connection_id: Default::default(),
111        })
112    }
113
114    #[instrument(skip_all)]
115    pub async fn add_connection<F, Fut, Out>(
116        self: &Arc<Self>,
117        connection: Connection,
118        create_timer: F,
119    ) -> (
120        ConnectionId,
121        impl Future<Output = anyhow::Result<()>> + Send,
122        BoxStream<'static, Box<dyn AnyTypedEnvelope>>,
123    )
124    where
125        F: Send + Fn(Duration) -> Fut,
126        Fut: Send + Future<Output = Out>,
127        Out: Send,
128    {
129        // For outgoing messages, use an unbounded channel so that application code
130        // can always send messages without yielding. For incoming messages, use a
131        // bounded channel so that other peers will receive backpressure if they send
132        // messages faster than this peer can process them.
133        #[cfg(any(test, feature = "test-support"))]
134        const INCOMING_BUFFER_SIZE: usize = 1;
135        #[cfg(not(any(test, feature = "test-support")))]
136        const INCOMING_BUFFER_SIZE: usize = 64;
137        let (mut incoming_tx, incoming_rx) = mpsc::channel(INCOMING_BUFFER_SIZE);
138        let (outgoing_tx, mut outgoing_rx) = mpsc::unbounded();
139
140        let connection_id = ConnectionId(self.next_connection_id.fetch_add(1, SeqCst));
141        let connection_state = ConnectionState {
142            outgoing_tx: outgoing_tx.clone(),
143            next_message_id: Default::default(),
144            response_channels: Arc::new(Mutex::new(Some(Default::default()))),
145        };
146        let mut writer = MessageStream::new(connection.tx);
147        let mut reader = MessageStream::new(connection.rx);
148
149        let this = self.clone();
150        let response_channels = connection_state.response_channels.clone();
151        let handle_io = async move {
152            tracing::debug!(%connection_id, "handle io future: start");
153
154            let _end_connection = util::defer(|| {
155                response_channels.lock().take();
156                this.connections.write().remove(&connection_id);
157                tracing::debug!(%connection_id, "handle io future: end");
158            });
159
160            // Send messages on this frequency so the connection isn't closed.
161            let keepalive_timer = create_timer(KEEPALIVE_INTERVAL).fuse();
162            futures::pin_mut!(keepalive_timer);
163
164            // Disconnect if we don't receive messages at least this frequently.
165            let receive_timeout = create_timer(RECEIVE_TIMEOUT).fuse();
166            futures::pin_mut!(receive_timeout);
167
168            loop {
169                tracing::debug!(%connection_id, "outer loop iteration start");
170                let read_message = reader.read().fuse();
171                futures::pin_mut!(read_message);
172
173                loop {
174                    tracing::debug!(%connection_id, "inner loop iteration start");
175                    futures::select_biased! {
176                        outgoing = outgoing_rx.next().fuse() => match outgoing {
177                            Some(outgoing) => {
178                                tracing::debug!(%connection_id, "outgoing rpc message: writing");
179                                futures::select_biased! {
180                                    result = writer.write(outgoing).fuse() => {
181                                        tracing::debug!(%connection_id, "outgoing rpc message: done writing");
182                                        result.context("failed to write RPC message")?;
183                                        tracing::debug!(%connection_id, "keepalive interval: resetting after sending message");
184                                        keepalive_timer.set(create_timer(KEEPALIVE_INTERVAL).fuse());
185                                    }
186                                    _ = create_timer(WRITE_TIMEOUT).fuse() => {
187                                        tracing::debug!(%connection_id, "outgoing rpc message: writing timed out");
188                                        Err(anyhow!("timed out writing message"))?;
189                                    }
190                                }
191                            }
192                            None => {
193                                tracing::debug!(%connection_id, "outgoing rpc message: channel closed");
194                                return Ok(())
195                            },
196                        },
197                        incoming = read_message => {
198                            let incoming = incoming.context("error reading rpc message from socket")?;
199                            tracing::debug!(%connection_id, "incoming rpc message: received");
200                            tracing::debug!(%connection_id, "receive timeout: resetting");
201                            receive_timeout.set(create_timer(RECEIVE_TIMEOUT).fuse());
202                            if let proto::Message::Envelope(incoming) = incoming {
203                                tracing::debug!(%connection_id, "incoming rpc message: processing");
204                                futures::select_biased! {
205                                    result = incoming_tx.send(incoming).fuse() => match result {
206                                        Ok(_) => {
207                                            tracing::debug!(%connection_id, "incoming rpc message: processed");
208                                        }
209                                        Err(_) => {
210                                            tracing::debug!(%connection_id, "incoming rpc message: channel closed");
211                                            return Ok(())
212                                        }
213                                    },
214                                    _ = create_timer(WRITE_TIMEOUT).fuse() => {
215                                        tracing::debug!(%connection_id, "incoming rpc message: processing timed out");
216                                        Err(anyhow!("timed out processing incoming message"))?
217                                    }
218                                }
219                            }
220                            break;
221                        },
222                        _ = keepalive_timer => {
223                            tracing::debug!(%connection_id, "keepalive interval: pinging");
224                            futures::select_biased! {
225                                result = writer.write(proto::Message::Ping).fuse() => {
226                                    tracing::debug!(%connection_id, "keepalive interval: done pinging");
227                                    result.context("failed to send keepalive")?;
228                                    tracing::debug!(%connection_id, "keepalive interval: resetting after pinging");
229                                    keepalive_timer.set(create_timer(KEEPALIVE_INTERVAL).fuse());
230                                }
231                                _ = create_timer(WRITE_TIMEOUT).fuse() => {
232                                    tracing::debug!(%connection_id, "keepalive interval: pinging timed out");
233                                    Err(anyhow!("timed out sending keepalive"))?;
234                                }
235                            }
236                        }
237                        _ = receive_timeout => {
238                            tracing::debug!(%connection_id, "receive timeout: delay between messages too long");
239                            Err(anyhow!("delay between messages too long"))?
240                        }
241                    }
242                }
243            }
244        };
245
246        let response_channels = connection_state.response_channels.clone();
247        self.connections
248            .write()
249            .insert(connection_id, connection_state);
250
251        let incoming_rx = incoming_rx.filter_map(move |incoming| {
252            let response_channels = response_channels.clone();
253            async move {
254                let message_id = incoming.id;
255                tracing::debug!(?incoming, "incoming message future: start");
256                let _end = util::defer(move || {
257                    tracing::debug!(
258                        %connection_id,
259                        message_id,
260                        "incoming message future: end"
261                    );
262                });
263
264                if let Some(responding_to) = incoming.responding_to {
265                    tracing::debug!(
266                        %connection_id,
267                        message_id,
268                        responding_to,
269                        "incoming response: received"
270                    );
271                    let channel = response_channels.lock().as_mut()?.remove(&responding_to);
272                    if let Some(tx) = channel {
273                        let requester_resumed = oneshot::channel();
274                        if let Err(error) = tx.send((incoming, requester_resumed.0)) {
275                            tracing::debug!(
276                                %connection_id,
277                                message_id,
278                                responding_to = responding_to,
279                                ?error,
280                                "incoming response: request future dropped",
281                            );
282                        }
283
284                        tracing::debug!(
285                            %connection_id,
286                            message_id,
287                            responding_to,
288                            "incoming response: waiting to resume requester"
289                        );
290                        let _ = requester_resumed.1.await;
291                        tracing::debug!(
292                            %connection_id,
293                            message_id,
294                            responding_to,
295                            "incoming response: requester resumed"
296                        );
297                    } else {
298                        tracing::warn!(
299                            %connection_id,
300                            message_id,
301                            responding_to,
302                            "incoming response: unknown request"
303                        );
304                    }
305
306                    None
307                } else {
308                    tracing::debug!(
309                        %connection_id,
310                        message_id,
311                        "incoming message: received"
312                    );
313                    proto::build_typed_envelope(connection_id, incoming).or_else(|| {
314                        tracing::error!(
315                            %connection_id,
316                            message_id,
317                            "unable to construct a typed envelope"
318                        );
319                        None
320                    })
321                }
322            }
323        });
324        (connection_id, handle_io, incoming_rx.boxed())
325    }
326
327    #[cfg(any(test, feature = "test-support"))]
328    pub async fn add_test_connection(
329        self: &Arc<Self>,
330        connection: Connection,
331        executor: Arc<gpui::executor::Background>,
332    ) -> (
333        ConnectionId,
334        impl Future<Output = anyhow::Result<()>> + Send,
335        BoxStream<'static, Box<dyn AnyTypedEnvelope>>,
336    ) {
337        let executor = executor.clone();
338        self.add_connection(connection, move |duration| executor.timer(duration))
339            .await
340    }
341
342    pub fn disconnect(&self, connection_id: ConnectionId) {
343        self.connections.write().remove(&connection_id);
344    }
345
346    pub fn reset(&self) {
347        self.connections.write().clear();
348    }
349
350    pub fn request<T: RequestMessage>(
351        &self,
352        receiver_id: ConnectionId,
353        request: T,
354    ) -> impl Future<Output = Result<T::Response>> {
355        self.request_internal(None, receiver_id, request)
356    }
357
358    pub fn forward_request<T: RequestMessage>(
359        &self,
360        sender_id: ConnectionId,
361        receiver_id: ConnectionId,
362        request: T,
363    ) -> impl Future<Output = Result<T::Response>> {
364        self.request_internal(Some(sender_id), receiver_id, request)
365    }
366
367    pub fn request_internal<T: RequestMessage>(
368        &self,
369        original_sender_id: Option<ConnectionId>,
370        receiver_id: ConnectionId,
371        request: T,
372    ) -> impl Future<Output = Result<T::Response>> {
373        let (tx, rx) = oneshot::channel();
374        let send = self.connection_state(receiver_id).and_then(|connection| {
375            let message_id = connection.next_message_id.fetch_add(1, SeqCst);
376            connection
377                .response_channels
378                .lock()
379                .as_mut()
380                .ok_or_else(|| anyhow!("connection was closed"))?
381                .insert(message_id, tx);
382            connection
383                .outgoing_tx
384                .unbounded_send(proto::Message::Envelope(request.into_envelope(
385                    message_id,
386                    None,
387                    original_sender_id.map(|id| id.0),
388                )))
389                .map_err(|_| anyhow!("connection was closed"))?;
390            Ok(())
391        });
392        async move {
393            send?;
394            let (response, _barrier) = rx.await.map_err(|_| anyhow!("connection was closed"))?;
395            if let Some(proto::envelope::Payload::Error(error)) = &response.payload {
396                Err(anyhow!("RPC request failed - {}", error.message))
397            } else {
398                T::Response::from_envelope(response)
399                    .ok_or_else(|| anyhow!("received response of the wrong type"))
400            }
401        }
402    }
403
404    pub fn send<T: EnvelopedMessage>(&self, receiver_id: ConnectionId, message: T) -> Result<()> {
405        let connection = self.connection_state(receiver_id)?;
406        let message_id = connection
407            .next_message_id
408            .fetch_add(1, atomic::Ordering::SeqCst);
409        connection
410            .outgoing_tx
411            .unbounded_send(proto::Message::Envelope(
412                message.into_envelope(message_id, None, None),
413            ))?;
414        Ok(())
415    }
416
417    pub fn forward_send<T: EnvelopedMessage>(
418        &self,
419        sender_id: ConnectionId,
420        receiver_id: ConnectionId,
421        message: T,
422    ) -> Result<()> {
423        let connection = self.connection_state(receiver_id)?;
424        let message_id = connection
425            .next_message_id
426            .fetch_add(1, atomic::Ordering::SeqCst);
427        connection
428            .outgoing_tx
429            .unbounded_send(proto::Message::Envelope(message.into_envelope(
430                message_id,
431                None,
432                Some(sender_id.0),
433            )))?;
434        Ok(())
435    }
436
437    pub fn respond<T: RequestMessage>(
438        &self,
439        receipt: Receipt<T>,
440        response: T::Response,
441    ) -> Result<()> {
442        let connection = self.connection_state(receipt.sender_id)?;
443        let message_id = connection
444            .next_message_id
445            .fetch_add(1, atomic::Ordering::SeqCst);
446        connection
447            .outgoing_tx
448            .unbounded_send(proto::Message::Envelope(response.into_envelope(
449                message_id,
450                Some(receipt.message_id),
451                None,
452            )))?;
453        Ok(())
454    }
455
456    pub fn respond_with_error<T: RequestMessage>(
457        &self,
458        receipt: Receipt<T>,
459        response: proto::Error,
460    ) -> Result<()> {
461        let connection = self.connection_state(receipt.sender_id)?;
462        let message_id = connection
463            .next_message_id
464            .fetch_add(1, atomic::Ordering::SeqCst);
465        connection
466            .outgoing_tx
467            .unbounded_send(proto::Message::Envelope(response.into_envelope(
468                message_id,
469                Some(receipt.message_id),
470                None,
471            )))?;
472        Ok(())
473    }
474
475    fn connection_state(&self, connection_id: ConnectionId) -> Result<ConnectionState> {
476        let connections = self.connections.read();
477        let connection = connections
478            .get(&connection_id)
479            .ok_or_else(|| anyhow!("no such connection: {}", connection_id))?;
480        Ok(connection.clone())
481    }
482}
483
484impl Serialize for Peer {
485    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
486    where
487        S: serde::Serializer,
488    {
489        let mut state = serializer.serialize_struct("Peer", 2)?;
490        state.serialize_field("connections", &*self.connections.read())?;
491        state.end()
492    }
493}
494
495#[cfg(test)]
496mod tests {
497    use super::*;
498    use crate::TypedEnvelope;
499    use async_tungstenite::tungstenite::Message as WebSocketMessage;
500    use gpui::TestAppContext;
501
502    #[gpui::test(iterations = 50)]
503    async fn test_request_response(cx: &mut TestAppContext) {
504        let executor = cx.foreground();
505
506        // create 2 clients connected to 1 server
507        let server = Peer::new();
508        let client1 = Peer::new();
509        let client2 = Peer::new();
510
511        let (client1_to_server_conn, server_to_client_1_conn, _kill) =
512            Connection::in_memory(cx.background());
513        let (client1_conn_id, io_task1, client1_incoming) = client1
514            .add_test_connection(client1_to_server_conn, cx.background())
515            .await;
516        let (_, io_task2, server_incoming1) = server
517            .add_test_connection(server_to_client_1_conn, cx.background())
518            .await;
519
520        let (client2_to_server_conn, server_to_client_2_conn, _kill) =
521            Connection::in_memory(cx.background());
522        let (client2_conn_id, io_task3, client2_incoming) = client2
523            .add_test_connection(client2_to_server_conn, cx.background())
524            .await;
525        let (_, io_task4, server_incoming2) = server
526            .add_test_connection(server_to_client_2_conn, cx.background())
527            .await;
528
529        executor.spawn(io_task1).detach();
530        executor.spawn(io_task2).detach();
531        executor.spawn(io_task3).detach();
532        executor.spawn(io_task4).detach();
533        executor
534            .spawn(handle_messages(server_incoming1, server.clone()))
535            .detach();
536        executor
537            .spawn(handle_messages(client1_incoming, client1.clone()))
538            .detach();
539        executor
540            .spawn(handle_messages(server_incoming2, server.clone()))
541            .detach();
542        executor
543            .spawn(handle_messages(client2_incoming, client2.clone()))
544            .detach();
545
546        assert_eq!(
547            client1
548                .request(client1_conn_id, proto::Ping {},)
549                .await
550                .unwrap(),
551            proto::Ack {}
552        );
553
554        assert_eq!(
555            client2
556                .request(client2_conn_id, proto::Ping {},)
557                .await
558                .unwrap(),
559            proto::Ack {}
560        );
561
562        assert_eq!(
563            client1
564                .request(client1_conn_id, proto::Test { id: 1 },)
565                .await
566                .unwrap(),
567            proto::Test { id: 1 }
568        );
569
570        assert_eq!(
571            client2
572                .request(client2_conn_id, proto::Test { id: 2 })
573                .await
574                .unwrap(),
575            proto::Test { id: 2 }
576        );
577
578        client1.disconnect(client1_conn_id);
579        client2.disconnect(client1_conn_id);
580
581        async fn handle_messages(
582            mut messages: BoxStream<'static, Box<dyn AnyTypedEnvelope>>,
583            peer: Arc<Peer>,
584        ) -> Result<()> {
585            while let Some(envelope) = messages.next().await {
586                let envelope = envelope.into_any();
587                if let Some(envelope) = envelope.downcast_ref::<TypedEnvelope<proto::Ping>>() {
588                    let receipt = envelope.receipt();
589                    peer.respond(receipt, proto::Ack {})?
590                } else if let Some(envelope) = envelope.downcast_ref::<TypedEnvelope<proto::Test>>()
591                {
592                    peer.respond(envelope.receipt(), envelope.payload.clone())?
593                } else {
594                    panic!("unknown message type");
595                }
596            }
597
598            Ok(())
599        }
600    }
601
602    #[gpui::test(iterations = 50)]
603    async fn test_order_of_response_and_incoming(cx: &mut TestAppContext) {
604        let executor = cx.foreground();
605        let server = Peer::new();
606        let client = Peer::new();
607
608        let (client_to_server_conn, server_to_client_conn, _kill) =
609            Connection::in_memory(cx.background());
610        let (client_to_server_conn_id, io_task1, mut client_incoming) = client
611            .add_test_connection(client_to_server_conn, cx.background())
612            .await;
613        let (server_to_client_conn_id, io_task2, mut server_incoming) = server
614            .add_test_connection(server_to_client_conn, cx.background())
615            .await;
616
617        executor.spawn(io_task1).detach();
618        executor.spawn(io_task2).detach();
619
620        executor
621            .spawn(async move {
622                let request = server_incoming
623                    .next()
624                    .await
625                    .unwrap()
626                    .into_any()
627                    .downcast::<TypedEnvelope<proto::Ping>>()
628                    .unwrap();
629
630                server
631                    .send(
632                        server_to_client_conn_id,
633                        proto::Error {
634                            message: "message 1".to_string(),
635                        },
636                    )
637                    .unwrap();
638                server
639                    .send(
640                        server_to_client_conn_id,
641                        proto::Error {
642                            message: "message 2".to_string(),
643                        },
644                    )
645                    .unwrap();
646                server.respond(request.receipt(), proto::Ack {}).unwrap();
647
648                // Prevent the connection from being dropped
649                server_incoming.next().await;
650            })
651            .detach();
652
653        let events = Arc::new(Mutex::new(Vec::new()));
654
655        let response = client.request(client_to_server_conn_id, proto::Ping {});
656        let response_task = executor.spawn({
657            let events = events.clone();
658            async move {
659                response.await.unwrap();
660                events.lock().push("response".to_string());
661            }
662        });
663
664        executor
665            .spawn({
666                let events = events.clone();
667                async move {
668                    let incoming1 = client_incoming
669                        .next()
670                        .await
671                        .unwrap()
672                        .into_any()
673                        .downcast::<TypedEnvelope<proto::Error>>()
674                        .unwrap();
675                    events.lock().push(incoming1.payload.message);
676                    let incoming2 = client_incoming
677                        .next()
678                        .await
679                        .unwrap()
680                        .into_any()
681                        .downcast::<TypedEnvelope<proto::Error>>()
682                        .unwrap();
683                    events.lock().push(incoming2.payload.message);
684
685                    // Prevent the connection from being dropped
686                    client_incoming.next().await;
687                }
688            })
689            .detach();
690
691        response_task.await;
692        assert_eq!(
693            &*events.lock(),
694            &[
695                "message 1".to_string(),
696                "message 2".to_string(),
697                "response".to_string()
698            ]
699        );
700    }
701
702    #[gpui::test(iterations = 50)]
703    async fn test_dropping_request_before_completion(cx: &mut TestAppContext) {
704        let executor = cx.foreground();
705        let server = Peer::new();
706        let client = Peer::new();
707
708        let (client_to_server_conn, server_to_client_conn, _kill) =
709            Connection::in_memory(cx.background());
710        let (client_to_server_conn_id, io_task1, mut client_incoming) = client
711            .add_test_connection(client_to_server_conn, cx.background())
712            .await;
713        let (server_to_client_conn_id, io_task2, mut server_incoming) = server
714            .add_test_connection(server_to_client_conn, cx.background())
715            .await;
716
717        executor.spawn(io_task1).detach();
718        executor.spawn(io_task2).detach();
719
720        executor
721            .spawn(async move {
722                let request1 = server_incoming
723                    .next()
724                    .await
725                    .unwrap()
726                    .into_any()
727                    .downcast::<TypedEnvelope<proto::Ping>>()
728                    .unwrap();
729                let request2 = server_incoming
730                    .next()
731                    .await
732                    .unwrap()
733                    .into_any()
734                    .downcast::<TypedEnvelope<proto::Ping>>()
735                    .unwrap();
736
737                server
738                    .send(
739                        server_to_client_conn_id,
740                        proto::Error {
741                            message: "message 1".to_string(),
742                        },
743                    )
744                    .unwrap();
745                server
746                    .send(
747                        server_to_client_conn_id,
748                        proto::Error {
749                            message: "message 2".to_string(),
750                        },
751                    )
752                    .unwrap();
753                server.respond(request1.receipt(), proto::Ack {}).unwrap();
754                server.respond(request2.receipt(), proto::Ack {}).unwrap();
755
756                // Prevent the connection from being dropped
757                server_incoming.next().await;
758            })
759            .detach();
760
761        let events = Arc::new(Mutex::new(Vec::new()));
762
763        let request1 = client.request(client_to_server_conn_id, proto::Ping {});
764        let request1_task = executor.spawn(request1);
765        let request2 = client.request(client_to_server_conn_id, proto::Ping {});
766        let request2_task = executor.spawn({
767            let events = events.clone();
768            async move {
769                request2.await.unwrap();
770                events.lock().push("response 2".to_string());
771            }
772        });
773
774        executor
775            .spawn({
776                let events = events.clone();
777                async move {
778                    let incoming1 = client_incoming
779                        .next()
780                        .await
781                        .unwrap()
782                        .into_any()
783                        .downcast::<TypedEnvelope<proto::Error>>()
784                        .unwrap();
785                    events.lock().push(incoming1.payload.message);
786                    let incoming2 = client_incoming
787                        .next()
788                        .await
789                        .unwrap()
790                        .into_any()
791                        .downcast::<TypedEnvelope<proto::Error>>()
792                        .unwrap();
793                    events.lock().push(incoming2.payload.message);
794
795                    // Prevent the connection from being dropped
796                    client_incoming.next().await;
797                }
798            })
799            .detach();
800
801        // Allow the request to make some progress before dropping it.
802        cx.background().simulate_random_delay().await;
803        drop(request1_task);
804
805        request2_task.await;
806        assert_eq!(
807            &*events.lock(),
808            &[
809                "message 1".to_string(),
810                "message 2".to_string(),
811                "response 2".to_string()
812            ]
813        );
814    }
815
816    #[gpui::test(iterations = 50)]
817    async fn test_disconnect(cx: &mut TestAppContext) {
818        let executor = cx.foreground();
819
820        let (client_conn, mut server_conn, _kill) = Connection::in_memory(cx.background());
821
822        let client = Peer::new();
823        let (connection_id, io_handler, mut incoming) = client
824            .add_test_connection(client_conn, cx.background())
825            .await;
826
827        let (io_ended_tx, io_ended_rx) = oneshot::channel();
828        executor
829            .spawn(async move {
830                io_handler.await.ok();
831                io_ended_tx.send(()).unwrap();
832            })
833            .detach();
834
835        let (messages_ended_tx, messages_ended_rx) = oneshot::channel();
836        executor
837            .spawn(async move {
838                incoming.next().await;
839                messages_ended_tx.send(()).unwrap();
840            })
841            .detach();
842
843        client.disconnect(connection_id);
844
845        let _ = io_ended_rx.await;
846        let _ = messages_ended_rx.await;
847        assert!(server_conn
848            .send(WebSocketMessage::Binary(vec![]))
849            .await
850            .is_err());
851    }
852
853    #[gpui::test(iterations = 50)]
854    async fn test_io_error(cx: &mut TestAppContext) {
855        let executor = cx.foreground();
856        let (client_conn, mut server_conn, _kill) = Connection::in_memory(cx.background());
857
858        let client = Peer::new();
859        let (connection_id, io_handler, mut incoming) = client
860            .add_test_connection(client_conn, cx.background())
861            .await;
862        executor.spawn(io_handler).detach();
863        executor
864            .spawn(async move { incoming.next().await })
865            .detach();
866
867        let response = executor.spawn(client.request(connection_id, proto::Ping {}));
868        let _request = server_conn.rx.next().await.unwrap().unwrap();
869
870        drop(server_conn);
871        assert_eq!(
872            response.await.unwrap_err().to_string(),
873            "connection was closed"
874        );
875    }
876}