Skip to main content

karyon_p2p/peer/
connection.rs

1use std::collections::{HashMap, HashSet};
2use std::sync::Arc;
3
4use async_channel::Sender;
5use async_trait::async_trait;
6use log::debug;
7
8use karyon_core::async_util::{AsyncQueue, TaskGroup, TaskResult};
9use karyon_net::{FramedReader, FramedWriter};
10
11#[cfg(feature = "quic")]
12use karyon_core::async_runtime::io::{AsyncReadExt, AsyncWriteExt};
13#[cfg(feature = "quic")]
14use karyon_net::{StreamMux, framed, quic::QuicConn};
15
16#[cfg(feature = "quic")]
17use crate::{conn_direction::ConnDirection, message::StreamInit, util::encode};
18
19use crate::{
20    Error, Result,
21    codec::PeerNetMsgCodec,
22    conn_queue::QueuedConn,
23    message::{PeerNetCmd, PeerNetMsg, ProtocolMsg, ShutdownMsg},
24    protocol::{ProtocolEvent, ProtocolID},
25    util::decode,
26};
27
28const SEND_QUEUE_SIZE: usize = 128;
29const RECV_QUEUE_SIZE: usize = 128;
30
31/// Per-peer wire abstraction. Hides single-pipe (TCP/TLS) vs.
32/// stream-mux (QUIC) framing from the layers above.
33#[async_trait]
34pub(crate) trait Wire: Send + Sync {
35    /// Send pre-encoded payload bytes for `proto_id`.
36    async fn send(&self, proto_id: &ProtocolID, payload: Vec<u8>) -> Result<()>;
37    /// Pop the next event for `proto_id`. Blocks until a message
38    /// arrives or shutdown is broadcast.
39    async fn recv(&self, proto_id: &ProtocolID) -> Result<ProtocolEvent>;
40    /// Graceful close. Pushes Shutdown to every recv queue and signals
41    /// the wire (transport-specific).
42    async fn shutdown(&self) -> Result<()>;
43    /// Opens what must be opened with the remote before the protocols
44    /// run, and starts its reader and writer tasks. Called once by
45    /// `Peer::run`. Only QUIC has such a step.
46    async fn setup(
47        &self,
48        _task_group: &TaskGroup,
49        _stop_reason_tx: Sender<Result<()>>,
50    ) -> Result<()> {
51        Ok(())
52    }
53}
54
55/// Build the right `Wire` for the post-handshake `QueuedConn`.
56/// Picks `MuxConnection` when QUIC is in use, `SingleConnection` otherwise.
57/// Only negotiated protocols get a recv queue, so a message for any other
58/// protocol is a protocol violation.
59///
60/// Never waits for the remote; see `Wire::setup`.
61pub(crate) fn from_queued(
62    queued: QueuedConn,
63    negotiated: &HashSet<ProtocolID>,
64    task_group: &TaskGroup,
65    stop_reason_tx: Sender<Result<()>>,
66) -> Result<Arc<dyn Wire>> {
67    #[cfg(feature = "quic")]
68    if queued.quic_conn.is_some() {
69        let conn = MuxConnection::from_queued(queued, negotiated)?;
70        return Ok(Arc::new(conn) as Arc<dyn Wire>);
71    }
72
73    let conn = SingleConnection::from_queued(queued, negotiated, task_group, stop_reason_tx);
74    Ok(Arc::new(conn))
75}
76
77/// TCP / TLS path: one shared writer drains `send_queue`; a single
78/// reader demuxes incoming `PeerNetMsg`s into the matching `recv_queues`.
79pub(crate) struct SingleConnection {
80    send_queue: Arc<AsyncQueue<PeerNetMsg>>,
81    recv_queues: HashMap<ProtocolID, Arc<AsyncQueue<ProtocolEvent>>>,
82}
83
84impl SingleConnection {
85    /// Spawn the writer + demux reader and return the connection.
86    pub(crate) fn from_queued(
87        queued: QueuedConn,
88        negotiated: &HashSet<ProtocolID>,
89        task_group: &TaskGroup,
90        stop_reason_tx: Sender<Result<()>>,
91    ) -> Self {
92        let send_queue = AsyncQueue::new(SEND_QUEUE_SIZE);
93        let recv_queues = build_recv_queues(negotiated);
94
95        spawn_writer_task(task_group, queued.writer, send_queue.clone());
96        spawn_demux_reader(
97            task_group,
98            queued.reader,
99            recv_queues.clone(),
100            stop_reason_tx,
101        );
102
103        Self {
104            send_queue,
105            recv_queues,
106        }
107    }
108}
109
110#[async_trait]
111impl Wire for SingleConnection {
112    async fn send(&self, proto_id: &ProtocolID, payload: Vec<u8>) -> Result<()> {
113        let proto_msg = ProtocolMsg {
114            protocol_id: proto_id.clone(),
115            payload,
116        };
117        let net_msg = PeerNetMsg::new(PeerNetCmd::Protocol, &proto_msg)?;
118        self.send_queue.push(net_msg).await;
119        Ok(())
120    }
121
122    async fn recv(&self, proto_id: &ProtocolID) -> Result<ProtocolEvent> {
123        match self.recv_queues.get(proto_id) {
124            Some(q) => Ok(q.recv().await),
125            None => Err(Error::UnsupportedProtocol(proto_id.clone())),
126        }
127    }
128
129    async fn shutdown(&self) -> Result<()> {
130        let m = PeerNetMsg::new(PeerNetCmd::Shutdown, ShutdownMsg(0))?;
131        self.send_queue.push(m).await;
132        broadcast_shutdown(&self.recv_queues).await;
133        Ok(())
134    }
135}
136
137/// QUIC path: one stream per protocol, each with its own writer task.
138/// `send_queues` and `recv_queues` are keyed by protocol id.
139#[cfg(feature = "quic")]
140pub(crate) struct MuxConnection {
141    send_queues: HashMap<ProtocolID, Arc<AsyncQueue<PeerNetMsg>>>,
142    recv_queues: HashMap<ProtocolID, Arc<AsyncQueue<ProtocolEvent>>>,
143    quic_conn: QuicConn,
144    direction: ConnDirection,
145}
146
147#[cfg(feature = "quic")]
148impl MuxConnection {
149    /// Create the queues of every negotiated protocol. The streams are
150    /// opened later by `setup`; until then messages wait in the queues.
151    pub(crate) fn from_queued(
152        queued: QueuedConn,
153        negotiated: &HashSet<ProtocolID>,
154    ) -> Result<Self> {
155        let quic_conn = queued
156            .quic_conn
157            .ok_or_else(|| Error::InvalidMsg("MuxConnection requires a QUIC conn".into()))?;
158
159        let send_queues = negotiated
160            .iter()
161            .map(|id| (id.clone(), AsyncQueue::new(SEND_QUEUE_SIZE)))
162            .collect();
163
164        Ok(Self {
165            send_queues,
166            recv_queues: build_recv_queues(negotiated),
167            quic_conn,
168            direction: queued.direction,
169        })
170    }
171}
172
173#[cfg(feature = "quic")]
174#[async_trait]
175impl Wire for MuxConnection {
176    async fn send(&self, proto_id: &ProtocolID, payload: Vec<u8>) -> Result<()> {
177        let proto_msg = ProtocolMsg {
178            protocol_id: proto_id.clone(),
179            payload,
180        };
181        let net_msg = PeerNetMsg::new(PeerNetCmd::Protocol, &proto_msg)?;
182        match self.send_queues.get(proto_id) {
183            Some(q) => {
184                q.push(net_msg).await;
185                Ok(())
186            }
187            None => Err(Error::UnsupportedProtocol(proto_id.clone())),
188        }
189    }
190
191    async fn recv(&self, proto_id: &ProtocolID) -> Result<ProtocolEvent> {
192        match self.recv_queues.get(proto_id) {
193            Some(q) => Ok(q.recv().await),
194            None => Err(Error::UnsupportedProtocol(proto_id.clone())),
195        }
196    }
197
198    async fn shutdown(&self) -> Result<()> {
199        self.quic_conn.close(0, b"shutdown");
200        broadcast_shutdown(&self.recv_queues).await;
201        Ok(())
202    }
203
204    /// Open / accept one QUIC stream per negotiated protocol and spawn a
205    /// reader and a writer task for each.
206    async fn setup(
207        &self,
208        task_group: &TaskGroup,
209        stop_reason_tx: Sender<Result<()>>,
210    ) -> Result<()> {
211        let negotiated: HashSet<ProtocolID> = self.send_queues.keys().cloned().collect();
212        let streams = setup_quic_streams(&self.quic_conn, &self.direction, &negotiated).await?;
213
214        for stream in streams {
215            let queues = (
216                self.send_queues.get(&stream.proto_id),
217                self.recv_queues.get(&stream.proto_id),
218            );
219            // `setup_quic_streams` only returns negotiated protocols.
220            let (Some(q_send), Some(q_recv)) = queues else {
221                continue;
222            };
223            spawn_writer_task(task_group, stream.writer, q_send.clone());
224            spawn_quic_reader(
225                task_group,
226                stream.reader,
227                q_recv.clone(),
228                stop_reason_tx.clone(),
229            );
230        }
231        Ok(())
232    }
233}
234
235/// One QUIC protocol stream's halves, returned from `setup_quic_streams`.
236#[cfg(feature = "quic")]
237struct MuxStream {
238    proto_id: ProtocolID,
239    reader: FramedReader<PeerNetMsgCodec>,
240    writer: FramedWriter<PeerNetMsgCodec>,
241}
242
243/// Spawn a task that drains `queue` into `writer`. Exits when the
244/// writer fails (peer hung up).
245fn spawn_writer_task(
246    task_group: &TaskGroup,
247    mut writer: FramedWriter<PeerNetMsgCodec>,
248    queue: Arc<AsyncQueue<PeerNetMsg>>,
249) {
250    task_group.spawn_then(
251        async move {
252            loop {
253                let msg = queue.recv().await;
254                if writer.send_msg(msg).await.is_err() {
255                    break;
256                }
257            }
258            Ok::<(), Error>(())
259        },
260        |res: TaskResult<Result<()>>| async move {
261            debug!("Peer writer task ended: {res}");
262        },
263    );
264}
265
266/// Spawn the single-pipe reader task. Reads `PeerNetMsg`s, routes
267/// `Protocol` payloads into the matching recv queue, signals the peer
268/// via `stop_reason_tx` on Shutdown / error.
269fn spawn_demux_reader(
270    task_group: &TaskGroup,
271    mut reader: FramedReader<PeerNetMsgCodec>,
272    recv_queues: HashMap<ProtocolID, Arc<AsyncQueue<ProtocolEvent>>>,
273    stop_reason_tx: Sender<Result<()>>,
274) {
275    task_group.spawn_then(
276        async move {
277            loop {
278                let msg = match reader.recv_msg().await {
279                    Ok(m) => m,
280                    Err(e) => {
281                        let _ = stop_reason_tx.try_send(Err(e.into()));
282                        break;
283                    }
284                };
285                match msg.header.command {
286                    PeerNetCmd::Protocol => {
287                        let proto_msg: ProtocolMsg = match decode(&msg.payload) {
288                            Ok((m, _)) => m,
289                            Err(e) => {
290                                let _ = stop_reason_tx.try_send(Err(e));
291                                break;
292                            }
293                        };
294                        match recv_queues.get(&proto_msg.protocol_id) {
295                            Some(q) => {
296                                q.push(ProtocolEvent::Message(proto_msg.payload)).await;
297                            }
298                            None => {
299                                // Not negotiated with this peer.
300                                let id = proto_msg.protocol_id;
301                                let _ =
302                                    stop_reason_tx.try_send(Err(Error::UnsupportedProtocol(id)));
303                                break;
304                            }
305                        }
306                    }
307                    PeerNetCmd::Shutdown => {
308                        let _ = stop_reason_tx.try_send(Err(Error::PeerShutdown));
309                        break;
310                    }
311                    command => {
312                        let _ = stop_reason_tx.try_send(Err(Error::InvalidMsg(format!(
313                            "Unexpected msg {command:?}"
314                        ))));
315                        break;
316                    }
317                }
318            }
319            Ok::<(), Error>(())
320        },
321        |res: TaskResult<Result<()>>| async move {
322            debug!("Peer reader task ended: {res}");
323        },
324    );
325}
326
327/// Spawn a per-stream QUIC reader task. The stream is already keyed
328/// by protocol id, so messages go straight into `recv_queue`. Signals
329/// the peer via `stop_reason_tx` on Shutdown / error, like the TCP reader.
330#[cfg(feature = "quic")]
331fn spawn_quic_reader(
332    task_group: &TaskGroup,
333    mut reader: FramedReader<PeerNetMsgCodec>,
334    recv_queue: Arc<AsyncQueue<ProtocolEvent>>,
335    stop_reason_tx: Sender<Result<()>>,
336) {
337    task_group.spawn_then(
338        async move {
339            loop {
340                let msg = match reader.recv_msg().await {
341                    Ok(m) => m,
342                    Err(e) => {
343                        let _ = stop_reason_tx.try_send(Err(e.into()));
344                        break;
345                    }
346                };
347                match msg.header.command {
348                    PeerNetCmd::Protocol => {
349                        let proto_msg: ProtocolMsg = match decode(&msg.payload) {
350                            Ok((m, _)) => m,
351                            Err(e) => {
352                                let _ = stop_reason_tx.try_send(Err(e));
353                                break;
354                            }
355                        };
356                        recv_queue
357                            .push(ProtocolEvent::Message(proto_msg.payload))
358                            .await;
359                    }
360                    PeerNetCmd::Shutdown => {
361                        let _ = stop_reason_tx.try_send(Err(Error::PeerShutdown));
362                        break;
363                    }
364                    command => {
365                        let _ = stop_reason_tx.try_send(Err(Error::InvalidMsg(format!(
366                            "Unexpected msg {command:?}"
367                        ))));
368                        break;
369                    }
370                }
371            }
372            Ok::<(), Error>(())
373        },
374        |res: TaskResult<Result<()>>| async move {
375            debug!("QUIC stream reader task ended: {res}");
376        },
377    );
378}
379
380/// Open / accept one QUIC stream per negotiated protocol and return
381/// the reader/writer halves of each.
382#[cfg(feature = "quic")]
383async fn setup_quic_streams(
384    quic_conn: &QuicConn,
385    direction: &ConnDirection,
386    negotiated: &HashSet<ProtocolID>,
387) -> Result<Vec<MuxStream>> {
388    let mut streams: Vec<MuxStream> = Vec::new();
389
390    match direction {
391        ConnDirection::Outbound => {
392            for proto_id in negotiated.iter() {
393                let mut stream = quic_conn.open_stream().await?;
394
395                let init = StreamInit {
396                    protocol_id: proto_id.clone(),
397                };
398                let encoded = encode(&init)?;
399                // Length-prefix the header so the reader takes exactly these
400                // bytes and leaves the protocol messages after it untouched.
401                let len = u16::try_from(encoded.len())
402                    .map_err(|_| Error::InvalidMsg("StreamInit too large".into()))?;
403                stream.write_all(&encode(&len)?).await?;
404                stream.write_all(&encoded).await?;
405                stream.flush().await?;
406
407                let conn = framed(stream, PeerNetMsgCodec::new());
408                let (reader, writer) = conn.split();
409
410                streams.push(MuxStream {
411                    proto_id: proto_id.clone(),
412                    reader,
413                    writer,
414                });
415            }
416        }
417        ConnDirection::Inbound => {
418            let expected = negotiated.len();
419            let mut received = 0;
420
421            while received < expected {
422                let mut stream = quic_conn.accept_stream().await?;
423
424                let mut len_buf = [0u8; 2];
425                stream.read_exact(&mut len_buf).await?;
426                let (len, _): (u16, _) = decode(&len_buf)?;
427                let mut header_buf = vec![0u8; usize::from(len)];
428                stream.read_exact(&mut header_buf).await?;
429
430                let (init, _): (StreamInit, _) = decode(&header_buf)?;
431
432                // Both sides negotiated the same set, so this is a violation.
433                if !negotiated.contains(&init.protocol_id) {
434                    return Err(Error::UnsupportedProtocol(init.protocol_id));
435                }
436                // One stream per protocol: a second one would get a second
437                // writer on the same queue.
438                if streams.iter().any(|s| s.proto_id == init.protocol_id) {
439                    return Err(Error::InvalidMsg("Duplicate protocol stream".into()));
440                }
441
442                let conn = framed(stream, PeerNetMsgCodec::new());
443                let (reader, writer) = conn.split();
444
445                streams.push(MuxStream {
446                    proto_id: init.protocol_id,
447                    reader,
448                    writer,
449                });
450
451                received += 1;
452            }
453        }
454    }
455
456    Ok(streams)
457}
458
459/// One bounded recv queue per protocol id.
460fn build_recv_queues(
461    proto_ids: &HashSet<ProtocolID>,
462) -> HashMap<ProtocolID, Arc<AsyncQueue<ProtocolEvent>>> {
463    proto_ids
464        .iter()
465        .map(|id| (id.clone(), AsyncQueue::new(RECV_QUEUE_SIZE)))
466        .collect()
467}
468
469/// Push `ProtocolEvent::Shutdown` into every recv queue, waking
470/// blocked `recv` callers.
471async fn broadcast_shutdown(queues: &HashMap<ProtocolID, Arc<AsyncQueue<ProtocolEvent>>>) {
472    for q in queues.values() {
473        q.push(ProtocolEvent::Shutdown).await;
474    }
475}