1use std::collections::{HashMap, HashSet};
2use std::sync::Arc;
3
4use async_channel::Sender;
5use async_trait::async_trait;
6use log::debug;
7
8use karyon_core::async_util::{AsyncQueue, TaskGroup, TaskResult};
9use karyon_net::{FramedReader, FramedWriter};
10
11#[cfg(feature = "quic")]
12use karyon_core::async_runtime::io::{AsyncReadExt, AsyncWriteExt};
13#[cfg(feature = "quic")]
14use karyon_net::{StreamMux, framed, quic::QuicConn};
15
16#[cfg(feature = "quic")]
17use crate::{conn_direction::ConnDirection, message::StreamInit, util::encode};
18
19use crate::{
20 Error, Result,
21 codec::PeerNetMsgCodec,
22 conn_queue::QueuedConn,
23 message::{PeerNetCmd, PeerNetMsg, ProtocolMsg, ShutdownMsg},
24 protocol::{ProtocolEvent, ProtocolID},
25 util::decode,
26};
27
28const SEND_QUEUE_SIZE: usize = 128;
29const RECV_QUEUE_SIZE: usize = 128;
30
31#[async_trait]
34pub(crate) trait Wire: Send + Sync {
35 async fn send(&self, proto_id: &ProtocolID, payload: Vec<u8>) -> Result<()>;
37 async fn recv(&self, proto_id: &ProtocolID) -> Result<ProtocolEvent>;
40 async fn shutdown(&self) -> Result<()>;
43 async fn setup(
47 &self,
48 _task_group: &TaskGroup,
49 _stop_reason_tx: Sender<Result<()>>,
50 ) -> Result<()> {
51 Ok(())
52 }
53}
54
55pub(crate) fn from_queued(
62 queued: QueuedConn,
63 negotiated: &HashSet<ProtocolID>,
64 task_group: &TaskGroup,
65 stop_reason_tx: Sender<Result<()>>,
66) -> Result<Arc<dyn Wire>> {
67 #[cfg(feature = "quic")]
68 if queued.quic_conn.is_some() {
69 let conn = MuxConnection::from_queued(queued, negotiated)?;
70 return Ok(Arc::new(conn) as Arc<dyn Wire>);
71 }
72
73 let conn = SingleConnection::from_queued(queued, negotiated, task_group, stop_reason_tx);
74 Ok(Arc::new(conn))
75}
76
77pub(crate) struct SingleConnection {
80 send_queue: Arc<AsyncQueue<PeerNetMsg>>,
81 recv_queues: HashMap<ProtocolID, Arc<AsyncQueue<ProtocolEvent>>>,
82}
83
84impl SingleConnection {
85 pub(crate) fn from_queued(
87 queued: QueuedConn,
88 negotiated: &HashSet<ProtocolID>,
89 task_group: &TaskGroup,
90 stop_reason_tx: Sender<Result<()>>,
91 ) -> Self {
92 let send_queue = AsyncQueue::new(SEND_QUEUE_SIZE);
93 let recv_queues = build_recv_queues(negotiated);
94
95 spawn_writer_task(task_group, queued.writer, send_queue.clone());
96 spawn_demux_reader(
97 task_group,
98 queued.reader,
99 recv_queues.clone(),
100 stop_reason_tx,
101 );
102
103 Self {
104 send_queue,
105 recv_queues,
106 }
107 }
108}
109
110#[async_trait]
111impl Wire for SingleConnection {
112 async fn send(&self, proto_id: &ProtocolID, payload: Vec<u8>) -> Result<()> {
113 let proto_msg = ProtocolMsg {
114 protocol_id: proto_id.clone(),
115 payload,
116 };
117 let net_msg = PeerNetMsg::new(PeerNetCmd::Protocol, &proto_msg)?;
118 self.send_queue.push(net_msg).await;
119 Ok(())
120 }
121
122 async fn recv(&self, proto_id: &ProtocolID) -> Result<ProtocolEvent> {
123 match self.recv_queues.get(proto_id) {
124 Some(q) => Ok(q.recv().await),
125 None => Err(Error::UnsupportedProtocol(proto_id.clone())),
126 }
127 }
128
129 async fn shutdown(&self) -> Result<()> {
130 let m = PeerNetMsg::new(PeerNetCmd::Shutdown, ShutdownMsg(0))?;
131 self.send_queue.push(m).await;
132 broadcast_shutdown(&self.recv_queues).await;
133 Ok(())
134 }
135}
136
137#[cfg(feature = "quic")]
140pub(crate) struct MuxConnection {
141 send_queues: HashMap<ProtocolID, Arc<AsyncQueue<PeerNetMsg>>>,
142 recv_queues: HashMap<ProtocolID, Arc<AsyncQueue<ProtocolEvent>>>,
143 quic_conn: QuicConn,
144 direction: ConnDirection,
145}
146
147#[cfg(feature = "quic")]
148impl MuxConnection {
149 pub(crate) fn from_queued(
152 queued: QueuedConn,
153 negotiated: &HashSet<ProtocolID>,
154 ) -> Result<Self> {
155 let quic_conn = queued
156 .quic_conn
157 .ok_or_else(|| Error::InvalidMsg("MuxConnection requires a QUIC conn".into()))?;
158
159 let send_queues = negotiated
160 .iter()
161 .map(|id| (id.clone(), AsyncQueue::new(SEND_QUEUE_SIZE)))
162 .collect();
163
164 Ok(Self {
165 send_queues,
166 recv_queues: build_recv_queues(negotiated),
167 quic_conn,
168 direction: queued.direction,
169 })
170 }
171}
172
173#[cfg(feature = "quic")]
174#[async_trait]
175impl Wire for MuxConnection {
176 async fn send(&self, proto_id: &ProtocolID, payload: Vec<u8>) -> Result<()> {
177 let proto_msg = ProtocolMsg {
178 protocol_id: proto_id.clone(),
179 payload,
180 };
181 let net_msg = PeerNetMsg::new(PeerNetCmd::Protocol, &proto_msg)?;
182 match self.send_queues.get(proto_id) {
183 Some(q) => {
184 q.push(net_msg).await;
185 Ok(())
186 }
187 None => Err(Error::UnsupportedProtocol(proto_id.clone())),
188 }
189 }
190
191 async fn recv(&self, proto_id: &ProtocolID) -> Result<ProtocolEvent> {
192 match self.recv_queues.get(proto_id) {
193 Some(q) => Ok(q.recv().await),
194 None => Err(Error::UnsupportedProtocol(proto_id.clone())),
195 }
196 }
197
198 async fn shutdown(&self) -> Result<()> {
199 self.quic_conn.close(0, b"shutdown");
200 broadcast_shutdown(&self.recv_queues).await;
201 Ok(())
202 }
203
204 async fn setup(
207 &self,
208 task_group: &TaskGroup,
209 stop_reason_tx: Sender<Result<()>>,
210 ) -> Result<()> {
211 let negotiated: HashSet<ProtocolID> = self.send_queues.keys().cloned().collect();
212 let streams = setup_quic_streams(&self.quic_conn, &self.direction, &negotiated).await?;
213
214 for stream in streams {
215 let queues = (
216 self.send_queues.get(&stream.proto_id),
217 self.recv_queues.get(&stream.proto_id),
218 );
219 let (Some(q_send), Some(q_recv)) = queues else {
221 continue;
222 };
223 spawn_writer_task(task_group, stream.writer, q_send.clone());
224 spawn_quic_reader(
225 task_group,
226 stream.reader,
227 q_recv.clone(),
228 stop_reason_tx.clone(),
229 );
230 }
231 Ok(())
232 }
233}
234
235#[cfg(feature = "quic")]
237struct MuxStream {
238 proto_id: ProtocolID,
239 reader: FramedReader<PeerNetMsgCodec>,
240 writer: FramedWriter<PeerNetMsgCodec>,
241}
242
243fn spawn_writer_task(
246 task_group: &TaskGroup,
247 mut writer: FramedWriter<PeerNetMsgCodec>,
248 queue: Arc<AsyncQueue<PeerNetMsg>>,
249) {
250 task_group.spawn_then(
251 async move {
252 loop {
253 let msg = queue.recv().await;
254 if writer.send_msg(msg).await.is_err() {
255 break;
256 }
257 }
258 Ok::<(), Error>(())
259 },
260 |res: TaskResult<Result<()>>| async move {
261 debug!("Peer writer task ended: {res}");
262 },
263 );
264}
265
266fn spawn_demux_reader(
270 task_group: &TaskGroup,
271 mut reader: FramedReader<PeerNetMsgCodec>,
272 recv_queues: HashMap<ProtocolID, Arc<AsyncQueue<ProtocolEvent>>>,
273 stop_reason_tx: Sender<Result<()>>,
274) {
275 task_group.spawn_then(
276 async move {
277 loop {
278 let msg = match reader.recv_msg().await {
279 Ok(m) => m,
280 Err(e) => {
281 let _ = stop_reason_tx.try_send(Err(e.into()));
282 break;
283 }
284 };
285 match msg.header.command {
286 PeerNetCmd::Protocol => {
287 let proto_msg: ProtocolMsg = match decode(&msg.payload) {
288 Ok((m, _)) => m,
289 Err(e) => {
290 let _ = stop_reason_tx.try_send(Err(e));
291 break;
292 }
293 };
294 match recv_queues.get(&proto_msg.protocol_id) {
295 Some(q) => {
296 q.push(ProtocolEvent::Message(proto_msg.payload)).await;
297 }
298 None => {
299 let id = proto_msg.protocol_id;
301 let _ =
302 stop_reason_tx.try_send(Err(Error::UnsupportedProtocol(id)));
303 break;
304 }
305 }
306 }
307 PeerNetCmd::Shutdown => {
308 let _ = stop_reason_tx.try_send(Err(Error::PeerShutdown));
309 break;
310 }
311 command => {
312 let _ = stop_reason_tx.try_send(Err(Error::InvalidMsg(format!(
313 "Unexpected msg {command:?}"
314 ))));
315 break;
316 }
317 }
318 }
319 Ok::<(), Error>(())
320 },
321 |res: TaskResult<Result<()>>| async move {
322 debug!("Peer reader task ended: {res}");
323 },
324 );
325}
326
327#[cfg(feature = "quic")]
331fn spawn_quic_reader(
332 task_group: &TaskGroup,
333 mut reader: FramedReader<PeerNetMsgCodec>,
334 recv_queue: Arc<AsyncQueue<ProtocolEvent>>,
335 stop_reason_tx: Sender<Result<()>>,
336) {
337 task_group.spawn_then(
338 async move {
339 loop {
340 let msg = match reader.recv_msg().await {
341 Ok(m) => m,
342 Err(e) => {
343 let _ = stop_reason_tx.try_send(Err(e.into()));
344 break;
345 }
346 };
347 match msg.header.command {
348 PeerNetCmd::Protocol => {
349 let proto_msg: ProtocolMsg = match decode(&msg.payload) {
350 Ok((m, _)) => m,
351 Err(e) => {
352 let _ = stop_reason_tx.try_send(Err(e));
353 break;
354 }
355 };
356 recv_queue
357 .push(ProtocolEvent::Message(proto_msg.payload))
358 .await;
359 }
360 PeerNetCmd::Shutdown => {
361 let _ = stop_reason_tx.try_send(Err(Error::PeerShutdown));
362 break;
363 }
364 command => {
365 let _ = stop_reason_tx.try_send(Err(Error::InvalidMsg(format!(
366 "Unexpected msg {command:?}"
367 ))));
368 break;
369 }
370 }
371 }
372 Ok::<(), Error>(())
373 },
374 |res: TaskResult<Result<()>>| async move {
375 debug!("QUIC stream reader task ended: {res}");
376 },
377 );
378}
379
380#[cfg(feature = "quic")]
383async fn setup_quic_streams(
384 quic_conn: &QuicConn,
385 direction: &ConnDirection,
386 negotiated: &HashSet<ProtocolID>,
387) -> Result<Vec<MuxStream>> {
388 let mut streams: Vec<MuxStream> = Vec::new();
389
390 match direction {
391 ConnDirection::Outbound => {
392 for proto_id in negotiated.iter() {
393 let mut stream = quic_conn.open_stream().await?;
394
395 let init = StreamInit {
396 protocol_id: proto_id.clone(),
397 };
398 let encoded = encode(&init)?;
399 let len = u16::try_from(encoded.len())
402 .map_err(|_| Error::InvalidMsg("StreamInit too large".into()))?;
403 stream.write_all(&encode(&len)?).await?;
404 stream.write_all(&encoded).await?;
405 stream.flush().await?;
406
407 let conn = framed(stream, PeerNetMsgCodec::new());
408 let (reader, writer) = conn.split();
409
410 streams.push(MuxStream {
411 proto_id: proto_id.clone(),
412 reader,
413 writer,
414 });
415 }
416 }
417 ConnDirection::Inbound => {
418 let expected = negotiated.len();
419 let mut received = 0;
420
421 while received < expected {
422 let mut stream = quic_conn.accept_stream().await?;
423
424 let mut len_buf = [0u8; 2];
425 stream.read_exact(&mut len_buf).await?;
426 let (len, _): (u16, _) = decode(&len_buf)?;
427 let mut header_buf = vec![0u8; usize::from(len)];
428 stream.read_exact(&mut header_buf).await?;
429
430 let (init, _): (StreamInit, _) = decode(&header_buf)?;
431
432 if !negotiated.contains(&init.protocol_id) {
434 return Err(Error::UnsupportedProtocol(init.protocol_id));
435 }
436 if streams.iter().any(|s| s.proto_id == init.protocol_id) {
439 return Err(Error::InvalidMsg("Duplicate protocol stream".into()));
440 }
441
442 let conn = framed(stream, PeerNetMsgCodec::new());
443 let (reader, writer) = conn.split();
444
445 streams.push(MuxStream {
446 proto_id: init.protocol_id,
447 reader,
448 writer,
449 });
450
451 received += 1;
452 }
453 }
454 }
455
456 Ok(streams)
457}
458
459fn build_recv_queues(
461 proto_ids: &HashSet<ProtocolID>,
462) -> HashMap<ProtocolID, Arc<AsyncQueue<ProtocolEvent>>> {
463 proto_ids
464 .iter()
465 .map(|id| (id.clone(), AsyncQueue::new(RECV_QUEUE_SIZE)))
466 .collect()
467}
468
469async fn broadcast_shutdown(queues: &HashMap<ProtocolID, Arc<AsyncQueue<ProtocolEvent>>>) {
472 for q in queues.values() {
473 q.push(ProtocolEvent::Shutdown).await;
474 }
475}