Skip to main content

karyon_p2p/
listener.rs

1use std::{future::Future, marker::PhantomData, sync::Arc, time::Duration};
2
3use log::{debug, error, info};
4
5use karyon_core::{
6    async_runtime::{Executor, lock::Semaphore},
7    async_util::{TaskGroup, TaskResult, timeout},
8    crypto::KeyPair,
9};
10use karyon_net::{
11    ByteBuffer, ByteStream, Endpoint, FramedConn, ServerLayer,
12    codec::Codec,
13    framed,
14    tcp::TcpListener,
15    tls::{ServerTlsConfig, TlsLayer},
16};
17
18use crate::{
19    Error, PeerID, Result,
20    access_control::{AccessControl, Action, Subject},
21    codec::PeerNetMsgCodec,
22    conn_direction::ConnDirection,
23    conn_queue::ConnQueue,
24    monitor::{ConnectionKind, Monitor},
25    tls_config::{peer_id_from_certs, tls_server_config},
26};
27
28/// Listener for byte-stream transports (TCP, TLS). Accepting is split in
29/// two phases: `accept` does only the cheap kernel accept, `handshake`
30/// runs the TLS upgrade. QUIC uses a separate `StreamMux` path.
31enum StreamListener {
32    Tcp(TcpListener),
33    Tls(TcpListener, Box<TlsLayer>),
34}
35
36impl StreamListener {
37    /// Kernel accept only. Returns the raw stream before any handshake.
38    async fn accept(&self) -> Result<Box<dyn ByteStream>> {
39        match self {
40            Self::Tcp(l) | Self::Tls(l, _) => Ok(l.accept().await?),
41        }
42    }
43
44    /// TLS upgrade, if this is a TLS listener. Runs off the accept loop.
45    async fn handshake(
46        &self,
47        stream: Box<dyn ByteStream>,
48        handshake_timeout: Duration,
49    ) -> Result<Box<dyn ByteStream>> {
50        match self {
51            Self::Tcp(_) => Ok(stream),
52            Self::Tls(_, layer) => Ok(timeout(
53                handshake_timeout,
54                ServerLayer::handshake(layer.as_ref(), stream),
55            )
56            .await??),
57        }
58    }
59
60    fn local_endpoint(&self) -> Result<Endpoint> {
61        match self {
62            Self::Tcp(l) => Ok(l.local_endpoint()?),
63            // The listener is plain TCP; report the TLS scheme so peers
64            // dialling this endpoint run the handshake.
65            Self::Tls(l, _) => {
66                let ep = l.local_endpoint()?;
67                Ok(Endpoint::Tls(ep.addr()?, ep.port()?))
68            }
69        }
70    }
71}
72
73#[cfg(feature = "quic")]
74use karyon_net::{StreamMux, quic};
75
76#[cfg(feature = "quic")]
77use crate::tls_config::quic_server_config;
78
79/// Creates inbound connections with other peers. Generic over the
80/// codec applied to the framed accepted streams so the same accept
81/// machinery serves the peer data-plane (`PeerNetMsgCodec`) and the
82/// kademlia lookup-plane (`KadNetMsgCodec`).
83pub struct Listener<C: Codec<ByteBuffer> + Default + Clone> {
84    key_pair: KeyPair,
85    task_group: TaskGroup,
86    connection_slots: Semaphore,
87    conn_queue: Option<Arc<ConnQueue>>,
88    monitor: Arc<Monitor>,
89    handshake_timeout: Duration,
90    access_control: Arc<dyn AccessControl>,
91    _codec: PhantomData<C>,
92}
93
94impl<C> Listener<C>
95where
96    C: Codec<ByteBuffer, Error = karyon_net::Error> + Default + Clone + Send + Sync + 'static,
97{
98    /// Create a new Listener (no auto-queue; use `start_with_callback`).
99    /// `handshake_timeout` is in seconds.
100    pub fn new(
101        key_pair: &KeyPair,
102        slots: usize,
103        monitor: Arc<Monitor>,
104        handshake_timeout: u64,
105        access_control: Arc<dyn AccessControl>,
106        ex: Executor,
107    ) -> Arc<Self> {
108        Arc::new(Self {
109            key_pair: key_pair.clone(),
110            connection_slots: Semaphore::new(slots),
111            conn_queue: None,
112            task_group: TaskGroup::with_executor(ex),
113            monitor,
114            handshake_timeout: Duration::from_secs(handshake_timeout),
115            access_control,
116            _codec: PhantomData,
117        })
118    }
119
120    /// Start with a user-provided callback for each connection.
121    pub async fn start_with_callback<Fut>(
122        self: &Arc<Self>,
123        endpoint: Endpoint,
124        callback: impl FnOnce(FramedConn<C>) -> Fut + Clone + Send + 'static,
125    ) -> Result<Endpoint>
126    where
127        Fut: Future<Output = Result<()>> + Send + 'static,
128    {
129        let listener = match self.listen(&endpoint).await {
130            Ok(l) => {
131                self.monitor
132                    .notify(ConnectionKind::Listening(endpoint.clone()))
133                    .await;
134                l
135            }
136            Err(err) => {
137                error!("Failed to listen on {endpoint}: {err}");
138                self.monitor
139                    .notify(ConnectionKind::ListenFailed(endpoint))
140                    .await;
141                return Err(err);
142            }
143        };
144
145        let resolved = listener.local_endpoint()?;
146        info!("Start listening on {resolved}");
147
148        self.task_group.spawn_then(
149            {
150                let this = self.clone();
151                async move {
152                    this.listen_loop_callback(Arc::new(listener), callback)
153                        .await
154                }
155            },
156            |res: TaskResult<()>| async move {
157                debug!("Listener callback loop ended: {res}");
158            },
159        );
160        Ok(resolved)
161    }
162
163    pub async fn shutdown(&self) {
164        self.task_group.cancel().await;
165    }
166
167    /// Runs the handshake and frames the stream. Called off the accept
168    /// loop. Also returns the peer id proved by the TLS client cert.
169    async fn upgrade(
170        &self,
171        listener: &StreamListener,
172        stream: Box<dyn ByteStream>,
173    ) -> Result<(FramedConn<C>, Option<PeerID>)> {
174        let stream = listener.handshake(stream, self.handshake_timeout).await?;
175        // Extract the peer cert before framing consumes the stream.
176        let vpid = stream
177            .peer_certificates()
178            .as_deref()
179            .and_then(peer_id_from_certs);
180        Ok((framed(stream, C::default()), vpid))
181    }
182
183    /// Accept loop (callback mode).
184    async fn listen_loop_callback<Fut>(
185        self: Arc<Self>,
186        listener: Arc<StreamListener>,
187        callback: impl FnOnce(FramedConn<C>) -> Fut + Clone + Send + 'static,
188    ) where
189        Fut: Future<Output = Result<()>> + Send + 'static,
190    {
191        loop {
192            // Released on `continue` or when the connection ends.
193            let slot = match self.connection_slots.acquire().await {
194                Ok(s) => s,
195                Err(err) => {
196                    error!("Failed to acquire a connection slot: {err}");
197                    return;
198                }
199            };
200
201            let stream = match listener.accept().await {
202                Ok(s) => s,
203                Err(err) => {
204                    error!("Failed to accept connection: {err}");
205                    self.monitor.notify(ConnectionKind::AcceptFailed).await;
206                    continue;
207                }
208            };
209
210            let endpoint = match stream.peer_endpoint() {
211                Some(ep) => ep,
212                None => {
213                    self.monitor.notify(ConnectionKind::AcceptFailed).await;
214                    error!("Failed to get peer endpoint");
215                    continue;
216                }
217            };
218
219            // Callback mode is the kademlia lookup plane.
220            if !self.access_control.allow(
221                &Subject::Endpoint(&endpoint),
222                Action::Lookup(ConnDirection::Inbound),
223            ) {
224                debug!("Rejected inbound lookup connection from {endpoint}");
225                continue;
226            }
227
228            self.monitor
229                .notify(ConnectionKind::Accepted(endpoint.clone()))
230                .await;
231
232            let on_disconnect = {
233                let this = self.clone();
234                |res| async move {
235                    if let TaskResult::Completed(Err(err)) = res {
236                        debug!("Inbound connection dropped: {err}");
237                    }
238                    this.monitor
239                        .notify(ConnectionKind::Disconnected(endpoint))
240                        .await;
241                    drop(slot);
242                }
243            };
244
245            let this = self.clone();
246            let listener = listener.clone();
247            let callback = callback.clone();
248            self.task_group.spawn_then(
249                async move {
250                    let (conn, _) = this.upgrade(&listener, stream).await?;
251                    callback(conn).await
252                },
253                on_disconnect,
254            );
255        }
256    }
257
258    /// Create a listener for TCP/TLS.
259    async fn listen(&self, endpoint: &Endpoint) -> Result<StreamListener> {
260        match endpoint {
261            Endpoint::Tcp(..) => {
262                let listener = TcpListener::bind(endpoint, Default::default()).await?;
263                Ok(StreamListener::Tcp(listener))
264            }
265            Endpoint::Tls(..) => {
266                let tls_config = ServerTlsConfig {
267                    server_config: tls_server_config(&self.key_pair)?,
268                };
269                let tcp_listener = TcpListener::bind(endpoint, Default::default()).await?;
270                Ok(StreamListener::Tls(
271                    tcp_listener,
272                    Box::new(TlsLayer::server(tls_config)),
273                ))
274            }
275            _ => Err(Error::UnsupportedEndpoint(endpoint.to_string())),
276        }
277    }
278}
279
280// Auto-queue paths only live on the peer-plane Listener. The kademlia
281// lookup plane uses `start_with_callback` and handles each connection
282// inline (no ConnQueue / handshake pipeline).
283impl Listener<PeerNetMsgCodec> {
284    /// Create a new Listener with a ConnQueue (auto-queue mode).
285    /// `handshake_timeout` is in seconds.
286    pub fn new_with_queue(
287        key_pair: &KeyPair,
288        slots: usize,
289        conn_queue: Arc<ConnQueue>,
290        monitor: Arc<Monitor>,
291        handshake_timeout: u64,
292        access_control: Arc<dyn AccessControl>,
293        ex: Executor,
294    ) -> Arc<Self> {
295        Arc::new(Self {
296            key_pair: key_pair.clone(),
297            connection_slots: Semaphore::new(slots),
298            conn_queue: Some(conn_queue),
299            task_group: TaskGroup::with_executor(ex),
300            monitor,
301            handshake_timeout: Duration::from_secs(handshake_timeout),
302            access_control,
303            _codec: PhantomData,
304        })
305    }
306
307    /// Start listening (auto-queue mode). Returns the resolved endpoint.
308    pub async fn start(self: &Arc<Self>, endpoint: Endpoint) -> Result<Endpoint> {
309        #[cfg(feature = "quic")]
310        if endpoint.is_quic() {
311            return self.start_quic(endpoint).await;
312        }
313
314        let listener = match self.listen(&endpoint).await {
315            Ok(l) => {
316                self.monitor
317                    .notify(ConnectionKind::Listening(endpoint.clone()))
318                    .await;
319                l
320            }
321            Err(err) => {
322                error!("Failed to listen on {endpoint}: {err}");
323                self.monitor
324                    .notify(ConnectionKind::ListenFailed(endpoint))
325                    .await;
326                return Err(err);
327            }
328        };
329
330        let resolved = listener.local_endpoint()?;
331        info!("Start listening on {resolved}");
332
333        self.task_group.spawn({
334            let this = self.clone();
335            async move { this.listen_loop(Arc::new(listener)).await }
336        });
337        Ok(resolved)
338    }
339
340    /// Accept loop (auto-queue mode).
341    async fn listen_loop(self: Arc<Self>, listener: Arc<StreamListener>) {
342        let conn_queue = self
343            .conn_queue
344            .as_ref()
345            .expect("listen_loop requires ConnQueue")
346            .clone();
347
348        loop {
349            // Released on `continue` or when the connection ends.
350            let slot = match self.connection_slots.acquire().await {
351                Ok(s) => s,
352                Err(err) => {
353                    error!("Failed to acquire a connection slot: {err}");
354                    return;
355                }
356            };
357
358            let stream = match listener.accept().await {
359                Ok(s) => s,
360                Err(err) => {
361                    error!("Failed to accept connection: {err}");
362                    self.monitor.notify(ConnectionKind::AcceptFailed).await;
363                    continue;
364                }
365            };
366
367            let endpoint = match stream.peer_endpoint() {
368                Some(ep) => ep,
369                None => {
370                    self.monitor.notify(ConnectionKind::AcceptFailed).await;
371                    error!("Failed to get peer endpoint");
372                    continue;
373                }
374            };
375
376            if !self.access_control.allow(
377                &Subject::Endpoint(&endpoint),
378                Action::Connect(ConnDirection::Inbound),
379            ) {
380                debug!("Rejected inbound connection from {endpoint}");
381                continue;
382            }
383
384            self.monitor
385                .notify(ConnectionKind::Accepted(endpoint.clone()))
386                .await;
387
388            let on_disconnect = {
389                let this = self.clone();
390                |res: TaskResult<Result<()>>| async move {
391                    if let TaskResult::Completed(Err(err)) = res {
392                        debug!("Inbound connection dropped: {err}");
393                    }
394                    this.monitor
395                        .notify(ConnectionKind::Disconnected(endpoint))
396                        .await;
397                    drop(slot);
398                }
399            };
400
401            let cq = conn_queue.clone();
402            let this = self.clone();
403            let listener = listener.clone();
404            self.task_group.spawn_then(
405                async move {
406                    let (conn, vpid) = this.upgrade(&listener, stream).await?;
407                    cq.handle(conn, ConnDirection::Inbound, vpid).await?;
408                    Ok(())
409                },
410                on_disconnect,
411            );
412        }
413    }
414
415    /// QUIC listener.
416    #[cfg(feature = "quic")]
417    async fn start_quic(self: &Arc<Self>, endpoint: Endpoint) -> Result<Endpoint> {
418        let rustls_config = quic_server_config(&self.key_pair)?;
419        let server_config = quic::ServerQuicConfig::from_rustls(rustls_config);
420
421        let quic_endpoint = match quic::QuicEndpoint::listen(&endpoint, server_config).await {
422            Ok(ep) => {
423                self.monitor
424                    .notify(ConnectionKind::Listening(endpoint.clone()))
425                    .await;
426                ep
427            }
428            Err(err) => {
429                error!("Failed to listen on {endpoint}: {err}");
430                self.monitor
431                    .notify(ConnectionKind::ListenFailed(endpoint))
432                    .await;
433                return Err(err.into());
434            }
435        };
436
437        let resolved: Endpoint = quic_endpoint.local_endpoint().map_err(Error::from)?;
438        info!("Start listening on {resolved}");
439
440        self.task_group.spawn_then(
441            {
442                let this = self.clone();
443                async move { this.listen_loop_quic(quic_endpoint).await }
444            },
445            |res: TaskResult<()>| async move {
446                debug!("QUIC listen loop ended: {res}");
447            },
448        );
449
450        Ok(resolved)
451    }
452
453    /// QUIC accept loop.
454    #[cfg(feature = "quic")]
455    async fn listen_loop_quic(self: Arc<Self>, quic_endpoint: quic::QuicEndpoint) {
456        loop {
457            // Released on `continue` or when the connection ends.
458            let slot = match self.connection_slots.acquire().await {
459                Ok(s) => s,
460                Err(err) => {
461                    error!("Failed to acquire a connection slot: {err}");
462                    return;
463                }
464            };
465
466            // Only the accept runs here. The handshake and the wait for
467            // the peer's first stream both happen in the spawned task,
468            // so a slow peer cannot stall the loop.
469            let incoming = match quic_endpoint.accept_incoming().await {
470                Ok(c) => c,
471                Err(err) => {
472                    error!("Failed to accept QUIC conn: {err}");
473                    self.monitor.notify(ConnectionKind::AcceptFailed).await;
474                    continue;
475                }
476            };
477
478            let peer_ep = incoming.peer_endpoint();
479
480            if !self.access_control.allow(
481                &Subject::Endpoint(&peer_ep),
482                Action::Connect(ConnDirection::Inbound),
483            ) {
484                debug!("Rejected inbound QUIC connection from {peer_ep}");
485                continue;
486            }
487
488            self.monitor
489                .notify(ConnectionKind::Accepted(peer_ep.clone()))
490                .await;
491
492            let on_disconnect = {
493                let this = self.clone();
494                |res: TaskResult<Result<()>>| async move {
495                    if let TaskResult::Completed(Err(err)) = res {
496                        debug!("Inbound QUIC conn dropped: {err}");
497                    }
498                    this.monitor
499                        .notify(ConnectionKind::Disconnected(peer_ep))
500                        .await;
501                    drop(slot);
502                }
503            };
504
505            let conn_queue = self
506                .conn_queue
507                .as_ref()
508                .expect("QUIC listener requires ConnQueue")
509                .clone();
510            let handshake_timeout = self.handshake_timeout;
511            self.task_group.spawn_then(
512                async move {
513                    // Without this the only bound is the QUIC idle
514                    // timeout, which holds the slot far longer.
515                    let quic_conn = timeout(handshake_timeout, incoming.handshake()).await??;
516                    let vpid = quic_conn
517                        .peer_certificates()
518                        .as_deref()
519                        .and_then(peer_id_from_certs);
520                    // A peer that handshakes and then never opens a
521                    // stream only wastes its own slot.
522                    let stream = timeout(handshake_timeout, quic_conn.accept_stream()).await??;
523                    let conn: FramedConn<PeerNetMsgCodec> = framed(stream, PeerNetMsgCodec::new());
524                    conn_queue
525                        .handle_quic(conn, quic_conn, ConnDirection::Inbound, vpid)
526                        .await?;
527                    Ok(())
528                },
529                on_disconnect,
530            );
531        }
532    }
533}