Skip to main content

karyon_p2p/
peer_pool.rs

1use std::{
2    collections::{HashMap, HashSet},
3    sync::{
4        Arc,
5        atomic::{AtomicU64, Ordering},
6    },
7};
8
9use log::{debug, error, info, warn};
10
11use karyon_core::{
12    async_runtime::{Executor, lock::RwLock},
13    async_util::{TaskGroup, TaskResult},
14};
15
16use karyon_eventemitter::{EventEmitter, EventListener, EventTopic, EventValue};
17
18use karyon_net::Endpoint;
19
20use crate::{
21    Error, PeerID, Result,
22    access_control::{Action, PeerCandidate, Subject},
23    config::Config,
24    conn_direction::ConnDirection,
25    conn_queue::{ConnQueue, QueuedConn},
26    handshake::{HandshakeParams, handshake},
27    monitor::{Monitor, PoolEvent},
28    peer::Peer,
29    protocol::{Protocol, ProtocolConstructor, ProtocolID, ProtocolMeta},
30};
31
32/// Topic key for the peer-lifecycle event emitter.
33#[derive(Hash, PartialEq, Eq, Debug, Clone)]
34pub enum PeerEventTopic {
35    Lifecycle,
36}
37
38/// Peer-lifecycle events. Each registered listener receives every
39/// event independently.
40#[derive(Debug, Clone, EventValue)]
41pub enum PeerEvent {
42    /// Peer added after a successful handshake.
43    Added(PeerID),
44    /// Previously-added peer removed.
45    Removed(PeerID),
46    /// Handshake failed before the peer was added.
47    HandshakeFailed(Option<PeerID>),
48}
49
50impl EventTopic for PeerEvent {
51    type Topic = PeerEventTopic;
52    fn topic() -> Self::Topic {
53        PeerEventTopic::Lifecycle
54    }
55}
56
57/// Listener returned by `register_peer_events`.
58pub type PeerEventListener = EventListener<PeerEventTopic, PeerEvent>;
59
60pub struct PeerPool {
61    /// Peer's ID
62    pub id: PeerID,
63
64    /// Connection queue
65    conn_queue: Arc<ConnQueue>,
66
67    /// Holds the running peers.
68    peers: RwLock<HashMap<PeerID, Arc<Peer>>>,
69
70    /// Next connection id, see `Peer::conn_id`.
71    next_conn_id: AtomicU64,
72
73    /// Hashmap contains protocol constructors.
74    pub(crate) protocols: RwLock<HashMap<ProtocolID, Box<ProtocolConstructor>>>,
75
76    /// Per-protocol metadata (version + kind, extensible). Keyed by
77    /// protocol id. Source of truth for the handshake's mandatory check
78    /// and version negotiation.
79    pub(crate) protocol_meta: RwLock<HashMap<ProtocolID, ProtocolMeta>>,
80
81    /// Peer-lifecycle event emitter. Each registered listener gets its
82    /// own copy of every event.
83    peer_emitter: Arc<EventEmitter<PeerEventTopic>>,
84
85    /// Managing spawned tasks.
86    task_group: TaskGroup,
87
88    /// A global Executor
89    pub(crate) executor: Executor,
90
91    /// The Configuration for the P2P network.
92    pub(crate) config: Arc<Config>,
93
94    /// Responsible for network and system monitoring.
95    monitor: Arc<Monitor>,
96}
97
98impl PeerPool {
99    /// Creates a new PeerPool
100    pub fn new(
101        id: &PeerID,
102        conn_queue: Arc<ConnQueue>,
103        config: Arc<Config>,
104        monitor: Arc<Monitor>,
105        executor: Executor,
106    ) -> Arc<Self> {
107        Arc::new(Self {
108            id: id.clone(),
109            conn_queue,
110            peers: RwLock::new(HashMap::new()),
111            next_conn_id: AtomicU64::new(0),
112            protocols: RwLock::new(HashMap::new()),
113            protocol_meta: RwLock::new(HashMap::new()),
114            peer_emitter: EventEmitter::new(),
115            task_group: TaskGroup::with_executor(executor.clone()),
116            executor,
117            monitor,
118            config,
119        })
120    }
121
122    /// Register a listener for the peer-lifecycle events.
123    pub fn register_peer_events(&self) -> EventListener<PeerEventTopic, PeerEvent> {
124        self.peer_emitter.register(&PeerEventTopic::Lifecycle)
125    }
126
127    /// Starts the [`PeerPool`]
128    pub async fn start(self: &Arc<Self>) -> Result<()> {
129        self.task_group.spawn(self.clone().run());
130        Ok(())
131    }
132
133    /// Shuts down
134    pub async fn shutdown(&self) {
135        // Cancel the task group first. This waits for the run loop to
136        // actually stop, so no new peer can be admitted while we close
137        // the ones already in the map.
138        self.task_group.cancel().await;
139
140        for peer in self.peers.read().await.values() {
141            let _ = peer.shutdown().await;
142        }
143    }
144
145    /// Register a protocol's user-supplied constructor and metadata.
146    /// Bloom advertising is handled by `Node::attach_protocol`.
147    pub async fn attach_protocol<P: Protocol>(&self, c: Box<ProtocolConstructor>) -> Result<()> {
148        let id = P::id();
149        self.protocols.write().await.insert(id.clone(), c);
150        self.protocol_meta.write().await.insert(
151            id,
152            ProtocolMeta {
153                version: P::version()?,
154                flags: P::flags(),
155            },
156        );
157        Ok(())
158    }
159
160    /// Broadcast a message to all connected peers.
161    pub async fn broadcast(&self, proto_id: &ProtocolID, msg: Vec<u8>) {
162        for (pid, peer) in self.peers.read().await.iter() {
163            if let Err(err) = peer.send(proto_id.to_string(), msg.clone()).await {
164                error!("failed to send msg to {pid}: {err}");
165                continue;
166            }
167        }
168    }
169
170    /// Broadcast a message to a specific set of peers.
171    pub async fn broadcast_to(
172        &self,
173        proto_id: &ProtocolID,
174        msg: Vec<u8>,
175        targets: &HashSet<PeerID>,
176    ) {
177        for (pid, peer) in self.peers.read().await.iter() {
178            if !targets.contains(pid) {
179                continue;
180            }
181            if let Err(err) = peer.send(proto_id.to_string(), msg.clone()).await {
182                error!("failed to send msg to {pid}: {err}");
183            }
184        }
185    }
186
187    /// Send a message to a specific peer on the given protocol. Returns
188    /// `PeerNotFound` if the peer is not currently in the pool.
189    pub async fn send_to(
190        &self,
191        peer_id: &PeerID,
192        proto_id: &ProtocolID,
193        msg: Vec<u8>,
194    ) -> Result<()> {
195        let peers = self.peers.read().await;
196        let peer = peers
197            .get(peer_id)
198            .ok_or_else(|| Error::PeerNotFound(peer_id.to_string()))?;
199        peer.send(proto_id.to_string(), msg).await
200    }
201
202    /// Returns the negotiated protocol set for a peer.
203    pub async fn peer_protocol_set(&self, pid: &PeerID) -> Option<HashSet<ProtocolID>> {
204        self.peers
205            .read()
206            .await
207            .get(pid)
208            .map(|p| p.negotiated_protocols().clone())
209    }
210
211    /// Checks if the peer list contains a peer with the given peer id
212    pub async fn contains_peer(&self, pid: &PeerID) -> bool {
213        self.peers.read().await.contains_key(pid)
214    }
215
216    /// Returns the number of currently connected peers.
217    pub async fn peers_len(&self) -> usize {
218        self.peers.read().await.len()
219    }
220
221    /// Returns a map of inbound peers with their endpoints.
222    pub async fn inbound_peers(&self) -> HashMap<PeerID, Endpoint> {
223        let mut peers = HashMap::new();
224        for (id, peer) in self.peers.read().await.iter() {
225            if peer.is_inbound() {
226                peers.insert(id.clone(), peer.remote_endpoint().clone());
227            }
228        }
229        peers
230    }
231
232    /// Returns a map of outbound peers with their endpoints.
233    pub async fn outbound_peers(&self) -> HashMap<PeerID, Endpoint> {
234        let mut peers = HashMap::new();
235        for (id, peer) in self.peers.read().await.iter() {
236            if !peer.is_inbound() {
237                peers.insert(id.clone(), peer.remote_endpoint().clone());
238            }
239        }
240        peers
241    }
242
243    async fn run(self: Arc<Self>) {
244        loop {
245            let queued = self.conn_queue.next().await;
246
247            // One task per connection, so a slow or silent peer does not
248            // hold back the others. Every queued connection holds a
249            // listener or connector slot, which bounds these tasks.
250            self.task_group.spawn(self.clone().admit(queued));
251        }
252    }
253
254    /// Handshake a queued connection and add it to the pool.
255    async fn admit(self: Arc<Self>, mut queued: QueuedConn) {
256        // Snapshot the protocol metadata so we don't hold a lock
257        // across the handshake. Drives both version negotiation
258        // and the mandatory-subset check.
259        let meta = self.protocol_meta.read().await.clone();
260
261        let params = HandshakeParams {
262            own_id: &self.id,
263            config_version: &self.config.version,
264            protocols: &meta,
265            timeout_secs: self.config.handshake_timeout,
266            verified_peer_id: queued.verified_peer_id.as_ref(),
267        };
268        let handshake = handshake(&mut queued.reader, &mut queued.writer, &params).await;
269
270        let (pid, negotiated) = match handshake {
271            Ok(v) => v,
272            Err(err) => {
273                let pid = queued.verified_peer_id.clone();
274                self.monitor
275                    .notify(PoolEvent::HandshakeFailed(pid.clone()))
276                    .await;
277                let _ = self
278                    .peer_emitter
279                    .emit(&PeerEvent::HandshakeFailed(pid))
280                    .await;
281                let _ = queued.conn_closed_tx.send(Err(err)).await;
282                return;
283            }
284        };
285
286        // Admission gate. Runs before the Peer is built, so a denied
287        // peer never gets tasks spawned or QUIC streams opened.
288        let candidate = PeerCandidate {
289            peer_id: &pid,
290            remote_endpoint: &queued.remote_endpoint,
291            negotiated_protocols: &negotiated,
292        };
293        let allowed = self.config.access_control.allow(
294            &Subject::Peer(&candidate),
295            Action::Connect(queued.direction.clone()),
296        );
297
298        if !allowed {
299            warn!("Access denied for peer {pid}");
300            self.monitor
301                .notify(PoolEvent::HandshakeFailed(Some(pid.clone())))
302                .await;
303            let _ = self
304                .peer_emitter
305                .emit(&PeerEvent::HandshakeFailed(Some(pid)))
306                .await;
307            let _ = queued.conn_closed_tx.send(Err(Error::AccessDenied)).await;
308            return;
309        }
310
311        // Wake the task waiting in `ConnQueue::handle`, or it keeps
312        // its connection slot forever.
313        let conn_closed_tx = queued.conn_closed_tx.clone();
314        if let Err(err) = self.new_peer(queued, pid, negotiated).await {
315            error!("new_peer failed: {err}");
316            let _ = conn_closed_tx.send(Err(err)).await;
317        }
318    }
319
320    /// Build a Peer from a post-handshake `QueuedConn` and run it.
321    async fn new_peer(
322        self: &Arc<Self>,
323        queued: QueuedConn,
324        pid: PeerID,
325        negotiated: Vec<ProtocolID>,
326    ) -> Result<()> {
327        let negotiated: HashSet<ProtocolID> = negotiated.into_iter().collect();
328
329        let res = self.insert_peer(queued, &pid, negotiated).await;
330        if matches!(res, Err(Error::PeerAlreadyConnected)) {
331            self.monitor
332                .notify(PoolEvent::PeerAlreadyConnected(pid.clone()))
333                .await;
334        }
335        let (peer, replaced) = res?;
336
337        let on_disconnect = {
338            let this = self.clone();
339            let peer = peer.clone();
340            |result| async move {
341                if let TaskResult::Completed(_) = result
342                    && let Err(err) = this.remove_peer(&peer).await
343                {
344                    error!("Failed to remove peer {}: {err}", peer.id());
345                }
346            }
347        };
348
349        self.task_group
350            .spawn_then(peer.clone().run(), on_disconnect);
351
352        // Same peer, now on the connection both nodes keep. It stays in
353        // the pool, so no `Removed`/`Added` events.
354        if let Some(old) = replaced {
355            debug!("Replace the connection of peer {pid}");
356            let _ = old.shutdown().await;
357            return Ok(());
358        }
359
360        info!("Add new peer {pid}");
361        self.monitor.notify(PoolEvent::NewPeer(pid.clone())).await;
362        let _ = self.peer_emitter.emit(&PeerEvent::Added(pid)).await;
363
364        Ok(())
365    }
366
367    /// Builds the peer and adds it. If a peer with this id is already in
368    /// the pool, the new connection either replaces it (returned as the
369    /// second value, for the caller to shut down) or is rejected; see
370    /// `new_conn_wins`. Connections are admitted in parallel, so the check
371    /// and the insert run under one write lock. `Peer::new` does not wait
372    /// for the remote, so the lock is held only briefly.
373    async fn insert_peer(
374        self: &Arc<Self>,
375        queued: QueuedConn,
376        pid: &PeerID,
377        negotiated: HashSet<ProtocolID>,
378    ) -> Result<(Arc<Peer>, Option<Arc<Peer>>)> {
379        let mut peers = self.peers.write().await;
380        if let Some(old) = peers.get(pid)
381            && !new_conn_wins(&self.id, pid, old.direction(), &queued.direction)
382        {
383            return Err(Error::PeerAlreadyConnected);
384        }
385        let conn_id = self.next_conn_id.fetch_add(1, Ordering::Relaxed);
386        let peer = Peer::new(self.clone(), queued, pid.clone(), conn_id, negotiated)?;
387        let replaced = peers.insert(pid.clone(), peer.clone());
388        Ok((peer, replaced))
389    }
390
391    /// Shuts down the peer and removes it from the peer list. Does nothing
392    /// if the pool holds another peer under this id (the connection was
393    /// replaced) or none.
394    async fn remove_peer(&self, peer: &Arc<Peer>) -> Result<()> {
395        let pid = peer.id();
396        {
397            let mut peers = self.peers.write().await;
398            match peers.get(pid) {
399                Some(p) if p.conn_id() == peer.conn_id() => peers.remove(pid),
400                _ => return Ok(()),
401            };
402        }
403
404        let _ = peer.shutdown().await;
405
406        self.monitor
407            .notify(PoolEvent::RemovePeer(pid.clone()))
408            .await;
409        let _ = self
410            .peer_emitter
411            .emit(&PeerEvent::Removed(pid.clone()))
412            .await;
413
414        warn!("Peer {pid} removed",);
415        Ok(())
416    }
417}
418
419/// Decides whether a new connection to a peer replaces the one already in
420/// the pool. When two nodes dial each other at the same time, each sees
421/// both connections in opposite directions. Both keep the one dialed by
422/// the node with the smaller PeerID, so they keep the same connection.
423/// Two connections in the same direction keep the first.
424fn new_conn_wins(own: &PeerID, remote: &PeerID, old: &ConnDirection, new: &ConnDirection) -> bool {
425    match (old, new) {
426        // The new one was dialed by us.
427        (ConnDirection::Inbound, ConnDirection::Outbound) => own.0 < remote.0,
428        // The new one was dialed by the remote.
429        (ConnDirection::Outbound, ConnDirection::Inbound) => remote.0 < own.0,
430        _ => false,
431    }
432}
433
434#[cfg(test)]
435mod tests {
436    use super::*;
437
438    #[test]
439    fn both_nodes_keep_the_same_connection() {
440        let a = PeerID([1; 32]);
441        let b = PeerID([2; 32]);
442        let (inb, outb) = (ConnDirection::Inbound, ConnDirection::Outbound);
443
444        // A has the smaller id, so both keep the connection A dialed:
445        // outbound on A, inbound on B. The other one never wins.
446        assert!(new_conn_wins(&a, &b, &inb, &outb));
447        assert!(!new_conn_wins(&a, &b, &outb, &inb));
448        assert!(new_conn_wins(&b, &a, &outb, &inb));
449        assert!(!new_conn_wins(&b, &a, &inb, &outb));
450
451        // Same direction: the first one stays.
452        assert!(!new_conn_wins(&a, &b, &outb, &outb));
453        assert!(!new_conn_wins(&a, &b, &inb, &inb));
454    }
455}