1use std::{
2 collections::{HashMap, HashSet},
3 sync::{
4 Arc,
5 atomic::{AtomicU64, Ordering},
6 },
7};
8
9use log::{debug, error, info, warn};
10
11use karyon_core::{
12 async_runtime::{Executor, lock::RwLock},
13 async_util::{TaskGroup, TaskResult},
14};
15
16use karyon_eventemitter::{EventEmitter, EventListener, EventTopic, EventValue};
17
18use karyon_net::Endpoint;
19
20use crate::{
21 Error, PeerID, Result,
22 access_control::{Action, PeerCandidate, Subject},
23 config::Config,
24 conn_direction::ConnDirection,
25 conn_queue::{ConnQueue, QueuedConn},
26 handshake::{HandshakeParams, handshake},
27 monitor::{Monitor, PoolEvent},
28 peer::Peer,
29 protocol::{Protocol, ProtocolConstructor, ProtocolID, ProtocolMeta},
30};
31
32#[derive(Hash, PartialEq, Eq, Debug, Clone)]
34pub enum PeerEventTopic {
35 Lifecycle,
36}
37
38#[derive(Debug, Clone, EventValue)]
41pub enum PeerEvent {
42 Added(PeerID),
44 Removed(PeerID),
46 HandshakeFailed(Option<PeerID>),
48}
49
50impl EventTopic for PeerEvent {
51 type Topic = PeerEventTopic;
52 fn topic() -> Self::Topic {
53 PeerEventTopic::Lifecycle
54 }
55}
56
57pub type PeerEventListener = EventListener<PeerEventTopic, PeerEvent>;
59
60pub struct PeerPool {
61 pub id: PeerID,
63
64 conn_queue: Arc<ConnQueue>,
66
67 peers: RwLock<HashMap<PeerID, Arc<Peer>>>,
69
70 next_conn_id: AtomicU64,
72
73 pub(crate) protocols: RwLock<HashMap<ProtocolID, Box<ProtocolConstructor>>>,
75
76 pub(crate) protocol_meta: RwLock<HashMap<ProtocolID, ProtocolMeta>>,
80
81 peer_emitter: Arc<EventEmitter<PeerEventTopic>>,
84
85 task_group: TaskGroup,
87
88 pub(crate) executor: Executor,
90
91 pub(crate) config: Arc<Config>,
93
94 monitor: Arc<Monitor>,
96}
97
98impl PeerPool {
99 pub fn new(
101 id: &PeerID,
102 conn_queue: Arc<ConnQueue>,
103 config: Arc<Config>,
104 monitor: Arc<Monitor>,
105 executor: Executor,
106 ) -> Arc<Self> {
107 Arc::new(Self {
108 id: id.clone(),
109 conn_queue,
110 peers: RwLock::new(HashMap::new()),
111 next_conn_id: AtomicU64::new(0),
112 protocols: RwLock::new(HashMap::new()),
113 protocol_meta: RwLock::new(HashMap::new()),
114 peer_emitter: EventEmitter::new(),
115 task_group: TaskGroup::with_executor(executor.clone()),
116 executor,
117 monitor,
118 config,
119 })
120 }
121
122 pub fn register_peer_events(&self) -> EventListener<PeerEventTopic, PeerEvent> {
124 self.peer_emitter.register(&PeerEventTopic::Lifecycle)
125 }
126
127 pub async fn start(self: &Arc<Self>) -> Result<()> {
129 self.task_group.spawn(self.clone().run());
130 Ok(())
131 }
132
133 pub async fn shutdown(&self) {
135 self.task_group.cancel().await;
139
140 for peer in self.peers.read().await.values() {
141 let _ = peer.shutdown().await;
142 }
143 }
144
145 pub async fn attach_protocol<P: Protocol>(&self, c: Box<ProtocolConstructor>) -> Result<()> {
148 let id = P::id();
149 self.protocols.write().await.insert(id.clone(), c);
150 self.protocol_meta.write().await.insert(
151 id,
152 ProtocolMeta {
153 version: P::version()?,
154 flags: P::flags(),
155 },
156 );
157 Ok(())
158 }
159
160 pub async fn broadcast(&self, proto_id: &ProtocolID, msg: Vec<u8>) {
162 for (pid, peer) in self.peers.read().await.iter() {
163 if let Err(err) = peer.send(proto_id.to_string(), msg.clone()).await {
164 error!("failed to send msg to {pid}: {err}");
165 continue;
166 }
167 }
168 }
169
170 pub async fn broadcast_to(
172 &self,
173 proto_id: &ProtocolID,
174 msg: Vec<u8>,
175 targets: &HashSet<PeerID>,
176 ) {
177 for (pid, peer) in self.peers.read().await.iter() {
178 if !targets.contains(pid) {
179 continue;
180 }
181 if let Err(err) = peer.send(proto_id.to_string(), msg.clone()).await {
182 error!("failed to send msg to {pid}: {err}");
183 }
184 }
185 }
186
187 pub async fn send_to(
190 &self,
191 peer_id: &PeerID,
192 proto_id: &ProtocolID,
193 msg: Vec<u8>,
194 ) -> Result<()> {
195 let peers = self.peers.read().await;
196 let peer = peers
197 .get(peer_id)
198 .ok_or_else(|| Error::PeerNotFound(peer_id.to_string()))?;
199 peer.send(proto_id.to_string(), msg).await
200 }
201
202 pub async fn peer_protocol_set(&self, pid: &PeerID) -> Option<HashSet<ProtocolID>> {
204 self.peers
205 .read()
206 .await
207 .get(pid)
208 .map(|p| p.negotiated_protocols().clone())
209 }
210
211 pub async fn contains_peer(&self, pid: &PeerID) -> bool {
213 self.peers.read().await.contains_key(pid)
214 }
215
216 pub async fn peers_len(&self) -> usize {
218 self.peers.read().await.len()
219 }
220
221 pub async fn inbound_peers(&self) -> HashMap<PeerID, Endpoint> {
223 let mut peers = HashMap::new();
224 for (id, peer) in self.peers.read().await.iter() {
225 if peer.is_inbound() {
226 peers.insert(id.clone(), peer.remote_endpoint().clone());
227 }
228 }
229 peers
230 }
231
232 pub async fn outbound_peers(&self) -> HashMap<PeerID, Endpoint> {
234 let mut peers = HashMap::new();
235 for (id, peer) in self.peers.read().await.iter() {
236 if !peer.is_inbound() {
237 peers.insert(id.clone(), peer.remote_endpoint().clone());
238 }
239 }
240 peers
241 }
242
243 async fn run(self: Arc<Self>) {
244 loop {
245 let queued = self.conn_queue.next().await;
246
247 self.task_group.spawn(self.clone().admit(queued));
251 }
252 }
253
254 async fn admit(self: Arc<Self>, mut queued: QueuedConn) {
256 let meta = self.protocol_meta.read().await.clone();
260
261 let params = HandshakeParams {
262 own_id: &self.id,
263 config_version: &self.config.version,
264 protocols: &meta,
265 timeout_secs: self.config.handshake_timeout,
266 verified_peer_id: queued.verified_peer_id.as_ref(),
267 };
268 let handshake = handshake(&mut queued.reader, &mut queued.writer, ¶ms).await;
269
270 let (pid, negotiated) = match handshake {
271 Ok(v) => v,
272 Err(err) => {
273 let pid = queued.verified_peer_id.clone();
274 self.monitor
275 .notify(PoolEvent::HandshakeFailed(pid.clone()))
276 .await;
277 let _ = self
278 .peer_emitter
279 .emit(&PeerEvent::HandshakeFailed(pid))
280 .await;
281 let _ = queued.conn_closed_tx.send(Err(err)).await;
282 return;
283 }
284 };
285
286 let candidate = PeerCandidate {
289 peer_id: &pid,
290 remote_endpoint: &queued.remote_endpoint,
291 negotiated_protocols: &negotiated,
292 };
293 let allowed = self.config.access_control.allow(
294 &Subject::Peer(&candidate),
295 Action::Connect(queued.direction.clone()),
296 );
297
298 if !allowed {
299 warn!("Access denied for peer {pid}");
300 self.monitor
301 .notify(PoolEvent::HandshakeFailed(Some(pid.clone())))
302 .await;
303 let _ = self
304 .peer_emitter
305 .emit(&PeerEvent::HandshakeFailed(Some(pid)))
306 .await;
307 let _ = queued.conn_closed_tx.send(Err(Error::AccessDenied)).await;
308 return;
309 }
310
311 let conn_closed_tx = queued.conn_closed_tx.clone();
314 if let Err(err) = self.new_peer(queued, pid, negotiated).await {
315 error!("new_peer failed: {err}");
316 let _ = conn_closed_tx.send(Err(err)).await;
317 }
318 }
319
320 async fn new_peer(
322 self: &Arc<Self>,
323 queued: QueuedConn,
324 pid: PeerID,
325 negotiated: Vec<ProtocolID>,
326 ) -> Result<()> {
327 let negotiated: HashSet<ProtocolID> = negotiated.into_iter().collect();
328
329 let res = self.insert_peer(queued, &pid, negotiated).await;
330 if matches!(res, Err(Error::PeerAlreadyConnected)) {
331 self.monitor
332 .notify(PoolEvent::PeerAlreadyConnected(pid.clone()))
333 .await;
334 }
335 let (peer, replaced) = res?;
336
337 let on_disconnect = {
338 let this = self.clone();
339 let peer = peer.clone();
340 |result| async move {
341 if let TaskResult::Completed(_) = result
342 && let Err(err) = this.remove_peer(&peer).await
343 {
344 error!("Failed to remove peer {}: {err}", peer.id());
345 }
346 }
347 };
348
349 self.task_group
350 .spawn_then(peer.clone().run(), on_disconnect);
351
352 if let Some(old) = replaced {
355 debug!("Replace the connection of peer {pid}");
356 let _ = old.shutdown().await;
357 return Ok(());
358 }
359
360 info!("Add new peer {pid}");
361 self.monitor.notify(PoolEvent::NewPeer(pid.clone())).await;
362 let _ = self.peer_emitter.emit(&PeerEvent::Added(pid)).await;
363
364 Ok(())
365 }
366
367 async fn insert_peer(
374 self: &Arc<Self>,
375 queued: QueuedConn,
376 pid: &PeerID,
377 negotiated: HashSet<ProtocolID>,
378 ) -> Result<(Arc<Peer>, Option<Arc<Peer>>)> {
379 let mut peers = self.peers.write().await;
380 if let Some(old) = peers.get(pid)
381 && !new_conn_wins(&self.id, pid, old.direction(), &queued.direction)
382 {
383 return Err(Error::PeerAlreadyConnected);
384 }
385 let conn_id = self.next_conn_id.fetch_add(1, Ordering::Relaxed);
386 let peer = Peer::new(self.clone(), queued, pid.clone(), conn_id, negotiated)?;
387 let replaced = peers.insert(pid.clone(), peer.clone());
388 Ok((peer, replaced))
389 }
390
391 async fn remove_peer(&self, peer: &Arc<Peer>) -> Result<()> {
395 let pid = peer.id();
396 {
397 let mut peers = self.peers.write().await;
398 match peers.get(pid) {
399 Some(p) if p.conn_id() == peer.conn_id() => peers.remove(pid),
400 _ => return Ok(()),
401 };
402 }
403
404 let _ = peer.shutdown().await;
405
406 self.monitor
407 .notify(PoolEvent::RemovePeer(pid.clone()))
408 .await;
409 let _ = self
410 .peer_emitter
411 .emit(&PeerEvent::Removed(pid.clone()))
412 .await;
413
414 warn!("Peer {pid} removed",);
415 Ok(())
416 }
417}
418
419fn new_conn_wins(own: &PeerID, remote: &PeerID, old: &ConnDirection, new: &ConnDirection) -> bool {
425 match (old, new) {
426 (ConnDirection::Inbound, ConnDirection::Outbound) => own.0 < remote.0,
428 (ConnDirection::Outbound, ConnDirection::Inbound) => remote.0 < own.0,
430 _ => false,
431 }
432}
433
434#[cfg(test)]
435mod tests {
436 use super::*;
437
438 #[test]
439 fn both_nodes_keep_the_same_connection() {
440 let a = PeerID([1; 32]);
441 let b = PeerID([2; 32]);
442 let (inb, outb) = (ConnDirection::Inbound, ConnDirection::Outbound);
443
444 assert!(new_conn_wins(&a, &b, &inb, &outb));
447 assert!(!new_conn_wins(&a, &b, &outb, &inb));
448 assert!(new_conn_wins(&b, &a, &outb, &inb));
449 assert!(!new_conn_wins(&b, &a, &inb, &outb));
450
451 assert!(!new_conn_wins(&a, &b, &outb, &outb));
453 assert!(!new_conn_wins(&a, &b, &inb, &inb));
454 }
455}