//! Implementation of the transport protocol. //! //! We use the Noise NN handshake pattern to establish an encrypted stream with a remote peer. //! The handshake itself is implemented in the external [`cyphernet`] and [`netservices`] crates. use std::collections::VecDeque; use std::os::unix::io::AsRawFd; use std::os::unix::prelude::RawFd; use std::sync::Arc; use std::time::{Duration, SystemTime}; use std::{fmt, io, net}; use amplify::Wrapper as _; use crossbeam_channel as chan; use cyphernet::{Cert, Digest, EcSign, Sha256}; use localtime::LocalTime; use netservices::resource::{ListenerEvent, NetAccept, NetTransport, SessionEvent}; use netservices::session::ProtocolArtifact; use netservices::session::{CypherReader, CypherSession, CypherWriter}; use netservices::{NetConnection, NetSession}; use radicle::collections::HashMap; use radicle::crypto::Signature; use radicle::node::NodeId; use radicle::storage::WriteStorage; use crate::crypto::Signer; use crate::service::reactor::{Fetch, Io}; use crate::service::{routing, session, DisconnectReason, Message, Service}; use crate::wire::{Decode, Encode}; use crate::worker::{WorkerReq, WorkerResp}; use crate::Link; use crate::{address, service}; #[allow(clippy::large_enum_variant)] /// Control message used internally between workers, users, and the service. pub enum Control { /// Message from the user to the service. User(service::Command), /// Message from a worker to the service. Worker(WorkerResp), } impl fmt::Debug for Control { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::User(cmd) => cmd.fmt(f), Self::Worker(resp) => resp.result.fmt(f), } } } /// Peer session type. pub type WireSession = CypherSession; /// Peer session type (read-only). pub type WireReader = CypherReader; /// Peer session type (write-only). pub type WireWriter = CypherWriter; /// Reactor action. type Action = reactor::Action>, NetTransport>>; /// Peer connection state machine. enum Peer { /// The initial state before handshake is completed. Connecting { link: Link }, /// The state after handshake is completed. /// Peers in this state are handled by the underlying service. Connected { link: Link, id: NodeId }, /// The state after a peer was disconnected, either during handshake, /// or once connected. Disconnected { id: Option, reason: DisconnectReason, }, /// The state after we've started the process of upgraded the peer for a fetch. /// The request to handover the socket was made to the reactor. Upgrading { fetch: Fetch, link: Link, id: NodeId, }, /// The peer is now upgraded and we are in control of the socket. Upgraded { link: Link, id: NodeId }, } impl std::fmt::Debug for Peer { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Self::Connecting { link } => write!(f, "Connecting({:?})", link), Self::Connected { link, id } => write!(f, "Connected({link:?}, {id})"), Self::Disconnected { reason, id } => write!(f, "Disconnected({reason}, {id:?})"), Self::Upgrading { fetch, link, id } => write!( f, "Upgrading(initiated={}, {link:?}, {id})", fetch.initiated ), Self::Upgraded { link, id, .. } => write!(f, "Upgraded({link:?}, {id})"), } } } impl Peer { /// Return the peer's id, if any. fn id(&self) -> Option<&NodeId> { match self { Peer::Connected { id, .. } | Peer::Disconnected { id: Some(id), .. } | Peer::Upgrading { id, .. } | Peer::Upgraded { id, .. } => Some(id), _ => None, } } /// Return a new connecting peer. fn connecting(link: Link) -> Self { Self::Connecting { link } } /// Switch to connected state. fn connected(&mut self, id: NodeId) { if let Self::Connecting { link } = self { *self = Self::Connected { link: *link, id }; } else { panic!("Peer::connected: session for {} is already established", id); } } /// Switch to disconnected state. fn disconnected(&mut self, reason: DisconnectReason) { if let Self::Connected { id, .. } = self { *self = Self::Disconnected { id: Some(*id), reason, }; } else if let Self::Connecting { .. } = self { *self = Self::Disconnected { id: None, reason }; } else { panic!("Peer::disconnected: session is not connected ({:?})", self); } } /// Switch to upgrading state. fn upgrading(&mut self, fetch: Fetch) { if let Self::Connected { id, link } = self { *self = Self::Upgrading { fetch, id: *id, link: *link, }; } else { panic!("Peer::upgrading: session is not connected"); } } /// Switch to upgraded state. fn upgraded(&mut self) -> Fetch { if let Self::Upgrading { fetch, id, link } = self { let fetch = fetch.clone(); log::debug!(target: "wire", "Peer {id} upgraded for fetch {}", fetch.repo); *self = Self::Upgraded { id: *id, link: *link, }; fetch } else { panic!("Peer::upgraded: can't upgrade before handover"); } } /// Switch back from upgraded to connected state. fn downgrade(&mut self) { if let Self::Upgraded { id, link, .. } = self { *self = Self::Connected { id: *id, link: *link, }; } else { panic!("Peer::downgrade: can't downgrade if not in upgraded state"); } } } /// Wire protocol implementation for a set of peers. pub struct Wire { /// Backing service instance. service: Service, /// Worker pool interface. worker: chan::Sender>, /// Used for authentication; keeps local identity. cert: Cert, /// Used for authentication. signer: G, /// Internal queue of actions to send to the reactor. actions: VecDeque>, /// Peer sessions. peers: HashMap, /// SOCKS5 proxy address. proxy: net::SocketAddr, /// Buffer for incoming peer data. read_queue: VecDeque, } impl Wire where R: routing::Store, S: address::Store, W: WriteStorage + 'static, G: Signer + EcSign, { pub fn new( mut service: Service, worker: chan::Sender>, cert: Cert, signer: G, proxy: net::SocketAddr, clock: LocalTime, ) -> Self { service .initialize(clock) .expect("Wire::new: error initializing service"); Self { service, worker, cert, signer, proxy, actions: VecDeque::new(), peers: HashMap::default(), read_queue: VecDeque::new(), } } pub fn listen(&mut self, socket: NetAccept>) { self.actions.push_back(Action::RegisterListener(socket)); } fn peer_mut_by_fd(&mut self, fd: RawFd) -> &mut Peer { self.peers.get_mut(&fd).unwrap_or_else(|| { log::error!(target: "wire", "Peer with fd {fd} was not found"); panic!("Peer with fd {fd} is not known"); }) } fn fd_by_id(&self, node_id: &NodeId) -> (RawFd, &Peer) { self.peers .iter() .find(|(_, peer)| peer.id() == Some(node_id)) .map(|(fd, peer)| (*fd, peer)) .unwrap_or_else(|| panic!("Peer {node_id} was expected to be known to the transport")) } fn connected_fd_by_id(&self, node_id: &NodeId) -> RawFd { match self.fd_by_id(node_id) { (fd, Peer::Connected { .. }) => fd, (fd, peer) => { panic!( "Peer {node_id} (fd={fd}) was expected to be in a connected state ({peer:?})" ) } } } fn connected(&self) -> impl Iterator { self.peers.iter().filter_map(|(fd, peer)| { if let Peer::Connected { id, .. } = peer { Some((*fd, id)) } else { None } }) } fn disconnect(&mut self, fd: RawFd, reason: DisconnectReason) { let peer = self.peer_mut_by_fd(fd); if let Peer::Disconnected { .. } = peer { log::error!(target: "wire", "Peer (fd={fd}) is already disconnected"); return; }; log::debug!(target: "wire", "Disconnecting peer (fd={fd}): {reason}"); peer.disconnected(reason); self.actions.push_back(Action::UnregisterTransport(fd)); } fn upgrade(&mut self, fd: RawFd, fetch: Fetch) { let peer = self.peer_mut_by_fd(fd); if let Peer::Disconnected { .. } = peer { log::error!(target: "wire", "Peer (fd={fd}) is already disconnected"); return; }; log::debug!(target: "wire", "Requesting transport handover from reactor for peer (fd={fd})"); peer.upgrading(fetch); self.actions.push_back(Action::UnregisterTransport(fd)); } fn upgraded(&mut self, transport: NetTransport>) { let fd = transport.as_raw_fd(); let peer = self.peer_mut_by_fd(fd); let fetch = peer.upgraded(); let session = match transport.into_session() { Ok(session) => session, Err(_) => panic!("Transport::upgraded: peer write buffer not empty on upgrade"), }; if self .worker .send(WorkerReq { fetch, session, drain: self.read_queue.drain(..).collect(), }) .is_err() { log::error!(target: "wire", "Worker pool is disconnected; cannot send fetch request"); } } fn worker_result(&mut self, resp: WorkerResp) { log::debug!(target: "wire", "Fetch completed: {:?}", resp.result); let session = resp.session; let fd = session.as_connection().as_raw_fd(); let peer = self.peer_mut_by_fd(fd); let session = if let Peer::Disconnected { .. } = peer { log::error!(target: "wire", "Peer with fd {fd} is already disconnected"); return; } else if let Peer::Upgraded { link, .. } = peer { match NetTransport::with_session(session, *link) { Ok(session) => session, Err(err) => { log::error!(target: "wire", "Session downgrade failed: {err}"); return; } } } else { todo!(); }; peer.downgrade(); self.actions.push_back(Action::RegisterTransport(session)); self.service.fetched(resp.result); } } impl reactor::Handler for Wire where R: routing::Store + Send, S: address::Store + Send, W: WriteStorage + Send + 'static, G: Signer + EcSign + Clone + Send, { type Listener = NetAccept>; type Transport = NetTransport>; type Command = Control; fn tick(&mut self, _time: Duration) { // FIXME: Change this once a proper timestamp is passed into the function. self.service.tick(LocalTime::from(SystemTime::now())); } fn handle_wakeup(&mut self) { self.service.wake() } fn handle_listener_event( &mut self, socket_addr: net::SocketAddr, event: ListenerEvent>, _: Duration, ) { match event { ListenerEvent::Accepted(connection) => { log::debug!( target: "wire", "Accepting inbound peer connection from {}..", connection.remote_addr() ); self.peers .insert(connection.as_raw_fd(), Peer::connecting(Link::Inbound)); let session = WireSession::accept::<{ Sha256::OUTPUT_LEN }>( connection, self.cert, vec![], self.signer.clone(), ); let transport = match NetTransport::with_session(session, Link::Inbound) { Ok(transport) => transport, Err(err) => { log::error!(target: "wire", "Failed to create transport for accepted connection: {err}"); return; } }; self.service.accepted(socket_addr); self.actions .push_back(reactor::Action::RegisterTransport(transport)) } ListenerEvent::Failure(err) => { log::error!(target: "wire", "Error listening for inbound connections: {err}"); } } } fn handle_transport_event( &mut self, fd: RawFd, event: SessionEvent>, _: Duration, ) { match event { SessionEvent::Established(ProtocolArtifact { state: Cert { pk: node_id, .. }, .. }) => { log::debug!(target: "wire", "Session established with {node_id}"); let conflicting = self .connected() .filter(|(_, id)| **id == node_id) .map(|(fd, _)| fd) .collect::>(); for fd in conflicting { log::warn!( target: "wire", "Closing conflicting session with {node_id} (fd={fd})" ); self.disconnect( fd, DisconnectReason::Dial(Arc::new(io::Error::from( io::ErrorKind::AlreadyExists, ))), ); } let Some(peer) = self.peers.get_mut(&fd) else { log::error!(target: "wire", "Session not found for fd {fd}"); return; }; let Peer::Connecting { link } = peer else { log::error!( target: "wire", "Session for {node_id} was either not found, or in an invalid state" ); return; }; let link = *link; peer.connected(node_id); self.service.connected(node_id, link); } SessionEvent::Data(data) => { if let Some(Peer::Connected { id, .. }) = self.peers.get(&fd) { self.read_queue.extend(data); loop { match Message::decode(&mut self.read_queue) { Ok(msg) => self.service.received_message(*id, msg), Err(err) if err.is_eof() => { // Buffer is empty, or message isn't complete. break; } Err(err) => { // TODO(cloudhead): Include error in reason. log::error!(target: "wire", "Invalid message from {}: {err}", id); self.disconnect( fd, DisconnectReason::Session(session::Error::Misbehavior), ); break; } } } } else { log::warn!(target: "wire", "Dropping message from unconnected peer with fd {fd}"); } } SessionEvent::Terminated(err) => { log::debug!(target: "wire", "Session for fd {fd} terminated: {err}"); self.disconnect(fd, DisconnectReason::Connection(Arc::new(err))); } } } fn handle_command(&mut self, cmd: Self::Command) { match cmd { Control::User(cmd) => self.service.command(cmd), Control::Worker(resp) => self.worker_result(resp), } } fn handle_error( &mut self, err: reactor::Error>, NetTransport>>, ) { match &err { reactor::Error::ListenerUnknown(id) => { // TODO: What are we supposed to do here? Remove this error. log::error!(target: "wire", "Received error: unknown listener {}", id); } reactor::Error::TransportUnknown(id) => { // TODO: What are we supposed to do here? Remove this error. log::error!(target: "wire", "Received error: unknown peer {}", id); } reactor::Error::Poll(err) => { // TODO: This should be a fatal error, there's nothing we can do here. log::error!(target: "wire", "Can't poll connections: {}", err); } reactor::Error::ListenerPollError(id, err) => { // TODO: This should be a fatal error, there's nothing we can do here. log::error!(target: "wire", "Received error: listener {} disconnected: {}", id, err); self.actions.push_back(Action::UnregisterListener(*id)); } reactor::Error::ListenerDisconnect(id, _, err) => { // TODO: This should be a fatal error, there's nothing we can do here. log::error!(target: "wire", "Received error: listener {} disconnected: {}", id, err); } reactor::Error::TransportPollError(id, err) => { log::error!(target: "wire", "Received error: peer {} disconnected: {}", id, err); self.actions.push_back(Action::UnregisterTransport(*id)); } reactor::Error::TransportDisconnect(id, _, err) => { log::error!(target: "wire", "Received error: peer {} disconnected: {}", id, err); } reactor::Error::WriteFailure(id, err) => { // TODO: Disconnect peer? log::error!(target: "wire", "Error during writing to peer {id}: {err}") } reactor::Error::WriteLogicError(id, _) => { // TODO: We shouldn't be receiving this error. There's nothing we can do. log::error!(target: "wire", "Write logic error for peer {id}: {err}") } } } fn handover_listener(&mut self, _listener: Self::Listener) { panic!("Transport::handover_listener: listener handover is not supported"); } fn handover_transport(&mut self, transport: Self::Transport) { let fd = transport.as_raw_fd(); match self.peers.get(&fd) { Some(Peer::Disconnected { id, reason }) => { // Disconnect TCP stream. drop(transport); if let Some(id) = id { self.service.disconnected(*id, reason); } else { // TODO: Handle this case by calling `disconnected` with the address instead of // the node id. } } Some(Peer::Upgrading { .. }) => { log::debug!(target: "wire", "Received handover of transport with fd {fd}"); self.upgraded(transport); } Some(_) => { panic!("Transport::handover_transport: Unexpected peer with fd {fd} handed over from the reactor"); } None => { panic!("Transport::handover_transport: Unknown peer with fd {fd} handed over"); } } } } impl Iterator for Wire where R: routing::Store, S: address::Store, W: WriteStorage + 'static, G: Signer + EcSign, { type Item = Action; fn next(&mut self) -> Option { while let Some(ev) = self.service.next() { match ev { Io::Write(node_id, msgs) => { log::trace!( target: "wire", "Writing {} message(s) to {}", msgs.len(), node_id ); let fd = self.connected_fd_by_id(&node_id); let mut data = Vec::new(); for msg in msgs { msg.encode(&mut data).expect("in-memory writes never fail"); } self.actions.push_back(reactor::Action::Send(fd, data)); } Io::Event(_e) => { log::warn!( target: "wire", "Events are not currently supported" ); } Io::Connect(node_id, addr) => { if self.connected().any(|(_, id)| id == &node_id) { log::error!( target: "wire", "Attempt to connect to already connected peer {node_id}" ); break; } match WireSession::connect_nonblocking::<{ Sha256::OUTPUT_LEN }>( addr.to_inner(), self.cert, vec![node_id], self.signer.clone(), self.proxy.into(), false, ) .and_then(|session| { NetTransport::>::with_session(session, Link::Outbound) }) { Ok(transport) => { self.service.attempted(node_id, &addr); // TODO: Keep track of peer address for when peer disconnects before // handshake is complete. self.peers .insert(transport.as_raw_fd(), Peer::connecting(Link::Outbound)); self.actions .push_back(reactor::Action::RegisterTransport(transport)); } Err(err) => { self.service .disconnected(node_id, &DisconnectReason::Dial(Arc::new(err))); break; } } } Io::Disconnect(node_id, reason) => { let fd = self.connected_fd_by_id(&node_id); self.disconnect(fd, reason); } Io::Wakeup(d) => { self.actions.push_back(reactor::Action::SetTimer(d.into())); } Io::Fetch(fetch) => { // TODO: Check that the node_id is connected, queue request otherwise. let fd = self.connected_fd_by_id(&fetch.remote); self.upgrade(fd, fetch); } } } self.actions.pop_front() } }