feat: integrate iroh-gossip for mesh networking, using ALPN for protocol routing and node events to manage gossip topics.
This commit is contained in:
+35
-8
@@ -146,26 +146,53 @@
|
|||||||
|
|
||||||
### Deliverables
|
### Deliverables
|
||||||
|
|
||||||
**Phase 1: LatticeServer Refactor**
|
**Phase 1: LatticeServer Refactor** ✓
|
||||||
- [x] `LatticeServer` struct in `lattice-net` wrapping `Arc<Node>` + `Endpoint`
|
- [x] `LatticeServer` struct in `lattice-net` wrapping `Arc<Node>` + `Endpoint`
|
||||||
- [x] Move `join_mesh`, `sync_with_peer`, `sync_all` to `LatticeServer` methods
|
- [x] Move `join_mesh`, `sync_with_peer`, `sync_all` to `LatticeServer` methods
|
||||||
- [x] Encapsulate `spawn_accept_loop` inside `LatticeServer`
|
- [x] Encapsulate accept loop inside `LatticeServer` (via Router + ProtocolHandler)
|
||||||
- [x] CLI uses `LatticeServer` instead of raw `Node` + `Endpoint`
|
- [x] CLI uses `LatticeServer` instead of raw `Node` + `Endpoint`
|
||||||
- [ ] Route sync command through `LatticeServer` (not raw functions)
|
|
||||||
- [ ] Integration test: invite → join → sync end-to-end
|
- [ ] Integration test: invite → join → sync end-to-end
|
||||||
- [ ] Periodic background sync with known peers
|
- [ ] Periodic background sync with known peers
|
||||||
- [ ] Track last sync time per peer
|
- [ ] Track last sync time per peer
|
||||||
|
|
||||||
**Phase 2: Gossip Protocol**
|
**Phase 2: Gossip Protocol** ✓ (iroh-gossip)
|
||||||
- [ ] Proto: `GossipAnnounce` message with author + latest seq + HLC
|
- [x] Router handles both `lattice-sync/1` and `/iroh-gossip/1` ALPNs
|
||||||
- [ ] `LatticeServer::spawn_gossip_loop` for periodic announcements
|
- [x] `NodeEvent::RootStoreActivated` emitted when root store opens
|
||||||
- [ ] On receiving announce: detect missing entries, trigger sync
|
- [x] Auto-join gossip topic on root store activation
|
||||||
- [ ] Track last-seen per peer for staleness detection
|
- [x] Broadcast local entries to gossip topic on commit
|
||||||
|
- [x] Receive gossip entries and apply to store
|
||||||
|
- [x] Topic ID via `blake3::hash("lattice/{store_id}")`
|
||||||
|
- [ ] Gossip bootstrap peers from `/peers/` (needs Prefix Watch)
|
||||||
|
|
||||||
|
**Next: Prefix Watch (reactive store updates)**
|
||||||
|
- [ ] `store.watch_prefix(prefix) -> Receiver<WatchEvent>`
|
||||||
|
- [ ] `WatchEvent::Put { key, value }` / `WatchEvent::Delete { key }`
|
||||||
|
- [ ] StoreActor tracks watchers per prefix, emits on matching put/delete
|
||||||
|
- [ ] LatticeServer uses `/peers/` watch to update gossip bootstrap peers dynamically
|
||||||
|
- [ ] Enables reactive patterns: config changes, presence, app-level subscriptions
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Technical Debt
|
||||||
|
|
||||||
|
**Logging**
|
||||||
|
- [ ] Replace `println!`/`eprintln!` with `tracing` crate (`tracing::info!`, `tracing::error!`)
|
||||||
|
- Standard in Rust async ecosystem, used by Iroh internally
|
||||||
|
|
||||||
|
**Lifecycle Management (Zombie Tasks)**
|
||||||
|
- [ ] Spawned infinite loops (`spawn_node_event_listener`, `spawn_entry_forward_loop`, gossip receive loop) keep running if `LatticeServer` is dropped
|
||||||
|
- [ ] Use `tokio_util::sync::CancellationToken` or keep `JoinHandle`s for graceful shutdown
|
||||||
|
|
||||||
|
**Error Handling**
|
||||||
|
- [ ] Replace `Result<..., String>` with `anyhow::Result` or define `LatticeNetError` enum
|
||||||
|
- String errors make it hard to handle specific failure cases
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Future
|
## Future
|
||||||
|
|
||||||
|
- offline nodes should not delay sync
|
||||||
|
- sync command should transitive sync all peers
|
||||||
- Gossip:
|
- Gossip:
|
||||||
- gossip new entries to peers
|
- gossip new entries to peers
|
||||||
- backfill missing entries from peers (how do peers notice missing entries?)
|
- backfill missing entries from peers (how do peers notice missing entries?)
|
||||||
|
|||||||
@@ -51,6 +51,7 @@ async fn main() {
|
|||||||
} else {
|
} else {
|
||||||
println!("Root: {}", open_info.store_id);
|
println!("Root: {}", open_info.store_id);
|
||||||
}
|
}
|
||||||
|
|
||||||
node.root_store().await.as_ref().cloned()
|
node.root_store().await.as_ref().cloned()
|
||||||
}
|
}
|
||||||
Ok(None) => {
|
Ok(None) => {
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ pub mod store_actor;
|
|||||||
pub const MAX_ENTRY_SIZE: usize = 16 * 1024 * 1024;
|
pub const MAX_ENTRY_SIZE: usize = 16 * 1024 * 1024;
|
||||||
|
|
||||||
pub use node_identity::{NodeIdentity, PeerStatus};
|
pub use node_identity::{NodeIdentity, PeerStatus};
|
||||||
pub use node::{Node, NodeBuilder, NodeInfo, StoreInfo, StoreHandle, NodeError, PeerInfo, JoinAcceptance};
|
pub use node::{Node, NodeBuilder, NodeInfo, StoreInfo, StoreHandle, NodeError, NodeEvent, PeerInfo, JoinAcceptance};
|
||||||
pub use sigchain::{SigChain, SigChainManager};
|
pub use sigchain::{SigChain, SigChainManager};
|
||||||
pub use entry::Entry;
|
pub use entry::Entry;
|
||||||
pub use sync_state::{SyncState, AuthorInfo, MissingRange};
|
pub use sync_state::{SyncState, AuthorInfo, MissingRange};
|
||||||
|
|||||||
@@ -8,9 +8,11 @@ use crate::{
|
|||||||
store::StoreError,
|
store::StoreError,
|
||||||
spawn_store_actor, StoreCmd,
|
spawn_store_actor, StoreCmd,
|
||||||
node_identity::NodeError as IdentityError,
|
node_identity::NodeError as IdentityError,
|
||||||
|
proto::SignedEntry,
|
||||||
};
|
};
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use thiserror::Error;
|
use thiserror::Error;
|
||||||
|
use tokio::sync::broadcast;
|
||||||
|
|
||||||
#[derive(Error, Debug)]
|
#[derive(Error, Debug)]
|
||||||
pub enum NodeError {
|
pub enum NodeError {
|
||||||
@@ -67,6 +69,13 @@ pub struct PeerInfo {
|
|||||||
pub status: PeerStatus,
|
pub status: PeerStatus,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Events emitted by Node for interested listeners (e.g., LatticeServer)
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
pub enum NodeEvent {
|
||||||
|
/// Root store was activated (opened or set)
|
||||||
|
RootStoreActivated(StoreHandle),
|
||||||
|
}
|
||||||
|
|
||||||
pub struct NodeBuilder {
|
pub struct NodeBuilder {
|
||||||
pub data_dir: DataDir,
|
pub data_dir: DataDir,
|
||||||
}
|
}
|
||||||
@@ -90,7 +99,6 @@ impl NodeBuilder {
|
|||||||
};
|
};
|
||||||
|
|
||||||
let meta = MetaStore::open(self.data_dir.meta_db())?;
|
let meta = MetaStore::open(self.data_dir.meta_db())?;
|
||||||
|
|
||||||
// Set hostname on first creation
|
// Set hostname on first creation
|
||||||
if is_new {
|
if is_new {
|
||||||
let hostname = hostname::get()
|
let hostname = hostname::get()
|
||||||
@@ -99,11 +107,15 @@ impl NodeBuilder {
|
|||||||
let _ = meta.set_name(&hostname);
|
let _ = meta.set_name(&hostname);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Create event channel
|
||||||
|
let (event_tx, _) = broadcast::channel(16);
|
||||||
|
|
||||||
Ok(Node {
|
Ok(Node {
|
||||||
data_dir: self.data_dir,
|
data_dir: self.data_dir,
|
||||||
node: std::sync::Arc::new(node),
|
node: std::sync::Arc::new(node),
|
||||||
meta,
|
meta,
|
||||||
root_store: tokio::sync::RwLock::new(None),
|
root_store: tokio::sync::RwLock::new(None),
|
||||||
|
event_tx,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -118,6 +130,7 @@ pub struct Node {
|
|||||||
node: std::sync::Arc<NodeIdentity>,
|
node: std::sync::Arc<NodeIdentity>,
|
||||||
meta: MetaStore,
|
meta: MetaStore,
|
||||||
root_store: tokio::sync::RwLock<Option<StoreHandle>>,
|
root_store: tokio::sync::RwLock<Option<StoreHandle>>,
|
||||||
|
event_tx: broadcast::Sender<NodeEvent>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Node {
|
impl Node {
|
||||||
@@ -133,6 +146,11 @@ impl Node {
|
|||||||
self.node.public_key_bytes()
|
self.node.public_key_bytes()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Subscribe to node events (e.g., root store activation)
|
||||||
|
pub fn subscribe_events(&self) -> broadcast::Receiver<NodeEvent> {
|
||||||
|
self.event_tx.subscribe()
|
||||||
|
}
|
||||||
|
|
||||||
/// Get the secret key bytes for Iroh integration (same Ed25519 key)
|
/// Get the secret key bytes for Iroh integration (same Ed25519 key)
|
||||||
pub fn secret_key_bytes(&self) -> [u8; 32] {
|
pub fn secret_key_bytes(&self) -> [u8; 32] {
|
||||||
self.node.secret_key_bytes()
|
self.node.secret_key_bytes()
|
||||||
@@ -184,7 +202,13 @@ impl Node {
|
|||||||
match self.meta.root_store()? {
|
match self.meta.root_store()? {
|
||||||
Some(id) => {
|
Some(id) => {
|
||||||
let (handle, info) = self.open_store(id).await?;
|
let (handle, info) = self.open_store(id).await?;
|
||||||
|
|
||||||
|
// Emit event for listeners (send clone, keep original)
|
||||||
|
let _ = self.event_tx.send(NodeEvent::RootStoreActivated(handle.clone()));
|
||||||
|
|
||||||
|
// Store original handle (owns actor thread)
|
||||||
*self.root_store.write().await = Some(handle);
|
*self.root_store.write().await = Some(handle);
|
||||||
|
|
||||||
Ok(Some(info))
|
Ok(Some(info))
|
||||||
}
|
}
|
||||||
None => Ok(None),
|
None => Ok(None),
|
||||||
@@ -234,14 +258,18 @@ impl Node {
|
|||||||
self.create_store_with_uuid(store_id)?;
|
self.create_store_with_uuid(store_id)?;
|
||||||
self.meta.set_root_store(store_id)?;
|
self.meta.set_root_store(store_id)?;
|
||||||
|
|
||||||
// Open and cache the handle
|
// Open and cache the handle (original stays in cache)
|
||||||
let (handle, _) = self.open_store(store_id).await?;
|
let (handle, _) = self.open_store(store_id).await?;
|
||||||
*self.root_store.write().await = Some(handle.clone());
|
let handle_clone = handle.clone();
|
||||||
|
*self.root_store.write().await = Some(handle);
|
||||||
|
|
||||||
|
// Emit event for listeners
|
||||||
|
let _ = self.event_tx.send(NodeEvent::RootStoreActivated(handle_clone.clone()));
|
||||||
|
|
||||||
// Publish our name to the store
|
// Publish our name to the store
|
||||||
let _ = self.publish_name().await;
|
let _ = self.publish_name().await;
|
||||||
|
|
||||||
Ok(handle)
|
Ok(handle_clone)
|
||||||
}
|
}
|
||||||
|
|
||||||
// --- Peer Management ---
|
// --- Peer Management ---
|
||||||
@@ -471,17 +499,19 @@ impl Node {
|
|||||||
let info = StoreInfo { store_id, entries_replayed };
|
let info = StoreInfo { store_id, entries_replayed };
|
||||||
|
|
||||||
// Spawn actor thread - actor owns store, sigchain, and node copy
|
// Spawn actor thread - actor owns store, sigchain, and node copy
|
||||||
let (tx, actor_handle) = spawn_store_actor(
|
let (tx, entry_tx, actor_handle) = spawn_store_actor(
|
||||||
store_id,
|
store_id,
|
||||||
store,
|
store,
|
||||||
sigchain,
|
sigchain,
|
||||||
(*self.node).clone(),
|
(*self.node).clone(),
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// Store the entry sender for gossip
|
||||||
let handle = StoreHandle {
|
let handle = StoreHandle {
|
||||||
store_id,
|
store_id,
|
||||||
tx,
|
tx,
|
||||||
actor_handle: Some(actor_handle),
|
actor_handle: Some(actor_handle),
|
||||||
|
entry_tx,
|
||||||
};
|
};
|
||||||
|
|
||||||
Ok((handle, info))
|
Ok((handle, info))
|
||||||
@@ -489,10 +519,12 @@ impl Node {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// A handle to a specific store - wraps channel to actor thread
|
/// A handle to a specific store - wraps channel to actor thread
|
||||||
|
#[derive(Debug)]
|
||||||
pub struct StoreHandle {
|
pub struct StoreHandle {
|
||||||
store_id: Uuid,
|
store_id: Uuid,
|
||||||
tx: tokio::sync::mpsc::Sender<StoreCmd>,
|
tx: tokio::sync::mpsc::Sender<StoreCmd>,
|
||||||
actor_handle: Option<std::thread::JoinHandle<()>>,
|
actor_handle: Option<std::thread::JoinHandle<()>>,
|
||||||
|
entry_tx: broadcast::Sender<SignedEntry>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Clone for StoreHandle {
|
impl Clone for StoreHandle {
|
||||||
@@ -501,6 +533,7 @@ impl Clone for StoreHandle {
|
|||||||
store_id: self.store_id,
|
store_id: self.store_id,
|
||||||
tx: self.tx.clone(),
|
tx: self.tx.clone(),
|
||||||
actor_handle: None, // Clones don't own the actor thread
|
actor_handle: None, // Clones don't own the actor thread
|
||||||
|
entry_tx: self.entry_tx.clone(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -508,6 +541,11 @@ impl Clone for StoreHandle {
|
|||||||
impl StoreHandle {
|
impl StoreHandle {
|
||||||
pub fn id(&self) -> Uuid { self.store_id }
|
pub fn id(&self) -> Uuid { self.store_id }
|
||||||
|
|
||||||
|
/// Subscribe to receive entries as they're committed locally
|
||||||
|
pub fn subscribe_entries(&self) -> broadcast::Receiver<SignedEntry> {
|
||||||
|
self.entry_tx.subscribe()
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn get(&self, key: &[u8]) -> Result<Option<Vec<u8>>, NodeError> {
|
pub async fn get(&self, key: &[u8]) -> Result<Option<Vec<u8>>, NodeError> {
|
||||||
use StoreCmd;
|
use StoreCmd;
|
||||||
let (resp_tx, resp_rx) = tokio::sync::oneshot::channel();
|
let (resp_tx, resp_rx) = tokio::sync::oneshot::channel();
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ use crate::{
|
|||||||
proto::SignedEntry,
|
proto::SignedEntry,
|
||||||
log,
|
log,
|
||||||
};
|
};
|
||||||
use tokio::sync::{mpsc, oneshot};
|
use tokio::sync::{mpsc, oneshot, broadcast};
|
||||||
use std::thread::{self, JoinHandle};
|
use std::thread::{self, JoinHandle};
|
||||||
|
|
||||||
/// Commands sent to the store actor
|
/// Commands sent to the store actor
|
||||||
@@ -103,9 +103,11 @@ impl std::error::Error for StoreActorError {}
|
|||||||
pub struct StoreActor {
|
pub struct StoreActor {
|
||||||
store_id: Uuid,
|
store_id: Uuid,
|
||||||
store: Store,
|
store: Store,
|
||||||
chain_manager: SigChainManager, // Manages all authors' sigchains
|
chain_manager: SigChainManager,
|
||||||
node: NodeIdentity,
|
node: NodeIdentity,
|
||||||
rx: mpsc::Receiver<StoreCmd>,
|
rx: mpsc::Receiver<StoreCmd>,
|
||||||
|
/// Broadcast sender for emitting entries after they're committed locally
|
||||||
|
entry_tx: broadcast::Sender<SignedEntry>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl StoreActor {
|
impl StoreActor {
|
||||||
@@ -116,6 +118,7 @@ impl StoreActor {
|
|||||||
sigchain: SigChain,
|
sigchain: SigChain,
|
||||||
node: NodeIdentity,
|
node: NodeIdentity,
|
||||||
rx: mpsc::Receiver<StoreCmd>,
|
rx: mpsc::Receiver<StoreCmd>,
|
||||||
|
entry_tx: broadcast::Sender<SignedEntry>,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
// Derive logs_dir from sigchain's log file path
|
// Derive logs_dir from sigchain's log file path
|
||||||
let logs_dir = sigchain.log_path()
|
let logs_dir = sigchain.log_path()
|
||||||
@@ -134,6 +137,7 @@ impl StoreActor {
|
|||||||
chain_manager,
|
chain_manager,
|
||||||
node,
|
node,
|
||||||
rx,
|
rx,
|
||||||
|
entry_tx,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -254,6 +258,9 @@ impl StoreActor {
|
|||||||
sigchain.append(&entry)?;
|
sigchain.append(&entry)?;
|
||||||
self.store.apply_entry(&entry)?;
|
self.store.apply_entry(&entry)?;
|
||||||
|
|
||||||
|
// Broadcast the entry to listeners (for gossip)
|
||||||
|
let _ = self.entry_tx.send(entry.clone());
|
||||||
|
|
||||||
Ok(seq)
|
Ok(seq)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -276,16 +283,17 @@ impl StoreActor {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Spawn a store actor in a new thread, returns (sender, join_handle)
|
/// Spawn a store actor in a new thread, returns (cmd_tx, entry_tx, join_handle)
|
||||||
/// Uses std::thread since redb is blocking
|
/// Uses std::thread since redb is blocking
|
||||||
pub fn spawn_store_actor(
|
pub fn spawn_store_actor(
|
||||||
store_id: Uuid,
|
store_id: Uuid,
|
||||||
store: Store,
|
store: Store,
|
||||||
sigchain: SigChain,
|
sigchain: SigChain,
|
||||||
node: NodeIdentity,
|
node: NodeIdentity,
|
||||||
) -> (mpsc::Sender<StoreCmd>, JoinHandle<()>) {
|
) -> (mpsc::Sender<StoreCmd>, broadcast::Sender<SignedEntry>, JoinHandle<()>) {
|
||||||
let (tx, rx) = mpsc::channel(32);
|
let (tx, rx) = mpsc::channel(32);
|
||||||
let actor = StoreActor::new(store_id, store, sigchain, node, rx);
|
let (entry_tx, _entry_rx) = broadcast::channel(64);
|
||||||
|
let actor = StoreActor::new(store_id, store, sigchain, node, rx, entry_tx.clone());
|
||||||
let handle = thread::spawn(move || actor.run());
|
let handle = thread::spawn(move || actor.run());
|
||||||
(tx, handle)
|
(tx, entry_tx, handle)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -17,6 +17,9 @@ bytes = { workspace = true }
|
|||||||
tokio-util = { workspace = true }
|
tokio-util = { workspace = true }
|
||||||
futures-util = { workspace = true }
|
futures-util = { workspace = true }
|
||||||
hex = { workspace = true }
|
hex = { workspace = true }
|
||||||
|
blake3.workspace = true
|
||||||
|
anyhow = "1.0.100"
|
||||||
|
futures-lite = "2.6.1"
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
tokio-test = { workspace = true }
|
tokio-test = { workspace = true }
|
||||||
|
|||||||
@@ -28,7 +28,10 @@ impl LatticeEndpoint {
|
|||||||
|
|
||||||
let endpoint = Endpoint::builder()
|
let endpoint = Endpoint::builder()
|
||||||
.secret_key(secret_key)
|
.secret_key(secret_key)
|
||||||
.alpns(vec![LATTICE_ALPN.to_vec()])
|
.alpns(vec![
|
||||||
|
LATTICE_ALPN.to_vec(),
|
||||||
|
iroh_gossip::ALPN.to_vec(), // Also accept gossip protocol
|
||||||
|
])
|
||||||
.discovery(mdns) // Add mDNS on top of default DNS
|
.discovery(mdns) // Add mDNS on top of default DNS
|
||||||
.bind()
|
.bind()
|
||||||
.await?;
|
.await?;
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ pub mod gossip;
|
|||||||
pub mod framing;
|
pub mod framing;
|
||||||
pub mod mesh;
|
pub mod mesh;
|
||||||
|
|
||||||
pub use endpoint::{LatticeEndpoint, PublicKey};
|
pub use endpoint::{LatticeEndpoint, PublicKey, LATTICE_ALPN};
|
||||||
pub use framing::{MessageSink, MessageStream};
|
pub use framing::{MessageSink, MessageStream};
|
||||||
pub use lattice_core::proto::{SyncRequest, SyncResponse, SyncEntry, SyncDone, SyncState, Frontier};
|
pub use lattice_core::proto::{SyncRequest, SyncResponse, SyncEntry, SyncDone, SyncState, Frontier};
|
||||||
pub use mesh::{LatticeServer, SyncResult};
|
pub use mesh::{LatticeServer, SyncResult};
|
||||||
|
|||||||
+286
-24
@@ -1,10 +1,16 @@
|
|||||||
//! Server - LatticeServer for mesh networking
|
//! Server - LatticeServer for mesh networking
|
||||||
|
|
||||||
use crate::{MessageSink, MessageStream, LatticeEndpoint, parse_node_id};
|
use crate::{MessageSink, MessageStream, LatticeEndpoint, parse_node_id, LATTICE_ALPN};
|
||||||
use lattice_core::{Node, NodeError, PeerStatus, Uuid, StoreHandle};
|
use lattice_core::{Node, NodeError, NodeEvent, PeerStatus, Uuid, StoreHandle};
|
||||||
use iroh::endpoint::Connection;
|
use iroh::endpoint::Connection;
|
||||||
|
use iroh::protocol::{Router, ProtocolHandler, AcceptError};
|
||||||
|
use iroh_gossip::Gossip;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use lattice_core::proto::{PeerMessage, peer_message, JoinRequest, JoinResponse};
|
use std::collections::HashMap;
|
||||||
|
use tokio::sync::RwLock;
|
||||||
|
use futures_util::StreamExt;
|
||||||
|
use lattice_core::proto::{PeerMessage, peer_message, JoinRequest, JoinResponse, SignedEntry};
|
||||||
|
use prost::Message;
|
||||||
use super::protocol;
|
use super::protocol;
|
||||||
|
|
||||||
/// Result of a sync operation with a peer
|
/// Result of a sync operation with a peer
|
||||||
@@ -13,11 +19,41 @@ pub struct SyncResult {
|
|||||||
pub entries_sent_by_peer: u64,
|
pub entries_sent_by_peer: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// LatticeServer wraps Node + Endpoint and provides mesh networking methods.
|
/// LatticeServer wraps Node + Endpoint + Gossip and provides mesh networking methods.
|
||||||
/// Spawns accept loop on creation to handle incoming connections.
|
/// Uses Router to handle incoming connections for both sync and gossip protocols.
|
||||||
pub struct LatticeServer {
|
pub struct LatticeServer {
|
||||||
node: Arc<Node>,
|
node: Arc<Node>,
|
||||||
endpoint: LatticeEndpoint,
|
endpoint: LatticeEndpoint,
|
||||||
|
gossip: Gossip,
|
||||||
|
#[allow(dead_code)]
|
||||||
|
router: Router,
|
||||||
|
/// Gossip senders per store topic
|
||||||
|
gossip_senders: Arc<RwLock<HashMap<Uuid, iroh_gossip::api::GossipSender>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Protocol handler for lattice sync connections
|
||||||
|
struct SyncProtocol {
|
||||||
|
node: Arc<Node>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Debug for SyncProtocol {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
f.debug_struct("SyncProtocol").finish()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
impl ProtocolHandler for SyncProtocol {
|
||||||
|
fn accept(&self, conn: Connection) -> impl std::future::Future<Output = Result<(), AcceptError>> + Send {
|
||||||
|
let node = self.node.clone();
|
||||||
|
Box::pin(async move {
|
||||||
|
if let Err(e) = handle_connection(node, conn).await {
|
||||||
|
eprintln!("[Accept] Error: {}", e);
|
||||||
|
// Log error but return Ok - protocol handled the connection
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl LatticeServer {
|
impl LatticeServer {
|
||||||
@@ -25,14 +61,175 @@ impl LatticeServer {
|
|||||||
pub async fn new_from_node(node: Arc<Node>) -> Result<Self, String> {
|
pub async fn new_from_node(node: Arc<Node>) -> Result<Self, String> {
|
||||||
let endpoint = LatticeEndpoint::new(node.secret_key_bytes()).await
|
let endpoint = LatticeEndpoint::new(node.secret_key_bytes()).await
|
||||||
.map_err(|e| format!("Failed to create endpoint: {}", e))?;
|
.map_err(|e| format!("Failed to create endpoint: {}", e))?;
|
||||||
Ok(Self::new(node, endpoint))
|
Self::new(node, endpoint).await
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Create a new LatticeServer with existing endpoint and spawn the accept loop.
|
/// Create a new LatticeServer with existing endpoint.
|
||||||
pub fn new(node: Arc<Node>, endpoint: LatticeEndpoint) -> Self {
|
pub async fn new(node: Arc<Node>, endpoint: LatticeEndpoint) -> Result<Self, String> {
|
||||||
let server = Self { node, endpoint };
|
// Create gossip instance
|
||||||
server.spawn_accept_loop();
|
let gossip = Gossip::builder().spawn(endpoint.endpoint().clone());
|
||||||
server
|
|
||||||
|
// Create sync protocol handler
|
||||||
|
let sync_protocol = SyncProtocol { node: node.clone() };
|
||||||
|
|
||||||
|
// Create router to handle both protocols
|
||||||
|
let router = Router::builder(endpoint.endpoint().clone())
|
||||||
|
.accept(LATTICE_ALPN, sync_protocol)
|
||||||
|
.accept(iroh_gossip::ALPN, gossip.clone())
|
||||||
|
.spawn();
|
||||||
|
|
||||||
|
let server = Self {
|
||||||
|
node,
|
||||||
|
endpoint,
|
||||||
|
gossip,
|
||||||
|
router,
|
||||||
|
gossip_senders: Arc::new(RwLock::new(HashMap::new())),
|
||||||
|
};
|
||||||
|
server.spawn_node_event_listener();
|
||||||
|
|
||||||
|
// If root store is already open, start gossip for it
|
||||||
|
if let Some(store) = (*server.node.root_store().await).clone() {
|
||||||
|
println!("[Gossip] Root store already open, starting gossip...");
|
||||||
|
server.join_gossip_topic(store.id()).await?;
|
||||||
|
server.spawn_entry_forward_loop(store);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(server)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Spawn a listener for Node events (auto-starts gossip when root store is activated)
|
||||||
|
fn spawn_node_event_listener(&self) {
|
||||||
|
let mut event_rx = self.node.subscribe_events();
|
||||||
|
let gossip_senders = self.gossip_senders.clone();
|
||||||
|
let gossip = self.gossip.clone();
|
||||||
|
let node = self.node.clone();
|
||||||
|
|
||||||
|
tokio::spawn(async move {
|
||||||
|
while let Ok(event) = event_rx.recv().await {
|
||||||
|
match event {
|
||||||
|
NodeEvent::RootStoreActivated(store) => {
|
||||||
|
println!("[Gossip] Root store activated: {}, starting gossip...", store.id());
|
||||||
|
|
||||||
|
let store_id = store.id();
|
||||||
|
|
||||||
|
// Get bootstrap peers from node's peer list
|
||||||
|
let bootstrap_peers: Vec<iroh::PublicKey> = match node.list_peers().await {
|
||||||
|
Ok(peers) => {
|
||||||
|
peers.iter()
|
||||||
|
.filter(|p| p.status == PeerStatus::Active)
|
||||||
|
.filter_map(|p| parse_node_id(&p.pubkey).ok())
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
eprintln!("[Gossip] Failed to list peers: {}, using empty list", e);
|
||||||
|
Vec::new()
|
||||||
|
}
|
||||||
|
};
|
||||||
|
println!("[Gossip] Bootstrap peers: {}", bootstrap_peers.len());
|
||||||
|
|
||||||
|
// Topic ID from hash of "lattice/{store_id}" for namespacing
|
||||||
|
let topic_bytes = blake3::hash(format!("lattice/{}", store_id).as_bytes());
|
||||||
|
let topic_id = iroh_gossip::TopicId::from_bytes(*topic_bytes.as_bytes());
|
||||||
|
|
||||||
|
// Use subscribe (non-blocking) - peers will connect when they sync
|
||||||
|
// subscribe_and_join would block waiting for peers we can't reach yet
|
||||||
|
match gossip.subscribe(topic_id, bootstrap_peers).await {
|
||||||
|
Ok(sub) => {
|
||||||
|
let (sender, receiver) = sub.split();
|
||||||
|
|
||||||
|
// Store sender
|
||||||
|
gossip_senders.write().await.insert(store_id, sender);
|
||||||
|
|
||||||
|
// Spawn receive loop
|
||||||
|
let store_recv = store.clone();
|
||||||
|
tokio::spawn(async move {
|
||||||
|
let mut receiver = receiver;
|
||||||
|
println!("[Gossip] Receive loop started for topic {:?}", topic_id);
|
||||||
|
|
||||||
|
while let Some(event) = futures_util::StreamExt::next(&mut receiver).await {
|
||||||
|
match event {
|
||||||
|
Ok(iroh_gossip::api::Event::Received(msg)) => {
|
||||||
|
println!("[Gossip] Received {} bytes", msg.content.len());
|
||||||
|
if let Ok(entry) = SignedEntry::decode(&msg.content[..]) {
|
||||||
|
if let Err(e) = store_recv.apply_entry(entry).await {
|
||||||
|
eprintln!("[Gossip] Failed to apply entry: {}", e);
|
||||||
|
} else {
|
||||||
|
println!("[Gossip] Applied entry successfully");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(other) => {
|
||||||
|
println!("[Gossip] Event: {:?}", other);
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
eprintln!("[Gossip] Error: {}", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Spawn entry forward loop
|
||||||
|
let gossip_senders = gossip_senders.clone();
|
||||||
|
let mut entry_rx = store.subscribe_entries();
|
||||||
|
tokio::spawn(async move {
|
||||||
|
println!("[Gossip] Entry forward loop started for store {}", store_id);
|
||||||
|
while let Ok(entry) = entry_rx.recv().await {
|
||||||
|
let senders = gossip_senders.read().await;
|
||||||
|
if let Some(sender) = senders.get(&store_id) {
|
||||||
|
let bytes = entry.encode_to_vec();
|
||||||
|
println!("[Gossip] Broadcasting {} bytes", bytes.len());
|
||||||
|
let _ = sender.broadcast(bytes.into()).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
println!("[Gossip] Gossip started for store {}", store_id);
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
eprintln!("[Gossip] Failed to subscribe to topic: {}", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Start gossip for a store (call after store is opened)
|
||||||
|
pub async fn start_gossip_for_store(&self, store: StoreHandle) -> Result<(), String> {
|
||||||
|
println!("[Gossip] Starting gossip for store {}", store.id());
|
||||||
|
self.join_gossip_topic(store.id()).await?;
|
||||||
|
self.spawn_entry_forward_loop(store);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Spawn a loop that forwards local store entry broadcasts to gossip
|
||||||
|
fn spawn_entry_forward_loop(&self, store: StoreHandle) {
|
||||||
|
let store_id = store.id();
|
||||||
|
let gossip_senders = self.gossip_senders.clone();
|
||||||
|
let mut entry_rx = store.subscribe_entries();
|
||||||
|
|
||||||
|
println!("[Gossip] Starting entry forward loop for store {}", store_id);
|
||||||
|
|
||||||
|
tokio::spawn(async move {
|
||||||
|
while let Ok(entry) = entry_rx.recv().await {
|
||||||
|
println!("[Gossip] Received local entry, forwarding to gossip...");
|
||||||
|
// Forward to gossip sender
|
||||||
|
let senders = gossip_senders.read().await;
|
||||||
|
if let Some(sender) = senders.get(&store_id) {
|
||||||
|
let bytes = entry.encode_to_vec();
|
||||||
|
println!("[Gossip] Broadcasting {} bytes to topic {}", bytes.len(), store_id);
|
||||||
|
if let Err(e) = sender.broadcast(bytes.into()).await {
|
||||||
|
eprintln!("[Gossip] Failed to broadcast entry: {}", e);
|
||||||
|
} else {
|
||||||
|
println!("[Gossip] Broadcast successful");
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
eprintln!("[Gossip] No gossip sender for store {}", store_id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
println!("[Gossip] Entry forward loop ended for store {}", store_id);
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Access the underlying node
|
/// Access the underlying node
|
||||||
@@ -45,27 +242,86 @@ impl LatticeServer {
|
|||||||
&self.endpoint
|
&self.endpoint
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Spawn the accept loop for incoming connections.
|
|
||||||
fn spawn_accept_loop(&self) {
|
/// Join gossip topic for a store (subscribes and spawns receive loop)
|
||||||
|
pub async fn join_gossip_topic(&self, store_id: Uuid) -> Result<(), String> {
|
||||||
|
// Get active peers to bootstrap gossip
|
||||||
|
let peers = self.node.list_peers().await
|
||||||
|
.map_err(|e| format!("Failed to list peers: {}", e))?;
|
||||||
|
|
||||||
|
let bootstrap_peers: Vec<iroh::PublicKey> = peers.iter()
|
||||||
|
.filter(|p| p.status == PeerStatus::Active)
|
||||||
|
.filter_map(|p| parse_node_id(&p.pubkey).ok())
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
println!("[Gossip] Joining topic {} with {} bootstrap peers", store_id, bootstrap_peers.len());
|
||||||
|
|
||||||
|
// Topic ID from store UUID bytes (padded to 32 bytes)
|
||||||
|
// Topic ID from hash of "lattice/{store_id}" for namespacing
|
||||||
|
let topic_bytes = blake3::hash(format!("lattice/{}", store_id).as_bytes());
|
||||||
|
let topic_id = iroh_gossip::TopicId::from_bytes(*topic_bytes.as_bytes());
|
||||||
|
|
||||||
|
// Subscribe to topic
|
||||||
|
let (sender, mut receiver) = self.gossip.subscribe(topic_id, bootstrap_peers).await
|
||||||
|
.map_err(|e| format!("Failed to subscribe to gossip topic: {}", e))?
|
||||||
|
.split();
|
||||||
|
|
||||||
|
// Store sender for broadcasting
|
||||||
|
{
|
||||||
|
let mut senders = self.gossip_senders.write().await;
|
||||||
|
senders.insert(store_id, sender);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Spawn receive loop
|
||||||
let node = self.node.clone();
|
let node = self.node.clone();
|
||||||
let endpoint = self.endpoint.endpoint().clone();
|
let topic = topic_id;
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
loop {
|
// StreamExt imported at module level
|
||||||
if let Some(incoming) = endpoint.accept().await {
|
println!("[Gossip] Receive loop started for topic {:?}", topic);
|
||||||
match incoming.await {
|
|
||||||
Ok(conn) => {
|
while let Some(event) = receiver.next().await {
|
||||||
let node = node.clone();
|
match event {
|
||||||
tokio::spawn(async move {
|
Ok(iroh_gossip::api::Event::Received(message)) => {
|
||||||
if let Err(e) = handle_connection(node, conn).await {
|
println!("[Gossip] Received gossip message: {} bytes", message.content.len());
|
||||||
eprintln!("[Accept] Error: {}", e);
|
// Decode SignedEntry and apply
|
||||||
|
match SignedEntry::decode(&message.content[..]) {
|
||||||
|
Ok(entry) => {
|
||||||
|
println!("[Gossip] Decoded entry, applying...");
|
||||||
|
// Find store and apply entry
|
||||||
|
if let Some(store) = (*node.root_store().await).clone() {
|
||||||
|
if let Err(e) = store.apply_entry(entry.into()).await {
|
||||||
|
eprintln!("[Gossip] Failed to apply entry: {}", e);
|
||||||
|
} else {
|
||||||
|
println!("[Gossip] Entry applied successfully");
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
eprintln!("[Gossip] No root store to apply entry to");
|
||||||
}
|
}
|
||||||
});
|
}
|
||||||
|
Err(e) => eprintln!("[Gossip] Failed to decode entry: {}", e),
|
||||||
}
|
}
|
||||||
Err(e) => eprintln!("[Accept] Handshake error: {:?}", e),
|
|
||||||
}
|
}
|
||||||
|
Ok(other) => {
|
||||||
|
println!("[Gossip] Other event: {:?}", other);
|
||||||
|
}
|
||||||
|
Err(e) => eprintln!("[Gossip] Receive error: {}", e),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
println!("[Gossip] Receive loop ended for topic");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Broadcast an entry to all gossip subscribers for a store
|
||||||
|
pub async fn broadcast_entry(&self, store_id: Uuid, entry: &SignedEntry) -> Result<(), String> {
|
||||||
|
let senders = self.gossip_senders.read().await;
|
||||||
|
if let Some(sender) = senders.get(&store_id) {
|
||||||
|
let bytes = entry.encode_to_vec();
|
||||||
|
sender.broadcast(bytes.into()).await
|
||||||
|
.map_err(|e| format!("Gossip broadcast failed: {}", e))?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Join an existing mesh by connecting to a peer.
|
/// Join an existing mesh by connecting to a peer.
|
||||||
@@ -163,6 +419,7 @@ impl LatticeServer {
|
|||||||
pub async fn sync_all(&self, store: &StoreHandle) -> Result<Vec<SyncResult>, NodeError> {
|
pub async fn sync_all(&self, store: &StoreHandle) -> Result<Vec<SyncResult>, NodeError> {
|
||||||
let peers = self.node.list_peers().await?;
|
let peers = self.node.list_peers().await?;
|
||||||
let mut results = Vec::new();
|
let mut results = Vec::new();
|
||||||
|
let my_pubkey = self.endpoint.public_key();
|
||||||
|
|
||||||
for peer in peers {
|
for peer in peers {
|
||||||
if peer.status != PeerStatus::Active {
|
if peer.status != PeerStatus::Active {
|
||||||
@@ -177,6 +434,11 @@ impl LatticeServer {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// Skip self
|
||||||
|
if peer_id == my_pubkey {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
println!("[Sync] Syncing with {}...", peer_id.fmt_short());
|
println!("[Sync] Syncing with {}...", peer_id.fmt_short());
|
||||||
match self.sync_with_peer(store, peer_id).await {
|
match self.sync_with_peer(store, peer_id).await {
|
||||||
Ok(result) => {
|
Ok(result) => {
|
||||||
|
|||||||
Reference in New Issue
Block a user