//! Store - persistent KV state with DAG-based conflict resolution //! //! Uses redb for efficient embedded storage. //! Tables: //! - kv: Vec → HeadList (multi-head DAG tips per key) //! - meta: String → Vec (system metadata: last_seq, last_hash, etc.) //! - author: [u8; 32] → AuthorState (per-author replay tracking) use crate::log::{read_entries, LogError}; use crate::proto::{operation, AuthorState, Entry, HeadInfo, HeadList, SignedEntry}; use crate::sigchain::SigChainError; use crate::signed_entry::hash_signed_entry; use prost::Message; use redb::{Database, ReadableTable, TableDefinition}; use std::path::Path; use thiserror::Error; // Table definitions const KV_TABLE: TableDefinition<&[u8], &[u8]> = TableDefinition::new("kv"); const AUTHOR_TABLE: TableDefinition<&[u8], &[u8]> = TableDefinition::new("author"); /// Errors that can occur during store operations #[derive(Error, Debug)] pub enum StoreError { #[error("Database error: {0}")] Database(#[from] redb::DatabaseError), #[error("Table error: {0}")] Table(#[from] redb::TableError), #[error("Transaction error: {0}")] Transaction(#[from] redb::TransactionError), #[error("Commit error: {0}")] Commit(#[from] redb::CommitError), #[error("Storage error: {0}")] Storage(#[from] redb::StorageError), #[error("Log error: {0}")] Log(#[from] LogError), #[error("Decode error: {0}")] Decode(#[from] prost::DecodeError), #[error("Sigchain error: {0}")] SigChain(#[from] SigChainError), } /// Persistent store for KV state with DAG conflict resolution pub struct Store { db: Database, } impl Store { /// Open or create a store at the given path pub fn open(path: impl AsRef) -> Result { let db = Database::create(path)?; // Ensure tables exist let write_txn = db.begin_write()?; { let _ = write_txn.open_table(KV_TABLE)?; let _ = write_txn.open_table(AUTHOR_TABLE)?; } write_txn.commit()?; Ok(Self { db }) } /// Replay a log file and apply all entries to the store (batched) /// Returns the number of newly applied entries (skipped entries not counted) pub fn replay_log(&self, log_path: impl AsRef) -> Result { let entries = read_entries(log_path)?; if entries.is_empty() { return Ok(0); } let write_txn = self.db.begin_write()?; let mut applied = 0u64; { let mut kv_table = write_txn.open_table(KV_TABLE)?; let mut author_table = write_txn.open_table(AUTHOR_TABLE)?; for signed_entry in &entries { if Self::apply_ops_to_tables(signed_entry, &mut kv_table, &mut author_table)? { applied += 1; } } } write_txn.commit()?; Ok(applied) } /// Apply a single signed entry to the store pub fn apply_entry(&self, signed_entry: &SignedEntry) -> Result<(), StoreError> { let write_txn = self.db.begin_write()?; { let mut kv_table = write_txn.open_table(KV_TABLE)?; let mut author_table = write_txn.open_table(AUTHOR_TABLE)?; Self::apply_ops_to_tables(signed_entry, &mut kv_table, &mut author_table)?; } write_txn.commit()?; Ok(()) } /// Internal: apply operations from a signed entry to tables /// Returns true if applied, false if skipped (already applied) fn apply_ops_to_tables( signed_entry: &SignedEntry, kv_table: &mut redb::Table<&[u8], &[u8]>, author_table: &mut redb::Table<&[u8], &[u8]>, ) -> Result { let entry = Entry::decode(&signed_entry.entry_bytes[..])?; let entry_hash = hash_signed_entry(signed_entry); let entry_hlc = entry.timestamp.as_ref().map(|t| (t.wall_time << 16) | t.counter as u64).unwrap_or(0); let author: [u8; 32] = signed_entry.author_id.clone().try_into().unwrap_or([0u8; 32]); // Check if entry was already applied (per-author seq check) if let Some(author_state_bytes) = author_table.get(&author[..])? { if let Ok(author_state) = AuthorState::decode(author_state_bytes.value()) { if entry.seq <= author_state.seq { return Ok(false); // Already applied, skip } } } for op in entry.ops { if let Some(op_type) = op.op_type { match op_type { operation::OpType::Put(put) => { let new_head = HeadInfo { value: put.value, hlc: entry_hlc, author: author.to_vec(), hash: entry_hash.to_vec(), tombstone: false, }; Self::apply_head(kv_table, &put.key, new_head, &entry.parent_hashes)?; } operation::OpType::Delete(del) => { let tombstone = HeadInfo { value: vec![], hlc: entry_hlc, author: author.to_vec(), hash: entry_hash.to_vec(), tombstone: true, }; Self::apply_head(kv_table, &del.key, tombstone, &entry.parent_hashes)?; } } } } // Update per-author state let author_state = AuthorState { seq: entry.seq, hash: entry_hash.to_vec(), log_offset: 0, // TODO: track actual log offset }; author_table.insert(&author[..], author_state.encode_to_vec().as_slice())?; Ok(true) } /// Apply a new head to a key, removing ancestor heads (idempotent) fn apply_head( kv_table: &mut redb::Table<&[u8], &[u8]>, key: &[u8], new_head: HeadInfo, parent_hashes: &[Vec], ) -> Result<(), StoreError> { let mut heads = match kv_table.get(key)? { Some(v) => HeadList::decode(v.value()).map(|h| h.heads).unwrap_or_default(), None => Vec::new(), }; // Idempotency: skip if this entry was already applied if heads.iter().any(|h| h.hash == new_head.hash) { return Ok(()); } // Remove any heads that are ancestors (their hash is in parent_hashes) heads.retain(|h| !parent_hashes.iter().any(|p| p == &h.hash)); // Add new head heads.push(new_head); let encoded = HeadList { heads }.encode_to_vec(); kv_table.insert(key, encoded.as_slice())?; Ok(()) } /// Get a value by key (returns deterministic winner from heads, None if tombstone) pub fn get(&self, key: &[u8]) -> Result>, StoreError> { let read_txn = self.db.begin_read()?; let table = read_txn.open_table(KV_TABLE)?; match table.get(key)? { Some(v) => { let heads = HeadList::decode(v.value())?.heads; match Self::pick_winner(&heads) { Some(winner) if winner.tombstone => Ok(None), Some(winner) => Ok(Some(winner.value.clone())), None => Ok(None), } } None => Ok(None), } } /// Get all heads for a key (for conflict inspection). /// Heads are sorted deterministically: highest HLC first, ties broken by author. pub fn get_heads(&self, key: &[u8]) -> Result, StoreError> { let read_txn = self.db.begin_read()?; let table = read_txn.open_table(KV_TABLE)?; match table.get(key)? { Some(v) => { let mut heads = HeadList::decode(v.value())?.heads; // Sort by winner criteria: highest HLC first, then highest author (deterministic) heads.sort_by(|a, b| { b.hlc.cmp(&a.hlc) .then_with(|| b.author.cmp(&a.author)) }); Ok(heads) } None => Ok(Vec::new()), } } /// Pick deterministic winner from heads: highest HLC, then highest author bytes. /// Heads should already be sorted by get_heads(), so winner is first. fn pick_winner(heads: &[HeadInfo]) -> Option<&HeadInfo> { // If heads are already sorted (via get_heads), first is winner // If not sorted, compute winner via max if heads.is_empty() { None } else { // Use max_by for correctness even on unsorted input heads.iter().max_by(|a, b| { a.hlc.cmp(&b.hlc) .then_with(|| a.author.cmp(&b.author)) }) } } /// List all key-value pairs (winner values only) /// If include_deleted is true, includes tombstoned entries pub fn list_all(&self, include_deleted: bool) -> Result, Vec)>, StoreError> { self.list_by_prefix(&[], include_deleted) } /// List all key-value pairs matching a prefix (winner values only) /// Uses efficient range query on redb's sorted B-tree /// If include_deleted is true, includes tombstoned entries pub fn list_by_prefix(&self, prefix: &[u8], include_deleted: bool) -> Result, Vec)>, StoreError> { let read_txn = self.db.begin_read()?; let table = read_txn.open_table(KV_TABLE)?; let mut result = Vec::new(); // Use range query: from prefix to first key that doesn't match for entry in table.range(prefix..)? { let (key, value) = entry?; let key_bytes = key.value(); // Stop when we've passed the prefix if !key_bytes.starts_with(prefix) { break; } let heads = HeadList::decode(value.value())?.heads; if let Some(winner) = Self::pick_winner(&heads) { // Skip tombstones unless include_deleted is true if include_deleted || !winner.tombstone { result.push((key_bytes.to_vec(), winner.value.clone())); } } } Ok(result) } /// Check if a put operation is needed given current heads /// Returns false if the winning head has the same value (idempotent) pub fn needs_put(heads: &[HeadInfo], value: &[u8]) -> bool { match Self::pick_winner(heads) { Some(winner) => winner.value != value, // Skip if winner already has value None => true, // No heads = need put } } /// Check if a delete operation is needed given current heads /// Returns false if no heads or winning head is already a tombstone (idempotent) pub fn needs_delete(heads: &[HeadInfo]) -> bool { match Self::pick_winner(heads) { Some(winner) => !winner.tombstone, // Skip if winner is already tombstone None => false, // No heads = nothing to delete } } /// Get author state for a specific author pub fn author_state(&self, author: &[u8; 32]) -> Result, StoreError> { let read_txn = self.db.begin_read()?; let table = read_txn.open_table(AUTHOR_TABLE)?; match table.get(&author[..])? { Some(v) => Ok(AuthorState::decode(v.value()).ok()), None => Ok(None), } } /// Get sync state for all authors (for reconciliation). /// /// Returns a SyncState with each author's highest seen sequence number and hash. pub fn sync_state(&self) -> Result { use crate::sync_state::SyncState; let read_txn = self.db.begin_read()?; let table = read_txn.open_table(AUTHOR_TABLE)?; let mut state = SyncState::new(); for entry in table.iter()? { let (key, value) = entry?; if key.value().len() == 32 { if let Ok(author_state) = AuthorState::decode(value.value()) { let mut author = [0u8; 32]; author.copy_from_slice(key.value()); let mut hash = [0u8; 32]; if author_state.hash.len() == 32 { hash.copy_from_slice(&author_state.hash); } state.set(author, author_state.seq, hash); } } } Ok(state) } } #[cfg(test)] mod tests { use super::*; use crate::clock::MockClock; use crate::hlc::HLC; use crate::node_identity::NodeIdentity; use crate::signed_entry::EntryBuilder; use std::env::temp_dir; fn temp_db_path(name: &str) -> std::path::PathBuf { let tid = std::thread::current().id(); temp_dir().join(format!("lattice_dag_store_test_{}_{:?}.db", name, tid)) } const TEST_STORE: [u8; 16] = [1u8; 16]; #[test] fn test_single_write_one_head() { let path = temp_db_path("single_write"); let _ = std::fs::remove_file(&path); let store = Store::open(&path).unwrap(); let node = NodeIdentity::generate(); let clock = MockClock::new(1000); let entry = EntryBuilder::new(1, HLC::now_with_clock(&clock)) .store_id(TEST_STORE.to_vec()) .prev_hash([0u8; 32].to_vec()) .put("/key", b"value".to_vec()) .sign(&node); store.apply_entry(&entry).unwrap(); let heads = store.get_heads(b"/key").unwrap(); assert_eq!(heads.len(), 1); assert_eq!(heads[0].value, b"value"); let _ = std::fs::remove_file(&path); } #[test] fn test_deterministic_winner() { // Test pick_winner logic directly (no store needed) let heads = HeadList { heads: vec![ HeadInfo { value: b"older".to_vec(), hlc: 100, author: [1u8; 32].to_vec(), hash: [1u8; 32].to_vec(), tombstone: false, }, HeadInfo { value: b"newer".to_vec(), hlc: 200, author: [2u8; 32].to_vec(), hash: [2u8; 32].to_vec(), tombstone: false, }, ], }; let winner = Store::pick_winner(&heads.heads).unwrap(); assert_eq!(winner.value, b"newer"); // Higher HLC wins } #[test] fn test_concurrent_writes_multiple_heads() { let path = temp_db_path("concurrent"); let _ = std::fs::remove_file(&path); let store = Store::open(&path).unwrap(); let node = NodeIdentity::generate(); let clock = MockClock::new(1000); // First write let entry1 = EntryBuilder::new(1, HLC::now_with_clock(&clock)) .store_id(TEST_STORE.to_vec()) .prev_hash([0u8; 32].to_vec()) .parent_hashes(vec![]) // No parent .put("/key", b"v1".to_vec()) .sign(&node); store.apply_entry(&entry1).unwrap(); // Second write with SAME parent (simulates concurrent/offline write) let clock2 = MockClock::new(2000); let entry2 = EntryBuilder::new(2, HLC::now_with_clock(&clock2)) .store_id(TEST_STORE.to_vec()) .prev_hash(hash_signed_entry(&entry1).to_vec()) .parent_hashes(vec![]) // Also no parent (doesn't know about entry1) .put("/key", b"v2".to_vec()) .sign(&node); store.apply_entry(&entry2).unwrap(); // Should have TWO heads now let heads = store.get_heads(b"/key").unwrap(); assert_eq!(heads.len(), 2); let _ = std::fs::remove_file(&path); } #[test] fn test_merge_write_single_head() { let path = temp_db_path("merge"); let _ = std::fs::remove_file(&path); let store = Store::open(&path).unwrap(); let node = NodeIdentity::generate(); // Create two heads let clock1 = MockClock::new(1000); let entry1 = EntryBuilder::new(1, HLC::now_with_clock(&clock1)) .store_id(TEST_STORE.to_vec()) .prev_hash([0u8; 32].to_vec()) .put("/key", b"v1".to_vec()) .sign(&node); store.apply_entry(&entry1).unwrap(); let clock2 = MockClock::new(2000); let entry2 = EntryBuilder::new(2, HLC::now_with_clock(&clock2)) .store_id(TEST_STORE.to_vec()) .prev_hash(hash_signed_entry(&entry1).to_vec()) .put("/key", b"v2".to_vec()) .sign(&node); store.apply_entry(&entry2).unwrap(); assert_eq!(store.get_heads(b"/key").unwrap().len(), 2); // Merge write citing BOTH heads as parents let hash1 = hash_signed_entry(&entry1); let hash2 = hash_signed_entry(&entry2); let clock3 = MockClock::new(3000); let entry3 = EntryBuilder::new(3, HLC::now_with_clock(&clock3)) .store_id(TEST_STORE.to_vec()) .prev_hash(hash2.to_vec()) .parent_hashes(vec![hash1.to_vec(), hash2.to_vec()]) .put("/key", b"merged".to_vec()) .sign(&node); store.apply_entry(&entry3).unwrap(); // Should now have ONE head let heads = store.get_heads(b"/key").unwrap(); assert_eq!(heads.len(), 1); assert_eq!(heads[0].value, b"merged"); let _ = std::fs::remove_file(&path); } #[test] fn test_delete_preserves_concurrent_heads() { let path = temp_db_path("delete_concurrent"); let _ = std::fs::remove_file(&path); let store = Store::open(&path).unwrap(); let node = NodeIdentity::generate(); // Create two concurrent heads let clock1 = MockClock::new(1000); let entry1 = EntryBuilder::new(1, HLC::now_with_clock(&clock1)) .store_id(TEST_STORE.to_vec()) .prev_hash([0u8; 32].to_vec()) .put("/key", b"v1".to_vec()) .sign(&node); store.apply_entry(&entry1).unwrap(); let clock2 = MockClock::new(2000); let entry2 = EntryBuilder::new(2, HLC::now_with_clock(&clock2)) .store_id(TEST_STORE.to_vec()) .prev_hash(hash_signed_entry(&entry1).to_vec()) // No parent_hashes = concurrent write .put("/key", b"v2".to_vec()) .sign(&node); store.apply_entry(&entry2).unwrap(); assert_eq!(store.get_heads(b"/key").unwrap().len(), 2); // Delete citing only entry1 as parent let hash1 = hash_signed_entry(&entry1); let clock3 = MockClock::new(3000); let entry3 = EntryBuilder::new(3, HLC::now_with_clock(&clock3)) .store_id(TEST_STORE.to_vec()) .prev_hash(hash1.to_vec()) .parent_hashes(vec![hash1.to_vec()]) // Only cites entry1 .delete("/key") .sign(&node); store.apply_entry(&entry3).unwrap(); // entry2 should survive (wasn't cited as parent), plus tombstone head let heads = store.get_heads(b"/key").unwrap(); assert_eq!(heads.len(), 2, "Expected tombstone + v2, got {}", heads.len()); // One should be a tombstone, one should be v2 let has_tombstone = heads.iter().any(|h| h.tombstone); let has_v2 = heads.iter().any(|h| h.value == b"v2"); assert!(has_tombstone); assert!(has_v2); let _ = std::fs::remove_file(&path); } #[test] fn test_delete_all_heads_removes_key() { let path = temp_db_path("delete_all"); let _ = std::fs::remove_file(&path); let store = Store::open(&path).unwrap(); let node = NodeIdentity::generate(); // Create a single head let clock1 = MockClock::new(1000); let entry1 = EntryBuilder::new(1, HLC::now_with_clock(&clock1)) .store_id(TEST_STORE.to_vec()) .prev_hash([0u8; 32].to_vec()) .put("/key", b"value".to_vec()) .sign(&node); store.apply_entry(&entry1).unwrap(); assert!(store.get(b"/key").unwrap().is_some()); // Delete citing the only head let hash1 = hash_signed_entry(&entry1); let clock2 = MockClock::new(2000); let entry2 = EntryBuilder::new(2, HLC::now_with_clock(&clock2)) .store_id(TEST_STORE.to_vec()) .prev_hash(hash1.to_vec()) .parent_hashes(vec![hash1.to_vec()]) .delete("/key") .sign(&node); store.apply_entry(&entry2).unwrap(); // Key should show as deleted (tombstone wins) assert!(store.get(b"/key").unwrap().is_none()); // Should have one tombstone head let heads = store.get_heads(b"/key").unwrap(); assert_eq!(heads.len(), 1); assert!(heads[0].tombstone); let _ = std::fs::remove_file(&path); } #[test] fn test_concurrent_delete_and_put() { // This test demonstrates that concurrent delete and put should both exist as heads // Scenario: // 1. Initial: K = v1 (head H1) // 2. Alice (offline): Delete K citing H1 // 3. Bob (offline): Put K = v2 citing H1 (doesn't know about delete) // 4. Result: Should have 2 heads (tombstone + v2), not just v2 let path = temp_db_path("concurrent_delete_put"); let _ = std::fs::remove_file(&path); let store = Store::open(&path).unwrap(); let alice = NodeIdentity::generate(); let bob = NodeIdentity::generate(); // Initial state: K = v1 let clock1 = MockClock::new(1000); let entry1 = EntryBuilder::new(1, HLC::now_with_clock(&clock1)) .store_id(TEST_STORE.to_vec()) .prev_hash([0u8; 32].to_vec()) .put(b"/key", b"v1".to_vec()) .sign(&alice); store.apply_entry(&entry1).unwrap(); let h1 = hash_signed_entry(&entry1); // Alice deletes K citing H1 let clock2 = MockClock::new(2000); let entry2 = EntryBuilder::new(2, HLC::now_with_clock(&clock2)) .store_id(TEST_STORE.to_vec()) .prev_hash(h1.to_vec()) .parent_hashes(vec![h1.to_vec()]) .delete(b"/key") .sign(&alice); store.apply_entry(&entry2).unwrap(); // Bob (concurrently) puts K = v2 citing H1 (doesn't know about Alice's delete) let clock3 = MockClock::new(2500); let entry3 = EntryBuilder::new(1, HLC::now_with_clock(&clock3)) .store_id(TEST_STORE.to_vec()) .prev_hash([0u8; 32].to_vec()) // Bob's own chain .parent_hashes(vec![h1.to_vec()]) // Cites H1 as parent .put(b"/key", b"v2".to_vec()) .sign(&bob); store.apply_entry(&entry3).unwrap(); // Should have 2 heads: Alice's tombstone and Bob's v2 let heads = store.get_heads(b"/key").unwrap(); assert_eq!(heads.len(), 2, "Expected 2 heads (tombstone + put), got {}", heads.len()); // One should be a tombstone, one should be v2 let has_tombstone = heads.iter().any(|h| h.tombstone); let has_v2 = heads.iter().any(|h| h.value == b"v2"); assert!(has_tombstone, "Expected a tombstone head"); assert!(has_v2, "Expected a v2 head"); let _ = std::fs::remove_file(&path); } #[test] fn test_two_authors_diverged_then_merge() { // Scenario: // 1. Alice creates K = v1 (head H1) // 2. Bob (offline, doesn't see H1) creates K = v2 (head H2) // 3. Result: 2 heads (conflict) // 4. Charlie (sees both) creates K = v3 citing H1 and H2 // 5. Result: 1 head (merged) let path = temp_db_path("two_authors_merge"); let _ = std::fs::remove_file(&path); let store = Store::open(&path).unwrap(); let alice = NodeIdentity::generate(); let bob = NodeIdentity::generate(); let charlie = NodeIdentity::generate(); // Alice creates K = v1 let clock1 = MockClock::new(1000); let entry1 = EntryBuilder::new(1, HLC::now_with_clock(&clock1)) .store_id(TEST_STORE.to_vec()) .prev_hash([0u8; 32].to_vec()) .put(b"/key", b"alice_v1".to_vec()) .sign(&alice); store.apply_entry(&entry1).unwrap(); let h1 = hash_signed_entry(&entry1); // Bob (offline, no parent_hashes) creates K = v2 let clock2 = MockClock::new(2000); let entry2 = EntryBuilder::new(1, HLC::now_with_clock(&clock2)) .store_id(TEST_STORE.to_vec()) .prev_hash([0u8; 32].to_vec()) // No parent_hashes = concurrent/diverged .put(b"/key", b"bob_v2".to_vec()) .sign(&bob); store.apply_entry(&entry2).unwrap(); let h2 = hash_signed_entry(&entry2); // Should have 2 heads now let heads = store.get_heads(b"/key").unwrap(); assert_eq!(heads.len(), 2, "Expected 2 diverged heads"); // Verify deterministic winner (higher HLC wins) let value = store.get(b"/key").unwrap().unwrap(); assert_eq!(value, b"bob_v2"); // Bob has higher HLC (2000 > 1000) // Charlie merges by citing both H1 and H2 let clock3 = MockClock::new(3000); let entry3 = EntryBuilder::new(1, HLC::now_with_clock(&clock3)) .store_id(TEST_STORE.to_vec()) .prev_hash([0u8; 32].to_vec()) .parent_hashes(vec![h1.to_vec(), h2.to_vec()]) .put(b"/key", b"charlie_merged".to_vec()) .sign(&charlie); store.apply_entry(&entry3).unwrap(); // Should have 1 head now (merged) let heads = store.get_heads(b"/key").unwrap(); assert_eq!(heads.len(), 1, "Expected 1 merged head"); assert_eq!(heads[0].value, b"charlie_merged"); let _ = std::fs::remove_file(&path); } #[test] fn test_apply_entry_is_idempotent() { // Applying the same entry twice should not duplicate the head // This is critical for log replay and network message deduplication let path = temp_db_path("idempotent"); let _ = std::fs::remove_file(&path); let store = Store::open(&path).unwrap(); let node = NodeIdentity::generate(); let clock1 = MockClock::new(1000); let entry1 = EntryBuilder::new(1, HLC::now_with_clock(&clock1)) .store_id(TEST_STORE.to_vec()) .prev_hash([0u8; 32].to_vec()) .put(b"/key", b"value".to_vec()) .sign(&node); // Apply once store.apply_entry(&entry1).unwrap(); assert_eq!(store.get_heads(b"/key").unwrap().len(), 1); // Apply again (e.g., log replay or duplicate message) store.apply_entry(&entry1).unwrap(); assert_eq!(store.get_heads(b"/key").unwrap().len(), 1, "Duplicate entry should not create duplicate head"); // Apply a third time for good measure store.apply_entry(&entry1).unwrap(); assert_eq!(store.get_heads(b"/key").unwrap().len(), 1); let _ = std::fs::remove_file(&path); } #[test] fn test_sequential_writes_then_replay() { // Simulates: put a=1, put a=2, then replay from log // After replay, should have only 1 head (the latest) let path = temp_db_path("sequential_replay"); let _ = std::fs::remove_file(&path); let store = Store::open(&path).unwrap(); let node = NodeIdentity::generate(); // First write: a = 1 let clock1 = MockClock::new(1000); let entry1 = EntryBuilder::new(1, HLC::now_with_clock(&clock1)) .store_id(TEST_STORE.to_vec()) .prev_hash([0u8; 32].to_vec()) .parent_hashes(vec![]) // No parents for first write .put(b"/key", b"1".to_vec()) .sign(&node); store.apply_entry(&entry1).unwrap(); let h1 = hash_signed_entry(&entry1); assert_eq!(store.get_heads(b"/key").unwrap().len(), 1); // Second write: a = 2, citing h1 as parent let clock2 = MockClock::new(2000); let entry2 = EntryBuilder::new(2, HLC::now_with_clock(&clock2)) .store_id(TEST_STORE.to_vec()) .prev_hash(h1.to_vec()) .parent_hashes(vec![h1.to_vec()]) // Cites h1 .put(b"/key", b"2".to_vec()) .sign(&node); store.apply_entry(&entry2).unwrap(); assert_eq!(store.get_heads(b"/key").unwrap().len(), 1, "After put 2, should have 1 head"); // Now simulate log replay: clear state and re-apply both entries drop(store); let _ = std::fs::remove_file(&path); let store = Store::open(&path).unwrap(); // Check what parent_hashes entry2 actually has let decoded_entry2 = Entry::decode(&entry2.entry_bytes[..]).unwrap(); eprintln!("Entry2 parent_hashes: {:?}", decoded_entry2.parent_hashes); eprintln!("H1: {:?}", h1); // Replay entry1 store.apply_entry(&entry1).unwrap(); assert_eq!(store.get_heads(b"/key").unwrap().len(), 1, "After replay entry1"); // Replay entry2 store.apply_entry(&entry2).unwrap(); let heads = store.get_heads(b"/key").unwrap(); assert_eq!(heads.len(), 1, "After replay entry2, should have 1 head, got {}: {:?}", heads.len(), heads.iter().map(|h| String::from_utf8_lossy(&h.value)).collect::>()); let _ = std::fs::remove_file(&path); } #[test] fn test_replay_to_existing_state_no_duplicates() { use crate::sigchain::SigChain; // This simulates: put a=1, put a=2, then restart and replay from log // The replay should skip already-applied entries let state_path = temp_db_path("replay_existing_state"); let log_path = temp_db_path("replay_existing_log"); let _ = std::fs::remove_file(&state_path); let _ = std::fs::remove_file(&log_path); let store = Store::open(&state_path).unwrap(); let node = NodeIdentity::generate(); let mut sigchain = SigChain::new(&log_path, TEST_STORE, node.public_key_bytes()); // First write: a = 1 let clock1 = MockClock::new(1000); let entry1 = EntryBuilder::new(1, HLC::now_with_clock(&clock1)) .store_id(TEST_STORE.to_vec()) .prev_hash([0u8; 32].to_vec()) .parent_hashes(vec![]) .put(b"/key", b"1".to_vec()) .sign(&node); sigchain.append(&entry1).unwrap(); store.apply_entry(&entry1).unwrap(); let h1 = hash_signed_entry(&entry1); // Second write: a = 2, citing h1 as parent let clock2 = MockClock::new(2000); let entry2 = EntryBuilder::new(2, HLC::now_with_clock(&clock2)) .store_id(TEST_STORE.to_vec()) .prev_hash(h1.to_vec()) .parent_hashes(vec![h1.to_vec()]) .put(b"/key", b"2".to_vec()) .sign(&node); sigchain.append(&entry2).unwrap(); store.apply_entry(&entry2).unwrap(); assert_eq!(store.get_heads(b"/key").unwrap().len(), 1, "Before restart"); let author = node.public_key_bytes(); assert_eq!(store.author_state(&author).unwrap().unwrap().seq, 2, "author seq should be 2"); // Simulate restart: reopen state.db (persisted) and replay log drop(store); drop(sigchain); let store = Store::open(&state_path).unwrap(); // Reopen existing state assert_eq!(store.author_state(&author).unwrap().unwrap().seq, 2, "author seq persisted"); // Replay log - entries already applied, skip all let replayed = store.replay_log(&log_path).unwrap(); assert_eq!(replayed, 0, "0 new entries (all skipped)"); let final_heads = store.get_heads(b"/key").unwrap(); assert_eq!(final_heads.len(), 1, "After replay, should have 1 head, got {}: {:?}", final_heads.len(), final_heads.iter().map(|h| String::from_utf8_lossy(&h.value)).collect::>()); let _ = std::fs::remove_file(&state_path); let _ = std::fs::remove_file(&log_path); } #[test] fn test_fast_resume_on_restart() { use crate::sigchain::SigChain; // Fast resume: entries already applied are skipped based on per-author seq let state_path = temp_db_path("fast_resume_state"); let log_path = temp_db_path("fast_resume_log"); let _ = std::fs::remove_file(&state_path); let _ = std::fs::remove_file(&log_path); let store = Store::open(&state_path).unwrap(); let node = NodeIdentity::generate(); let author = node.public_key_bytes(); let mut sigchain = SigChain::new(&log_path, TEST_STORE, node.public_key_bytes()); // Apply 3 entries with proper chaining for i in 1u64..=3 { let clock = MockClock::new(i * 1000); let prev = sigchain.last_hash().to_vec(); let entry = EntryBuilder::new(i, HLC::now_with_clock(&clock)) .store_id(TEST_STORE.to_vec()) .prev_hash(prev) .put(format!("/key{}", i).as_bytes(), format!("v{}", i).into_bytes()) .sign(&node); sigchain.append(&entry).unwrap(); store.apply_entry(&entry).unwrap(); } assert_eq!(store.author_state(&author).unwrap().unwrap().seq, 3); assert_eq!(store.get_heads(b"/key3").unwrap().len(), 1); // Restart and replay - should skip all entries drop(store); drop(sigchain); let store = Store::open(&state_path).unwrap(); let replayed = store.replay_log(&log_path).unwrap(); // All 3 entries were read but skipped (already applied) assert_eq!(replayed, 0, "0 new entries (all skipped)"); assert_eq!(store.author_state(&author).unwrap().unwrap().seq, 3, "seq unchanged"); assert_eq!(store.get_heads(b"/key3").unwrap().len(), 1, "heads unchanged"); let _ = std::fs::remove_file(&state_path); let _ = std::fs::remove_file(&log_path); } #[test] fn test_partial_replay_after_crash() { use crate::sigchain::SigChain; // Simulates: log has 5 entries, state.db only has first 3 applied (crash) // Replay should apply entries 4 and 5 let state_path = temp_db_path("partial_replay_state"); let log_path = temp_db_path("partial_replay_log"); let _ = std::fs::remove_file(&state_path); let _ = std::fs::remove_file(&log_path); let store = Store::open(&state_path).unwrap(); let node = NodeIdentity::generate(); let author = node.public_key_bytes(); let mut sigchain = SigChain::new(&log_path, TEST_STORE, node.public_key_bytes()); // Write 5 entries to log with proper chaining for i in 1u64..=5 { let clock = MockClock::new(i * 1000); let prev = sigchain.last_hash().to_vec(); let entry = EntryBuilder::new(i, HLC::now_with_clock(&clock)) .store_id(TEST_STORE.to_vec()) .prev_hash(prev) .put(format!("/key{}", i).as_bytes(), format!("v{}", i).into_bytes()) .sign(&node); sigchain.append(&entry).unwrap(); // Only apply first 3 to state.db (simulating crash after 3rd) if i <= 3 { store.apply_entry(&entry).unwrap(); } } assert_eq!(store.author_state(&author).unwrap().unwrap().seq, 3); assert!(store.get_heads(b"/key4").unwrap().is_empty(), "key4 not applied yet"); // Simulate restart and replay drop(store); drop(sigchain); let store = Store::open(&state_path).unwrap(); let replayed = store.replay_log(&log_path).unwrap(); assert_eq!(replayed, 2, "Only 2 new entries applied (3 skipped)"); assert_eq!(store.author_state(&author).unwrap().unwrap().seq, 5, "seq updated to 5"); assert_eq!(store.get_heads(b"/key4").unwrap().len(), 1, "key4 now applied"); assert_eq!(store.get_heads(b"/key5").unwrap().len(), 1, "key5 now applied"); let _ = std::fs::remove_file(&state_path); let _ = std::fs::remove_file(&log_path); } #[test] fn test_state_db_rollback_and_replay() { use crate::sigchain::SigChain; // Simulates: // 1. Apply entries 1-3 // 2. Copy state.db (backup) // 3. Apply entries 4-5 // 4. Restore state.db from backup // 5. Restart and replay - should apply entries 4-5 let state_path = temp_db_path("rollback_state"); let backup_path = temp_db_path("rollback_backup"); let log_path = temp_db_path("rollback_log"); let _ = std::fs::remove_file(&state_path); let _ = std::fs::remove_file(&backup_path); let _ = std::fs::remove_file(&log_path); let store = Store::open(&state_path).unwrap(); let node = NodeIdentity::generate(); let author = node.public_key_bytes(); let mut sigchain = SigChain::new(&log_path, TEST_STORE, node.public_key_bytes()); // Apply first 3 entries for i in 1u64..=3 { let clock = MockClock::new(i * 1000); let prev = sigchain.last_hash().to_vec(); let entry = EntryBuilder::new(i, HLC::now_with_clock(&clock)) .store_id(TEST_STORE.to_vec()) .prev_hash(prev) .put(format!("/key{}", i).as_bytes(), format!("v{}", i).into_bytes()) .sign(&node); sigchain.append(&entry).unwrap(); store.apply_entry(&entry).unwrap(); } assert_eq!(store.author_state(&author).unwrap().unwrap().seq, 3); // Close and backup state.db drop(store); std::fs::copy(&state_path, &backup_path).unwrap(); // Reopen and apply entries 4-5 let store = Store::open(&state_path).unwrap(); for i in 4u64..=5 { let clock = MockClock::new(i * 1000); let prev = sigchain.last_hash().to_vec(); let entry = EntryBuilder::new(i, HLC::now_with_clock(&clock)) .store_id(TEST_STORE.to_vec()) .prev_hash(prev) .put(format!("/key{}", i).as_bytes(), format!("v{}", i).into_bytes()) .sign(&node); sigchain.append(&entry).unwrap(); store.apply_entry(&entry).unwrap(); } assert_eq!(store.author_state(&author).unwrap().unwrap().seq, 5); assert_eq!(store.get_heads(b"/key5").unwrap().len(), 1); // Now restore state.db from backup (simulating crash/rollback) drop(store); drop(sigchain); std::fs::copy(&backup_path, &state_path).unwrap(); // Restart and replay let store = Store::open(&state_path).unwrap(); // State should be at seq 3 (restored from backup) assert_eq!(store.author_state(&author).unwrap().unwrap().seq, 3, "Restored to seq 3"); assert!(store.get_heads(b"/key4").unwrap().is_empty(), "key4 not in restored state"); // Replay log - should apply entries 4 and 5 (skip 1-3) let replayed = store.replay_log(&log_path).unwrap(); assert_eq!(replayed, 2, "Only 2 new entries applied (3 skipped)"); // Now seq should be 5 and keys 4-5 should exist assert_eq!(store.author_state(&author).unwrap().unwrap().seq, 5, "seq updated to 5"); assert_eq!(store.get_heads(b"/key4").unwrap().len(), 1, "key4 now applied"); assert_eq!(store.get_heads(b"/key5").unwrap().len(), 1, "key5 now applied"); let _ = std::fs::remove_file(&state_path); let _ = std::fs::remove_file(&backup_path); let _ = std::fs::remove_file(&log_path); } #[test] fn test_needs_put_empty_heads() { // No heads = need put let heads: Vec = vec![]; assert!(Store::needs_put(&heads, b"value")); } #[test] fn test_needs_put_same_value() { // Single head with same value = idempotent, no put needed let heads = vec![HeadInfo { value: b"hello".to_vec(), hlc: 1000, author: [1u8; 32].to_vec(), hash: [2u8; 32].to_vec(), tombstone: false, }]; assert!(!Store::needs_put(&heads, b"hello")); } #[test] fn test_needs_put_different_value() { // Single head with different value = need put let heads = vec![HeadInfo { value: b"hello".to_vec(), hlc: 1000, author: [1u8; 32].to_vec(), hash: [2u8; 32].to_vec(), tombstone: false, }]; assert!(Store::needs_put(&heads, b"world")); } #[test] fn test_needs_put_multiple_heads_winner_matches() { // Multiple heads where WINNER has our value = idempotent // Winner is highest HLC (1001), value "v2" let heads = vec![ HeadInfo { value: b"v1".to_vec(), hlc: 1000, author: [1u8; 32].to_vec(), hash: [2u8; 32].to_vec(), tombstone: false, }, HeadInfo { value: b"v2".to_vec(), hlc: 1001, // Winner (highest HLC) author: [3u8; 32].to_vec(), hash: [4u8; 32].to_vec(), tombstone: false, }, ]; assert!(!Store::needs_put(&heads, b"v2")); // Winner has value = skip assert!(Store::needs_put(&heads, b"v1")); // Winner doesn't have value = put } #[test] fn test_needs_delete_empty_heads() { // No heads = idempotent, no delete needed let heads: Vec = vec![]; assert!(!Store::needs_delete(&heads)); } #[test] fn test_needs_delete_with_heads() { // Has non-tombstone heads = need delete let heads = vec![HeadInfo { value: b"data".to_vec(), hlc: 1000, author: [1u8; 32].to_vec(), hash: [2u8; 32].to_vec(), tombstone: false, }]; assert!(Store::needs_delete(&heads)); } #[test] fn test_needs_delete_tombstone_is_winner() { // Winning head is already tombstone = no delete needed let heads = vec![HeadInfo { value: vec![], hlc: 1000, author: [1u8; 32].to_vec(), hash: [2u8; 32].to_vec(), tombstone: true, }]; assert!(!Store::needs_delete(&heads)); } #[test] fn test_sync_state_diff_and_apply() { // Test that two stores can compute diff and sync entries let path_a = temp_db_path("sync_a"); let path_b = temp_db_path("sync_b"); let log_path_a = temp_db_path("sync_a_log"); let _ = std::fs::remove_file(&path_a); let _ = std::fs::remove_file(&path_b); let _ = std::fs::remove_file(&log_path_a); // Node A writes some entries let store_a = Store::open(&path_a).unwrap(); let node_a = NodeIdentity::generate(); // Write 3 entries on node A for i in 1u64..=3 { let clock = MockClock::new(1000 + i * 100); let entry = EntryBuilder::new(i, HLC::now_with_clock(&clock)) .store_id(TEST_STORE.to_vec()) .prev_hash([0u8; 32].to_vec()) .put(format!("/key{}", i), format!("value{}", i).into_bytes()) .sign(&node_a); store_a.apply_entry(&entry).unwrap(); crate::log::append_entry(&log_path_a, &entry).unwrap(); } // Node B is empty let store_b = Store::open(&path_b).unwrap(); // Get sync states let sync_a = store_a.sync_state().unwrap(); let sync_b = store_b.sync_state().unwrap(); // Compute diff: B needs entries from A let missing = sync_b.diff(&sync_a); // Should need entries for author A assert_eq!(missing.len(), 1); assert_eq!(missing[0].author, node_a.public_key_bytes()); assert_eq!(missing[0].from_seq, 0); // B has nothing assert_eq!(missing[0].to_seq, 3); // A has 3 entries // Fetch entries from A's log (using from_hash = 0 means read all) let entries = crate::log::read_entries_after( &log_path_a, if missing[0].from_hash == [0u8; 32] { None } else { Some(missing[0].from_hash) } ).unwrap(); assert_eq!(entries.len(), 3); // Apply entries to B for entry in &entries { store_b.apply_entry(entry).unwrap(); } // Verify B has same KV state as A assert_eq!(store_b.get(b"/key1").unwrap(), Some(b"value1".to_vec())); assert_eq!(store_b.get(b"/key2").unwrap(), Some(b"value2".to_vec())); assert_eq!(store_b.get(b"/key3").unwrap(), Some(b"value3".to_vec())); // Verify sync states now match let sync_a_after = store_a.sync_state().unwrap(); let sync_b_after = store_b.sync_state().unwrap(); assert!(sync_b_after.diff(&sync_a_after).is_empty()); let _ = std::fs::remove_file(path_a); let _ = std::fs::remove_file(path_b); let _ = std::fs::remove_file(log_path_a); } #[test] fn test_bidirectional_sync() { // Test that two stores can sync in both directions let path_a = temp_db_path("bidir_a"); let path_b = temp_db_path("bidir_b"); let log_path_a = temp_db_path("bidir_log_a"); let log_path_b = temp_db_path("bidir_log_b"); let _ = std::fs::remove_file(&path_a); let _ = std::fs::remove_file(&path_b); let _ = std::fs::remove_file(&log_path_a); let _ = std::fs::remove_file(&log_path_b); let store_a = Store::open(&path_a).unwrap(); let store_b = Store::open(&path_b).unwrap(); let node_a = NodeIdentity::generate(); let node_b = NodeIdentity::generate(); // Node A writes entries for i in 1u64..=2 { let clock = MockClock::new(1000 + i * 100); let entry = EntryBuilder::new(i, HLC::now_with_clock(&clock)) .store_id(TEST_STORE.to_vec()) .prev_hash([0u8; 32].to_vec()) .put(format!("/a{}", i), format!("from_a{}", i).into_bytes()) .sign(&node_a); store_a.apply_entry(&entry).unwrap(); crate::log::append_entry(&log_path_a, &entry).unwrap(); } // Node B writes different entries for i in 1u64..=2 { let clock = MockClock::new(2000 + i * 100); let entry = EntryBuilder::new(i, HLC::now_with_clock(&clock)) .store_id(TEST_STORE.to_vec()) .prev_hash([0u8; 32].to_vec()) .put(format!("/b{}", i), format!("from_b{}", i).into_bytes()) .sign(&node_b); store_b.apply_entry(&entry).unwrap(); crate::log::append_entry(&log_path_b, &entry).unwrap(); } // Get sync states let sync_a = store_a.sync_state().unwrap(); let sync_b = store_b.sync_state().unwrap(); // A needs B's entries let a_needs = sync_a.diff(&sync_b); assert_eq!(a_needs.len(), 1); assert_eq!(a_needs[0].author, node_b.public_key_bytes()); // B needs A's entries let b_needs = sync_b.diff(&sync_a); assert_eq!(b_needs.len(), 1); assert_eq!(b_needs[0].author, node_a.public_key_bytes()); // Sync A → B let entries_a = crate::log::read_entries(&log_path_a).unwrap(); for entry in &entries_a { store_b.apply_entry(entry).unwrap(); } // Sync B → A let entries_b = crate::log::read_entries(&log_path_b).unwrap(); for entry in &entries_b { store_a.apply_entry(entry).unwrap(); } // Both should now have all 4 keys assert_eq!(store_a.get(b"/a1").unwrap(), Some(b"from_a1".to_vec())); assert_eq!(store_a.get(b"/b1").unwrap(), Some(b"from_b1".to_vec())); assert_eq!(store_b.get(b"/a1").unwrap(), Some(b"from_a1".to_vec())); assert_eq!(store_b.get(b"/b1").unwrap(), Some(b"from_b1".to_vec())); // Sync states should match let sync_a_after = store_a.sync_state().unwrap(); let sync_b_after = store_b.sync_state().unwrap(); assert!(sync_a_after.diff(&sync_b_after).is_empty()); assert!(sync_b_after.diff(&sync_a_after).is_empty()); let _ = std::fs::remove_file(path_a); let _ = std::fs::remove_file(path_b); let _ = std::fs::remove_file(log_path_a); let _ = std::fs::remove_file(log_path_b); } #[test] fn test_three_way_sync() { // Test that three stores can all sync with each other let path_a = temp_db_path("three_a"); let path_b = temp_db_path("three_b"); let path_c = temp_db_path("three_c"); let log_path_a = temp_db_path("three_log_a"); let log_path_b = temp_db_path("three_log_b"); let log_path_c = temp_db_path("three_log_c"); for p in [&path_a, &path_b, &path_c, &log_path_a, &log_path_b, &log_path_c] { let _ = std::fs::remove_file(p); } let store_a = Store::open(&path_a).unwrap(); let store_b = Store::open(&path_b).unwrap(); let store_c = Store::open(&path_c).unwrap(); let node_a = NodeIdentity::generate(); let node_b = NodeIdentity::generate(); let node_c = NodeIdentity::generate(); // Each node writes one entry let entry_a = EntryBuilder::new(1, HLC::now_with_clock(&MockClock::new(1000))) .store_id(TEST_STORE.to_vec()) .prev_hash([0u8; 32].to_vec()) .put("/key_a", b"from_a".to_vec()) .sign(&node_a); store_a.apply_entry(&entry_a).unwrap(); crate::log::append_entry(&log_path_a, &entry_a).unwrap(); let entry_b = EntryBuilder::new(1, HLC::now_with_clock(&MockClock::new(2000))) .store_id(TEST_STORE.to_vec()) .prev_hash([0u8; 32].to_vec()) .put("/key_b", b"from_b".to_vec()) .sign(&node_b); store_b.apply_entry(&entry_b).unwrap(); crate::log::append_entry(&log_path_b, &entry_b).unwrap(); let entry_c = EntryBuilder::new(1, HLC::now_with_clock(&MockClock::new(3000))) .store_id(TEST_STORE.to_vec()) .prev_hash([0u8; 32].to_vec()) .put("/key_c", b"from_c".to_vec()) .sign(&node_c); store_c.apply_entry(&entry_c).unwrap(); crate::log::append_entry(&log_path_c, &entry_c).unwrap(); // Sync A ↔ B for entry in crate::log::read_entries(&log_path_a).unwrap() { store_b.apply_entry(&entry).unwrap(); } for entry in crate::log::read_entries(&log_path_b).unwrap() { store_a.apply_entry(&entry).unwrap(); } // Sync B ↔ C for entry in crate::log::read_entries(&log_path_b).unwrap() { store_c.apply_entry(&entry).unwrap(); } for entry in crate::log::read_entries(&log_path_c).unwrap() { store_b.apply_entry(&entry).unwrap(); } // Sync A ↔ C (A should get C's entry, C should get A's entry) for entry in crate::log::read_entries(&log_path_a).unwrap() { store_c.apply_entry(&entry).unwrap(); } for entry in crate::log::read_entries(&log_path_c).unwrap() { store_a.apply_entry(&entry).unwrap(); } // All three stores should have all three keys for store in [&store_a, &store_b, &store_c] { assert_eq!(store.get(b"/key_a").unwrap(), Some(b"from_a".to_vec())); assert_eq!(store.get(b"/key_b").unwrap(), Some(b"from_b".to_vec())); assert_eq!(store.get(b"/key_c").unwrap(), Some(b"from_c".to_vec())); } // All sync states should match let sync_a = store_a.sync_state().unwrap(); let sync_b = store_b.sync_state().unwrap(); let sync_c = store_c.sync_state().unwrap(); assert!(sync_a.diff(&sync_b).is_empty()); assert!(sync_b.diff(&sync_c).is_empty()); assert!(sync_c.diff(&sync_a).is_empty()); for p in [path_a, path_b, path_c, log_path_a, log_path_b, log_path_c] { let _ = std::fs::remove_file(p); } } #[test] fn test_conflict_deterministic_resolution() { // Test that two nodes writing the same key resolve deterministically let path_a = temp_db_path("conflict_a"); let path_b = temp_db_path("conflict_b"); let log_path_a = temp_db_path("conflict_log_a"); let log_path_b = temp_db_path("conflict_log_b"); for p in [&path_a, &path_b, &log_path_a, &log_path_b] { let _ = std::fs::remove_file(p); } let store_a = Store::open(&path_a).unwrap(); let store_b = Store::open(&path_b).unwrap(); let node_a = NodeIdentity::generate(); let node_b = NodeIdentity::generate(); // Both nodes write to the SAME key with different values // Use same HLC to force conflict (tie-break on author) let entry_a = EntryBuilder::new(1, HLC::now_with_clock(&MockClock::new(1000))) .store_id(TEST_STORE.to_vec()) .prev_hash([0u8; 32].to_vec()) .put("/shared_key", b"value_from_a".to_vec()) .sign(&node_a); store_a.apply_entry(&entry_a).unwrap(); crate::log::append_entry(&log_path_a, &entry_a).unwrap(); let entry_b = EntryBuilder::new(1, HLC::now_with_clock(&MockClock::new(1000))) // Same HLC! .store_id(TEST_STORE.to_vec()) .prev_hash([0u8; 32].to_vec()) .put("/shared_key", b"value_from_b".to_vec()) .sign(&node_b); store_b.apply_entry(&entry_b).unwrap(); crate::log::append_entry(&log_path_b, &entry_b).unwrap(); // Before sync: A has A's value, B has B's value assert_eq!(store_a.get(b"/shared_key").unwrap(), Some(b"value_from_a".to_vec())); assert_eq!(store_b.get(b"/shared_key").unwrap(), Some(b"value_from_b".to_vec())); // Sync A → B and B → A for entry in crate::log::read_entries(&log_path_a).unwrap() { store_b.apply_entry(&entry).unwrap(); } for entry in crate::log::read_entries(&log_path_b).unwrap() { store_a.apply_entry(&entry).unwrap(); } // After sync: both should have SAME value (deterministic winner) let value_a = store_a.get(b"/shared_key").unwrap(); let value_b = store_b.get(b"/shared_key").unwrap(); assert_eq!(value_a, value_b, "Conflict should resolve deterministically"); // Both should have 2 heads for this key (conflict) let heads_a = store_a.get_heads(b"/shared_key").unwrap(); let heads_b = store_b.get_heads(b"/shared_key").unwrap(); assert_eq!(heads_a.len(), 2, "Should have 2 heads (conflict)"); assert_eq!(heads_b.len(), 2, "Should have 2 heads (conflict)"); // Both stores have the same heads in same order (deterministic) assert_eq!(heads_a[0].value, heads_b[0].value, "Winner should be same"); assert_eq!(heads_a[0].author, heads_b[0].author, "Winner author should be same"); // Verify tie-breaker: winner is the one with higher author bytes (deterministic) // Since HLC is the same, the author with lexicographically higher bytes wins let winner_author = &heads_a[0].author; let loser_author = &heads_a[1].author; assert!(winner_author > loser_author, "Winner should have higher author bytes"); for p in [path_a, path_b, log_path_a, log_path_b] { let _ = std::fs::remove_file(p); } } #[test] fn test_hlc_tiebreak_explicit() { // Explicit test: equal HLC, winner determined by node ID (author bytes) let path = temp_db_path("tiebreak"); let _ = std::fs::remove_file(&path); let store = Store::open(&path).unwrap(); let node_low = NodeIdentity::generate(); let node_high = NodeIdentity::generate(); // Determine which node has "higher" author bytes let (high_node, low_node) = if node_high.public_key_bytes() > node_low.public_key_bytes() { (&node_high, &node_low) } else { (&node_low, &node_high) }; // Both entries have SAME HLC let clock = MockClock::new(5000); let entry_low = EntryBuilder::new(1, HLC::now_with_clock(&clock)) .store_id(TEST_STORE.to_vec()) .prev_hash([0u8; 32].to_vec()) .put("/tiebreak_key", b"from_low".to_vec()) .sign(low_node); store.apply_entry(&entry_low).unwrap(); let entry_high = EntryBuilder::new(1, HLC::now_with_clock(&clock)) .store_id(TEST_STORE.to_vec()) .prev_hash([0u8; 32].to_vec()) .put("/tiebreak_key", b"from_high".to_vec()) .sign(high_node); store.apply_entry(&entry_high).unwrap(); // Winner should be the one with higher author bytes let value = store.get(b"/tiebreak_key").unwrap(); assert_eq!(value, Some(b"from_high".to_vec()), "Higher author bytes should win"); let heads = store.get_heads(b"/tiebreak_key").unwrap(); assert_eq!(heads.len(), 2); assert_eq!(heads[0].value, b"from_high".to_vec(), "heads[0] should be winner"); assert_eq!(heads[0].author, high_node.public_key_bytes().to_vec()); let _ = std::fs::remove_file(path); } /// Test case for multi-node sync: 3 nodes create multi-heads, then merge, then sync to new node. /// /// Scenario: /// 1. Node A, B, C each write to key "/a" independently (creating 3 heads) /// 2. Node A does a final put to merge all heads /// 3. After merge, node A should have only 1 head /// 4. Simulate sync to new node D using SyncState diff /// 5. Node D should end up with same state as A (1 head, not 3) #[test] fn test_multinode_sync_after_merge() { let path_a = temp_db_path("multinode_a"); let path_d = temp_db_path("multinode_d"); let _ = std::fs::remove_file(&path_a); let _ = std::fs::remove_file(&path_d); // Create stores let store_a = Store::open(&path_a).unwrap(); let store_d = Store::open(&path_d).unwrap(); // Create 3 nodes (virtual peers) let node_a = NodeIdentity::generate(); let node_b = NodeIdentity::generate(); let node_c = NodeIdentity::generate(); let clock = MockClock::new(1000); // 1. Each node writes to "/a" independently (simulating offline concurrent writes) // Node A: seq 1 let entry_a = EntryBuilder::new(1, HLC::now_with_clock(&clock)) .store_id(TEST_STORE.to_vec()) .prev_hash([0u8; 32].to_vec()) .put("/a", b"from_a".to_vec()) .sign(&node_a); store_a.apply_entry(&entry_a).unwrap(); // Node B: seq 1 (different author, same key - creates fork) let entry_b = EntryBuilder::new(1, HLC::now_with_clock(&clock)) .store_id(TEST_STORE.to_vec()) .prev_hash([0u8; 32].to_vec()) .put("/a", b"from_b".to_vec()) .sign(&node_b); store_a.apply_entry(&entry_b).unwrap(); // Node C: seq 1 (third author, same key - creates third fork) let entry_c = EntryBuilder::new(1, HLC::now_with_clock(&clock)) .store_id(TEST_STORE.to_vec()) .prev_hash([0u8; 32].to_vec()) .put("/a", b"from_c".to_vec()) .sign(&node_c); store_a.apply_entry(&entry_c).unwrap(); // After applying all 3 entries, store_a has 3 heads for "/a" let heads_before_merge = store_a.get_heads(b"/a").unwrap(); assert_eq!(heads_before_merge.len(), 3, "Should have 3 heads before merge"); // 2. Node A does a final put referencing all heads (merge) // Get the hashes of all current heads as parent_hashes let parent_hashes: Vec> = heads_before_merge.iter() .map(|h| h.hash.clone()) .collect(); let merge_entry = EntryBuilder::new(2, HLC::now_with_clock(&clock)) .store_id(TEST_STORE.to_vec()) .prev_hash(hash_signed_entry(&entry_a).to_vec()) // Continues A's chain .parent_hashes(parent_hashes) // References all heads .put("/a", b"merged".to_vec()) .sign(&node_a); store_a.apply_entry(&merge_entry).unwrap(); // After merge, should have only 1 head let heads_after_merge = store_a.get_heads(b"/a").unwrap(); assert_eq!(heads_after_merge.len(), 1, "Should have 1 head after merge"); assert_eq!(heads_after_merge[0].value, b"merged"); // 3. Get sync state from store_a let sync_state_a = store_a.sync_state().unwrap(); println!("Store A sync state:"); for (author, info) in sync_state_a.authors() { println!(" author {:?}: seq={}, heads={:?}", hex::encode(&author[..4]), info.seq, info.heads.iter().map(|h| hex::encode(&h[..4])).collect::>()); } // 4. Store D is empty, compute diff let sync_state_d = store_d.sync_state().unwrap(); let missing = sync_state_d.diff(&sync_state_a); println!("Missing ranges: {:?}", missing.len()); for m in &missing { println!(" author {:?}: from_seq={}, to_seq={}", hex::encode(&m.author[..4]), m.from_seq, m.to_seq); } // We should get missing ranges for all authors that have entries assert!(!missing.is_empty(), "Should have missing entries to sync"); // 5. Apply all entries to store_d (simulating sync) // In a real sync, we'd read entries from logs, but for this test, // we just apply the same entries in order store_d.apply_entry(&entry_a).unwrap(); store_d.apply_entry(&entry_b).unwrap(); store_d.apply_entry(&entry_c).unwrap(); store_d.apply_entry(&merge_entry).unwrap(); // 6. Check state on store_d let heads_d = store_d.get_heads(b"/a").unwrap(); println!("Store D heads count: {}", heads_d.len()); for (i, h) in heads_d.iter().enumerate() { println!(" head[{}]: value={:?}, author={}", i, String::from_utf8_lossy(&h.value), hex::encode(&h.author[..4])); } // BUG CHECK: Store D should have same state as Store A (1 head, not 3) assert_eq!(heads_d.len(), 1, "BUG: Store D should have 1 head (merged) but has {} heads", heads_d.len()); assert_eq!(heads_d[0].value, b"merged"); let _ = std::fs::remove_file(&path_a); let _ = std::fs::remove_file(&path_d); } /// Test what happens when entries are applied in "wrong" order. /// This simulates the real sync bug where: /// - Sync iterates by author /// - Author A's entries (including merge) are sent first /// - Author B and C's entries are sent after /// - The merge entry arrives BEFORE the entries it merges! #[test] fn test_multinode_sync_wrong_order() { let path = temp_db_path("wrongorder"); let _ = std::fs::remove_file(&path); let store = Store::open(&path).unwrap(); // Create 3 nodes let node_a = NodeIdentity::generate(); let node_b = NodeIdentity::generate(); let node_c = NodeIdentity::generate(); let clock = MockClock::new(1000); // Create entries (same as before) let entry_a = EntryBuilder::new(1, HLC::now_with_clock(&clock)) .store_id(TEST_STORE.to_vec()) .prev_hash([0u8; 32].to_vec()) .put("/a", b"from_a".to_vec()) .sign(&node_a); let entry_b = EntryBuilder::new(1, HLC::now_with_clock(&clock)) .store_id(TEST_STORE.to_vec()) .prev_hash([0u8; 32].to_vec()) .put("/a", b"from_b".to_vec()) .sign(&node_b); let entry_c = EntryBuilder::new(1, HLC::now_with_clock(&clock)) .store_id(TEST_STORE.to_vec()) .prev_hash([0u8; 32].to_vec()) .put("/a", b"from_c".to_vec()) .sign(&node_c); // We need the hashes for parent_hashes - compute them let hash_a = hash_signed_entry(&entry_a); let hash_b = hash_signed_entry(&entry_b); let hash_c = hash_signed_entry(&entry_c); let merge_entry = EntryBuilder::new(2, HLC::now_with_clock(&clock)) .store_id(TEST_STORE.to_vec()) .prev_hash(hash_a.to_vec()) .parent_hashes(vec![hash_a.to_vec(), hash_b.to_vec(), hash_c.to_vec()]) .put("/a", b"merged".to_vec()) .sign(&node_a); // Apply in WRONG order: A's chain first (entry_a + merge), then B, then C // This is what happens in sync when iterating by author println!("Applying entry_a (A seq 1)..."); store.apply_entry(&entry_a).unwrap(); println!("Applying merge_entry (A seq 2) BEFORE B and C..."); store.apply_entry(&merge_entry).unwrap(); println!("Applying entry_b (B seq 1)..."); store.apply_entry(&entry_b).unwrap(); println!("Applying entry_c (C seq 1)..."); store.apply_entry(&entry_c).unwrap(); // Check final state let heads = store.get_heads(b"/a").unwrap(); println!("Final heads count: {}", heads.len()); for (i, h) in heads.iter().enumerate() { println!(" head[{}]: value={:?}", i, String::from_utf8_lossy(&h.value)); } assert_eq!(heads.len(), 3, "Wrong order application creates 3 heads (expected - sync handles ordering)"); let _ = std::fs::remove_file(&path); } #[test] fn test_list_by_prefix_filters_tombstones() { let path = temp_db_path("list_tombstones"); let _ = std::fs::remove_file(&path); let store = Store::open(&path).unwrap(); let node = NodeIdentity::generate(); // Create a key under /test/ prefix let clock1 = MockClock::new(1000); let entry1 = EntryBuilder::new(1, HLC::now_with_clock(&clock1)) .store_id(TEST_STORE.to_vec()) .prev_hash([0u8; 32].to_vec()) .put("/test/key1", b"value1".to_vec()) .sign(&node); store.apply_entry(&entry1).unwrap(); // Create another key let clock2 = MockClock::new(2000); let entry2 = EntryBuilder::new(2, HLC::now_with_clock(&clock2)) .store_id(TEST_STORE.to_vec()) .prev_hash(hash_signed_entry(&entry1).to_vec()) .put("/test/key2", b"value2".to_vec()) .sign(&node); store.apply_entry(&entry2).unwrap(); // Delete key1 let clock3 = MockClock::new(3000); let entry3 = EntryBuilder::new(3, HLC::now_with_clock(&clock3)) .store_id(TEST_STORE.to_vec()) .prev_hash(hash_signed_entry(&entry2).to_vec()) .parent_hashes(vec![hash_signed_entry(&entry1).to_vec()]) .delete(b"/test/key1") .sign(&node); store.apply_entry(&entry3).unwrap(); // list_by_prefix without include_deleted should only show key2 let entries = store.list_by_prefix(b"/test/", false).unwrap(); assert_eq!(entries.len(), 1); assert_eq!(entries[0].0, b"/test/key2"); // list_by_prefix with include_deleted should show both (key1 as tombstone) let entries_all = store.list_by_prefix(b"/test/", true).unwrap(); assert_eq!(entries_all.len(), 2); // Verify list_all also respects the flag let all_entries = store.list_all(false).unwrap(); assert_eq!(all_entries.len(), 1); let all_entries_incl_deleted = store.list_all(true).unwrap(); assert_eq!(all_entries_incl_deleted.len(), 2); let _ = std::fs::remove_file(&path); } }