feat: Introduce signed, verifiable entries with protobuf definitions, cryptographic signing, and hashing, updating the data model and architecture.

This commit is contained in:
2025-12-20 14:29:08 +01:00
parent 2f48483d0e
commit f153cba267
10 changed files with 667 additions and 6 deletions
+2
View File
@@ -37,6 +37,8 @@ tokio = { version = "1", features = ["full"] }
thiserror = "2"
tracing = "0.1"
bytes = "1"
dirs = "5"
blake3 = "1"
# Testing
tokio-test = "0.4"
+9 -5
View File
@@ -52,6 +52,10 @@ Networking modes:
- `disabled`: Permanently removed from mesh.
- Sync priority: Low-power clients prefer peers marked as `server` or recently active.
Future:
- Key rotation: Allow nodes to rotate their keypair. Old key signs a "rotation" entry pointing to new key.
- Secure storage: Support platform keystores (macOS Keychain, Linux Secret Service, TPM) for private key protection.
### Data Model
- Keys are flat strings using path conventions (e.g., `/nodes/{pubkey}`, `/config/sync/interval`).
@@ -95,12 +99,12 @@ data/
#### state.db Tables (redb)
```
Table Key Value Purpose
Table Key Value Purpose
─────────────────────────────────────────────────────────────────────────────
kv String (path) Vec<u8> Replicated key-value data
vector_clocks [u8; 32] (author_id) u64 (max seq) Track sync state per author
entry_index (author_id, seq) u64 (offset) Fast entry lookup by position
meta String Vec<u8> System metadata (own_seq, watermark, etc.)
kv String (path) Vec<u8> Replicated key-value data
vector_clocks [u8; 32] (author_id) (u64 seq, [u8; 32] hash) Track sync state + chain verification
entry_index (author_id, seq) u64 (offset) Fast entry lookup by position
meta String Vec<u8> System metadata (own_seq, watermark, etc.)
```
### Watermarks
+5
View File
@@ -11,6 +11,11 @@ rand = { workspace = true }
prost = { workspace = true }
thiserror = { workspace = true }
bytes = { workspace = true }
dirs = { workspace = true }
blake3 = { workspace = true }
[build-dependencies]
prost-build = { workspace = true }
[dev-dependencies]
tokio-test = { workspace = true }
+9
View File
@@ -0,0 +1,9 @@
use std::io::Result;
fn main() -> Result<()> {
prost_build::compile_protos(
&["../proto/lattice.proto"],
&["../proto/"],
)?;
Ok(())
}
+111
View File
@@ -0,0 +1,111 @@
//! Data directory management
//!
//! Provides platform-specific paths for Lattice data storage:
//! - `identity.key` — Ed25519 private key
//! - `logs/` — Append-only log files per author
//! - `state.db` — KV snapshot and indexes
use std::path::{Path, PathBuf};
const APP_NAME: &str = "lattice";
/// Data directory configuration.
///
/// Handles paths for:
/// - `identity.key` — node's private key
/// - `logs/{author_id}.log` — per-author log files
/// - `state.db` — redb database
#[derive(Debug, Clone)]
pub struct DataDir {
base: PathBuf,
}
impl DataDir {
/// Create a DataDir with a custom base path.
pub fn new(base: impl Into<PathBuf>) -> Self {
Self { base: base.into() }
}
/// Create a DataDir using the platform-specific data directory.
///
/// - Linux: `~/.local/share/lattice/`
/// - macOS: `~/Library/Application Support/lattice/`
/// - Windows: `C:\Users\<user>\AppData\Roaming\lattice\`
pub fn default_location() -> Option<Self> {
dirs::data_dir().map(|d| Self::new(d.join(APP_NAME)))
}
/// Get the base directory path.
pub fn base(&self) -> &Path {
&self.base
}
/// Get the path to the identity key file.
pub fn identity_key(&self) -> PathBuf {
self.base.join("identity.key")
}
/// Get the path to the logs directory.
pub fn logs_dir(&self) -> PathBuf {
self.base.join("logs")
}
/// Get the path to a specific author's log file.
pub fn log_file(&self, author_id_hex: &str) -> PathBuf {
self.logs_dir().join(format!("{}.log", author_id_hex))
}
/// Get the path to the state database.
pub fn state_db(&self) -> PathBuf {
self.base.join("state.db")
}
/// Ensure all required directories exist.
pub fn ensure_dirs(&self) -> std::io::Result<()> {
std::fs::create_dir_all(&self.base)?;
std::fs::create_dir_all(self.logs_dir())?;
Ok(())
}
}
impl Default for DataDir {
fn default() -> Self {
Self::default_location().unwrap_or_else(|| Self::new("./data"))
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_custom_path() {
let dd = DataDir::new("/custom/path");
assert_eq!(dd.base(), Path::new("/custom/path"));
assert_eq!(dd.identity_key(), PathBuf::from("/custom/path/identity.key"));
assert_eq!(dd.logs_dir(), PathBuf::from("/custom/path/logs"));
assert_eq!(dd.state_db(), PathBuf::from("/custom/path/state.db"));
}
#[test]
fn test_log_file_path() {
let dd = DataDir::new("/data");
let path = dd.log_file("abc123");
assert_eq!(path, PathBuf::from("/data/logs/abc123.log"));
}
#[test]
fn test_default_location_exists() {
// On most systems, default_location should return Some
let location = DataDir::default_location();
// Just verify it doesn't panic - actual path varies by platform
assert!(location.is_some() || true);
}
#[test]
fn test_default_impl() {
let dd = DataDir::default();
// Should either be platform default or ./data fallback
assert!(dd.base().to_str().is_some());
}
}
+8
View File
@@ -7,6 +7,9 @@
//! - **VectorClock**: Causality tracking for reconciliation
//! - **HLC**: Hybrid Logical Clock for ordering
//! - **Clock**: Time abstraction for testability
//! - **Proto**: Generated protobuf types from lattice.proto
//! - **DataDir**: Platform-specific data directory paths
//! - **SignedEntry**: Entry creation, signing, and verification
pub mod node;
pub mod sigchain;
@@ -14,6 +17,9 @@ pub mod entry;
pub mod vector_clock;
pub mod hlc;
pub mod clock;
pub mod proto;
pub mod data_dir;
pub mod signed_entry;
pub use node::Node;
pub use sigchain::SigChain;
@@ -21,3 +27,5 @@ pub use entry::Entry;
pub use vector_clock::VectorClock;
pub use hlc::HLC;
pub use clock::{Clock, SystemClock, MockClock};
pub use data_dir::DataDir;
pub use signed_entry::{EntryBuilder, sign_entry, verify_signed_entry, hash_signed_entry};
+186 -1
View File
@@ -1,7 +1,28 @@
//! Node identity and cryptographic keys
//!
//! Each node has an Ed25519 keypair:
//! - Private key: stored locally in `identity.key` (never replicated)
//! - Public key: serves as the node's identity (32 bytes)
use ed25519_dalek::{SigningKey, VerifyingKey};
use ed25519_dalek::{Signature, Signer, SigningKey, Verifier, VerifyingKey};
use rand::rngs::OsRng;
use std::fs;
use std::io::{self, Read, Write};
use std::path::Path;
use thiserror::Error;
/// Errors that can occur during node operations
#[derive(Error, Debug)]
pub enum NodeError {
#[error("IO error: {0}")]
Io(#[from] io::Error),
#[error("Invalid key length: expected 32 bytes, got {0}")]
InvalidKeyLength(usize),
#[error("Invalid signature")]
InvalidSignature,
}
/// A node in the Lattice mesh.
///
@@ -18,13 +39,177 @@ impl Node {
Self { signing_key }
}
/// Create a node from an existing signing key.
pub fn from_signing_key(signing_key: SigningKey) -> Self {
Self { signing_key }
}
/// Load a node's identity from a key file, or generate and save if it doesn't exist.
pub fn load_or_generate(path: impl AsRef<Path>) -> Result<Self, NodeError> {
let path = path.as_ref();
if path.exists() {
Self::load(path)
} else {
let node = Self::generate();
node.save(path)?;
Ok(node)
}
}
/// Load a node's identity from a key file.
pub fn load(path: impl AsRef<Path>) -> Result<Self, NodeError> {
let mut file = fs::File::open(path)?;
let mut bytes = Vec::new();
file.read_to_end(&mut bytes)?;
if bytes.len() != 32 {
return Err(NodeError::InvalidKeyLength(bytes.len()));
}
let key_bytes: [u8; 32] = bytes.try_into().unwrap();
let signing_key = SigningKey::from_bytes(&key_bytes);
Ok(Self { signing_key })
}
/// Save the node's private key to a file.
pub fn save(&self, path: impl AsRef<Path>) -> Result<(), NodeError> {
let path = path.as_ref();
// Create parent directories if they don't exist
if let Some(parent) = path.parent() {
fs::create_dir_all(parent)?;
}
let mut file = fs::File::create(path)?;
file.write_all(self.signing_key.as_bytes())?;
Ok(())
}
/// Get the node's public key (identity).
pub fn public_key(&self) -> VerifyingKey {
self.signing_key.verifying_key()
}
/// Get the node's public key as bytes (32 bytes).
pub fn public_key_bytes(&self) -> [u8; 32] {
self.signing_key.verifying_key().to_bytes()
}
/// Get the signing key for creating signatures.
pub fn signing_key(&self) -> &SigningKey {
&self.signing_key
}
/// Sign a message.
pub fn sign(&self, message: &[u8]) -> Signature {
self.signing_key.sign(message)
}
/// Verify a signature against this node's public key.
pub fn verify(&self, message: &[u8], signature: &Signature) -> Result<(), NodeError> {
self.public_key()
.verify(message, signature)
.map_err(|_| NodeError::InvalidSignature)
}
/// Verify a signature using a raw public key.
pub fn verify_with_key(
public_key: &VerifyingKey,
message: &[u8],
signature: &Signature,
) -> Result<(), NodeError> {
public_key
.verify(message, signature)
.map_err(|_| NodeError::InvalidSignature)
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::env::temp_dir;
#[test]
fn test_generate() {
let node = Node::generate();
let pk = node.public_key_bytes();
assert_eq!(pk.len(), 32);
}
#[test]
fn test_sign_and_verify() {
let node = Node::generate();
let message = b"hello lattice";
let signature = node.sign(message);
assert!(node.verify(message, &signature).is_ok());
}
#[test]
fn test_verify_wrong_message() {
let node = Node::generate();
let signature = node.sign(b"original");
assert!(node.verify(b"tampered", &signature).is_err());
}
#[test]
fn test_verify_with_different_key() {
let node1 = Node::generate();
let node2 = Node::generate();
let signature = node1.sign(b"message");
assert!(node2.verify(b"message", &signature).is_err());
}
#[test]
fn test_save_and_load() {
let temp_path = temp_dir().join("lattice_test_identity.key");
// Generate and save
let node1 = Node::generate();
let pk1 = node1.public_key_bytes();
node1.save(&temp_path).unwrap();
// Load and verify same key
let node2 = Node::load(&temp_path).unwrap();
let pk2 = node2.public_key_bytes();
assert_eq!(pk1, pk2);
// Cleanup
fs::remove_file(&temp_path).ok();
}
#[test]
fn test_load_or_generate() {
let temp_path = temp_dir().join("lattice_test_identity2.key");
// Remove if exists
fs::remove_file(&temp_path).ok();
// First call: generates
let node1 = Node::load_or_generate(&temp_path).unwrap();
let pk1 = node1.public_key_bytes();
// Second call: loads existing
let node2 = Node::load_or_generate(&temp_path).unwrap();
let pk2 = node2.public_key_bytes();
assert_eq!(pk1, pk2);
// Cleanup
fs::remove_file(&temp_path).ok();
}
#[test]
fn test_verify_with_key_static() {
let node = Node::generate();
let pk = node.public_key();
let message = b"test message";
let signature = node.sign(message);
assert!(Node::verify_with_key(&pk, message, &signature).is_ok());
}
}
+76
View File
@@ -0,0 +1,76 @@
//! Generated protobuf types for Lattice
//!
//! This module re-exports types generated from `proto/lattice.proto`
// Include the generated code from prost-build
include!(concat!(env!("OUT_DIR"), "/lattice.rs"));
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_hlc_roundtrip() {
let hlc = Hlc {
wall_time: 1234567890,
counter: 42,
};
// Encode
let mut buf = Vec::new();
prost::Message::encode(&hlc, &mut buf).unwrap();
// Decode
let decoded: Hlc = prost::Message::decode(&buf[..]).unwrap();
assert_eq!(decoded.wall_time, 1234567890);
assert_eq!(decoded.counter, 42);
}
#[test]
fn test_entry_with_ops() {
let entry = Entry {
version: 1,
prev_hash: vec![0u8; 32],
seq: 5,
timestamp: Some(Hlc {
wall_time: 1000,
counter: 0,
}),
ops: vec![
Operation {
op_type: Some(operation::OpType::Put(PutOp {
key: "/nodes/abc".to_string(),
value: b"hello".to_vec(),
})),
},
],
};
// Encode
let mut buf = Vec::new();
prost::Message::encode(&entry, &mut buf).unwrap();
// Decode
let decoded: Entry = prost::Message::decode(&buf[..]).unwrap();
assert_eq!(decoded.version, 1);
assert_eq!(decoded.seq, 5);
assert_eq!(decoded.ops.len(), 1);
}
#[test]
fn test_signed_entry() {
let signed = SignedEntry {
entry_bytes: vec![1, 2, 3, 4],
signature: vec![0u8; 64],
author_id: vec![0u8; 32],
};
let mut buf = Vec::new();
prost::Message::encode(&signed, &mut buf).unwrap();
let decoded: SignedEntry = prost::Message::decode(&buf[..]).unwrap();
assert_eq!(decoded.entry_bytes, vec![1, 2, 3, 4]);
}
}
+260
View File
@@ -0,0 +1,260 @@
//! Signed entry creation and verification
//!
//! Provides utilities for:
//! - Building Entry messages with operations
//! - Signing entries to create SignedEntry
//! - Verifying signatures
//! - Computing entry hashes for prev_hash linking
use crate::hlc::HLC;
use crate::node::{Node, NodeError};
use crate::proto::{Entry, Hlc, Operation, PutOp, DeleteOp, SignedEntry, operation};
use ed25519_dalek::{Signature, VerifyingKey};
use prost::Message;
use thiserror::Error;
/// Errors that can occur during entry operations
#[derive(Error, Debug)]
pub enum EntryError {
#[error("Signature verification failed: {0}")]
Signature(#[from] NodeError),
#[error("Proto decode error: {0}")]
Decode(#[from] prost::DecodeError),
#[error("Invalid signature length: expected 64 bytes, got {0}")]
InvalidSignatureLength(usize),
#[error("Invalid public key length: expected 32 bytes, got {0}")]
InvalidPublicKeyLength(usize),
}
/// Builder for creating Entry messages
pub struct EntryBuilder {
version: u32,
prev_hash: Vec<u8>,
seq: u64,
timestamp: HLC,
ops: Vec<Operation>,
}
impl EntryBuilder {
/// Create a new entry builder with the given sequence number and timestamp
pub fn new(seq: u64, timestamp: HLC) -> Self {
Self {
version: 1,
prev_hash: vec![0u8; 32], // Genesis or will be set
seq,
timestamp,
ops: Vec::new(),
}
}
/// Set the previous entry hash (for chaining)
pub fn prev_hash(mut self, hash: impl Into<Vec<u8>>) -> Self {
self.prev_hash = hash.into();
self
}
/// Add a Put operation
pub fn put(mut self, key: impl Into<String>, value: impl Into<Vec<u8>>) -> Self {
self.ops.push(Operation {
op_type: Some(operation::OpType::Put(PutOp {
key: key.into(),
value: value.into(),
})),
});
self
}
/// Add a Delete operation
pub fn delete(mut self, key: impl Into<String>) -> Self {
self.ops.push(Operation {
op_type: Some(operation::OpType::Delete(DeleteOp {
key: key.into(),
})),
});
self
}
/// Build the Entry proto message
pub fn build(self) -> Entry {
Entry {
version: self.version,
prev_hash: self.prev_hash,
seq: self.seq,
timestamp: Some(Hlc {
wall_time: self.timestamp.wall_time,
counter: self.timestamp.counter,
}),
ops: self.ops,
}
}
/// Build and sign the entry, returning a SignedEntry
pub fn sign(self, node: &Node) -> SignedEntry {
let entry = self.build();
sign_entry(&entry, node)
}
}
/// Sign an Entry to create a SignedEntry
pub fn sign_entry(entry: &Entry, node: &Node) -> SignedEntry {
let entry_bytes = entry.encode_to_vec();
let signature = node.sign(&entry_bytes);
SignedEntry {
entry_bytes,
signature: signature.to_bytes().to_vec(),
author_id: node.public_key_bytes().to_vec(),
}
}
/// Verify a SignedEntry's signature
pub fn verify_signed_entry(signed: &SignedEntry) -> Result<Entry, EntryError> {
// Parse public key
if signed.author_id.len() != 32 {
return Err(EntryError::InvalidPublicKeyLength(signed.author_id.len()));
}
let pk_bytes: [u8; 32] = signed.author_id.clone().try_into().unwrap();
let public_key = VerifyingKey::from_bytes(&pk_bytes)
.map_err(|_| NodeError::InvalidSignature)?;
// Parse signature
if signed.signature.len() != 64 {
return Err(EntryError::InvalidSignatureLength(signed.signature.len()));
}
let sig_bytes: [u8; 64] = signed.signature.clone().try_into().unwrap();
let signature = Signature::from_bytes(&sig_bytes);
// Verify
Node::verify_with_key(&public_key, &signed.entry_bytes, &signature)?;
// Decode entry
let entry = Entry::decode(&signed.entry_bytes[..])?;
Ok(entry)
}
/// Compute the BLAKE3 hash of a SignedEntry (for prev_hash linking)
pub fn hash_signed_entry(signed: &SignedEntry) -> [u8; 32] {
let bytes = signed.encode_to_vec();
blake3::hash(&bytes).into()
}
/// Compute the BLAKE3 hash of entry_bytes (alternative for lighter hashing)
pub fn hash_entry_bytes(entry_bytes: &[u8]) -> [u8; 32] {
blake3::hash(entry_bytes).into()
}
#[cfg(test)]
mod tests {
use super::*;
use crate::clock::MockClock;
#[test]
fn test_entry_builder() {
let clock = MockClock::new(1000);
let hlc = HLC::now_with_clock(&clock);
let entry = EntryBuilder::new(1, hlc)
.put("/test/key", b"value".to_vec())
.delete("/test/old")
.build();
assert_eq!(entry.version, 1);
assert_eq!(entry.seq, 1);
assert_eq!(entry.ops.len(), 2);
}
#[test]
fn test_sign_and_verify() {
let node = Node::generate();
let clock = MockClock::new(1000);
let hlc = HLC::now_with_clock(&clock);
let signed = EntryBuilder::new(1, hlc)
.put("/nodes/abc", b"test".to_vec())
.sign(&node);
assert_eq!(signed.author_id.len(), 32);
assert_eq!(signed.signature.len(), 64);
// Verify
let entry = verify_signed_entry(&signed).unwrap();
assert_eq!(entry.seq, 1);
assert_eq!(entry.ops.len(), 1);
}
#[test]
fn test_verify_tampered_fails() {
let node = Node::generate();
let clock = MockClock::new(1000);
let hlc = HLC::now_with_clock(&clock);
let mut signed = EntryBuilder::new(1, hlc)
.put("/key", b"value".to_vec())
.sign(&node);
// Tamper with entry bytes
signed.entry_bytes[0] ^= 0xFF;
assert!(verify_signed_entry(&signed).is_err());
}
#[test]
fn test_verify_wrong_key_fails() {
let node1 = Node::generate();
let node2 = Node::generate();
let clock = MockClock::new(1000);
let hlc = HLC::now_with_clock(&clock);
let mut signed = EntryBuilder::new(1, hlc)
.put("/key", b"value".to_vec())
.sign(&node1);
// Replace author with different key
signed.author_id = node2.public_key_bytes().to_vec();
assert!(verify_signed_entry(&signed).is_err());
}
#[test]
fn test_hash_signed_entry() {
let node = Node::generate();
let clock = MockClock::new(1000);
let hlc = HLC::now_with_clock(&clock);
let signed = EntryBuilder::new(1, hlc)
.put("/key", b"value".to_vec())
.sign(&node);
let hash = hash_signed_entry(&signed);
assert_eq!(hash.len(), 32);
// Same entry should produce same hash
let hash2 = hash_signed_entry(&signed);
assert_eq!(hash, hash2);
}
#[test]
fn test_prev_hash_chaining() {
let node = Node::generate();
let clock = MockClock::new(1000);
// First entry
let entry1 = EntryBuilder::new(1, HLC::now_with_clock(&clock))
.put("/key", b"v1".to_vec())
.sign(&node);
let hash1 = hash_signed_entry(&entry1);
// Second entry links to first
let entry2 = EntryBuilder::new(2, HLC::now_with_clock(&clock))
.prev_hash(hash1)
.put("/key", b"v2".to_vec())
.sign(&node);
let decoded = verify_signed_entry(&entry2).unwrap();
assert_eq!(decoded.prev_hash, hash1.to_vec());
}
}
+1
View File
@@ -63,4 +63,5 @@ message SyncState {
message Frontier {
bytes author_id = 1; // Ed25519 public key (32 bytes)
uint64 max_seq = 2; // Highest sequence number seen from this author
bytes last_hash = 3; // Hash of the last entry (for chain verification)
}