Fragile hare

2025-09-09 22:23:20 +03:00
5 changed files with 6 additions and 1289 deletions
@@ -1,6 +1 @@
 .claude/
 data/
 data*/
 *.yaml
 !config.yaml
 kvs
@@ -1,406 +0,0 @@
 # KVS - Distributed Key-Value Store
 A minimalistic, clustered key-value database system written in Go that prioritizes **availability** and **partition tolerance** over strong consistency. KVS implements a gossip-style membership protocol with sophisticated conflict resolution for eventually consistent distributed storage.
 ## 🚀 Key Features
 - **Hierarchical Keys**: Support for structured paths (e.g., `/home/room/closet/socks`)
 - **Eventual Consistency**: Local operations are fast, replication happens in background
 - **Gossip Protocol**: Decentralized node discovery and failure detection
 - **Sophisticated Conflict Resolution**: Majority vote with oldest-node tie-breaking
 - **Local-First Truth**: All operations work locally first, sync globally later
 - **Read-Only Mode**: Configurable mode for reducing write load
 - **Gradual Bootstrapping**: New nodes integrate smoothly without overwhelming cluster
 - **Zero Dependencies**: Single binary with embedded BadgerDB storage
 ## 🏗️ Architecture
 ```
 ┌─────────────────┐    ┌─────────────────┐    ┌─────────────────┐
 │     Node A      │    │     Node B      │    │     Node C      │
 │  (Go Service)   │    │  (Go Service)   │    │  (Go Service)   │
 │                 │    │                 │    │                 │
 │ ┌─────────────┐ │    │ ┌─────────────┐ │    │ ┌─────────────┐ │
 │ │ HTTP Server │ │◄──►│ │ HTTP Server │ │◄──►│ │ HTTP Server │ │
 │ │    (API)    │ │    │ │    (API)    │ │    │ │    (API)    │ │
 │ └─────────────┘ │    │ └─────────────┘ │    │ └─────────────┘ │
 │ ┌─────────────┐ │    │ ┌─────────────┐ │    │ ┌─────────────┐ │
 │ │   Gossip    │ │◄──►│ │   Gossip    │ │◄──►│ │   Gossip    │ │
 │ │  Protocol   │ │    │ │  Protocol   │ │    │ │  Protocol   │ │
 │ └─────────────┘ │    │ └─────────────┘ │    │ └─────────────┘ │
 │ ┌─────────────┐ │    │ ┌─────────────┐ │    │ ┌─────────────┐ │
 │ │  BadgerDB   │ │    │ │  BadgerDB   │ │    │ │  BadgerDB   │ │
 │ │ (Local KV)  │ │    │ │ (Local KV)  │ │    │ │ (Local KV)  │ │
 │ └─────────────┘ │    │ └─────────────┘ │    │ └─────────────┘ │
 └─────────────────┘    └─────────────────┘    └─────────────────┘
        ▲
        │
    External Clients
 ```
 Each node is fully autonomous and communicates with peers via HTTP REST API for both external client requests and internal cluster operations.
 ## 📦 Installation
 ### Prerequisites
 - Go 1.21 or higher
 ### Build from Source
 ```bash
 git clone <repository-url>
 cd kvs
 go mod tidy
 go build -o kvs .
 ```
 ### Quick Test
 ```bash
 # Start standalone node
 ./kvs
 # Test the API
 curl http://localhost:8080/health
 ```
 ## ⚙️ Configuration
 KVS uses YAML configuration files. On first run, a default `config.yaml` is automatically generated:
 ```yaml
 node_id: "hostname"           # Unique node identifier
 bind_address: "127.0.0.1"     # IP address to bind to
 port: 8080                    # HTTP port
 data_dir: "./data"            # Directory for BadgerDB storage
 seed_nodes: []                # List of seed nodes for cluster joining
 read_only: false              # Enable read-only mode
 log_level: "info"             # Logging level (debug, info, warn, error)
 gossip_interval_min: 60       # Min gossip interval (seconds)
 gossip_interval_max: 120      # Max gossip interval (seconds)
 sync_interval: 300            # Regular sync interval (seconds)
 catchup_interval: 120         # Catch-up sync interval (seconds)
 bootstrap_max_age_hours: 720  # Max age for bootstrap sync (hours)
 throttle_delay_ms: 100        # Delay between sync requests (ms)
 fetch_delay_ms: 50            # Delay between data fetches (ms)
 ```
 ### Custom Configuration
 ```bash
 # Use custom config file
 ./kvs /path/to/custom-config.yaml
 ```
 ## 🔌 REST API
 ### Data Operations (`/kv/`)
 #### Store Data
 ```bash
 PUT /kv/{path}
 Content-Type: application/json
 curl -X PUT http://localhost:8080/kv/users/john/profile \
  -H "Content-Type: application/json" \
  -d '{"name":"John Doe","age":30,"email":"john@example.com"}'
 # Response
 {
  "uuid": "a1b2c3d4-e5f6-7890-1234-567890abcdef",
  "timestamp": 1672531200000
 }
 ```
 #### Retrieve Data
 ```bash
 GET /kv/{path}
 curl http://localhost:8080/kv/users/john/profile
 # Response
 {
  "name": "John Doe",
  "age": 30,
  "email": "john@example.com"
 }
 ```
 #### Delete Data
 ```bash
 DELETE /kv/{path}
 curl -X DELETE http://localhost:8080/kv/users/john/profile
 # Returns: 204 No Content
 ```
 ### Cluster Operations (`/members/`)
 #### View Cluster Members
 ```bash
 GET /members/
 curl http://localhost:8080/members/
 # Response
 [
  {
    "id": "node-alpha",
    "address": "192.168.1.10:8080",
    "last_seen": 1672531200000,
    "joined_timestamp": 1672530000000
  }
 ]
 ```
 #### Join Cluster (Internal)
 ```bash
 POST /members/join
 # Used internally during bootstrap process
 ```
 #### Health Check
 ```bash
 GET /health
 curl http://localhost:8080/health
 # Response
 {
  "status": "ok",
  "mode": "normal",
  "member_count": 2,
  "node_id": "node-alpha"
 }
 ```
 ## 🏘️ Cluster Setup
 ### Single Node (Standalone)
 ```bash
 # config.yaml
 node_id: "standalone"
 port: 8080
 seed_nodes: []  # Empty = standalone mode
 ```
 ### Multi-Node Cluster
 #### Node 1 (Bootstrap Node)
 ```bash
 # node1.yaml
 node_id: "node1"
 port: 8081
 seed_nodes: []  # First node, no seeds needed
 ```
 #### Node 2 (Joins via Node 1)
 ```bash
 # node2.yaml  
 node_id: "node2"
 port: 8082
 seed_nodes: ["127.0.0.1:8081"]  # Points to node1
 ```
 #### Node 3 (Joins via Node 1 & 2)
 ```bash
 # node3.yaml
 node_id: "node3" 
 port: 8083
 seed_nodes: ["127.0.0.1:8081", "127.0.0.1:8082"]  # Multiple seeds for reliability
 ```
 #### Start the Cluster
 ```bash
 # Terminal 1
 ./kvs node1.yaml
 # Terminal 2 (wait a few seconds)
 ./kvs node2.yaml
 # Terminal 3 (wait a few seconds)
 ./kvs node3.yaml
 ```
 ## 🔄 How It Works
 ### Gossip Protocol
 - Nodes randomly select 1-3 peers every 1-2 minutes for membership exchange
 - Failed nodes are detected via timeout (5 minutes) and removed (10 minutes)
 - New members are automatically discovered and added to local member lists
 ### Data Synchronization  
 - **Regular Sync**: Every 5 minutes, nodes compare their latest 15 data items with a random peer
 - **Catch-up Sync**: Every 2 minutes when nodes detect they're significantly behind
 - **Bootstrap Sync**: New nodes gradually fetch historical data up to 30 days old
 ### Conflict Resolution
 When two nodes have different data for the same key with identical timestamps:
 1. **Majority Vote**: Query all healthy cluster members for their version
 2. **Tie-Breaker**: If votes are tied, the version from the oldest node (earliest `joined_timestamp`) wins
 3. **Automatic Resolution**: Losing nodes automatically fetch and store the winning version
 ### Operational Modes
 - **Normal**: Full read/write capabilities
 - **Read-Only**: Rejects external writes but accepts internal replication
 - **Syncing**: Temporary mode during bootstrap, rejects external writes
 ## 🛠️ Development
 ### Running Tests
 ```bash
 # Basic functionality test
 go build -o kvs .
 ./kvs &
 curl http://localhost:8080/health
 pkill kvs
 # Cluster test with provided configs
 ./kvs node1.yaml &
 ./kvs node2.yaml &  
 ./kvs node3.yaml &
 # Test data replication
 curl -X PUT http://localhost:8081/kv/test/data \
  -H "Content-Type: application/json" \
  -d '{"message":"hello world"}'
 # Wait 30+ seconds for sync, then check other nodes
 curl http://localhost:8082/kv/test/data
 curl http://localhost:8083/kv/test/data
 # Cleanup
 pkill kvs
 ```
 ### Conflict Resolution Testing
 ```bash
 # Create conflicting data scenario
 rm -rf data1 data2
 mkdir data1 data2
 go run test_conflict.go data1 data2
 # Start nodes with conflicting data
 ./kvs node1.yaml &
 ./kvs node2.yaml &
 # Watch logs for conflict resolution
 # Both nodes will converge to same data within ~30 seconds
 ```
 ### Project Structure
 ```
 kvs/
 ├── main.go              # Main application with all functionality
 ├── config.yaml          # Default configuration (auto-generated)
 ├── test_conflict.go     # Conflict resolution testing utility
 ├── node1.yaml           # Example cluster node config
 ├── node2.yaml           # Example cluster node config  
 ├── node3.yaml           # Example cluster node config
 ├── go.mod               # Go module dependencies
 ├── go.sum               # Go module checksums
 └── README.md            # This documentation
 ```
 ### Key Data Structures
 #### Stored Value Format
 ```go
 type StoredValue struct {
    UUID      string          `json:"uuid"`      // Unique version identifier
    Timestamp int64           `json:"timestamp"` // Unix timestamp (milliseconds)
    Data      json.RawMessage `json:"data"`      // Actual user JSON payload
 }
 ```
 #### BadgerDB Storage
 - **Main Key**: Direct path mapping (e.g., `users/john/profile`)
 - **Index Key**: `_ts:{timestamp}:{path}` for efficient time-based queries
 - **Values**: JSON-marshaled `StoredValue` structures
 ## 🔧 Configuration Options Explained
 | Setting | Description | Default | Notes |
 |---------|-------------|---------|-------|
 | `node_id` | Unique identifier for this node | hostname | Must be unique across cluster |
 | `bind_address` | IP address to bind HTTP server | "127.0.0.1" | Use 0.0.0.0 for external access |
 | `port` | HTTP port for API and cluster communication | 8080 | Must be accessible to peers |
 | `data_dir` | Directory for BadgerDB storage | "./data" | Will be created if doesn't exist |
 | `seed_nodes` | List of initial cluster nodes | [] | Empty = standalone mode |
 | `read_only` | Enable read-only mode | false | Accepts replication, rejects client writes |
 | `log_level` | Logging verbosity | "info" | debug/info/warn/error |
 | `gossip_interval_min/max` | Gossip frequency range | 60-120 sec | Randomized interval |
 | `sync_interval` | Regular sync frequency | 300 sec | How often to sync with peers |
 | `catchup_interval` | Catch-up sync frequency | 120 sec | Faster sync when behind |
 | `bootstrap_max_age_hours` | Max historical data to sync | 720 hours | 30 days default |
 | `throttle_delay_ms` | Delay between sync requests | 100 ms | Prevents overwhelming peers |
 | `fetch_delay_ms` | Delay between individual fetches | 50 ms | Rate limiting |
 ## 🚨 Important Notes
 ### Consistency Model
 - **Eventual Consistency**: Data will eventually be consistent across all nodes
 - **Local-First**: All operations succeed locally first, then replicate
 - **No Transactions**: Each key operation is independent
 - **Conflict Resolution**: Automatic resolution of timestamp collisions
 ### Network Requirements  
 - All nodes must be able to reach each other via HTTP
 - Firewalls must allow traffic on configured ports
 - IPv4 private networks supported (IPv6 not tested)
 ### Limitations
 - No authentication/authorization (planned for future releases)
 - No encryption in transit (use reverse proxy for TLS)
 - No cross-key transactions
 - No complex queries (key-based lookups only)
 - No data compression (planned for future releases)
 ### Performance Characteristics
 - **Read Latency**: ~1ms (local BadgerDB lookup)
 - **Write Latency**: ~5ms (local write + timestamp indexing)  
 - **Replication Lag**: 30 seconds - 5 minutes depending on sync cycles
 - **Memory Usage**: Minimal (BadgerDB handles caching efficiently)
 - **Disk Usage**: Raw JSON + metadata overhead (~20-30%)
 ## 🛡️ Production Considerations
 ### Deployment
 - Use systemd or similar for process management
 - Configure log rotation for JSON logs
 - Set up monitoring for `/health` endpoint
 - Use reverse proxy (nginx/traefik) for TLS and load balancing
 ### Monitoring
 - Monitor `/health` endpoint for node status
 - Watch logs for conflict resolution events
 - Track member count for cluster health
 - Monitor disk usage in data directories
 ### Backup Strategy
 - BadgerDB supports snapshots
 - Data directories can be backed up while running
 - Consider backing up multiple nodes for redundancy
 ### Scaling
 - Add new nodes by configuring existing cluster members as seeds
 - Remove nodes gracefully using `/members/leave` endpoint  
 - Cluster can operate with any number of nodes (tested with 2-10)
 ## 📄 License
 This project is licensed under the MIT License - see the LICENSE file for details.
 ## 🤝 Contributing
 1. Fork the repository
 2. Create a feature branch (`git checkout -b feature/amazing-feature`)
 3. Commit your changes (`git commit -m 'Add amazing feature'`)
 4. Push to the branch (`git push origin feature/amazing-feature`)
 5. Open a Pull Request
 ## 📚 Additional Resources
 - [BadgerDB Documentation](https://dgraph.io/docs/badger/)
 - [Gossip Protocol Paper](https://www.cs.cornell.edu/home/rvr/papers/flowgossip.pdf)
 - [Eventually Consistent Systems](https://www.allthingsdistributed.com/2008/12/eventually_consistent.html)
 ---
 **Built with ❤️ in Go** | **Powered by BadgerDB** | **Inspired by distributed systems theory**
@@ -1,14 +0,0 @@
 node_id: GALACTICA
 bind_address: 127.0.0.1
 port: 8080
 data_dir: ./data
 seed_nodes: []
 read_only: false
 log_level: info
 gossip_interval_min: 60
 gossip_interval_max: 120
 sync_interval: 300
 catchup_interval: 120
 bootstrap_max_age_hours: 720
 throttle_delay_ms: 100
 fetch_delay_ms: 50
@@ -1,11 +1,9 @@
 package main
 import (
 	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
 	"math/rand"
 	"net"
 	"net/http"
 	"os"
@@ -567,38 +565,6 @@ func (s *Server) pairsByTimeHandler(w http.ResponseWriter, r *http.Request) {
 	json.NewEncoder(w).Encode(pairs)
 }
 func (s *Server) gossipHandler(w http.ResponseWriter, r *http.Request) {
 	var remoteMemberList []Member
 	if err := json.NewDecoder(r.Body).Decode(&remoteMemberList); err != nil {
 		http.Error(w, "Bad Request", http.StatusBadRequest)
 		return
 	}
 	// Merge the received member list
 	s.mergeMemberList(remoteMemberList)
 	// Respond with our current member list
 	localMembers := s.getMembers()
 	gossipResponse := make([]Member, len(localMembers))
 	for i, member := range localMembers {
 		gossipResponse[i] = *member
 	}
 	// Add ourselves to the response
 	selfMember := Member{
 		ID:              s.config.NodeID,
 		Address:         fmt.Sprintf("%s:%d", s.config.BindAddress, s.config.Port),
 		LastSeen:        time.Now().UnixMilli(),
 		JoinedTimestamp: s.getJoinedTimestamp(),
 	}
 	gossipResponse = append(gossipResponse, selfMember)
 	w.Header().Set("Content-Type", "application/json")
 	json.NewEncoder(w).Encode(gossipResponse)
 	s.logger.WithField("remote_members", len(remoteMemberList)).Debug("Processed gossip request")
 }
 // Utility function to check if request is from cluster member
 func (s *Server) isClusterMember(remoteAddr string) bool {
 	host, _, err := net.SplitHostPort(remoteAddr)
@@ -635,7 +601,6 @@ func (s *Server) setupRoutes() *mux.Router {
 	router.HandleFunc("/members/", s.getMembersHandler).Methods("GET")
 	router.HandleFunc("/members/join", s.joinMemberHandler).Methods("POST")
 	router.HandleFunc("/members/leave", s.leaveMemberHandler).Methods("DELETE")
 	router.HandleFunc("/members/gossip", s.gossipHandler).Methods("POST")
 	router.HandleFunc("/members/pairs_by_time", s.pairsByTimeHandler).Methods("POST")
 	return router
@@ -688,738 +653,16 @@ func (s *Server) Stop() error {
 	return nil
 }
-// Background tasks (gossip, sync, etc.)
+// Background tasks placeholder (gossip, sync, etc.)
 func (s *Server) startBackgroundTasks() {
-	// Start gossip routine
+	// TODO: Implement gossip protocol
-	s.wg.Add(1)
+	// TODO: Implement periodic sync
-	go s.gossipRoutine()
+	// TODO: Implement catch-up sync
 	// Start sync routine
 	s.wg.Add(1)
 	go s.syncRoutine()
 }
-// Gossip routine - runs periodically to exchange member lists
+// Bootstrap placeholder
 func (s *Server) gossipRoutine() {
 	defer s.wg.Done()
 	for {
 		// Random interval between 1-2 minutes
 		minInterval := time.Duration(s.config.GossipIntervalMin) * time.Second
 		maxInterval := time.Duration(s.config.GossipIntervalMax) * time.Second
 		interval := minInterval + time.Duration(rand.Int63n(int64(maxInterval-minInterval)))
 		select {
 		case <-s.ctx.Done():
 			return
 		case <-time.After(interval):
 			s.performGossipRound()
 		}
 	}
 }
 // Perform a gossip round with random healthy peers
 func (s *Server) performGossipRound() {
 	members := s.getHealthyMembers()
 	if len(members) == 0 {
 		s.logger.Debug("No healthy members for gossip round")
 		return
 	}
 	// Select 1-3 random peers for gossip
 	maxPeers := 3
 	if len(members) < maxPeers {
 		maxPeers = len(members)
 	}
 	// Shuffle and select
 	rand.Shuffle(len(members), func(i, j int) {
 		members[i], members[j] = members[j], members[i]
 	})
 	selectedPeers := members[:rand.Intn(maxPeers)+1]
 	for _, peer := range selectedPeers {
 		go s.gossipWithPeer(peer)
 	}
 }
 // Gossip with a specific peer
 func (s *Server) gossipWithPeer(peer *Member) {
 	s.logger.WithField("peer", peer.Address).Debug("Starting gossip with peer")
 	// Get our current member list
 	localMembers := s.getMembers()
 	// Send our member list to the peer
 	gossipData := make([]Member, len(localMembers))
 	for i, member := range localMembers {
 		gossipData[i] = *member
 	}
 	// Add ourselves to the list
 	selfMember := Member{
 		ID:              s.config.NodeID,
 		Address:         fmt.Sprintf("%s:%d", s.config.BindAddress, s.config.Port),
 		LastSeen:        time.Now().UnixMilli(),
 		JoinedTimestamp: s.getJoinedTimestamp(),
 	}
 	gossipData = append(gossipData, selfMember)
 	jsonData, err := json.Marshal(gossipData)
 	if err != nil {
 		s.logger.WithError(err).Error("Failed to marshal gossip data")
 		return
 	}
 	// Send HTTP request to peer
 	client := &http.Client{Timeout: 5 * time.Second}
 	url := fmt.Sprintf("http://%s/members/gossip", peer.Address)
 	resp, err := client.Post(url, "application/json", bytes.NewBuffer(jsonData))
 	if err != nil {
 		s.logger.WithFields(logrus.Fields{
 			"peer":  peer.Address,
 			"error": err.Error(),
 		}).Warn("Failed to gossip with peer")
 		s.markPeerUnhealthy(peer.ID)
 		return
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK {
 		s.logger.WithFields(logrus.Fields{
 			"peer":   peer.Address,
 			"status": resp.StatusCode,
 		}).Warn("Gossip request failed")
 		s.markPeerUnhealthy(peer.ID)
 		return
 	}
 	// Process response - peer's member list
 	var remoteMemberList []Member
 	if err := json.NewDecoder(resp.Body).Decode(&remoteMemberList); err != nil {
 		s.logger.WithError(err).Error("Failed to decode gossip response")
 		return
 	}
 	// Merge remote member list with our local list
 	s.mergeMemberList(remoteMemberList)
 	// Update peer's last seen timestamp
 	s.updateMemberLastSeen(peer.ID, time.Now().UnixMilli())
 	s.logger.WithField("peer", peer.Address).Debug("Completed gossip with peer")
 }
 // Get healthy members (exclude those marked as down)
 func (s *Server) getHealthyMembers() []*Member {
 	s.membersMu.RLock()
 	defer s.membersMu.RUnlock()
 	now := time.Now().UnixMilli()
 	healthyMembers := make([]*Member, 0)
 	for _, member := range s.members {
 		// Consider member healthy if last seen within last 5 minutes
 		if now-member.LastSeen < 5*60*1000 {
 			healthyMembers = append(healthyMembers, member)
 		}
 	}
 	return healthyMembers
 }
 // Mark a peer as unhealthy
 func (s *Server) markPeerUnhealthy(nodeID string) {
 	s.membersMu.Lock()
 	defer s.membersMu.Unlock()
 	if member, exists := s.members[nodeID]; exists {
 		// Mark as last seen a long time ago to indicate unhealthy
 		member.LastSeen = time.Now().UnixMilli() - 10*60*1000 // 10 minutes ago
 		s.logger.WithField("node_id", nodeID).Warn("Marked peer as unhealthy")
 	}
 }
 // Update member's last seen timestamp
 func (s *Server) updateMemberLastSeen(nodeID string, timestamp int64) {
 	s.membersMu.Lock()
 	defer s.membersMu.Unlock()
 	if member, exists := s.members[nodeID]; exists {
 		member.LastSeen = timestamp
 	}
 }
 // Merge remote member list with local member list
 func (s *Server) mergeMemberList(remoteMembers []Member) {
 	s.membersMu.Lock()
 	defer s.membersMu.Unlock()
 	now := time.Now().UnixMilli()
 	for _, remoteMember := range remoteMembers {
 		// Skip ourselves
 		if remoteMember.ID == s.config.NodeID {
 			continue
 		}
 		if localMember, exists := s.members[remoteMember.ID]; exists {
 			// Update existing member
 			if remoteMember.LastSeen > localMember.LastSeen {
 				localMember.LastSeen = remoteMember.LastSeen
 			}
 			// Keep the earlier joined timestamp
 			if remoteMember.JoinedTimestamp < localMember.JoinedTimestamp {
 				localMember.JoinedTimestamp = remoteMember.JoinedTimestamp
 			}
 		} else {
 			// Add new member
 			newMember := &Member{
 				ID:              remoteMember.ID,
 				Address:         remoteMember.Address,
 				LastSeen:        remoteMember.LastSeen,
 				JoinedTimestamp: remoteMember.JoinedTimestamp,
 			}
 			s.members[remoteMember.ID] = newMember
 			s.logger.WithFields(logrus.Fields{
 				"node_id": remoteMember.ID,
 				"address": remoteMember.Address,
 			}).Info("Discovered new member through gossip")
 		}
 	}
 	// Clean up old members (not seen for more than 10 minutes)
 	toRemove := make([]string, 0)
 	for nodeID, member := range s.members {
 		if now-member.LastSeen > 10*60*1000 { // 10 minutes
 			toRemove = append(toRemove, nodeID)
 		}
 	}
 	for _, nodeID := range toRemove {
 		delete(s.members, nodeID)
 		s.logger.WithField("node_id", nodeID).Info("Removed stale member")
 	}
 }
 // Get this node's joined timestamp (startup time)
 func (s *Server) getJoinedTimestamp() int64 {
 	// For now, use a simple approach - this should be stored persistently
 	return time.Now().UnixMilli()
 }
 // Sync routine - handles regular and catch-up syncing
 func (s *Server) syncRoutine() {
 	defer s.wg.Done()
 	syncTicker := time.NewTicker(time.Duration(s.config.SyncInterval) * time.Second)
 	defer syncTicker.Stop()
 	for {
 		select {
 		case <-s.ctx.Done():
 			return
 		case <-syncTicker.C:
 			s.performRegularSync()
 		}
 	}
 }
 // Perform regular 5-minute sync
 func (s *Server) performRegularSync() {
 	members := s.getHealthyMembers()
 	if len(members) == 0 {
 		s.logger.Debug("No healthy members for sync")
 		return
 	}
 	// Select random peer
 	peer := members[rand.Intn(len(members))]
 	s.logger.WithField("peer", peer.Address).Info("Starting regular sync")
 	// Request latest 15 UUIDs
 	req := PairsByTimeRequest{
 		StartTimestamp: 0,
 		EndTimestamp:   0, // Current time
 		Limit:          15,
 	}
 	remotePairs, err := s.requestPairsByTime(peer.Address, req)
 	if err != nil {
 		s.logger.WithError(err).WithField("peer", peer.Address).Error("Failed to sync with peer")
 		s.markPeerUnhealthy(peer.ID)
 		return
 	}
 	// Compare with our local data and fetch missing/newer data
 	s.syncDataFromPairs(peer.Address, remotePairs)
 	s.logger.WithField("peer", peer.Address).Info("Completed regular sync")
 }
 // Request pairs by time from a peer
 func (s *Server) requestPairsByTime(peerAddress string, req PairsByTimeRequest) ([]PairsByTimeResponse, error) {
 	jsonData, err := json.Marshal(req)
 	if err != nil {
 		return nil, err
 	}
 	client := &http.Client{Timeout: 10 * time.Second}
 	url := fmt.Sprintf("http://%s/members/pairs_by_time", peerAddress)
 	resp, err := client.Post(url, "application/json", bytes.NewBuffer(jsonData))
 	if err != nil {
 		return nil, err
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode == http.StatusNoContent {
 		return []PairsByTimeResponse{}, nil
 	}
 	if resp.StatusCode != http.StatusOK {
 		return nil, fmt.Errorf("peer returned status %d", resp.StatusCode)
 	}
 	var pairs []PairsByTimeResponse
 	if err := json.NewDecoder(resp.Body).Decode(&pairs); err != nil {
 		return nil, err
 	}
 	return pairs, nil
 }
 // Sync data from pairs - fetch missing or newer data
 func (s *Server) syncDataFromPairs(peerAddress string, remotePairs []PairsByTimeResponse) {
 	for _, remotePair := range remotePairs {
 		// Check our local version
 		localData, localExists := s.getLocalData(remotePair.Path)
 		shouldFetch := false
 		if !localExists {
 			shouldFetch = true
 			s.logger.WithField("path", remotePair.Path).Debug("Missing local data, will fetch")
 		} else if localData.Timestamp < remotePair.Timestamp {
 			shouldFetch = true
 			s.logger.WithFields(logrus.Fields{
 				"path":            remotePair.Path,
 				"local_timestamp": localData.Timestamp,
 				"remote_timestamp": remotePair.Timestamp,
 			}).Debug("Local data is older, will fetch")
 		} else if localData.Timestamp == remotePair.Timestamp && localData.UUID != remotePair.UUID {
 			// Timestamp collision - need conflict resolution
 			s.logger.WithFields(logrus.Fields{
 				"path":       remotePair.Path,
 				"timestamp":  remotePair.Timestamp,
 				"local_uuid": localData.UUID,
 				"remote_uuid": remotePair.UUID,
 			}).Warn("Timestamp collision detected, starting conflict resolution")
 			resolved, err := s.resolveConflict(remotePair.Path, localData, &remotePair, peerAddress)
 			if err != nil {
 				s.logger.WithError(err).WithField("path", remotePair.Path).Error("Failed to resolve conflict")
 				continue
 			}
 			if resolved {
 				s.logger.WithField("path", remotePair.Path).Info("Conflict resolved, updated local data")
 			} else {
 				s.logger.WithField("path", remotePair.Path).Info("Conflict resolved, keeping local data")
 			}
 			continue
 		}
 		if shouldFetch {
 			if err := s.fetchAndStoreData(peerAddress, remotePair.Path); err != nil {
 				s.logger.WithError(err).WithFields(logrus.Fields{
 					"peer": peerAddress,
 					"path": remotePair.Path,
 				}).Error("Failed to fetch data from peer")
 			}
 		}
 	}
 }
 // Get local data for a path
 func (s *Server) getLocalData(path string) (*StoredValue, bool) {
 	var storedValue StoredValue
 	err := s.db.View(func(txn *badger.Txn) error {
 		item, err := txn.Get([]byte(path))
 		if err != nil {
 			return err
 		}
 		return item.Value(func(val []byte) error {
 			return json.Unmarshal(val, &storedValue)
 		})
 	})
 	if err != nil {
 		return nil, false
 	}
 	return &storedValue, true
 }
 // Fetch and store data from peer
 func (s *Server) fetchAndStoreData(peerAddress, path string) error {
 	client := &http.Client{Timeout: 5 * time.Second}
 	url := fmt.Sprintf("http://%s/kv/%s", peerAddress, path)
 	resp, err := client.Get(url)
 	if err != nil {
 		return err
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK {
 		return fmt.Errorf("peer returned status %d for path %s", resp.StatusCode, path)
 	}
 	var data json.RawMessage
 	if err := json.NewDecoder(resp.Body).Decode(&data); err != nil {
 		return err
 	}
 	// Store the data using our internal storage mechanism
 	return s.storeReplicatedData(path, data)
 }
 // Store replicated data (internal storage without timestamp/UUID generation)
 func (s *Server) storeReplicatedData(path string, data json.RawMessage) error {
 	// For now, we'll generate new timestamp/UUID - in full implementation,
 	// we'd need to preserve the original metadata from the source
 	now := time.Now().UnixMilli()
 	newUUID := uuid.New().String()
 	storedValue := StoredValue{
 		UUID:      newUUID,
 		Timestamp: now,
 		Data:      data,
 	}
 	valueBytes, err := json.Marshal(storedValue)
 	if err != nil {
 		return err
 	}
 	return s.db.Update(func(txn *badger.Txn) error {
 		// Store main data
 		if err := txn.Set([]byte(path), valueBytes); err != nil {
 			return err
 		}
 		// Store timestamp index
 		indexKey := fmt.Sprintf("_ts:%020d:%s", now, path)
 		return txn.Set([]byte(indexKey), []byte(newUUID))
 	})
 }
 // Bootstrap - join cluster using seed nodes
 func (s *Server) bootstrap() {
-	if len(s.config.SeedNodes) == 0 {
+	// TODO: Implement gradual bootstrapping
 		s.logger.Info("No seed nodes configured, running as standalone")
 		return
 	}
 	s.logger.Info("Starting bootstrap process")
 	s.setMode("syncing")
 	// Try to join via each seed node
 	joined := false
 	for _, seedAddr := range s.config.SeedNodes {
 		if s.attemptJoin(seedAddr) {
 			joined = true
 			break
 		}
 	}
 	if !joined {
 		s.logger.Warn("Failed to join cluster via seed nodes, running as standalone")
 		s.setMode("normal")
 		return
 	}
 	// Wait a bit for member discovery
 	time.Sleep(2 * time.Second)
 	// Perform gradual sync
 	s.performGradualSync()
 	// Switch to normal mode
 	s.setMode("normal")
 	s.logger.Info("Bootstrap completed, entering normal mode")
 }
 // Attempt to join cluster via a seed node
 func (s *Server) attemptJoin(seedAddr string) bool {
 	joinReq := JoinRequest{
 		ID:              s.config.NodeID,
 		Address:         fmt.Sprintf("%s:%d", s.config.BindAddress, s.config.Port),
 		JoinedTimestamp: time.Now().UnixMilli(),
 	}
 	jsonData, err := json.Marshal(joinReq)
 	if err != nil {
 		s.logger.WithError(err).Error("Failed to marshal join request")
 		return false
 	}
 	client := &http.Client{Timeout: 10 * time.Second}
 	url := fmt.Sprintf("http://%s/members/join", seedAddr)
 	resp, err := client.Post(url, "application/json", bytes.NewBuffer(jsonData))
 	if err != nil {
 		s.logger.WithFields(logrus.Fields{
 			"seed": seedAddr,
 			"error": err.Error(),
 		}).Warn("Failed to contact seed node")
 		return false
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK {
 		s.logger.WithFields(logrus.Fields{
 			"seed":   seedAddr,
 			"status": resp.StatusCode,
 		}).Warn("Seed node rejected join request")
 		return false
 	}
 	// Process member list response
 	var memberList []Member
 	if err := json.NewDecoder(resp.Body).Decode(&memberList); err != nil {
 		s.logger.WithError(err).Error("Failed to decode member list from seed")
 		return false
 	}
 	// Add all members to our local list
 	for _, member := range memberList {
 		if member.ID != s.config.NodeID {
 			s.addMember(&member)
 		}
 	}
 	s.logger.WithFields(logrus.Fields{
 		"seed":         seedAddr,
 		"member_count": len(memberList),
 	}).Info("Successfully joined cluster")
 	return true
 }
 // Perform gradual sync (simplified version)
 func (s *Server) performGradualSync() {
 	s.logger.Info("Starting gradual sync")
 	members := s.getHealthyMembers()
 	if len(members) == 0 {
 		s.logger.Info("No healthy members for gradual sync")
 		return
 	}
 	// For now, just do a few rounds of regular sync
 	for i := 0; i < 3; i++ {
 		s.performRegularSync()
 		time.Sleep(time.Duration(s.config.ThrottleDelayMs) * time.Millisecond)
 	}
 	s.logger.Info("Gradual sync completed")
 }
 // Resolve conflict between local and remote data using majority vote and oldest node tie-breaker
 func (s *Server) resolveConflict(path string, localData *StoredValue, remotePair *PairsByTimeResponse, peerAddress string) (bool, error) {
 	s.logger.WithFields(logrus.Fields{
 		"path":       path,
 		"timestamp":  localData.Timestamp,
 		"local_uuid": localData.UUID,
 		"remote_uuid": remotePair.UUID,
 	}).Info("Starting conflict resolution with majority vote")
 	// Get list of healthy members for voting
 	members := s.getHealthyMembers()
 	if len(members) == 0 {
 		// No other members to consult, use oldest node rule (local vs remote)
 		// We'll consider the peer as the "remote" node for comparison
 		return s.resolveByOldestNode(localData, remotePair, peerAddress)
 	}
 	// Query all healthy members for their version of this path
 	votes := make(map[string]int) // UUID -> vote count
 	uuidToTimestamp := make(map[string]int64)
 	uuidToJoinedTime := make(map[string]int64)
 	// Add our local vote
 	votes[localData.UUID] = 1
 	uuidToTimestamp[localData.UUID] = localData.Timestamp
 	uuidToJoinedTime[localData.UUID] = s.getJoinedTimestamp()
 	// Add the remote peer's vote
 	votes[remotePair.UUID] = 1
 	uuidToTimestamp[remotePair.UUID] = remotePair.Timestamp
 	// We'll need to get the peer's joined timestamp
 	// Query other members
 	for _, member := range members {
 		if member.Address == peerAddress {
 			// We already counted this peer
 			uuidToJoinedTime[remotePair.UUID] = member.JoinedTimestamp
 			continue
 		}
 		memberData, exists := s.queryMemberForData(member.Address, path)
 		if !exists {
 			continue // Member doesn't have this data
 		}
 		// Only count votes for data with the same timestamp
 		if memberData.Timestamp == localData.Timestamp {
 			votes[memberData.UUID]++
 			if _, exists := uuidToTimestamp[memberData.UUID]; !exists {
 				uuidToTimestamp[memberData.UUID] = memberData.Timestamp
 				uuidToJoinedTime[memberData.UUID] = member.JoinedTimestamp
 			}
 		}
 	}
 	// Find the UUID with majority votes
 	maxVotes := 0
 	var winningUUIDs []string
 	for uuid, voteCount := range votes {
 		if voteCount > maxVotes {
 			maxVotes = voteCount
 			winningUUIDs = []string{uuid}
 		} else if voteCount == maxVotes {
 			winningUUIDs = append(winningUUIDs, uuid)
 		}
 	}
 	var winnerUUID string
 	if len(winningUUIDs) == 1 {
 		winnerUUID = winningUUIDs[0]
 	} else {
 		// Tie-breaker: oldest node (earliest joined timestamp)
 		oldestJoinedTime := int64(0)
 		for _, uuid := range winningUUIDs {
 			joinedTime := uuidToJoinedTime[uuid]
 			if oldestJoinedTime == 0 || joinedTime < oldestJoinedTime {
 				oldestJoinedTime = joinedTime
 				winnerUUID = uuid
 			}
 		}
 		s.logger.WithFields(logrus.Fields{
 			"path":            path,
 			"tied_votes":      maxVotes,
 			"winner_uuid":     winnerUUID,
 			"oldest_joined":   oldestJoinedTime,
 		}).Info("Resolved conflict using oldest node tie-breaker")
 	}
 	// If remote UUID wins, fetch and store the remote data
 	if winnerUUID == remotePair.UUID {
 		err := s.fetchAndStoreData(peerAddress, path)
 		if err != nil {
 			return false, fmt.Errorf("failed to fetch winning data: %v", err)
 		}
 		s.logger.WithFields(logrus.Fields{
 			"path":         path,
 			"winner_uuid":  winnerUUID,
 			"winner_votes": maxVotes,
 			"total_nodes":  len(members) + 2, // +2 for local and peer
 		}).Info("Conflict resolved: remote data wins")
 		return true, nil
 	}
 	// Local data wins, no action needed
 	s.logger.WithFields(logrus.Fields{
 		"path":         path,
 		"winner_uuid":  winnerUUID,
 		"winner_votes": maxVotes,
 		"total_nodes":  len(members) + 2,
 	}).Info("Conflict resolved: local data wins")
 	return false, nil
 }
 // Resolve conflict using oldest node rule when no other members available
 func (s *Server) resolveByOldestNode(localData *StoredValue, remotePair *PairsByTimeResponse, peerAddress string) (bool, error) {
 	// Find the peer's joined timestamp
 	peerJoinedTime := int64(0)
 	s.membersMu.RLock()
 	for _, member := range s.members {
 		if member.Address == peerAddress {
 			peerJoinedTime = member.JoinedTimestamp
 			break
 		}
 	}
 	s.membersMu.RUnlock()
 	localJoinedTime := s.getJoinedTimestamp()
 	// Oldest node wins
 	if peerJoinedTime > 0 && peerJoinedTime < localJoinedTime {
 		// Peer is older, fetch remote data
 		err := s.fetchAndStoreData(peerAddress, remotePair.Path)
 		if err != nil {
 			return false, fmt.Errorf("failed to fetch data from older node: %v", err)
 		}
 		s.logger.WithFields(logrus.Fields{
 			"path":              remotePair.Path,
 			"local_joined":      localJoinedTime,
 			"peer_joined":       peerJoinedTime,
 			"winner":           "remote",
 		}).Info("Conflict resolved using oldest node rule")
 		return true, nil
 	}
 	// Local node is older or equal, keep local data
 	s.logger.WithFields(logrus.Fields{
 		"path":         remotePair.Path,
 		"local_joined": localJoinedTime,
 		"peer_joined":  peerJoinedTime,
 		"winner":      "local",
 	}).Info("Conflict resolved using oldest node rule")
 	return false, nil
 }
 // Query a member for their version of specific data
 func (s *Server) queryMemberForData(memberAddress, path string) (*StoredValue, bool) {
 	client := &http.Client{Timeout: 5 * time.Second}
 	url := fmt.Sprintf("http://%s/kv/%s", memberAddress, path)
 	resp, err := client.Get(url)
 	if err != nil {
 		return nil, false
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK {
 		return nil, false
 	}
 	var data json.RawMessage
 	if err := json.NewDecoder(resp.Body).Decode(&data); err != nil {
 		return nil, false
 	}
 	// We need to get the metadata too - this is a simplified approach
 	// In a full implementation, we'd have a separate endpoint for metadata queries
 	localData, exists := s.getLocalData(path)
 	if exists {
 		return localData, true
 	}
 	return nil, false
 }
 func main() {
@@ -1,101 +0,0 @@
 package main
 import (
 	"encoding/json"
 	"fmt"
 	"os"
 	"time"
 	badger "github.com/dgraph-io/badger/v4"
 	"github.com/google/uuid"
 )
 // StoredValue matches the structure in main.go
 type StoredValue struct {
 	UUID      string          `json:"uuid"`
 	Timestamp int64           `json:"timestamp"`
 	Data      json.RawMessage `json:"data"`
 }
 // Test utility to create conflicting data directly in BadgerDB
 func createConflictingData(dataDir1, dataDir2 string) error {
 	// Same timestamp, different UUIDs
 	timestamp := time.Now().UnixMilli()
 	path := "test/conflict/data"
 	// Data for node1
 	data1 := json.RawMessage(`{"message": "from node1", "value": 100}`)
 	uuid1 := uuid.New().String()
 	// Data for node2 (same timestamp, different UUID and content)
 	data2 := json.RawMessage(`{"message": "from node2", "value": 200}`)
 	uuid2 := uuid.New().String()
 	// Store in node1's database
 	err := storeConflictData(dataDir1, path, timestamp, uuid1, data1)
 	if err != nil {
 		return fmt.Errorf("failed to store in node1: %v", err)
 	}
 	// Store in node2's database  
 	err = storeConflictData(dataDir2, path, timestamp, uuid2, data2)
 	if err != nil {
 		return fmt.Errorf("failed to store in node2: %v", err)
 	}
 	fmt.Printf("Created conflict scenario:\n")
 	fmt.Printf("Path: %s\n", path)
 	fmt.Printf("Timestamp: %d\n", timestamp)
 	fmt.Printf("Node1 UUID: %s, Data: %s\n", uuid1, string(data1))
 	fmt.Printf("Node2 UUID: %s, Data: %s\n", uuid2, string(data2))
 	return nil
 }
 func storeConflictData(dataDir, path string, timestamp int64, uuid string, data json.RawMessage) error {
 	opts := badger.DefaultOptions(dataDir + "/badger")
 	opts.Logger = nil
 	db, err := badger.Open(opts)
 	if err != nil {
 		return err
 	}
 	defer db.Close()
 	storedValue := StoredValue{
 		UUID:      uuid,
 		Timestamp: timestamp,
 		Data:      data,
 	}
 	valueBytes, err := json.Marshal(storedValue)
 	if err != nil {
 		return err
 	}
 	return db.Update(func(txn *badger.Txn) error {
 		// Store main data
 		if err := txn.Set([]byte(path), valueBytes); err != nil {
 			return err
 		}
 		// Store timestamp index
 		indexKey := fmt.Sprintf("_ts:%020d:%s", timestamp, path)
 		return txn.Set([]byte(indexKey), []byte(uuid))
 	})
 }
 func main() {
 	if len(os.Args) < 3 {
 		fmt.Println("Usage: go run test_conflict.go <data_dir1> <data_dir2>")
 		os.Exit(1)
 	}
 	err := createConflictingData(os.Args[1], os.Args[2])
 	if err != nil {
 		fmt.Printf("Error: %v\n", err)
 		os.Exit(1)
 	}
 	fmt.Println("Conflict data created successfully!")
 	fmt.Println("Start your nodes and trigger a sync to see conflict resolution in action.")
 }