kalzu-value-store/integration_test.sh

#!/bin/bash

# KVS Integration Test Suite - Adapted for Merkle Tree Sync
# Tests all critical features of the distributed key-value store with Merkle Tree replication

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Test configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
TEST_DIR="$SCRIPT_DIR/integration_test"
BINARY="$SCRIPT_DIR/kvs"

# Counters
TESTS_PASSED=0
TESTS_FAILED=0
TOTAL_TESTS=0

# Helper functions
log_info() {
    echo -e "${BLUE}[INFO]${NC} $1"
}

log_success() {
    echo -e "${GREEN}[PASS]${NC} $1"
    ((TESTS_PASSED++))
}

log_error() {
    echo -e "${RED}[FAIL]${NC} $1"
    ((TESTS_FAILED++))
}

test_start() {
    ((TOTAL_TESTS++))
    log_info "Test $TOTAL_TESTS: $1"
}

# Cleanup function
cleanup() {
    log_info "Cleaning up test environment..."
    pkill -f "$BINARY" 2>/dev/null || true
    rm -rf "$TEST_DIR" 2>/dev/null || true
    sleep 2  # Allow processes to fully terminate
}

# Wait for service to be ready
wait_for_service() {
    local port=$1
    local timeout=${2:-30}
    local count=0
    
    while [ $count -lt $timeout ]; do
        if curl -s "http://localhost:$port/health" >/dev/null 2>&1; then
            return 0
        fi
        sleep 1
        ((count++))
    done
    return 1
}

# Test 1: Build verification
test_build() {
    test_start "Binary build verification"
    
    cd "$SCRIPT_DIR"
    if go build -o kvs . >/dev/null 2>&1; then
        log_success "Binary builds successfully"
    else
        log_error "Binary build failed"
        return 1
    fi
    # Ensure we are back in TEST_DIR for subsequent tests
    cd "$TEST_DIR"
}

# Test 2: Basic functionality
test_basic_functionality() {
    test_start "Basic functionality test"
    
    # Create basic config
    cat > basic.yaml <<EOF
node_id: "basic-test"
bind_address: "127.0.0.1"
port: 8090
data_dir: "./basic_data"
seed_nodes: []
log_level: "error"
allow_anonymous_read: true
allow_anonymous_write: true
EOF
    
    # Start node
    $BINARY basic.yaml >/dev/null 2>&1 &
    local pid=$!
    
    if wait_for_service 8090; then
        # Test basic CRUD
        local put_result=$(curl -s -X PUT http://localhost:8090/kv/test/basic \
            -H "Content-Type: application/json" \
            -d '{"message":"hello world"}')
        
        local get_result=$(curl -s http://localhost:8090/kv/test/basic)
        local message=$(echo "$get_result" | jq -r '.data.message' 2>/dev/null) # Adjusted jq path
        
        if [ "$message" = "hello world" ]; then
            log_success "Basic CRUD operations work"
        else
            log_error "Basic CRUD failed: Expected 'hello world', got '$message' from $get_result"
        fi
    else
        log_error "Basic test node failed to start"
    fi
    
    kill $pid 2>/dev/null || true
    sleep 2
}

# Test 3: Cluster formation
test_cluster_formation() {
    test_start "2-node cluster formation and Merkle Tree replication"

    # Shared cluster secret for authentication (Issue #13)
    local CLUSTER_SECRET="test-cluster-secret-12345678901234567890"

    # Node 1 config
    cat > cluster1.yaml <<EOF
node_id: "cluster-1"
bind_address: "127.0.0.1"
port: 8101
data_dir: "./cluster1_data"
seed_nodes: []
log_level: "error"
gossip_interval_min: 5
gossip_interval_max: 10
sync_interval: 10
allow_anonymous_read: true
allow_anonymous_write: true
cluster_secret: "$CLUSTER_SECRET"
EOF

    # Node 2 config
    cat > cluster2.yaml <<EOF
node_id: "cluster-2"
bind_address: "127.0.0.1"
port: 8102
data_dir: "./cluster2_data"
seed_nodes: ["127.0.0.1:8101"]
log_level: "error"
gossip_interval_min: 5
gossip_interval_max: 10
sync_interval: 10
allow_anonymous_read: true
allow_anonymous_write: true
cluster_secret: "$CLUSTER_SECRET"
EOF
    
    # Start nodes
    $BINARY cluster1.yaml >/dev/null 2>&1 &
    local pid1=$!
    
    if ! wait_for_service 8101; then
        log_error "Cluster node 1 failed to start"
        kill $pid1 2>/dev/null || true
        return 1
    fi
    
    sleep 2 # Give node 1 a moment to fully initialize
    $BINARY cluster2.yaml >/dev/null 2>&1 &
    local pid2=$!
    
    if ! wait_for_service 8102; then
        log_error "Cluster node 2 failed to start"
        kill $pid1 $pid2 2>/dev/null || true
        return 1
    fi
    
    # Wait for cluster formation and initial Merkle sync
    sleep 15
    
    # Check if nodes see each other
    local node1_members=$(curl -s http://localhost:8101/members/ | jq length 2>/dev/null || echo 0)
    local node2_members=$(curl -s http://localhost:8102/members/ | jq length 2>/dev/null || echo 0)
    
    if [ "$node1_members" -ge 1 ] && [ "$node2_members" -ge 1 ]; then
        log_success "2-node cluster formed successfully (N1 members: $node1_members, N2 members: $node2_members)"
        
        # Test data replication
        log_info "Putting data on Node 1, waiting for Merkle sync..."
        curl -s -X PUT http://localhost:8101/kv/cluster/test \
            -H "Content-Type: application/json" \
            -d '{"source":"node1", "value": 1}' >/dev/null
        
        # Wait for Merkle sync cycle to complete
        sleep 12
        
        local node2_data_full=$(curl -s http://localhost:8102/kv/cluster/test)
        local node2_data_source=$(echo "$node2_data_full" | jq -r '.data.source' 2>/dev/null)
        local node2_data_value=$(echo "$node2_data_full" | jq -r '.data.value' 2>/dev/null)
        local node1_data_full=$(curl -s http://localhost:8101/kv/cluster/test)
        
        if [ "$node2_data_source" = "node1" ] && [ "$node2_data_value" = "1" ]; then
            log_success "Data replication works correctly (Node 2 has data from Node 1)"

            # Verify UUIDs and Timestamps are identical (crucial for Merkle sync correctness)
            local node1_uuid=$(echo "$node1_data_full" | jq -r '.uuid' 2>/dev/null)
            local node1_timestamp=$(echo "$node1_data_full" | jq -r '.timestamp' 2>/dev/null)
            local node2_uuid=$(echo "$node2_data_full" | jq -r '.uuid' 2>/dev/null)
            local node2_timestamp=$(echo "$node2_data_full" | jq -r '.timestamp' 2>/dev/null)

            if [ "$node1_uuid" = "$node2_uuid" ] && [ "$node1_timestamp" = "$node2_timestamp" ]; then
                log_success "Replicated data retains original UUID and Timestamp"
            else
                log_error "Replicated data changed UUID/Timestamp: N1_UUID=$node1_uuid, N1_TS=$node1_timestamp, N2_UUID=$node2_uuid, N2_TS=$node2_timestamp"
            fi
        else
            log_error "Data replication failed: Node 2 data: $node2_data_full"
        fi
    else
        log_error "Cluster formation failed (N1 members: $node1_members, N2 members: $node2_members)"
    fi
    
    kill $pid1 $pid2 2>/dev/null || true
    sleep 2
}

# Test 4: Conflict resolution (Merkle Tree based)
# This test assumes 'test_conflict.go' creates two BadgerDBs with a key
# that has the same path and timestamp but different UUIDs, or different timestamps
# but same path. The Merkle tree sync should then trigger conflict resolution.
test_conflict_resolution() {
    test_start "Conflict resolution test (Merkle Tree based)"

    # Create conflicting data using our utility
    rm -rf conflict1_data conflict2_data 2>/dev/null || true
    mkdir -p conflict1_data conflict2_data

    cd "$SCRIPT_DIR"
    if go run test_conflict.go "$TEST_DIR/conflict1_data" "$TEST_DIR/conflict2_data"; then
        cd "$TEST_DIR"

        # Shared cluster secret for authentication (Issue #13)
        local CLUSTER_SECRET="conflict-cluster-secret-1234567890123"

        # Create configs
        cat > conflict1.yaml <<EOF
node_id: "conflict-1"
bind_address: "127.0.0.1"
port: 8111
data_dir: "./conflict1_data"
seed_nodes: []
log_level: "info"
sync_interval: 3
allow_anonymous_read: true
allow_anonymous_write: true
cluster_secret: "$CLUSTER_SECRET"
EOF

        cat > conflict2.yaml <<EOF
node_id: "conflict-2"
bind_address: "127.0.0.1"
port: 8112
data_dir: "./conflict2_data"
seed_nodes: ["127.0.0.1:8111"]
log_level: "info"
sync_interval: 3
allow_anonymous_read: true
allow_anonymous_write: true
cluster_secret: "$CLUSTER_SECRET"
EOF
        
        # Start nodes
        # Node 1 started first, making it "older" for tie-breaker if timestamps are equal
        "$BINARY" conflict1.yaml >conflict1.log 2>&1 &
        local pid1=$!
        
        if wait_for_service 8111; then
            sleep 2
            $BINARY conflict2.yaml >conflict2.log 2>&1 &
            local pid2=$!
            
            if wait_for_service 8112; then
                # Get initial data (full StoredValue)
                local node1_initial_full=$(curl -s http://localhost:8111/kv/test/conflict/data)
                local node2_initial_full=$(curl -s http://localhost:8112/kv/test/conflict/data)
                
                local node1_initial_msg=$(echo "$node1_initial_full" | jq -r '.data.message' 2>/dev/null)
                local node2_initial_msg=$(echo "$node2_initial_full" | jq -r '.data.message' 2>/dev/null)
                
                log_info "Initial conflict state: Node1='$node1_initial_msg', Node2='$node2_initial_msg'"

                # Allow time for cluster formation and gossip protocol to stabilize
                log_info "Waiting for cluster formation and gossip stabilization..."
                sleep 20
                
                # Wait for conflict resolution with retry logic (up to 60 seconds)
                local max_attempts=20
                local attempt=1
                local node1_final_msg=""
                local node2_final_msg=""
                local node1_final_full=""
                local node2_final_full=""
                
                log_info "Waiting for conflict resolution (checking every 3 seconds, max 60 seconds)..."
                
                while [ $attempt -le $max_attempts ]; do
                    sleep 3
                    
                    # Get current data from both nodes
                    node1_final_full=$(curl -s http://localhost:8111/kv/test/conflict/data)
                    node2_final_full=$(curl -s http://localhost:8112/kv/test/conflict/data)
                    
                    node1_final_msg=$(echo "$node1_final_full" | jq -r '.data.message' 2>/dev/null)
                    node2_final_msg=$(echo "$node2_final_full" | jq -r '.data.message' 2>/dev/null)
                    
                    # Check if they've converged
                    if [ "$node1_final_msg" = "$node2_final_msg" ] && [ -n "$node1_final_msg" ] && [ "$node1_final_msg" != "null" ]; then
                        log_info "Conflict resolution achieved after $((attempt * 3)) seconds"
                        break
                    fi
                    
                    log_info "Attempt $attempt/$max_attempts: Node1='$node1_final_msg', Node2='$node2_final_msg' (not converged yet)"
                    attempt=$((attempt + 1))
                done
                
                # Check if they converged
                if [ "$node1_final_msg" = "$node2_final_msg" ] && [ -n "$node1_final_msg" ]; then
                    log_success "Conflict resolution converged to: '$node1_final_msg'"
                    
                    # Verify UUIDs and Timestamps are identical after resolution
                    local node1_final_uuid=$(echo "$node1_final_full" | jq -r '.uuid' 2>/dev/null)
                    local node1_final_timestamp=$(echo "$node1_final_full" | jq -r '.timestamp' 2>/dev/null)
                    local node2_final_uuid=$(echo "$node2_final_full" | jq -r '.uuid' 2>/dev/null)
                    local node2_final_timestamp=$(echo "$node2_final_full" | jq -r '.timestamp' 2>/dev/null)

                    if [ "$node1_final_uuid" = "$node2_final_uuid" ] && [ "$node1_final_timestamp" = "$node2_final_timestamp" ]; then
                        log_success "Resolved data retains consistent UUID and Timestamp across nodes"
                    else
                        log_error "Resolved data has inconsistent UUID/Timestamp: N1_UUID=$node1_final_uuid, N1_TS=$node1_final_timestamp, N2_UUID=$node2_final_uuid, N2_TS=$node2_final_timestamp"
                    fi

                    # Optionally, check logs for conflict resolution messages
                    if grep -q "Conflict resolved" conflict1.log conflict2.log 2>/dev/null; then
                        log_success "Conflict resolution messages found in logs"
                    else
                        log_error "No 'Conflict resolved' messages found in logs, but data converged."
                    fi

                else
                    log_error "Conflict resolution failed: N1_final='$node1_final_msg', N2_final='$node2_final_msg'"
                fi
            else
                log_error "Conflict node 2 failed to start"
            fi
            
            kill $pid2 2>/dev/null || true
        else
            log_error "Conflict node 1 failed to start"
        fi
        
        kill $pid1 2>/dev/null || true
        sleep 2
    else
        cd "$TEST_DIR"
        log_error "Failed to create conflict test data. Ensure test_conflict.go is correct."
    fi
}

# Test 5: Authentication middleware (Issue #4)
test_authentication_middleware() {
    test_start "Authentication middleware test (Issue #4)"
    
    # Create auth test config
    cat > auth_test.yaml <<EOF
node_id: "auth-test"
bind_address: "127.0.0.1"
port: 8095
data_dir: "./auth_test_data"
seed_nodes: []
log_level: "error"
auth_enabled: true
allow_anonymous_read: false
allow_anonymous_write: false
EOF
    
    # Start node
    $BINARY auth_test.yaml >auth_test.log 2>&1 &
    local pid=$!
    
    if wait_for_service 8095; then
        sleep 2  # Allow root account creation
        
        # Extract the token from logs
        local token=$(grep "Token:" auth_test.log | sed 's/.*Token: //' | tr -d '\n\r')
        
        if [ -z "$token" ]; then
            log_error "Failed to extract authentication token from logs"
            kill $pid 2>/dev/null || true
            return
        fi
        
        # Test 1: Admin endpoints should fail without authentication
        local no_auth_response=$(curl -s -X POST http://localhost:8095/api/users -H "Content-Type: application/json" -d '{"nickname":"test","password":"test"}')
        if echo "$no_auth_response" | grep -q "Unauthorized"; then
            log_success "Admin endpoints properly reject unauthenticated requests"
        else
            log_error "Admin endpoints should reject unauthenticated requests, got: $no_auth_response"
        fi
        
        # Test 2: Admin endpoints should work with valid authentication
        local auth_response=$(curl -s -X POST http://localhost:8095/api/users -H "Content-Type: application/json" -H "Authorization: Bearer $token" -d '{"nickname":"authtest","password":"authtest"}')
        if echo "$auth_response" | grep -q "uuid"; then
            log_success "Admin endpoints work with valid authentication"
        else
            log_error "Admin endpoints should work with authentication, got: $auth_response"
        fi
        
        # Test 3: KV endpoints should require auth when anonymous access is disabled
        local kv_no_auth=$(curl -s -X PUT http://localhost:8095/kv/test/auth -H "Content-Type: application/json" -d '{"test":"auth"}')
        if echo "$kv_no_auth" | grep -q "Unauthorized"; then
            log_success "KV endpoints properly require authentication when anonymous access disabled"
        else
            log_error "KV endpoints should require auth when anonymous access disabled, got: $kv_no_auth"
        fi
        
        # Test 4: KV endpoints should work with valid authentication
        local kv_auth=$(curl -s -X PUT http://localhost:8095/kv/test/auth -H "Content-Type: application/json" -H "Authorization: Bearer $token" -d '{"test":"auth"}')
        if echo "$kv_auth" | grep -q "uuid\|timestamp" || [ -z "$kv_auth" ]; then
            log_success "KV endpoints work with valid authentication"
        else
            log_error "KV endpoints should work with authentication, got: $kv_auth"
        fi
        
        kill $pid 2>/dev/null || true
        sleep 2
    else
        log_error "Auth test node failed to start"
        kill $pid 2>/dev/null || true
    fi
}

# Test 6: Resource Metadata Management (Issue #12)
test_metadata_management() {
    test_start "Resource Metadata Management test (Issue #12)"

    # Create metadata test config
    cat > metadata_test.yaml <<EOF
node_id: "metadata-test"
bind_address: "127.0.0.1"
port: 8096
data_dir: "./metadata_test_data"
seed_nodes: []
log_level: "error"
auth_enabled: true
allow_anonymous_read: false
allow_anonymous_write: false
EOF

    # Start node
    $BINARY metadata_test.yaml >metadata_test.log 2>&1 &
    local pid=$!

    if wait_for_service 8096; then
        sleep 2  # Allow root account creation

        # Extract the token from logs
        local token=$(grep "Token:" metadata_test.log | sed 's/.*Token: //' | tr -d '\n\r')

        if [ -z "$token" ]; then
            log_error "Failed to extract authentication token from logs"
            kill $pid 2>/dev/null || true
            return
        fi

        # First, create a KV resource
        curl -s -X PUT http://localhost:8096/kv/test/resource -H "Content-Type: application/json" -H "Authorization: Bearer $token" -d '{"data":"test"}' >/dev/null
        sleep 1

        # Test 1: Get metadata should fail for non-existent metadata (initially no metadata exists)
        local get_response=$(curl -s -w "\n%{http_code}" -X GET http://localhost:8096/kv/test/resource/metadata -H "Authorization: Bearer $token")
        local get_body=$(echo "$get_response" | head -n -1)
        local get_code=$(echo "$get_response" | tail -n 1)

        if [ "$get_code" = "404" ]; then
            log_success "GET metadata returns 404 for non-existent metadata"
        else
            log_error "GET metadata should return 404 for non-existent metadata, got code: $get_code, body: $get_body"
        fi

        # Test 2: Update metadata should create new metadata
        local update_response=$(curl -s -X PUT http://localhost:8096/kv/test/resource/metadata -H "Content-Type: application/json" -H "Authorization: Bearer $token" -d '{"owner_uuid":"test-owner-123","permissions":3840}')
        if echo "$update_response" | grep -q "owner_uuid"; then
            log_success "PUT metadata creates metadata successfully"
        else
            log_error "PUT metadata should create metadata, got: $update_response"
        fi

        # Test 3: Get metadata should now return the created metadata
        local get_response2=$(curl -s -X GET http://localhost:8096/kv/test/resource/metadata -H "Authorization: Bearer $token")
        if echo "$get_response2" | grep -q "test-owner-123" && echo "$get_response2" | grep -q "3840"; then
            log_success "GET metadata returns created metadata"
        else
            log_error "GET metadata should return created metadata, got: $get_response2"
        fi

        # Test 4: Update metadata should modify existing metadata
        local update_response2=$(curl -s -X PUT http://localhost:8096/kv/test/resource/metadata -H "Content-Type: application/json" -H "Authorization: Bearer $token" -d '{"owner_uuid":"new-owner-456"}')
        if echo "$update_response2" | grep -q "new-owner-456"; then
            log_success "PUT metadata updates existing metadata"
        else
            log_error "PUT metadata should update metadata, got: $update_response2"
        fi

        # Test 5: Metadata endpoints should require authentication
        local no_auth=$(curl -s -w "\n%{http_code}" -X GET http://localhost:8096/kv/test/resource/metadata)
        local no_auth_code=$(echo "$no_auth" | tail -n 1)
        if [ "$no_auth_code" = "401" ]; then
            log_success "Metadata endpoints properly require authentication"
        else
            log_error "Metadata endpoints should require authentication, got code: $no_auth_code"
        fi

        kill $pid 2>/dev/null || true
        sleep 2
    else
        log_error "Metadata test node failed to start"
        kill $pid 2>/dev/null || true
    fi
}

# Main test execution
main() {
    echo "=================================================="
    echo "         KVS Integration Test Suite (Merkle Tree)"
    echo "=================================================="
    
    # Setup
    log_info "Setting up test environment..."
    cleanup
    mkdir -p "$TEST_DIR"
    cd "$TEST_DIR"
    
    # Run core tests
    test_build
    test_basic_functionality
    test_cluster_formation
    test_conflict_resolution
    test_authentication_middleware
    test_metadata_management

    # Results
    echo "=================================================="
    echo "              Test Results"
    echo "=================================================="
    echo -e "Total Tests: $TOTAL_TESTS"
    echo -e "${GREEN}Passed: $TESTS_PASSED${NC}"
    echo -e "${RED}Failed: $TESTS_FAILED${NC}"
    echo "=================================================="
    
    if [ $TESTS_FAILED -eq 0 ]; then
        echo -e "${GREEN}🎉 All tests passed! KVS with Merkle Tree sync is working correctly.${NC}"
        cleanup
        exit 0
    else
        echo -e "${RED}❌ Some tests failed. Please check the output above.${NC}"
        cleanup
        exit 1
    fi
}

# Handle interruption
trap cleanup INT TERM

# Run tests
main "$@"