#!/usr/bin/env bash # ============================================================================== # repl-test.sh: Integration harness for Records API & Replication (mTLS) # Starts Primary (:3000), Replica (:3001), and Dashboard (:8080). # ============================================================================== set -euo pipefail # --- Configuration & Overrides --- ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" BIN="${BIN:-$ROOT_DIR/target/debug/rust_test}" WORK="${WORK:-$ROOT_DIR/repl-test}" PRIMARY_PORT="${PRIMARY_PORT:-3000}" REPLICA_PORT="${REPLICA_PORT:-3001}" DASHBOARD_PORT="${DASHBOARD_PORT:-8080}" TOKEN="${TOKEN:-change-me}" # Replication bearer token CORS_ORIGIN="${CORS_ORIGIN:-http://127.0.0.1:$DASHBOARD_PORT}" START_DASHBOARD="${START_DASHBOARD:-1}" PIDS=() # --- Helper Utilities --- check_port() { local port=$1 if command -v lsof >/dev/null 2>&1; then lsof -iTCP:"$port" -sTCP:LISTEN -n -P >/dev/null 2>&1 elif command -v nc >/dev/null 2>&1; then nc -z 127.0.0.1 "$port" >/dev/null 2>&1 else (exec 3<>/dev/tcp/127.0.0.1/"$port") 2>/dev/null && { exec 3>&-; exec 3<&-; return 0; } || return 1 fi } cleanup() { trap - EXIT INT TERM echo -e "\n\033[1;33mStopping all instances and dashboard...\033[0m" for pid in "${PIDS[@]}"; do if kill -0 "$pid" 2>/dev/null; then kill "$pid" 2>/dev/null || true fi done wait 2>/dev/null || true echo "Cleaned up." } trap cleanup EXIT INT TERM # --- Pre-flight Checks --- for port in "$PRIMARY_PORT" "$REPLICA_PORT"; do if check_port "$port"; then echo "Error: Port $port is already in use by another process." >&2 exit 1 fi done if [[ -f "$ROOT_DIR/Cargo.toml" ]]; then echo "Ensuring debug binary is up to date with cargo..." ( cd "$ROOT_DIR" && cargo build ) elif [[ ! -x "$BIN" ]]; then echo "Error: Binary not found or not executable: $BIN" >&2 exit 1 fi # --- Workspace & Schema Setup --- rm -rf "$WORK" mkdir -p "$WORK/primary" "$WORK/replica" # Schema allows "msg" and "message", requiring integer "version" for d in primary replica; do cat > "$WORK/$d/config.json" <<'EOF' { "mode": "loose", "fields": { "version": { "type": "integer" } } } EOF done start_node() { local name=$1 port=$2 shift 2 ( cd "$WORK/$name" && exec env \ APP_CONFIG=config.json \ APP_BIND="127.0.0.1:$port" \ APP_REPL_TOKEN="$TOKEN" \ APP_SYNC_INTERVAL_SECS=1 \ APP_CORS_ORIGIN="$CORS_ORIGIN" \ "$@" "$BIN" ) >"$WORK/$name/log.txt" 2>&1 & PIDS+=("$!") } wait_up() { local name=$1 port=$2 for _ in $(seq 1 60); do # Note: -k used since cluster cert is generated on boot if curl -k -sf "https://127.0.0.1:$port/api/now" >/dev/null 2>&1; then return 0 fi sleep 0.1 done echo -e "\n\033[1;31mError: $name failed to start on :$port\033[0m" >&2 if [[ -f "$WORK/$name/log.txt" ]]; then echo "--- Last 20 lines of $WORK/$name/log.txt ---" >&2 tail -n 20 "$WORK/$name/log.txt" >&2 fi exit 1 } # ============================================================================== # 1. Launch Primary (Cluster CA & Server Cert generated automatically) # ============================================================================== echo -e "\033[1;32m==> Starting Primary on :$PRIMARY_PORT (HTTPS/mTLS)...\033[0m" start_node primary "$PRIMARY_PORT" APP_ROLE=primary wait_up "primary" "$PRIMARY_PORT" # ============================================================================== # 2. Test Dynamic Discovery: Retrieve Cluster Public CA # ============================================================================== echo -e "\n\033[1;34m==> Testing dynamic discovery: Fetching Cluster Public CA...\033[0m" ca_resp=$(curl -k -sf "https://127.0.0.1:$PRIMARY_PORT/api/cluster/ca") if echo "$ca_resp" | grep -q 'BEGIN CERTIFICATE'; then echo -e " \033[1;32m✔ Cluster CA certificate downloaded successfully.\033[0m" else echo -e " \033[1;31m✖ Failed to retrieve Cluster CA.\033[0m" exit 1 fi # ============================================================================== # 3. Mint Short-Lived Join Token via Primary # ============================================================================== echo -e "\n\033[1;34m==> Minting short-lived Join Token from Primary...\033[0m" token_resp=$(curl -k -sf -X POST "https://127.0.0.1:$PRIMARY_PORT/api/cluster/tokens" \ -H 'Content-Type: application/json' \ -d '{"ttl_secs": 120}') if command -v jq >/dev/null 2>&1; then JOIN_TOKEN=$(echo "$token_resp" | jq -r '.token') else JOIN_TOKEN=$(echo "$token_resp" | grep -o '"token":"[^"]*' | cut -d'"' -f4) fi echo -e " \033[1;32m✔ Obtained one-time join token: $JOIN_TOKEN\033[0m" # ============================================================================== # 4. Launch Replica (Bootstraps mTLS identity using the Join Token) # ============================================================================== echo -e "\n\033[1;32m==> Starting Replica on :$REPLICA_PORT (Dynamic Join Enrollment)...\033[0m" start_node replica "$REPLICA_PORT" \ APP_ROLE=replica \ APP_PRIMARY_URL="https://127.0.0.1:$PRIMARY_PORT" \ APP_JOIN_TOKEN="$JOIN_TOKEN" wait_up "replica" "$REPLICA_PORT" # Verify credentials were saved to replica's SQLite database if [[ -f "$WORK/replica/messages.db" ]]; then echo -e " \033[1;32m✔ Replica enrolled and saved mTLS credentials in SQLite.\033[0m" fi # --- Launch Dashboard (if available) --- if [[ "$START_DASHBOARD" -eq 1 && -f "$ROOT_DIR/dashboard.py" ]]; then if check_port "$DASHBOARD_PORT"; then echo -e "\033[1;33mDashboard port :$DASHBOARD_PORT is in use. Skipping dashboard.\033[0m" else echo -e "\033[1;32m==> Starting Dashboard on :$DASHBOARD_PORT\033[0m" ( cd "$ROOT_DIR" && exec python3 dashboard.py ) >"$WORK/dashboard.log" 2>&1 & PIDS+=("$!") fi fi # ============================================================================== # 5. Write Smoke Test Records to Primary # ============================================================================== echo -e "\n\033[1;34m==> Writing 3 test records to Primary via HTTPS\033[0m" for i in 1 2 3; do curl -k -s -X POST "https://127.0.0.1:$PRIMARY_PORT/api/records" \ -H 'Content-Type: application/json' \ -d "{\"data\":{\"version\":1,\"msg\":\"hello $i\"}}" >/dev/null echo " Record #$i written" done # ============================================================================== # 6. Await Replica mTLS Sync # ============================================================================== echo -e "\n\033[1;34m==> Verifying replica mTLS sync stream...\033[0m" caught_up=0 for _ in $(seq 1 40); do status_output=$(curl -k -s "https://127.0.0.1:$REPLICA_PORT/api/replication/status" || true) if echo "$status_output" | grep -q '"last_applied_seq":3'; then caught_up=1 break fi sleep 0.25 done if [[ $caught_up -eq 1 ]]; then echo -e "\033[1;32m✔ SUCCESS: Replica caught up to seq 3 over mTLS!\033[0m" else echo -e "\033[1;31m✖ WARNING: Replica did not reach seq 3 in time.\033[0m" echo "Replica Status Output: $status_output" fi # ============================================================================== # 7. Verify Read-Only Safety Guard on Replica # ============================================================================== echo -e "\n\033[1;34m==> Testing Replica read-only safety guard:\033[0m" http_code=$(curl -k -s -o /dev/null -w "%{http_code}" -X POST "https://127.0.0.1:$REPLICA_PORT/api/records" \ -H 'Content-Type: application/json' \ -d '{"data":{"version":1,"msg":"should fail"}}') if [[ "$http_code" == "403" ]]; then echo -e " \033[1;32m✔ HTTP 403 Forbidden received as expected.\033[0m" else echo -e " \033[1;31m✖ Expected HTTP 403, got $http_code.\033[0m" fi # --- Ready Banner --- cat <