Support mTLS clustering on the repl testing.

repl-test.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 # ==============================================================================
-# repl-test.sh: Integration harness for Records API & Replication
+# repl-test.sh: Integration harness for Records API & Replication (mTLS)
 # Starts Primary (:3000), Replica (:3001), and Dashboard (:8080).
 # ==============================================================================
 set -euo pipefail
@@ -12,7 +12,7 @@ WORK="${WORK:-$ROOT_DIR/repl-test}"
 PRIMARY_PORT="${PRIMARY_PORT:-3000}"
 REPLICA_PORT="${REPLICA_PORT:-3001}"
 DASHBOARD_PORT="${DASHBOARD_PORT:-8080}"
-TOKEN="${TOKEN:-change-me}" # Matches index.html default
+TOKEN="${TOKEN:-change-me}" # Replication bearer token
 CORS_ORIGIN="${CORS_ORIGIN:-http://127.0.0.1:$DASHBOARD_PORT}"
 START_DASHBOARD="${START_DASHBOARD:-1}"
 
@@ -66,7 +66,6 @@ rm -rf "$WORK"
 mkdir -p "$WORK/primary" "$WORK/replica"
 
 # Schema allows "msg" and "message", requiring integer "version"
-# This aligns with index.html tests and manual curl inputs.
 for d in primary replica; do
   cat > "$WORK/$d/config.json" <<'EOF'
 {
@@ -93,8 +92,9 @@ start_node() {
 
 wait_up() {
   local name=$1 port=$2
-  for _ in $(seq 1 50); do
+  for _ in $(seq 1 60); do
-    if curl -sf "http://127.0.0.1:$port/api/now" >/dev/null 2>&1; then
+    # Note: -k used since cluster cert is generated on boot
+    if curl -k -sf "https://127.0.0.1:$port/api/now" >/dev/null 2>&1; then
       return 0
     fi
     sleep 0.1
@@ -107,17 +107,56 @@ wait_up() {
   exit 1
 }
 
-# --- Launch Cluster ---
+# ==============================================================================
-echo -e "\033[1;32m==> Starting Primary on :$PRIMARY_PORT\033[0m"
+# 1. Launch Primary (Cluster CA & Server Cert generated automatically)
+# ==============================================================================
+echo -e "\033[1;32m==> Starting Primary on :$PRIMARY_PORT (HTTPS/mTLS)...\033[0m"
 start_node primary "$PRIMARY_PORT" APP_ROLE=primary
 wait_up "primary" "$PRIMARY_PORT"
 
-echo -e "\033[1;32m==> Starting Replica on :$REPLICA_PORT\033[0m"
+# ==============================================================================
+# 2. Test Dynamic Discovery: Retrieve Cluster Public CA
+# ==============================================================================
+echo -e "\n\033[1;34m==> Testing dynamic discovery: Fetching Cluster Public CA...\033[0m"
+ca_resp=$(curl -k -sf "https://127.0.0.1:$PRIMARY_PORT/api/cluster/ca")
+if echo "$ca_resp" | grep -q 'BEGIN CERTIFICATE'; then
+  echo -e "  \033[1;32m✔ Cluster CA certificate downloaded successfully.\033[0m"
+else
+  echo -e "  \033[1;31m✖ Failed to retrieve Cluster CA.\033[0m"
+  exit 1
+fi
+
+# ==============================================================================
+# 3. Mint Short-Lived Join Token via Primary
+# ==============================================================================
+echo -e "\n\033[1;34m==> Minting short-lived Join Token from Primary...\033[0m"
+token_resp=$(curl -k -sf -X POST "https://127.0.0.1:$PRIMARY_PORT/api/cluster/tokens" \
+  -H 'Content-Type: application/json' \
+  -d '{"ttl_secs": 120}')
+
+if command -v jq >/dev/null 2>&1; then
+  JOIN_TOKEN=$(echo "$token_resp" | jq -r '.token')
+else
+  JOIN_TOKEN=$(echo "$token_resp" | grep -o '"token":"[^"]*' | cut -d'"' -f4)
+fi
+
+echo -e "  \033[1;32m✔ Obtained one-time join token: $JOIN_TOKEN\033[0m"
+
+# ==============================================================================
+# 4. Launch Replica (Bootstraps mTLS identity using the Join Token)
+# ==============================================================================
+echo -e "\n\033[1;32m==> Starting Replica on :$REPLICA_PORT (Dynamic Join Enrollment)...\033[0m"
 start_node replica "$REPLICA_PORT" \
   APP_ROLE=replica \
-  APP_PRIMARY_URL="http://127.0.0.1:$PRIMARY_PORT"
+  APP_PRIMARY_URL="https://127.0.0.1:$PRIMARY_PORT" \
+  APP_JOIN_TOKEN="$JOIN_TOKEN"
 wait_up "replica" "$REPLICA_PORT"
 
+# Verify credentials were saved to replica's SQLite database
+if [[ -f "$WORK/replica/messages.db" ]]; then
+  echo -e "  \033[1;32m✔ Replica enrolled and saved mTLS credentials in SQLite.\033[0m"
+fi
+
 # --- Launch Dashboard (if available) ---
 if [[ "$START_DASHBOARD" -eq 1 && -f "$ROOT_DIR/dashboard.py" ]]; then
   if check_port "$DASHBOARD_PORT"; then
@@ -129,20 +168,24 @@ if [[ "$START_DASHBOARD" -eq 1 && -f "$ROOT_DIR/dashboard.py" ]]; then
   fi
 fi
 
-# --- Write Smoke Test Records ---
+# ==============================================================================
-echo -e "\n\033[1;34m==> Writing 3 test records to Primary\033[0m"
+# 5. Write Smoke Test Records to Primary
+# ==============================================================================
+echo -e "\n\033[1;34m==> Writing 3 test records to Primary via HTTPS\033[0m"
 for i in 1 2 3; do
-  curl -s -X POST "http://127.0.0.1:$PRIMARY_PORT/api/records" \
+  curl -k -s -X POST "https://127.0.0.1:$PRIMARY_PORT/api/records" \
     -H 'Content-Type: application/json' \
     -d "{\"data\":{\"version\":1,\"msg\":\"hello $i\"}}" >/dev/null
   echo "  Record #$i written"
 done
 
-# --- Await Replication Sync ---
+# ==============================================================================
-echo -e "\n\033[1;34m==> Verifying replica sync...\033[0m"
+# 6. Await Replica mTLS Sync
+# ==============================================================================
+echo -e "\n\033[1;34m==> Verifying replica mTLS sync stream...\033[0m"
 caught_up=0
 for _ in $(seq 1 40); do
-  status_output=$(curl -s "http://127.0.0.1:$REPLICA_PORT/api/replication/status" || true)
+  status_output=$(curl -k -s "https://127.0.0.1:$REPLICA_PORT/api/replication/status" || true)
   if echo "$status_output" | grep -q '"last_applied_seq":3'; then
     caught_up=1
     break
@@ -151,14 +194,17 @@ for _ in $(seq 1 40); do
 done
 
 if [[ $caught_up -eq 1 ]]; then
-  echo -e "\033[1;32m✔ SUCCESS: Replica caught up to seq 3!\033[0m"
+  echo -e "\033[1;32m✔ SUCCESS: Replica caught up to seq 3 over mTLS!\033[0m"
 else
   echo -e "\033[1;31m✖ WARNING: Replica did not reach seq 3 in time.\033[0m"
+  echo "Replica Status Output: $status_output"
 fi
 
-# --- Verify Read-Only Guard on Replica ---
+# ==============================================================================
+# 7. Verify Read-Only Safety Guard on Replica
+# ==============================================================================
 echo -e "\n\033[1;34m==> Testing Replica read-only safety guard:\033[0m"
-http_code=$(curl -s -o /dev/null -w "%{http_code}" -X POST "http://127.0.0.1:$REPLICA_PORT/api/records" \
+http_code=$(curl -k -s -o /dev/null -w "%{http_code}" -X POST "https://127.0.0.1:$REPLICA_PORT/api/records" \
   -H 'Content-Type: application/json' \
   -d '{"data":{"version":1,"msg":"should fail"}}')
 
@@ -172,9 +218,9 @@ fi
 cat <<EOF
 
 ======================================================================
-  Cluster is healthy and running:
+  Cluster is healthy and running over mTLS:
-  - Primary API:   http://127.0.0.1:$PRIMARY_PORT
+  - Primary API:   https://127.0.0.1:$PRIMARY_PORT
-  - Replica API:   http://127.0.0.1:$REPLICA_PORT
+  - Replica API:   https://127.0.0.1:$REPLICA_PORT
   - Web Dashboard: http://127.0.0.1:$DASHBOARD_PORT
   - Logs Dir:      $WORK/