231 lines
8.1 KiB
Bash
231 lines
8.1 KiB
Bash
#!/usr/bin/env bash
|
|
# ==============================================================================
|
|
# repl-test.sh: Integration harness for Records API & Replication (mTLS)
|
|
# Starts Primary (:3000), Replica (:3001), and Dashboard (:8080).
|
|
# ==============================================================================
|
|
set -euo pipefail
|
|
|
|
# --- Configuration & Overrides ---
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
BIN="${BIN:-$ROOT_DIR/target/debug/rust_test}"
|
|
WORK="${WORK:-$ROOT_DIR/repl-test}"
|
|
PRIMARY_PORT="${PRIMARY_PORT:-3000}"
|
|
REPLICA_PORT="${REPLICA_PORT:-3001}"
|
|
DASHBOARD_PORT="${DASHBOARD_PORT:-8080}"
|
|
TOKEN="${TOKEN:-change-me}" # Replication bearer token
|
|
CORS_ORIGIN="${CORS_ORIGIN:-http://127.0.0.1:$DASHBOARD_PORT}"
|
|
START_DASHBOARD="${START_DASHBOARD:-1}"
|
|
|
|
PIDS=()
|
|
|
|
# --- Helper Utilities ---
|
|
check_port() {
|
|
local port=$1
|
|
if command -v lsof >/dev/null 2>&1; then
|
|
lsof -iTCP:"$port" -sTCP:LISTEN -n -P >/dev/null 2>&1
|
|
elif command -v nc >/dev/null 2>&1; then
|
|
nc -z 127.0.0.1 "$port" >/dev/null 2>&1
|
|
else
|
|
(exec 3<>/dev/tcp/127.0.0.1/"$port") 2>/dev/null && { exec 3>&-; exec 3<&-; return 0; } || return 1
|
|
fi
|
|
}
|
|
|
|
cleanup() {
|
|
trap - EXIT INT TERM
|
|
echo -e "\n\033[1;33mStopping all instances and dashboard...\033[0m"
|
|
for pid in "${PIDS[@]}"; do
|
|
if kill -0 "$pid" 2>/dev/null; then
|
|
kill "$pid" 2>/dev/null || true
|
|
fi
|
|
done
|
|
wait 2>/dev/null || true
|
|
echo "Cleaned up."
|
|
}
|
|
trap cleanup EXIT INT TERM
|
|
|
|
# --- Pre-flight Checks ---
|
|
for port in "$PRIMARY_PORT" "$REPLICA_PORT"; do
|
|
if check_port "$port"; then
|
|
echo "Error: Port $port is already in use by another process." >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
if [[ ! -x "$BIN" ]]; then
|
|
if [[ -f "$ROOT_DIR/Cargo.toml" ]]; then
|
|
echo "Binary not found at $BIN. Building debug binary with cargo..."
|
|
( cd "$ROOT_DIR" && cargo build )
|
|
else
|
|
echo "Error: Binary not found or not executable: $BIN" >&2
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
# --- Workspace & Schema Setup ---
|
|
rm -rf "$WORK"
|
|
mkdir -p "$WORK/primary" "$WORK/replica"
|
|
|
|
# Schema allows "msg" and "message", requiring integer "version"
|
|
for d in primary replica; do
|
|
cat > "$WORK/$d/config.json" <<'EOF'
|
|
{
|
|
"mode": "loose",
|
|
"fields": {
|
|
"version": { "type": "integer" }
|
|
}
|
|
}
|
|
EOF
|
|
done
|
|
|
|
start_node() {
|
|
local name=$1 port=$2
|
|
shift 2
|
|
( cd "$WORK/$name" && exec env \
|
|
APP_CONFIG=config.json \
|
|
APP_BIND="127.0.0.1:$port" \
|
|
APP_REPL_TOKEN="$TOKEN" \
|
|
APP_SYNC_INTERVAL_SECS=1 \
|
|
APP_CORS_ORIGIN="$CORS_ORIGIN" \
|
|
"$@" "$BIN" ) >"$WORK/$name/log.txt" 2>&1 &
|
|
PIDS+=("$!")
|
|
}
|
|
|
|
wait_up() {
|
|
local name=$1 port=$2
|
|
for _ in $(seq 1 60); do
|
|
# Note: -k used since cluster cert is generated on boot
|
|
if curl -k -sf "https://127.0.0.1:$port/api/now" >/dev/null 2>&1; then
|
|
return 0
|
|
fi
|
|
sleep 0.1
|
|
done
|
|
echo -e "\n\033[1;31mError: $name failed to start on :$port\033[0m" >&2
|
|
if [[ -f "$WORK/$name/log.txt" ]]; then
|
|
echo "--- Last 20 lines of $WORK/$name/log.txt ---" >&2
|
|
tail -n 20 "$WORK/$name/log.txt" >&2
|
|
fi
|
|
exit 1
|
|
}
|
|
|
|
# ==============================================================================
|
|
# 1. Launch Primary (Cluster CA & Server Cert generated automatically)
|
|
# ==============================================================================
|
|
echo -e "\033[1;32m==> Starting Primary on :$PRIMARY_PORT (HTTPS/mTLS)...\033[0m"
|
|
start_node primary "$PRIMARY_PORT" APP_ROLE=primary
|
|
wait_up "primary" "$PRIMARY_PORT"
|
|
|
|
# ==============================================================================
|
|
# 2. Test Dynamic Discovery: Retrieve Cluster Public CA
|
|
# ==============================================================================
|
|
echo -e "\n\033[1;34m==> Testing dynamic discovery: Fetching Cluster Public CA...\033[0m"
|
|
ca_resp=$(curl -k -sf "https://127.0.0.1:$PRIMARY_PORT/api/cluster/ca")
|
|
if echo "$ca_resp" | grep -q 'BEGIN CERTIFICATE'; then
|
|
echo -e " \033[1;32m✔ Cluster CA certificate downloaded successfully.\033[0m"
|
|
else
|
|
echo -e " \033[1;31m✖ Failed to retrieve Cluster CA.\033[0m"
|
|
exit 1
|
|
fi
|
|
|
|
# ==============================================================================
|
|
# 3. Mint Short-Lived Join Token via Primary
|
|
# ==============================================================================
|
|
echo -e "\n\033[1;34m==> Minting short-lived Join Token from Primary...\033[0m"
|
|
token_resp=$(curl -k -sf -X POST "https://127.0.0.1:$PRIMARY_PORT/api/cluster/tokens" \
|
|
-H 'Content-Type: application/json' \
|
|
-d '{"ttl_secs": 120}')
|
|
|
|
if command -v jq >/dev/null 2>&1; then
|
|
JOIN_TOKEN=$(echo "$token_resp" | jq -r '.token')
|
|
else
|
|
JOIN_TOKEN=$(echo "$token_resp" | grep -o '"token":"[^"]*' | cut -d'"' -f4)
|
|
fi
|
|
|
|
echo -e " \033[1;32m✔ Obtained one-time join token: $JOIN_TOKEN\033[0m"
|
|
|
|
# ==============================================================================
|
|
# 4. Launch Replica (Bootstraps mTLS identity using the Join Token)
|
|
# ==============================================================================
|
|
echo -e "\n\033[1;32m==> Starting Replica on :$REPLICA_PORT (Dynamic Join Enrollment)...\033[0m"
|
|
start_node replica "$REPLICA_PORT" \
|
|
APP_ROLE=replica \
|
|
APP_PRIMARY_URL="https://127.0.0.1:$PRIMARY_PORT" \
|
|
APP_JOIN_TOKEN="$JOIN_TOKEN"
|
|
wait_up "replica" "$REPLICA_PORT"
|
|
|
|
# Verify credentials were saved to replica's SQLite database
|
|
if [[ -f "$WORK/replica/messages.db" ]]; then
|
|
echo -e " \033[1;32m✔ Replica enrolled and saved mTLS credentials in SQLite.\033[0m"
|
|
fi
|
|
|
|
# --- Launch Dashboard (if available) ---
|
|
if [[ "$START_DASHBOARD" -eq 1 && -f "$ROOT_DIR/dashboard.py" ]]; then
|
|
if check_port "$DASHBOARD_PORT"; then
|
|
echo -e "\033[1;33mDashboard port :$DASHBOARD_PORT is in use. Skipping dashboard.\033[0m"
|
|
else
|
|
echo -e "\033[1;32m==> Starting Dashboard on :$DASHBOARD_PORT\033[0m"
|
|
( cd "$ROOT_DIR" && exec python3 dashboard.py ) >"$WORK/dashboard.log" 2>&1 &
|
|
PIDS+=("$!")
|
|
fi
|
|
fi
|
|
|
|
# ==============================================================================
|
|
# 5. Write Smoke Test Records to Primary
|
|
# ==============================================================================
|
|
echo -e "\n\033[1;34m==> Writing 3 test records to Primary via HTTPS\033[0m"
|
|
for i in 1 2 3; do
|
|
curl -k -s -X POST "https://127.0.0.1:$PRIMARY_PORT/api/records" \
|
|
-H 'Content-Type: application/json' \
|
|
-d "{\"data\":{\"version\":1,\"msg\":\"hello $i\"}}" >/dev/null
|
|
echo " Record #$i written"
|
|
done
|
|
|
|
# ==============================================================================
|
|
# 6. Await Replica mTLS Sync
|
|
# ==============================================================================
|
|
echo -e "\n\033[1;34m==> Verifying replica mTLS sync stream...\033[0m"
|
|
caught_up=0
|
|
for _ in $(seq 1 40); do
|
|
status_output=$(curl -k -s "https://127.0.0.1:$REPLICA_PORT/api/replication/status" || true)
|
|
if echo "$status_output" | grep -q '"last_applied_seq":3'; then
|
|
caught_up=1
|
|
break
|
|
fi
|
|
sleep 0.25
|
|
done
|
|
|
|
if [[ $caught_up -eq 1 ]]; then
|
|
echo -e "\033[1;32m✔ SUCCESS: Replica caught up to seq 3 over mTLS!\033[0m"
|
|
else
|
|
echo -e "\033[1;31m✖ WARNING: Replica did not reach seq 3 in time.\033[0m"
|
|
echo "Replica Status Output: $status_output"
|
|
fi
|
|
|
|
# ==============================================================================
|
|
# 7. Verify Read-Only Safety Guard on Replica
|
|
# ==============================================================================
|
|
echo -e "\n\033[1;34m==> Testing Replica read-only safety guard:\033[0m"
|
|
http_code=$(curl -k -s -o /dev/null -w "%{http_code}" -X POST "https://127.0.0.1:$REPLICA_PORT/api/records" \
|
|
-H 'Content-Type: application/json' \
|
|
-d '{"data":{"version":1,"msg":"should fail"}}')
|
|
|
|
if [[ "$http_code" == "403" ]]; then
|
|
echo -e " \033[1;32m✔ HTTP 403 Forbidden received as expected.\033[0m"
|
|
else
|
|
echo -e " \033[1;31m✖ Expected HTTP 403, got $http_code.\033[0m"
|
|
fi
|
|
|
|
# --- Ready Banner ---
|
|
cat <<EOF
|
|
|
|
======================================================================
|
|
Cluster is healthy and running over mTLS:
|
|
- Primary API: https://127.0.0.1:$PRIMARY_PORT
|
|
- Replica API: https://127.0.0.1:$REPLICA_PORT
|
|
- Web Dashboard: http://127.0.0.1:$DASHBOARD_PORT
|
|
- Logs Dir: $WORK/
|
|
|
|
Press Ctrl+C to stop all instances.
|
|
======================================================================
|
|
EOF
|
|
|
|
wait |